1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2014 by Delphix. All rights reserved. 24 */ 25 26 #include <sys/dsl_scan.h> 27 #include <sys/dsl_pool.h> 28 #include <sys/dsl_dataset.h> 29 #include <sys/dsl_prop.h> 30 #include <sys/dsl_dir.h> 31 #include <sys/dsl_synctask.h> 32 #include <sys/dnode.h> 33 #include <sys/dmu_tx.h> 34 #include <sys/dmu_objset.h> 35 #include <sys/arc.h> 36 #include <sys/zap.h> 37 #include <sys/zio.h> 38 #include <sys/zfs_context.h> 39 #include <sys/fs/zfs.h> 40 #include <sys/zfs_znode.h> 41 #include <sys/spa_impl.h> 42 #include <sys/vdev_impl.h> 43 #include <sys/zil_impl.h> 44 #include <sys/zio_checksum.h> 45 #include <sys/ddt.h> 46 #include <sys/sa.h> 47 #include <sys/sa_impl.h> 48 #include <sys/zfeature.h> 49 #ifdef _KERNEL 50 #include <sys/zfs_vfsops.h> 51 #endif 52 53 typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, 54 const zbookmark_phys_t *); 55 56 static scan_cb_t dsl_scan_scrub_cb; 57 static void dsl_scan_cancel_sync(void *, dmu_tx_t *); 58 static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *tx); 59 60 int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ 61 int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ 62 int zfs_scrub_delay = 4; /* number of ticks to delay scrub */ 63 int zfs_scan_idle = 50; /* idle window in clock ticks */ 64 65 int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */ 66 int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */ 67 int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */ 68 boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ 69 boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ 70 enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; 71 int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */ 72 /* max number of blocks to free in a single TXG */ 73 uint64_t zfs_free_max_blocks = UINT64_MAX; 74 75 #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ 76 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \ 77 (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER) 78 79 extern int zfs_txg_timeout; 80 81 /* the order has to match pool_scan_type */ 82 static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = { 83 NULL, 84 dsl_scan_scrub_cb, /* POOL_SCAN_SCRUB */ 85 dsl_scan_scrub_cb, /* POOL_SCAN_RESILVER */ 86 }; 87 88 int 89 dsl_scan_init(dsl_pool_t *dp, uint64_t txg) 90 { 91 int err; 92 dsl_scan_t *scn; 93 spa_t *spa = dp->dp_spa; 94 uint64_t f; 95 96 scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); 97 scn->scn_dp = dp; 98 99 /* 100 * It's possible that we're resuming a scan after a reboot so 101 * make sure that the scan_async_destroying flag is initialized 102 * appropriately. 103 */ 104 ASSERT(!scn->scn_async_destroying); 105 scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, 106 SPA_FEATURE_ASYNC_DESTROY); 107 108 err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 109 "scrub_func", sizeof (uint64_t), 1, &f); 110 if (err == 0) { 111 /* 112 * There was an old-style scrub in progress. Restart a 113 * new-style scrub from the beginning. 114 */ 115 scn->scn_restart_txg = txg; 116 zfs_dbgmsg("old-style scrub was in progress; " 117 "restarting new-style scrub in txg %llu", 118 scn->scn_restart_txg); 119 120 /* 121 * Load the queue obj from the old location so that it 122 * can be freed by dsl_scan_done(). 123 */ 124 (void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 125 "scrub_queue", sizeof (uint64_t), 1, 126 &scn->scn_phys.scn_queue_obj); 127 } else { 128 err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 129 DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 130 &scn->scn_phys); 131 if (err == ENOENT) 132 return (0); 133 else if (err) 134 return (err); 135 136 if (scn->scn_phys.scn_state == DSS_SCANNING && 137 spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) { 138 /* 139 * A new-type scrub was in progress on an old 140 * pool, and the pool was accessed by old 141 * software. Restart from the beginning, since 142 * the old software may have changed the pool in 143 * the meantime. 144 */ 145 scn->scn_restart_txg = txg; 146 zfs_dbgmsg("new-style scrub was modified " 147 "by old software; restarting in txg %llu", 148 scn->scn_restart_txg); 149 } 150 } 151 152 spa_scan_stat_init(spa); 153 return (0); 154 } 155 156 void 157 dsl_scan_fini(dsl_pool_t *dp) 158 { 159 if (dp->dp_scan) { 160 kmem_free(dp->dp_scan, sizeof (dsl_scan_t)); 161 dp->dp_scan = NULL; 162 } 163 } 164 165 /* ARGSUSED */ 166 static int 167 dsl_scan_setup_check(void *arg, dmu_tx_t *tx) 168 { 169 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 170 171 if (scn->scn_phys.scn_state == DSS_SCANNING) 172 return (SET_ERROR(EBUSY)); 173 174 return (0); 175 } 176 177 static void 178 dsl_scan_setup_sync(void *arg, dmu_tx_t *tx) 179 { 180 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 181 pool_scan_func_t *funcp = arg; 182 dmu_object_type_t ot = 0; 183 dsl_pool_t *dp = scn->scn_dp; 184 spa_t *spa = dp->dp_spa; 185 186 ASSERT(scn->scn_phys.scn_state != DSS_SCANNING); 187 ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS); 188 bzero(&scn->scn_phys, sizeof (scn->scn_phys)); 189 scn->scn_phys.scn_func = *funcp; 190 scn->scn_phys.scn_state = DSS_SCANNING; 191 scn->scn_phys.scn_min_txg = 0; 192 scn->scn_phys.scn_max_txg = tx->tx_txg; 193 scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ 194 scn->scn_phys.scn_start_time = gethrestime_sec(); 195 scn->scn_phys.scn_errors = 0; 196 scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; 197 scn->scn_restart_txg = 0; 198 scn->scn_done_txg = 0; 199 spa_scan_stat_init(spa); 200 201 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 202 scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; 203 204 /* rewrite all disk labels */ 205 vdev_config_dirty(spa->spa_root_vdev); 206 207 if (vdev_resilver_needed(spa->spa_root_vdev, 208 &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { 209 spa_event_notify(spa, NULL, ESC_ZFS_RESILVER_START); 210 } else { 211 spa_event_notify(spa, NULL, ESC_ZFS_SCRUB_START); 212 } 213 214 spa->spa_scrub_started = B_TRUE; 215 /* 216 * If this is an incremental scrub, limit the DDT scrub phase 217 * to just the auto-ditto class (for correctness); the rest 218 * of the scrub should go faster using top-down pruning. 219 */ 220 if (scn->scn_phys.scn_min_txg > TXG_INITIAL) 221 scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; 222 223 } 224 225 /* back to the generic stuff */ 226 227 if (dp->dp_blkstats == NULL) { 228 dp->dp_blkstats = 229 kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP); 230 } 231 bzero(dp->dp_blkstats, sizeof (zfs_all_blkstats_t)); 232 233 if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) 234 ot = DMU_OT_ZAP_OTHER; 235 236 scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, 237 ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx); 238 239 dsl_scan_sync_state(scn, tx); 240 241 spa_history_log_internal(spa, "scan setup", tx, 242 "func=%u mintxg=%llu maxtxg=%llu", 243 *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); 244 } 245 246 /* ARGSUSED */ 247 static void 248 dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) 249 { 250 static const char *old_names[] = { 251 "scrub_bookmark", 252 "scrub_ddt_bookmark", 253 "scrub_ddt_class_max", 254 "scrub_queue", 255 "scrub_min_txg", 256 "scrub_max_txg", 257 "scrub_func", 258 "scrub_errors", 259 NULL 260 }; 261 262 dsl_pool_t *dp = scn->scn_dp; 263 spa_t *spa = dp->dp_spa; 264 int i; 265 266 /* Remove any remnants of an old-style scrub. */ 267 for (i = 0; old_names[i]; i++) { 268 (void) zap_remove(dp->dp_meta_objset, 269 DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx); 270 } 271 272 if (scn->scn_phys.scn_queue_obj != 0) { 273 VERIFY(0 == dmu_object_free(dp->dp_meta_objset, 274 scn->scn_phys.scn_queue_obj, tx)); 275 scn->scn_phys.scn_queue_obj = 0; 276 } 277 278 /* 279 * If we were "restarted" from a stopped state, don't bother 280 * with anything else. 281 */ 282 if (scn->scn_phys.scn_state != DSS_SCANNING) 283 return; 284 285 if (complete) 286 scn->scn_phys.scn_state = DSS_FINISHED; 287 else 288 scn->scn_phys.scn_state = DSS_CANCELED; 289 290 spa_history_log_internal(spa, "scan done", tx, 291 "complete=%u", complete); 292 293 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 294 mutex_enter(&spa->spa_scrub_lock); 295 while (spa->spa_scrub_inflight > 0) { 296 cv_wait(&spa->spa_scrub_io_cv, 297 &spa->spa_scrub_lock); 298 } 299 mutex_exit(&spa->spa_scrub_lock); 300 spa->spa_scrub_started = B_FALSE; 301 spa->spa_scrub_active = B_FALSE; 302 303 /* 304 * If the scrub/resilver completed, update all DTLs to 305 * reflect this. Whether it succeeded or not, vacate 306 * all temporary scrub DTLs. 307 */ 308 vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 309 complete ? scn->scn_phys.scn_max_txg : 0, B_TRUE); 310 if (complete) { 311 spa_event_notify(spa, NULL, scn->scn_phys.scn_min_txg ? 312 ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH); 313 } 314 spa_errlog_rotate(spa); 315 316 /* 317 * We may have finished replacing a device. 318 * Let the async thread assess this and handle the detach. 319 */ 320 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 321 } 322 323 scn->scn_phys.scn_end_time = gethrestime_sec(); 324 } 325 326 /* ARGSUSED */ 327 static int 328 dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) 329 { 330 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 331 332 if (scn->scn_phys.scn_state != DSS_SCANNING) 333 return (SET_ERROR(ENOENT)); 334 return (0); 335 } 336 337 /* ARGSUSED */ 338 static void 339 dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) 340 { 341 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 342 343 dsl_scan_done(scn, B_FALSE, tx); 344 dsl_scan_sync_state(scn, tx); 345 } 346 347 int 348 dsl_scan_cancel(dsl_pool_t *dp) 349 { 350 return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check, 351 dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED)); 352 } 353 354 static void dsl_scan_visitbp(blkptr_t *bp, 355 const zbookmark_phys_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf, 356 dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype, 357 dmu_tx_t *tx); 358 static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds, 359 dmu_objset_type_t ostype, 360 dnode_phys_t *dnp, arc_buf_t *buf, uint64_t object, dmu_tx_t *tx); 361 362 void 363 dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp) 364 { 365 zio_free(dp->dp_spa, txg, bp); 366 } 367 368 void 369 dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp) 370 { 371 ASSERT(dsl_pool_sync_context(dp)); 372 zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags)); 373 } 374 375 static uint64_t 376 dsl_scan_ds_maxtxg(dsl_dataset_t *ds) 377 { 378 uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg; 379 if (dsl_dataset_is_snapshot(ds)) 380 return (MIN(smt, ds->ds_phys->ds_creation_txg)); 381 return (smt); 382 } 383 384 static void 385 dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx) 386 { 387 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 388 DMU_POOL_DIRECTORY_OBJECT, 389 DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 390 &scn->scn_phys, tx)); 391 } 392 393 static boolean_t 394 dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_phys_t *zb) 395 { 396 uint64_t elapsed_nanosecs; 397 int mintime; 398 399 /* we never skip user/group accounting objects */ 400 if (zb && (int64_t)zb->zb_object < 0) 401 return (B_FALSE); 402 403 if (scn->scn_pausing) 404 return (B_TRUE); /* we're already pausing */ 405 406 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) 407 return (B_FALSE); /* we're resuming */ 408 409 /* We only know how to resume from level-0 blocks. */ 410 if (zb && zb->zb_level != 0) 411 return (B_FALSE); 412 413 mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 414 zfs_resilver_min_time_ms : zfs_scan_min_time_ms; 415 elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 416 if (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 417 (NSEC2MSEC(elapsed_nanosecs) > mintime && 418 txg_sync_waiting(scn->scn_dp)) || 419 spa_shutting_down(scn->scn_dp->dp_spa)) { 420 if (zb) { 421 dprintf("pausing at bookmark %llx/%llx/%llx/%llx\n", 422 (longlong_t)zb->zb_objset, 423 (longlong_t)zb->zb_object, 424 (longlong_t)zb->zb_level, 425 (longlong_t)zb->zb_blkid); 426 scn->scn_phys.scn_bookmark = *zb; 427 } 428 dprintf("pausing at DDT bookmark %llx/%llx/%llx/%llx\n", 429 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 430 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 431 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 432 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 433 scn->scn_pausing = B_TRUE; 434 return (B_TRUE); 435 } 436 return (B_FALSE); 437 } 438 439 typedef struct zil_scan_arg { 440 dsl_pool_t *zsa_dp; 441 zil_header_t *zsa_zh; 442 } zil_scan_arg_t; 443 444 /* ARGSUSED */ 445 static int 446 dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 447 { 448 zil_scan_arg_t *zsa = arg; 449 dsl_pool_t *dp = zsa->zsa_dp; 450 dsl_scan_t *scn = dp->dp_scan; 451 zil_header_t *zh = zsa->zsa_zh; 452 zbookmark_phys_t zb; 453 454 if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 455 return (0); 456 457 /* 458 * One block ("stubby") can be allocated a long time ago; we 459 * want to visit that one because it has been allocated 460 * (on-disk) even if it hasn't been claimed (even though for 461 * scrub there's nothing to do to it). 462 */ 463 if (claim_txg == 0 && bp->blk_birth >= spa_first_txg(dp->dp_spa)) 464 return (0); 465 466 SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 467 ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 468 469 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 470 return (0); 471 } 472 473 /* ARGSUSED */ 474 static int 475 dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg) 476 { 477 if (lrc->lrc_txtype == TX_WRITE) { 478 zil_scan_arg_t *zsa = arg; 479 dsl_pool_t *dp = zsa->zsa_dp; 480 dsl_scan_t *scn = dp->dp_scan; 481 zil_header_t *zh = zsa->zsa_zh; 482 lr_write_t *lr = (lr_write_t *)lrc; 483 blkptr_t *bp = &lr->lr_blkptr; 484 zbookmark_phys_t zb; 485 486 if (BP_IS_HOLE(bp) || 487 bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 488 return (0); 489 490 /* 491 * birth can be < claim_txg if this record's txg is 492 * already txg sync'ed (but this log block contains 493 * other records that are not synced) 494 */ 495 if (claim_txg == 0 || bp->blk_birth < claim_txg) 496 return (0); 497 498 SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 499 lr->lr_foid, ZB_ZIL_LEVEL, 500 lr->lr_offset / BP_GET_LSIZE(bp)); 501 502 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 503 } 504 return (0); 505 } 506 507 static void 508 dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh) 509 { 510 uint64_t claim_txg = zh->zh_claim_txg; 511 zil_scan_arg_t zsa = { dp, zh }; 512 zilog_t *zilog; 513 514 /* 515 * We only want to visit blocks that have been claimed but not yet 516 * replayed (or, in read-only mode, blocks that *would* be claimed). 517 */ 518 if (claim_txg == 0 && spa_writeable(dp->dp_spa)) 519 return; 520 521 zilog = zil_alloc(dp->dp_meta_objset, zh); 522 523 (void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa, 524 claim_txg); 525 526 zil_free(zilog); 527 } 528 529 /* ARGSUSED */ 530 static void 531 dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp, 532 uint64_t objset, uint64_t object, uint64_t blkid) 533 { 534 zbookmark_phys_t czb; 535 uint32_t flags = ARC_NOWAIT | ARC_PREFETCH; 536 537 if (zfs_no_scrub_prefetch) 538 return; 539 540 if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_min_txg || 541 (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE)) 542 return; 543 544 SET_BOOKMARK(&czb, objset, object, BP_GET_LEVEL(bp), blkid); 545 546 (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, bp, 547 NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 548 ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD, &flags, &czb); 549 } 550 551 static boolean_t 552 dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, 553 const zbookmark_phys_t *zb) 554 { 555 /* 556 * We never skip over user/group accounting objects (obj<0) 557 */ 558 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && 559 (int64_t)zb->zb_object >= 0) { 560 /* 561 * If we already visited this bp & everything below (in 562 * a prior txg sync), don't bother doing it again. 563 */ 564 if (zbookmark_is_before(dnp, zb, &scn->scn_phys.scn_bookmark)) 565 return (B_TRUE); 566 567 /* 568 * If we found the block we're trying to resume from, or 569 * we went past it to a different object, zero it out to 570 * indicate that it's OK to start checking for pausing 571 * again. 572 */ 573 if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || 574 zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { 575 dprintf("resuming at %llx/%llx/%llx/%llx\n", 576 (longlong_t)zb->zb_objset, 577 (longlong_t)zb->zb_object, 578 (longlong_t)zb->zb_level, 579 (longlong_t)zb->zb_blkid); 580 bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); 581 } 582 } 583 return (B_FALSE); 584 } 585 586 /* 587 * Return nonzero on i/o error. 588 * Return new buf to write out in *bufp. 589 */ 590 static int 591 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, 592 dnode_phys_t *dnp, const blkptr_t *bp, 593 const zbookmark_phys_t *zb, dmu_tx_t *tx, arc_buf_t **bufp) 594 { 595 dsl_pool_t *dp = scn->scn_dp; 596 int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 597 int err; 598 599 if (BP_GET_LEVEL(bp) > 0) { 600 uint32_t flags = ARC_WAIT; 601 int i; 602 blkptr_t *cbp; 603 int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 604 605 err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 606 ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 607 if (err) { 608 scn->scn_phys.scn_errors++; 609 return (err); 610 } 611 for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) { 612 dsl_scan_prefetch(scn, *bufp, cbp, zb->zb_objset, 613 zb->zb_object, zb->zb_blkid * epb + i); 614 } 615 for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) { 616 zbookmark_phys_t czb; 617 618 SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 619 zb->zb_level - 1, 620 zb->zb_blkid * epb + i); 621 dsl_scan_visitbp(cbp, &czb, dnp, 622 *bufp, ds, scn, ostype, tx); 623 } 624 } else if (BP_GET_TYPE(bp) == DMU_OT_USERGROUP_USED) { 625 uint32_t flags = ARC_WAIT; 626 627 err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 628 ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 629 if (err) { 630 scn->scn_phys.scn_errors++; 631 return (err); 632 } 633 } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 634 uint32_t flags = ARC_WAIT; 635 dnode_phys_t *cdnp; 636 int i, j; 637 int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 638 639 err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 640 ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 641 if (err) { 642 scn->scn_phys.scn_errors++; 643 return (err); 644 } 645 for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) { 646 for (j = 0; j < cdnp->dn_nblkptr; j++) { 647 blkptr_t *cbp = &cdnp->dn_blkptr[j]; 648 dsl_scan_prefetch(scn, *bufp, cbp, 649 zb->zb_objset, zb->zb_blkid * epb + i, j); 650 } 651 } 652 for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) { 653 dsl_scan_visitdnode(scn, ds, ostype, 654 cdnp, *bufp, zb->zb_blkid * epb + i, tx); 655 } 656 657 } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 658 uint32_t flags = ARC_WAIT; 659 objset_phys_t *osp; 660 661 err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 662 ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 663 if (err) { 664 scn->scn_phys.scn_errors++; 665 return (err); 666 } 667 668 osp = (*bufp)->b_data; 669 670 dsl_scan_visitdnode(scn, ds, osp->os_type, 671 &osp->os_meta_dnode, *bufp, DMU_META_DNODE_OBJECT, tx); 672 673 if (OBJSET_BUF_HAS_USERUSED(*bufp)) { 674 /* 675 * We also always visit user/group accounting 676 * objects, and never skip them, even if we are 677 * pausing. This is necessary so that the space 678 * deltas from this txg get integrated. 679 */ 680 dsl_scan_visitdnode(scn, ds, osp->os_type, 681 &osp->os_groupused_dnode, *bufp, 682 DMU_GROUPUSED_OBJECT, tx); 683 dsl_scan_visitdnode(scn, ds, osp->os_type, 684 &osp->os_userused_dnode, *bufp, 685 DMU_USERUSED_OBJECT, tx); 686 } 687 } 688 689 return (0); 690 } 691 692 static void 693 dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, 694 dmu_objset_type_t ostype, dnode_phys_t *dnp, arc_buf_t *buf, 695 uint64_t object, dmu_tx_t *tx) 696 { 697 int j; 698 699 for (j = 0; j < dnp->dn_nblkptr; j++) { 700 zbookmark_phys_t czb; 701 702 SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 703 dnp->dn_nlevels - 1, j); 704 dsl_scan_visitbp(&dnp->dn_blkptr[j], 705 &czb, dnp, buf, ds, scn, ostype, tx); 706 } 707 708 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 709 zbookmark_phys_t czb; 710 SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 711 0, DMU_SPILL_BLKID); 712 dsl_scan_visitbp(&dnp->dn_spill, 713 &czb, dnp, buf, ds, scn, ostype, tx); 714 } 715 } 716 717 /* 718 * The arguments are in this order because mdb can only print the 719 * first 5; we want them to be useful. 720 */ 721 static void 722 dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 723 dnode_phys_t *dnp, arc_buf_t *pbuf, 724 dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype, 725 dmu_tx_t *tx) 726 { 727 dsl_pool_t *dp = scn->scn_dp; 728 arc_buf_t *buf = NULL; 729 blkptr_t bp_toread = *bp; 730 731 /* ASSERT(pbuf == NULL || arc_released(pbuf)); */ 732 733 if (dsl_scan_check_pause(scn, zb)) 734 return; 735 736 if (dsl_scan_check_resume(scn, dnp, zb)) 737 return; 738 739 if (BP_IS_HOLE(bp)) 740 return; 741 742 scn->scn_visited_this_txg++; 743 744 dprintf_bp(bp, 745 "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx buf=%p bp=%p", 746 ds, ds ? ds->ds_object : 0, 747 zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid, 748 pbuf, bp); 749 750 if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 751 return; 752 753 if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx, 754 &buf) != 0) 755 return; 756 757 /* 758 * If dsl_scan_ddt() has aready visited this block, it will have 759 * already done any translations or scrubbing, so don't call the 760 * callback again. 761 */ 762 if (ddt_class_contains(dp->dp_spa, 763 scn->scn_phys.scn_ddt_class_max, bp)) { 764 ASSERT(buf == NULL); 765 return; 766 } 767 768 /* 769 * If this block is from the future (after cur_max_txg), then we 770 * are doing this on behalf of a deleted snapshot, and we will 771 * revisit the future block on the next pass of this dataset. 772 * Don't scan it now unless we need to because something 773 * under it was modified. 774 */ 775 if (BP_PHYSICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_max_txg) { 776 scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); 777 } 778 if (buf) 779 (void) arc_buf_remove_ref(buf, &buf); 780 } 781 782 static void 783 dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, 784 dmu_tx_t *tx) 785 { 786 zbookmark_phys_t zb; 787 788 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 789 ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); 790 dsl_scan_visitbp(bp, &zb, NULL, NULL, 791 ds, scn, DMU_OST_NONE, tx); 792 793 dprintf_ds(ds, "finished scan%s", ""); 794 } 795 796 void 797 dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx) 798 { 799 dsl_pool_t *dp = ds->ds_dir->dd_pool; 800 dsl_scan_t *scn = dp->dp_scan; 801 uint64_t mintxg; 802 803 if (scn->scn_phys.scn_state != DSS_SCANNING) 804 return; 805 806 if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 807 if (dsl_dataset_is_snapshot(ds)) { 808 /* Note, scn_cur_{min,max}_txg stays the same. */ 809 scn->scn_phys.scn_bookmark.zb_objset = 810 ds->ds_phys->ds_next_snap_obj; 811 zfs_dbgmsg("destroying ds %llu; currently traversing; " 812 "reset zb_objset to %llu", 813 (u_longlong_t)ds->ds_object, 814 (u_longlong_t)ds->ds_phys->ds_next_snap_obj); 815 scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN; 816 } else { 817 SET_BOOKMARK(&scn->scn_phys.scn_bookmark, 818 ZB_DESTROYED_OBJSET, 0, 0, 0); 819 zfs_dbgmsg("destroying ds %llu; currently traversing; " 820 "reset bookmark to -1,0,0,0", 821 (u_longlong_t)ds->ds_object); 822 } 823 } else if (zap_lookup_int_key(dp->dp_meta_objset, 824 scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 825 ASSERT3U(ds->ds_phys->ds_num_children, <=, 1); 826 VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 827 scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 828 if (dsl_dataset_is_snapshot(ds)) { 829 /* 830 * We keep the same mintxg; it could be > 831 * ds_creation_txg if the previous snapshot was 832 * deleted too. 833 */ 834 VERIFY(zap_add_int_key(dp->dp_meta_objset, 835 scn->scn_phys.scn_queue_obj, 836 ds->ds_phys->ds_next_snap_obj, mintxg, tx) == 0); 837 zfs_dbgmsg("destroying ds %llu; in queue; " 838 "replacing with %llu", 839 (u_longlong_t)ds->ds_object, 840 (u_longlong_t)ds->ds_phys->ds_next_snap_obj); 841 } else { 842 zfs_dbgmsg("destroying ds %llu; in queue; removing", 843 (u_longlong_t)ds->ds_object); 844 } 845 } else { 846 zfs_dbgmsg("destroying ds %llu; ignoring", 847 (u_longlong_t)ds->ds_object); 848 } 849 850 /* 851 * dsl_scan_sync() should be called after this, and should sync 852 * out our changed state, but just to be safe, do it here. 853 */ 854 dsl_scan_sync_state(scn, tx); 855 } 856 857 void 858 dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx) 859 { 860 dsl_pool_t *dp = ds->ds_dir->dd_pool; 861 dsl_scan_t *scn = dp->dp_scan; 862 uint64_t mintxg; 863 864 if (scn->scn_phys.scn_state != DSS_SCANNING) 865 return; 866 867 ASSERT(ds->ds_phys->ds_prev_snap_obj != 0); 868 869 if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 870 scn->scn_phys.scn_bookmark.zb_objset = 871 ds->ds_phys->ds_prev_snap_obj; 872 zfs_dbgmsg("snapshotting ds %llu; currently traversing; " 873 "reset zb_objset to %llu", 874 (u_longlong_t)ds->ds_object, 875 (u_longlong_t)ds->ds_phys->ds_prev_snap_obj); 876 } else if (zap_lookup_int_key(dp->dp_meta_objset, 877 scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 878 VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 879 scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 880 VERIFY(zap_add_int_key(dp->dp_meta_objset, 881 scn->scn_phys.scn_queue_obj, 882 ds->ds_phys->ds_prev_snap_obj, mintxg, tx) == 0); 883 zfs_dbgmsg("snapshotting ds %llu; in queue; " 884 "replacing with %llu", 885 (u_longlong_t)ds->ds_object, 886 (u_longlong_t)ds->ds_phys->ds_prev_snap_obj); 887 } 888 dsl_scan_sync_state(scn, tx); 889 } 890 891 void 892 dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) 893 { 894 dsl_pool_t *dp = ds1->ds_dir->dd_pool; 895 dsl_scan_t *scn = dp->dp_scan; 896 uint64_t mintxg; 897 898 if (scn->scn_phys.scn_state != DSS_SCANNING) 899 return; 900 901 if (scn->scn_phys.scn_bookmark.zb_objset == ds1->ds_object) { 902 scn->scn_phys.scn_bookmark.zb_objset = ds2->ds_object; 903 zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 904 "reset zb_objset to %llu", 905 (u_longlong_t)ds1->ds_object, 906 (u_longlong_t)ds2->ds_object); 907 } else if (scn->scn_phys.scn_bookmark.zb_objset == ds2->ds_object) { 908 scn->scn_phys.scn_bookmark.zb_objset = ds1->ds_object; 909 zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 910 "reset zb_objset to %llu", 911 (u_longlong_t)ds2->ds_object, 912 (u_longlong_t)ds1->ds_object); 913 } 914 915 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 916 ds1->ds_object, &mintxg) == 0) { 917 int err; 918 919 ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg); 920 ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg); 921 VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 922 scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); 923 err = zap_add_int_key(dp->dp_meta_objset, 924 scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); 925 VERIFY(err == 0 || err == EEXIST); 926 if (err == EEXIST) { 927 /* Both were there to begin with */ 928 VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 929 scn->scn_phys.scn_queue_obj, 930 ds1->ds_object, mintxg, tx)); 931 } 932 zfs_dbgmsg("clone_swap ds %llu; in queue; " 933 "replacing with %llu", 934 (u_longlong_t)ds1->ds_object, 935 (u_longlong_t)ds2->ds_object); 936 } else if (zap_lookup_int_key(dp->dp_meta_objset, 937 scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) { 938 ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg); 939 ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg); 940 VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 941 scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); 942 VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 943 scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); 944 zfs_dbgmsg("clone_swap ds %llu; in queue; " 945 "replacing with %llu", 946 (u_longlong_t)ds2->ds_object, 947 (u_longlong_t)ds1->ds_object); 948 } 949 950 dsl_scan_sync_state(scn, tx); 951 } 952 953 struct enqueue_clones_arg { 954 dmu_tx_t *tx; 955 uint64_t originobj; 956 }; 957 958 /* ARGSUSED */ 959 static int 960 enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 961 { 962 struct enqueue_clones_arg *eca = arg; 963 dsl_dataset_t *ds; 964 int err; 965 dsl_scan_t *scn = dp->dp_scan; 966 967 if (hds->ds_dir->dd_phys->dd_origin_obj != eca->originobj) 968 return (0); 969 970 err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 971 if (err) 972 return (err); 973 974 while (ds->ds_phys->ds_prev_snap_obj != eca->originobj) { 975 dsl_dataset_t *prev; 976 err = dsl_dataset_hold_obj(dp, 977 ds->ds_phys->ds_prev_snap_obj, FTAG, &prev); 978 979 dsl_dataset_rele(ds, FTAG); 980 if (err) 981 return (err); 982 ds = prev; 983 } 984 VERIFY(zap_add_int_key(dp->dp_meta_objset, 985 scn->scn_phys.scn_queue_obj, ds->ds_object, 986 ds->ds_phys->ds_prev_snap_txg, eca->tx) == 0); 987 dsl_dataset_rele(ds, FTAG); 988 return (0); 989 } 990 991 static void 992 dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) 993 { 994 dsl_pool_t *dp = scn->scn_dp; 995 dsl_dataset_t *ds; 996 objset_t *os; 997 998 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 999 1000 if (dmu_objset_from_ds(ds, &os)) 1001 goto out; 1002 1003 /* 1004 * Only the ZIL in the head (non-snapshot) is valid. Even though 1005 * snapshots can have ZIL block pointers (which may be the same 1006 * BP as in the head), they must be ignored. So we traverse the 1007 * ZIL here, rather than in scan_recurse(), because the regular 1008 * snapshot block-sharing rules don't apply to it. 1009 */ 1010 if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds)) 1011 dsl_scan_zil(dp, &os->os_zil_header); 1012 1013 /* 1014 * Iterate over the bps in this ds. 1015 */ 1016 dmu_buf_will_dirty(ds->ds_dbuf, tx); 1017 dsl_scan_visit_rootbp(scn, ds, &ds->ds_phys->ds_bp, tx); 1018 1019 char *dsname = kmem_alloc(ZFS_MAXNAMELEN, KM_SLEEP); 1020 dsl_dataset_name(ds, dsname); 1021 zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " 1022 "pausing=%u", 1023 (longlong_t)dsobj, dsname, 1024 (longlong_t)scn->scn_phys.scn_cur_min_txg, 1025 (longlong_t)scn->scn_phys.scn_cur_max_txg, 1026 (int)scn->scn_pausing); 1027 kmem_free(dsname, ZFS_MAXNAMELEN); 1028 1029 if (scn->scn_pausing) 1030 goto out; 1031 1032 /* 1033 * We've finished this pass over this dataset. 1034 */ 1035 1036 /* 1037 * If we did not completely visit this dataset, do another pass. 1038 */ 1039 if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { 1040 zfs_dbgmsg("incomplete pass; visiting again"); 1041 scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; 1042 VERIFY(zap_add_int_key(dp->dp_meta_objset, 1043 scn->scn_phys.scn_queue_obj, ds->ds_object, 1044 scn->scn_phys.scn_cur_max_txg, tx) == 0); 1045 goto out; 1046 } 1047 1048 /* 1049 * Add descendent datasets to work queue. 1050 */ 1051 if (ds->ds_phys->ds_next_snap_obj != 0) { 1052 VERIFY(zap_add_int_key(dp->dp_meta_objset, 1053 scn->scn_phys.scn_queue_obj, ds->ds_phys->ds_next_snap_obj, 1054 ds->ds_phys->ds_creation_txg, tx) == 0); 1055 } 1056 if (ds->ds_phys->ds_num_children > 1) { 1057 boolean_t usenext = B_FALSE; 1058 if (ds->ds_phys->ds_next_clones_obj != 0) { 1059 uint64_t count; 1060 /* 1061 * A bug in a previous version of the code could 1062 * cause upgrade_clones_cb() to not set 1063 * ds_next_snap_obj when it should, leading to a 1064 * missing entry. Therefore we can only use the 1065 * next_clones_obj when its count is correct. 1066 */ 1067 int err = zap_count(dp->dp_meta_objset, 1068 ds->ds_phys->ds_next_clones_obj, &count); 1069 if (err == 0 && 1070 count == ds->ds_phys->ds_num_children - 1) 1071 usenext = B_TRUE; 1072 } 1073 1074 if (usenext) { 1075 VERIFY0(zap_join_key(dp->dp_meta_objset, 1076 ds->ds_phys->ds_next_clones_obj, 1077 scn->scn_phys.scn_queue_obj, 1078 ds->ds_phys->ds_creation_txg, tx)); 1079 } else { 1080 struct enqueue_clones_arg eca; 1081 eca.tx = tx; 1082 eca.originobj = ds->ds_object; 1083 1084 VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 1085 enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); 1086 } 1087 } 1088 1089 out: 1090 dsl_dataset_rele(ds, FTAG); 1091 } 1092 1093 /* ARGSUSED */ 1094 static int 1095 enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 1096 { 1097 dmu_tx_t *tx = arg; 1098 dsl_dataset_t *ds; 1099 int err; 1100 dsl_scan_t *scn = dp->dp_scan; 1101 1102 err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 1103 if (err) 1104 return (err); 1105 1106 while (ds->ds_phys->ds_prev_snap_obj != 0) { 1107 dsl_dataset_t *prev; 1108 err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj, 1109 FTAG, &prev); 1110 if (err) { 1111 dsl_dataset_rele(ds, FTAG); 1112 return (err); 1113 } 1114 1115 /* 1116 * If this is a clone, we don't need to worry about it for now. 1117 */ 1118 if (prev->ds_phys->ds_next_snap_obj != ds->ds_object) { 1119 dsl_dataset_rele(ds, FTAG); 1120 dsl_dataset_rele(prev, FTAG); 1121 return (0); 1122 } 1123 dsl_dataset_rele(ds, FTAG); 1124 ds = prev; 1125 } 1126 1127 VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 1128 ds->ds_object, ds->ds_phys->ds_prev_snap_txg, tx) == 0); 1129 dsl_dataset_rele(ds, FTAG); 1130 return (0); 1131 } 1132 1133 /* 1134 * Scrub/dedup interaction. 1135 * 1136 * If there are N references to a deduped block, we don't want to scrub it 1137 * N times -- ideally, we should scrub it exactly once. 1138 * 1139 * We leverage the fact that the dde's replication class (enum ddt_class) 1140 * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest 1141 * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order. 1142 * 1143 * To prevent excess scrubbing, the scrub begins by walking the DDT 1144 * to find all blocks with refcnt > 1, and scrubs each of these once. 1145 * Since there are two replication classes which contain blocks with 1146 * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first. 1147 * Finally the top-down scrub begins, only visiting blocks with refcnt == 1. 1148 * 1149 * There would be nothing more to say if a block's refcnt couldn't change 1150 * during a scrub, but of course it can so we must account for changes 1151 * in a block's replication class. 1152 * 1153 * Here's an example of what can occur: 1154 * 1155 * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1 1156 * when visited during the top-down scrub phase, it will be scrubbed twice. 1157 * This negates our scrub optimization, but is otherwise harmless. 1158 * 1159 * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1 1160 * on each visit during the top-down scrub phase, it will never be scrubbed. 1161 * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's 1162 * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to 1163 * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1 1164 * while a scrub is in progress, it scrubs the block right then. 1165 */ 1166 static void 1167 dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) 1168 { 1169 ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; 1170 ddt_entry_t dde = { 0 }; 1171 int error; 1172 uint64_t n = 0; 1173 1174 while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { 1175 ddt_t *ddt; 1176 1177 if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) 1178 break; 1179 dprintf("visiting ddb=%llu/%llu/%llu/%llx\n", 1180 (longlong_t)ddb->ddb_class, 1181 (longlong_t)ddb->ddb_type, 1182 (longlong_t)ddb->ddb_checksum, 1183 (longlong_t)ddb->ddb_cursor); 1184 1185 /* There should be no pending changes to the dedup table */ 1186 ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; 1187 ASSERT(avl_first(&ddt->ddt_tree) == NULL); 1188 1189 dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); 1190 n++; 1191 1192 if (dsl_scan_check_pause(scn, NULL)) 1193 break; 1194 } 1195 1196 zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; pausing=%u", 1197 (longlong_t)n, (int)scn->scn_phys.scn_ddt_class_max, 1198 (int)scn->scn_pausing); 1199 1200 ASSERT(error == 0 || error == ENOENT); 1201 ASSERT(error != ENOENT || 1202 ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); 1203 } 1204 1205 /* ARGSUSED */ 1206 void 1207 dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, 1208 ddt_entry_t *dde, dmu_tx_t *tx) 1209 { 1210 const ddt_key_t *ddk = &dde->dde_key; 1211 ddt_phys_t *ddp = dde->dde_phys; 1212 blkptr_t bp; 1213 zbookmark_phys_t zb = { 0 }; 1214 1215 if (scn->scn_phys.scn_state != DSS_SCANNING) 1216 return; 1217 1218 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1219 if (ddp->ddp_phys_birth == 0 || 1220 ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) 1221 continue; 1222 ddt_bp_create(checksum, ddk, ddp, &bp); 1223 1224 scn->scn_visited_this_txg++; 1225 scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); 1226 } 1227 } 1228 1229 static void 1230 dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) 1231 { 1232 dsl_pool_t *dp = scn->scn_dp; 1233 zap_cursor_t zc; 1234 zap_attribute_t za; 1235 1236 if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 1237 scn->scn_phys.scn_ddt_class_max) { 1238 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 1239 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 1240 dsl_scan_ddt(scn, tx); 1241 if (scn->scn_pausing) 1242 return; 1243 } 1244 1245 if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { 1246 /* First do the MOS & ORIGIN */ 1247 1248 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 1249 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 1250 dsl_scan_visit_rootbp(scn, NULL, 1251 &dp->dp_meta_rootbp, tx); 1252 spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp); 1253 if (scn->scn_pausing) 1254 return; 1255 1256 if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) { 1257 VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 1258 enqueue_cb, tx, DS_FIND_CHILDREN)); 1259 } else { 1260 dsl_scan_visitds(scn, 1261 dp->dp_origin_snap->ds_object, tx); 1262 } 1263 ASSERT(!scn->scn_pausing); 1264 } else if (scn->scn_phys.scn_bookmark.zb_objset != 1265 ZB_DESTROYED_OBJSET) { 1266 /* 1267 * If we were paused, continue from here. Note if the 1268 * ds we were paused on was deleted, the zb_objset may 1269 * be -1, so we will skip this and find a new objset 1270 * below. 1271 */ 1272 dsl_scan_visitds(scn, scn->scn_phys.scn_bookmark.zb_objset, tx); 1273 if (scn->scn_pausing) 1274 return; 1275 } 1276 1277 /* 1278 * In case we were paused right at the end of the ds, zero the 1279 * bookmark so we don't think that we're still trying to resume. 1280 */ 1281 bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); 1282 1283 /* keep pulling things out of the zap-object-as-queue */ 1284 while (zap_cursor_init(&zc, dp->dp_meta_objset, 1285 scn->scn_phys.scn_queue_obj), 1286 zap_cursor_retrieve(&zc, &za) == 0) { 1287 dsl_dataset_t *ds; 1288 uint64_t dsobj; 1289 1290 dsobj = strtonum(za.za_name, NULL); 1291 VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 1292 scn->scn_phys.scn_queue_obj, dsobj, tx)); 1293 1294 /* Set up min/max txg */ 1295 VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 1296 if (za.za_first_integer != 0) { 1297 scn->scn_phys.scn_cur_min_txg = 1298 MAX(scn->scn_phys.scn_min_txg, 1299 za.za_first_integer); 1300 } else { 1301 scn->scn_phys.scn_cur_min_txg = 1302 MAX(scn->scn_phys.scn_min_txg, 1303 ds->ds_phys->ds_prev_snap_txg); 1304 } 1305 scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); 1306 dsl_dataset_rele(ds, FTAG); 1307 1308 dsl_scan_visitds(scn, dsobj, tx); 1309 zap_cursor_fini(&zc); 1310 if (scn->scn_pausing) 1311 return; 1312 } 1313 zap_cursor_fini(&zc); 1314 } 1315 1316 static boolean_t 1317 dsl_scan_free_should_pause(dsl_scan_t *scn) 1318 { 1319 uint64_t elapsed_nanosecs; 1320 1321 if (zfs_recover) 1322 return (B_FALSE); 1323 1324 if (scn->scn_visited_this_txg >= zfs_free_max_blocks) 1325 return (B_TRUE); 1326 1327 elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 1328 return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 1329 (NSEC2MSEC(elapsed_nanosecs) > zfs_free_min_time_ms && 1330 txg_sync_waiting(scn->scn_dp)) || 1331 spa_shutting_down(scn->scn_dp->dp_spa)); 1332 } 1333 1334 static int 1335 dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 1336 { 1337 dsl_scan_t *scn = arg; 1338 1339 if (!scn->scn_is_bptree || 1340 (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) { 1341 if (dsl_scan_free_should_pause(scn)) 1342 return (SET_ERROR(ERESTART)); 1343 } 1344 1345 zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, 1346 dmu_tx_get_txg(tx), bp, 0)); 1347 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 1348 -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), 1349 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 1350 scn->scn_visited_this_txg++; 1351 return (0); 1352 } 1353 1354 boolean_t 1355 dsl_scan_active(dsl_scan_t *scn) 1356 { 1357 spa_t *spa = scn->scn_dp->dp_spa; 1358 uint64_t used = 0, comp, uncomp; 1359 1360 if (spa->spa_load_state != SPA_LOAD_NONE) 1361 return (B_FALSE); 1362 if (spa_shutting_down(spa)) 1363 return (B_FALSE); 1364 if (scn->scn_phys.scn_state == DSS_SCANNING || 1365 (scn->scn_async_destroying && !scn->scn_async_stalled)) 1366 return (B_TRUE); 1367 1368 if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1369 (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, 1370 &used, &comp, &uncomp); 1371 } 1372 return (used != 0); 1373 } 1374 1375 void 1376 dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) 1377 { 1378 dsl_scan_t *scn = dp->dp_scan; 1379 spa_t *spa = dp->dp_spa; 1380 int err = 0; 1381 1382 /* 1383 * Check for scn_restart_txg before checking spa_load_state, so 1384 * that we can restart an old-style scan while the pool is being 1385 * imported (see dsl_scan_init). 1386 */ 1387 if (scn->scn_restart_txg != 0 && 1388 scn->scn_restart_txg <= tx->tx_txg) { 1389 pool_scan_func_t func = POOL_SCAN_SCRUB; 1390 dsl_scan_done(scn, B_FALSE, tx); 1391 if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) 1392 func = POOL_SCAN_RESILVER; 1393 zfs_dbgmsg("restarting scan func=%u txg=%llu", 1394 func, tx->tx_txg); 1395 dsl_scan_setup_sync(&func, tx); 1396 } 1397 1398 /* 1399 * If the scan is inactive due to a stalled async destroy, try again. 1400 */ 1401 if ((!scn->scn_async_stalled && !dsl_scan_active(scn)) || 1402 spa_sync_pass(dp->dp_spa) > 1) 1403 return; 1404 1405 scn->scn_visited_this_txg = 0; 1406 scn->scn_pausing = B_FALSE; 1407 scn->scn_sync_start_time = gethrtime(); 1408 spa->spa_scrub_active = B_TRUE; 1409 1410 /* 1411 * First process the async destroys. If we pause, don't do 1412 * any scrubbing or resilvering. This ensures that there are no 1413 * async destroys while we are scanning, so the scan code doesn't 1414 * have to worry about traversing it. It is also faster to free the 1415 * blocks than to scrub them. 1416 */ 1417 if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1418 scn->scn_is_bptree = B_FALSE; 1419 scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1420 NULL, ZIO_FLAG_MUSTSUCCEED); 1421 err = bpobj_iterate(&dp->dp_free_bpobj, 1422 dsl_scan_free_block_cb, scn, tx); 1423 VERIFY3U(0, ==, zio_wait(scn->scn_zio_root)); 1424 1425 if (err != 0 && err != ERESTART) 1426 zfs_panic_recover("error %u from bpobj_iterate()", err); 1427 } 1428 1429 if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) { 1430 ASSERT(scn->scn_async_destroying); 1431 scn->scn_is_bptree = B_TRUE; 1432 scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1433 NULL, ZIO_FLAG_MUSTSUCCEED); 1434 err = bptree_iterate(dp->dp_meta_objset, 1435 dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); 1436 VERIFY0(zio_wait(scn->scn_zio_root)); 1437 1438 if (err == EIO || err == ECKSUM) { 1439 err = 0; 1440 } else if (err != 0 && err != ERESTART) { 1441 zfs_panic_recover("error %u from " 1442 "traverse_dataset_destroyed()", err); 1443 } 1444 1445 /* 1446 * If we didn't make progress, mark the async destroy as 1447 * stalled, so that we will not initiate a spa_sync() on 1448 * its behalf. 1449 */ 1450 scn->scn_async_stalled = (scn->scn_visited_this_txg == 0); 1451 1452 if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) { 1453 /* finished; deactivate async destroy feature */ 1454 spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx); 1455 ASSERT(!spa_feature_is_active(spa, 1456 SPA_FEATURE_ASYNC_DESTROY)); 1457 VERIFY0(zap_remove(dp->dp_meta_objset, 1458 DMU_POOL_DIRECTORY_OBJECT, 1459 DMU_POOL_BPTREE_OBJ, tx)); 1460 VERIFY0(bptree_free(dp->dp_meta_objset, 1461 dp->dp_bptree_obj, tx)); 1462 dp->dp_bptree_obj = 0; 1463 scn->scn_async_destroying = B_FALSE; 1464 } 1465 } 1466 if (scn->scn_visited_this_txg) { 1467 zfs_dbgmsg("freed %llu blocks in %llums from " 1468 "free_bpobj/bptree txg %llu; err=%u", 1469 (longlong_t)scn->scn_visited_this_txg, 1470 (longlong_t) 1471 NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), 1472 (longlong_t)tx->tx_txg, err); 1473 scn->scn_visited_this_txg = 0; 1474 1475 /* 1476 * Write out changes to the DDT that may be required as a 1477 * result of the blocks freed. This ensures that the DDT 1478 * is clean when a scrub/resilver runs. 1479 */ 1480 ddt_sync(spa, tx->tx_txg); 1481 } 1482 if (err != 0) 1483 return; 1484 if (!scn->scn_async_destroying && zfs_free_leak_on_eio && 1485 (dp->dp_free_dir->dd_phys->dd_used_bytes != 0 || 1486 dp->dp_free_dir->dd_phys->dd_compressed_bytes != 0 || 1487 dp->dp_free_dir->dd_phys->dd_uncompressed_bytes != 0)) { 1488 /* 1489 * We have finished background destroying, but there is still 1490 * some space left in the dp_free_dir. Transfer this leaked 1491 * space to the dp_leak_dir. 1492 */ 1493 if (dp->dp_leak_dir == NULL) { 1494 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 1495 (void) dsl_dir_create_sync(dp, dp->dp_root_dir, 1496 LEAK_DIR_NAME, tx); 1497 VERIFY0(dsl_pool_open_special_dir(dp, 1498 LEAK_DIR_NAME, &dp->dp_leak_dir)); 1499 rrw_exit(&dp->dp_config_rwlock, FTAG); 1500 } 1501 dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD, 1502 dp->dp_free_dir->dd_phys->dd_used_bytes, 1503 dp->dp_free_dir->dd_phys->dd_compressed_bytes, 1504 dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx); 1505 dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD, 1506 -dp->dp_free_dir->dd_phys->dd_used_bytes, 1507 -dp->dp_free_dir->dd_phys->dd_compressed_bytes, 1508 -dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx); 1509 } 1510 if (!scn->scn_async_destroying) { 1511 /* finished; verify that space accounting went to zero */ 1512 ASSERT0(dp->dp_free_dir->dd_phys->dd_used_bytes); 1513 ASSERT0(dp->dp_free_dir->dd_phys->dd_compressed_bytes); 1514 ASSERT0(dp->dp_free_dir->dd_phys->dd_uncompressed_bytes); 1515 } 1516 1517 if (scn->scn_phys.scn_state != DSS_SCANNING) 1518 return; 1519 1520 if (scn->scn_done_txg == tx->tx_txg) { 1521 ASSERT(!scn->scn_pausing); 1522 /* finished with scan. */ 1523 zfs_dbgmsg("txg %llu scan complete", tx->tx_txg); 1524 dsl_scan_done(scn, B_TRUE, tx); 1525 ASSERT3U(spa->spa_scrub_inflight, ==, 0); 1526 dsl_scan_sync_state(scn, tx); 1527 return; 1528 } 1529 1530 if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 1531 scn->scn_phys.scn_ddt_class_max) { 1532 zfs_dbgmsg("doing scan sync txg %llu; " 1533 "ddt bm=%llu/%llu/%llu/%llx", 1534 (longlong_t)tx->tx_txg, 1535 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 1536 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 1537 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 1538 (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 1539 ASSERT(scn->scn_phys.scn_bookmark.zb_objset == 0); 1540 ASSERT(scn->scn_phys.scn_bookmark.zb_object == 0); 1541 ASSERT(scn->scn_phys.scn_bookmark.zb_level == 0); 1542 ASSERT(scn->scn_phys.scn_bookmark.zb_blkid == 0); 1543 } else { 1544 zfs_dbgmsg("doing scan sync txg %llu; bm=%llu/%llu/%llu/%llu", 1545 (longlong_t)tx->tx_txg, 1546 (longlong_t)scn->scn_phys.scn_bookmark.zb_objset, 1547 (longlong_t)scn->scn_phys.scn_bookmark.zb_object, 1548 (longlong_t)scn->scn_phys.scn_bookmark.zb_level, 1549 (longlong_t)scn->scn_phys.scn_bookmark.zb_blkid); 1550 } 1551 1552 scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1553 NULL, ZIO_FLAG_CANFAIL); 1554 dsl_pool_config_enter(dp, FTAG); 1555 dsl_scan_visit(scn, tx); 1556 dsl_pool_config_exit(dp, FTAG); 1557 (void) zio_wait(scn->scn_zio_root); 1558 scn->scn_zio_root = NULL; 1559 1560 zfs_dbgmsg("visited %llu blocks in %llums", 1561 (longlong_t)scn->scn_visited_this_txg, 1562 (longlong_t)NSEC2MSEC(gethrtime() - scn->scn_sync_start_time)); 1563 1564 if (!scn->scn_pausing) { 1565 scn->scn_done_txg = tx->tx_txg + 1; 1566 zfs_dbgmsg("txg %llu traversal complete, waiting till txg %llu", 1567 tx->tx_txg, scn->scn_done_txg); 1568 } 1569 1570 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 1571 mutex_enter(&spa->spa_scrub_lock); 1572 while (spa->spa_scrub_inflight > 0) { 1573 cv_wait(&spa->spa_scrub_io_cv, 1574 &spa->spa_scrub_lock); 1575 } 1576 mutex_exit(&spa->spa_scrub_lock); 1577 } 1578 1579 dsl_scan_sync_state(scn, tx); 1580 } 1581 1582 /* 1583 * This will start a new scan, or restart an existing one. 1584 */ 1585 void 1586 dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg) 1587 { 1588 if (txg == 0) { 1589 dmu_tx_t *tx; 1590 tx = dmu_tx_create_dd(dp->dp_mos_dir); 1591 VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT)); 1592 1593 txg = dmu_tx_get_txg(tx); 1594 dp->dp_scan->scn_restart_txg = txg; 1595 dmu_tx_commit(tx); 1596 } else { 1597 dp->dp_scan->scn_restart_txg = txg; 1598 } 1599 zfs_dbgmsg("restarting resilver txg=%llu", txg); 1600 } 1601 1602 boolean_t 1603 dsl_scan_resilvering(dsl_pool_t *dp) 1604 { 1605 return (dp->dp_scan->scn_phys.scn_state == DSS_SCANNING && 1606 dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER); 1607 } 1608 1609 /* 1610 * scrub consumers 1611 */ 1612 1613 static void 1614 count_block(zfs_all_blkstats_t *zab, const blkptr_t *bp) 1615 { 1616 int i; 1617 1618 /* 1619 * If we resume after a reboot, zab will be NULL; don't record 1620 * incomplete stats in that case. 1621 */ 1622 if (zab == NULL) 1623 return; 1624 1625 for (i = 0; i < 4; i++) { 1626 int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS; 1627 int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL; 1628 if (t & DMU_OT_NEWTYPE) 1629 t = DMU_OT_OTHER; 1630 zfs_blkstat_t *zb = &zab->zab_type[l][t]; 1631 int equal; 1632 1633 zb->zb_count++; 1634 zb->zb_asize += BP_GET_ASIZE(bp); 1635 zb->zb_lsize += BP_GET_LSIZE(bp); 1636 zb->zb_psize += BP_GET_PSIZE(bp); 1637 zb->zb_gangs += BP_COUNT_GANG(bp); 1638 1639 switch (BP_GET_NDVAS(bp)) { 1640 case 2: 1641 if (DVA_GET_VDEV(&bp->blk_dva[0]) == 1642 DVA_GET_VDEV(&bp->blk_dva[1])) 1643 zb->zb_ditto_2_of_2_samevdev++; 1644 break; 1645 case 3: 1646 equal = (DVA_GET_VDEV(&bp->blk_dva[0]) == 1647 DVA_GET_VDEV(&bp->blk_dva[1])) + 1648 (DVA_GET_VDEV(&bp->blk_dva[0]) == 1649 DVA_GET_VDEV(&bp->blk_dva[2])) + 1650 (DVA_GET_VDEV(&bp->blk_dva[1]) == 1651 DVA_GET_VDEV(&bp->blk_dva[2])); 1652 if (equal == 1) 1653 zb->zb_ditto_2_of_3_samevdev++; 1654 else if (equal == 3) 1655 zb->zb_ditto_3_of_3_samevdev++; 1656 break; 1657 } 1658 } 1659 } 1660 1661 static void 1662 dsl_scan_scrub_done(zio_t *zio) 1663 { 1664 spa_t *spa = zio->io_spa; 1665 1666 zio_data_buf_free(zio->io_data, zio->io_size); 1667 1668 mutex_enter(&spa->spa_scrub_lock); 1669 spa->spa_scrub_inflight--; 1670 cv_broadcast(&spa->spa_scrub_io_cv); 1671 1672 if (zio->io_error && (zio->io_error != ECKSUM || 1673 !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) { 1674 spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors++; 1675 } 1676 mutex_exit(&spa->spa_scrub_lock); 1677 } 1678 1679 static int 1680 dsl_scan_scrub_cb(dsl_pool_t *dp, 1681 const blkptr_t *bp, const zbookmark_phys_t *zb) 1682 { 1683 dsl_scan_t *scn = dp->dp_scan; 1684 size_t size = BP_GET_PSIZE(bp); 1685 spa_t *spa = dp->dp_spa; 1686 uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp); 1687 boolean_t needs_io; 1688 int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL; 1689 int scan_delay = 0; 1690 1691 if (phys_birth <= scn->scn_phys.scn_min_txg || 1692 phys_birth >= scn->scn_phys.scn_max_txg) 1693 return (0); 1694 1695 count_block(dp->dp_blkstats, bp); 1696 1697 if (BP_IS_EMBEDDED(bp)) 1698 return (0); 1699 1700 ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); 1701 if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { 1702 zio_flags |= ZIO_FLAG_SCRUB; 1703 needs_io = B_TRUE; 1704 scan_delay = zfs_scrub_delay; 1705 } else { 1706 ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); 1707 zio_flags |= ZIO_FLAG_RESILVER; 1708 needs_io = B_FALSE; 1709 scan_delay = zfs_resilver_delay; 1710 } 1711 1712 /* If it's an intent log block, failure is expected. */ 1713 if (zb->zb_level == ZB_ZIL_LEVEL) 1714 zio_flags |= ZIO_FLAG_SPECULATIVE; 1715 1716 for (int d = 0; d < BP_GET_NDVAS(bp); d++) { 1717 vdev_t *vd = vdev_lookup_top(spa, 1718 DVA_GET_VDEV(&bp->blk_dva[d])); 1719 1720 /* 1721 * Keep track of how much data we've examined so that 1722 * zpool(1M) status can make useful progress reports. 1723 */ 1724 scn->scn_phys.scn_examined += DVA_GET_ASIZE(&bp->blk_dva[d]); 1725 spa->spa_scan_pass_exam += DVA_GET_ASIZE(&bp->blk_dva[d]); 1726 1727 /* if it's a resilver, this may not be in the target range */ 1728 if (!needs_io) { 1729 if (DVA_GET_GANG(&bp->blk_dva[d])) { 1730 /* 1731 * Gang members may be spread across multiple 1732 * vdevs, so the best estimate we have is the 1733 * scrub range, which has already been checked. 1734 * XXX -- it would be better to change our 1735 * allocation policy to ensure that all 1736 * gang members reside on the same vdev. 1737 */ 1738 needs_io = B_TRUE; 1739 } else { 1740 needs_io = vdev_dtl_contains(vd, DTL_PARTIAL, 1741 phys_birth, 1); 1742 } 1743 } 1744 } 1745 1746 if (needs_io && !zfs_no_scrub_io) { 1747 vdev_t *rvd = spa->spa_root_vdev; 1748 uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight; 1749 void *data = zio_data_buf_alloc(size); 1750 1751 mutex_enter(&spa->spa_scrub_lock); 1752 while (spa->spa_scrub_inflight >= maxinflight) 1753 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 1754 spa->spa_scrub_inflight++; 1755 mutex_exit(&spa->spa_scrub_lock); 1756 1757 /* 1758 * If we're seeing recent (zfs_scan_idle) "important" I/Os 1759 * then throttle our workload to limit the impact of a scan. 1760 */ 1761 if (ddi_get_lbolt64() - spa->spa_last_io <= zfs_scan_idle) 1762 delay(scan_delay); 1763 1764 zio_nowait(zio_read(NULL, spa, bp, data, size, 1765 dsl_scan_scrub_done, NULL, ZIO_PRIORITY_SCRUB, 1766 zio_flags, zb)); 1767 } 1768 1769 /* do not relocate this block */ 1770 return (0); 1771 } 1772 1773 int 1774 dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) 1775 { 1776 spa_t *spa = dp->dp_spa; 1777 1778 /* 1779 * Purge all vdev caches and probe all devices. We do this here 1780 * rather than in sync context because this requires a writer lock 1781 * on the spa_config lock, which we can't do from sync context. The 1782 * spa_scrub_reopen flag indicates that vdev_open() should not 1783 * attempt to start another scrub. 1784 */ 1785 spa_vdev_state_enter(spa, SCL_NONE); 1786 spa->spa_scrub_reopen = B_TRUE; 1787 vdev_reopen(spa->spa_root_vdev); 1788 spa->spa_scrub_reopen = B_FALSE; 1789 (void) spa_vdev_state_exit(spa, NULL, 0); 1790 1791 return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check, 1792 dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_NONE)); 1793 } 1794