Lines Matching refs:scn
123 static void scan_ds_queue_clear(dsl_scan_t *scn);
124 static boolean_t scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj,
126 static void scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg);
127 static void scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj);
128 static void scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx);
209 #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ argument
210 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
211 (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
338 static void scan_io_queues_destroy(dsl_scan_t *scn);
394 dsl_scan_is_running(const dsl_scan_t *scn) in dsl_scan_is_running() argument
396 return (scn->scn_phys.scn_state == DSS_SCANNING); in dsl_scan_is_running()
446 dsl_scan_t *scn; in dsl_scan_init() local
450 scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); in dsl_scan_init()
451 scn->scn_dp = dp; in dsl_scan_init()
458 ASSERT(!scn->scn_async_destroying); in dsl_scan_init()
459 scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, in dsl_scan_init()
462 avl_create(&scn->scn_queue, scan_ds_queue_compare, sizeof (scan_ds_t), in dsl_scan_init()
464 avl_create(&scn->scn_prefetch_queue, scan_prefetch_queue_compare, in dsl_scan_init()
475 scn->scn_restart_txg = txg; in dsl_scan_init()
478 (longlong_t)scn->scn_restart_txg); in dsl_scan_init()
486 &scn->scn_phys.scn_queue_obj); in dsl_scan_init()
490 &scn->scn_phys); in dsl_scan_init()
510 scn->scn_async_destroying) { in dsl_scan_init()
516 bcopy(zaptmp, &scn->scn_phys, in dsl_scan_init()
518 scn->scn_phys.scn_flags = overflow; in dsl_scan_init()
521 if (scn->scn_phys.scn_state == DSS_FINISHED || in dsl_scan_init()
522 scn->scn_phys.scn_state == DSS_CANCELED) in dsl_scan_init()
538 scn->scn_issued_before_pass = scn->scn_phys.scn_examined; in dsl_scan_init()
540 if (dsl_scan_is_running(scn) && in dsl_scan_init()
549 scn->scn_restart_txg = txg; in dsl_scan_init()
552 (longlong_t)scn->scn_restart_txg); in dsl_scan_init()
560 if (scn->scn_phys.scn_errors > 0) { in dsl_scan_init()
561 scn->scn_restart_txg = txg; in dsl_scan_init()
564 (u_longlong_t)scn->scn_restart_txg); in dsl_scan_init()
572 bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); in dsl_scan_init()
575 if (scn->scn_phys.scn_queue_obj != 0) { in dsl_scan_init()
580 scn->scn_phys.scn_queue_obj); in dsl_scan_init()
583 scan_ds_queue_insert(scn, in dsl_scan_init()
598 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_fini() local
600 if (scn->scn_taskq != NULL) in dsl_scan_fini()
601 taskq_destroy(scn->scn_taskq); in dsl_scan_fini()
602 scan_ds_queue_clear(scn); in dsl_scan_fini()
603 avl_destroy(&scn->scn_queue); in dsl_scan_fini()
604 avl_destroy(&scn->scn_prefetch_queue); in dsl_scan_fini()
612 dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx) in dsl_scan_restarting() argument
614 return (scn->scn_restart_txg != 0 && in dsl_scan_restarting()
615 scn->scn_restart_txg <= tx->tx_txg); in dsl_scan_restarting()
635 dsl_scan_is_paused_scrub(const dsl_scan_t *scn) in dsl_scan_is_paused_scrub() argument
637 return (dsl_scan_scrubbing(scn->scn_dp) && in dsl_scan_is_paused_scrub()
638 scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED); in dsl_scan_is_paused_scrub()
664 dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx, state_sync_type_t sync_type) in dsl_scan_sync_state() argument
667 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_sync_state()
669 ASSERT(sync_type != SYNC_MANDATORY || scn->scn_bytes_pending == 0); in dsl_scan_sync_state()
670 if (scn->scn_bytes_pending == 0) { in dsl_scan_sync_state()
686 if (scn->scn_phys.scn_queue_obj != 0) in dsl_scan_sync_state()
687 scan_ds_queue_sync(scn, tx); in dsl_scan_sync_state()
688 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, in dsl_scan_sync_state()
691 &scn->scn_phys, tx)); in dsl_scan_sync_state()
692 bcopy(&scn->scn_phys, &scn->scn_phys_cached, in dsl_scan_sync_state()
693 sizeof (scn->scn_phys)); in dsl_scan_sync_state()
695 if (scn->scn_checkpointing) in dsl_scan_sync_state()
698 scn->scn_checkpointing = B_FALSE; in dsl_scan_sync_state()
699 scn->scn_last_checkpoint = ddi_get_lbolt(); in dsl_scan_sync_state()
701 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, in dsl_scan_sync_state()
704 &scn->scn_phys_cached, tx)); in dsl_scan_sync_state()
712 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; in dsl_scan_setup_check() local
714 if (dsl_scan_is_running(scn)) in dsl_scan_setup_check()
723 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; in dsl_scan_setup_sync() local
726 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_setup_sync()
729 ASSERT(!dsl_scan_is_running(scn)); in dsl_scan_setup_sync()
731 bzero(&scn->scn_phys, sizeof (scn->scn_phys)); in dsl_scan_setup_sync()
732 scn->scn_phys.scn_func = *funcp; in dsl_scan_setup_sync()
733 scn->scn_phys.scn_state = DSS_SCANNING; in dsl_scan_setup_sync()
734 scn->scn_phys.scn_min_txg = 0; in dsl_scan_setup_sync()
735 scn->scn_phys.scn_max_txg = tx->tx_txg; in dsl_scan_setup_sync()
736 scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ in dsl_scan_setup_sync()
737 scn->scn_phys.scn_start_time = gethrestime_sec(); in dsl_scan_setup_sync()
738 scn->scn_phys.scn_errors = 0; in dsl_scan_setup_sync()
739 scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; in dsl_scan_setup_sync()
740 scn->scn_issued_before_pass = 0; in dsl_scan_setup_sync()
741 scn->scn_restart_txg = 0; in dsl_scan_setup_sync()
742 scn->scn_done_txg = 0; in dsl_scan_setup_sync()
743 scn->scn_last_checkpoint = 0; in dsl_scan_setup_sync()
744 scn->scn_checkpointing = B_FALSE; in dsl_scan_setup_sync()
747 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { in dsl_scan_setup_sync()
748 scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; in dsl_scan_setup_sync()
754 &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { in dsl_scan_setup_sync()
767 if (scn->scn_phys.scn_min_txg > TXG_INITIAL) in dsl_scan_setup_sync()
768 scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; in dsl_scan_setup_sync()
785 scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, in dsl_scan_setup_sync()
788 bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); in dsl_scan_setup_sync()
790 dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); in dsl_scan_setup_sync()
794 *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); in dsl_scan_setup_sync()
805 dsl_scan_t *scn = dp->dp_scan; in dsl_scan() local
825 if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) { in dsl_scan()
827 int err = dsl_scrub_set_pause_resume(scn->scn_dp, in dsl_scan()
842 dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) in dsl_scan_done() argument
856 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_done()
866 if (scn->scn_phys.scn_queue_obj != 0) { in dsl_scan_done()
868 scn->scn_phys.scn_queue_obj, tx)); in dsl_scan_done()
869 scn->scn_phys.scn_queue_obj = 0; in dsl_scan_done()
871 scan_ds_queue_clear(scn); in dsl_scan_done()
873 scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; in dsl_scan_done()
879 if (!dsl_scan_is_running(scn)) { in dsl_scan_done()
880 ASSERT(!scn->scn_is_sorted); in dsl_scan_done()
884 if (scn->scn_is_sorted) { in dsl_scan_done()
885 scan_io_queues_destroy(scn); in dsl_scan_done()
886 scn->scn_is_sorted = B_FALSE; in dsl_scan_done()
888 if (scn->scn_taskq != NULL) { in dsl_scan_done()
889 taskq_destroy(scn->scn_taskq); in dsl_scan_done()
890 scn->scn_taskq = NULL; in dsl_scan_done()
894 scn->scn_phys.scn_state = complete ? DSS_FINISHED : DSS_CANCELED; in dsl_scan_done()
896 if (dsl_scan_restarting(scn, tx)) in dsl_scan_done()
906 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { in dsl_scan_done()
922 scn->scn_phys.scn_max_txg, B_TRUE); in dsl_scan_done()
925 scn->scn_phys.scn_min_txg ? in dsl_scan_done()
964 scn->scn_phys.scn_end_time = gethrestime_sec(); in dsl_scan_done()
966 ASSERT(!dsl_scan_is_running(scn)); in dsl_scan_done()
973 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; in dsl_scan_cancel_check() local
975 if (!dsl_scan_is_running(scn)) in dsl_scan_cancel_check()
984 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; in dsl_scan_cancel_sync() local
986 dsl_scan_done(scn, B_FALSE, tx); in dsl_scan_cancel_sync()
987 dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); in dsl_scan_cancel_sync()
988 spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL, ESC_ZFS_SCRUB_ABORT); in dsl_scan_cancel_sync()
1003 dsl_scan_t *scn = dp->dp_scan; in dsl_scrub_pause_resume_check() local
1011 if (dsl_scan_is_paused_scrub(scn)) in dsl_scrub_pause_resume_check()
1026 dsl_scan_t *scn = dp->dp_scan; in dsl_scrub_pause_resume_sync() local
1031 scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED; in dsl_scrub_pause_resume_sync()
1032 scn->scn_phys_cached.scn_flags |= DSF_SCRUB_PAUSED; in dsl_scrub_pause_resume_sync()
1033 dsl_scan_sync_state(scn, tx, SYNC_CACHED); in dsl_scrub_pause_resume_sync()
1037 if (dsl_scan_is_paused_scrub(scn)) { in dsl_scrub_pause_resume_sync()
1046 scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; in dsl_scrub_pause_resume_sync()
1047 scn->scn_phys_cached.scn_flags &= ~DSF_SCRUB_PAUSED; in dsl_scrub_pause_resume_sync()
1048 dsl_scan_sync_state(scn, tx, SYNC_CACHED); in dsl_scrub_pause_resume_sync()
1109 scan_ds_queue_clear(dsl_scan_t *scn) in scan_ds_queue_clear() argument
1113 while ((sds = avl_destroy_nodes(&scn->scn_queue, &cookie)) != NULL) { in scan_ds_queue_clear()
1119 scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, uint64_t *txg) in scan_ds_queue_contains() argument
1124 sds = avl_find(&scn->scn_queue, &srch, NULL); in scan_ds_queue_contains()
1131 scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg) in scan_ds_queue_insert() argument
1140 VERIFY3P(avl_find(&scn->scn_queue, sds, &where), ==, NULL); in scan_ds_queue_insert()
1141 avl_insert(&scn->scn_queue, sds, where); in scan_ds_queue_insert()
1145 scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj) in scan_ds_queue_remove() argument
1151 sds = avl_find(&scn->scn_queue, &srch, NULL); in scan_ds_queue_remove()
1153 avl_remove(&scn->scn_queue, sds); in scan_ds_queue_remove()
1158 scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx) in scan_ds_queue_sync() argument
1160 dsl_pool_t *dp = scn->scn_dp; in scan_ds_queue_sync()
1165 ASSERT0(scn->scn_bytes_pending); in scan_ds_queue_sync()
1166 ASSERT(scn->scn_phys.scn_queue_obj != 0); in scan_ds_queue_sync()
1169 scn->scn_phys.scn_queue_obj, tx)); in scan_ds_queue_sync()
1170 scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, ot, in scan_ds_queue_sync()
1172 for (scan_ds_t *sds = avl_first(&scn->scn_queue); in scan_ds_queue_sync()
1173 sds != NULL; sds = AVL_NEXT(&scn->scn_queue, sds)) { in scan_ds_queue_sync()
1175 scn->scn_phys.scn_queue_obj, sds->sds_dsobj, in scan_ds_queue_sync()
1213 dsl_scan_should_clear(dsl_scan_t *scn) in dsl_scan_should_clear() argument
1215 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_should_clear()
1216 vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; in dsl_scan_should_clear()
1254 ASSERT0(scn->scn_bytes_pending); in dsl_scan_should_clear()
1266 return (scn->scn_clearing); in dsl_scan_should_clear()
1270 dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb) in dsl_scan_check_suspend() argument
1276 if (scn->scn_suspending) in dsl_scan_check_suspend()
1279 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) in dsl_scan_check_suspend()
1300 uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; in dsl_scan_check_suspend()
1302 scn->scn_dp->dp_spa->spa_sync_starttime; in dsl_scan_check_suspend()
1304 int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; in dsl_scan_check_suspend()
1305 int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? in dsl_scan_check_suspend()
1310 txg_sync_waiting(scn->scn_dp) || in dsl_scan_check_suspend()
1312 spa_shutting_down(scn->scn_dp->dp_spa) || in dsl_scan_check_suspend()
1313 (zfs_scan_strict_mem_lim && dsl_scan_should_clear(scn))) { in dsl_scan_check_suspend()
1320 scn->scn_phys.scn_bookmark = *zb; in dsl_scan_check_suspend()
1322 dsl_scan_phys_t *scnp = &scn->scn_phys; in dsl_scan_check_suspend()
1331 scn->scn_suspending = B_TRUE; in dsl_scan_check_suspend()
1348 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_zil_block() local
1352 if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) in dsl_scan_zil_block()
1367 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); in dsl_scan_zil_block()
1378 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_zil_record() local
1385 bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) in dsl_scan_zil_record()
1400 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); in dsl_scan_zil_record()
1455 scan_prefetch_ctx_create(dsl_scan_t *scn, dnode_phys_t *dnp, void *tag) in scan_prefetch_ctx_create() argument
1462 spc->spc_scn = scn; in scan_prefetch_ctx_create()
1508 dsl_scan_t *scn = spc->spc_scn; in dsl_scan_prefetch() local
1509 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_prefetch()
1515 if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg || in dsl_scan_prefetch()
1523 scan_prefetch_ctx_add_ref(spc, scn); in dsl_scan_prefetch()
1535 if (avl_find(&scn->scn_prefetch_queue, spic, &idx) != NULL) { in dsl_scan_prefetch()
1538 scan_prefetch_ctx_rele(spc, scn); in dsl_scan_prefetch()
1543 avl_insert(&scn->scn_prefetch_queue, spic, idx); in dsl_scan_prefetch()
1549 dsl_scan_prefetch_dnode(dsl_scan_t *scn, dnode_phys_t *dnp, in dsl_scan_prefetch_dnode() argument
1561 spc = scan_prefetch_ctx_create(scn, dnp, FTAG); in dsl_scan_prefetch_dnode()
1583 dsl_scan_t *scn = spc->spc_scn; in dsl_scan_prefetch_cb() local
1584 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_prefetch_cb()
1594 if (buf == NULL || scn->scn_suspending) in dsl_scan_prefetch_cb()
1616 dsl_scan_prefetch_dnode(scn, cdnp, in dsl_scan_prefetch_cb()
1622 dsl_scan_prefetch_dnode(scn, &osp->os_meta_dnode, in dsl_scan_prefetch_cb()
1626 dsl_scan_prefetch_dnode(scn, in dsl_scan_prefetch_cb()
1629 dsl_scan_prefetch_dnode(scn, in dsl_scan_prefetch_cb()
1638 scan_prefetch_ctx_rele(spc, scn); in dsl_scan_prefetch_cb()
1645 dsl_scan_t *scn = arg; in dsl_scan_prefetch_thread() local
1646 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_prefetch_thread()
1652 while (!scn->scn_prefetch_stop) { in dsl_scan_prefetch_thread()
1663 while (!scn->scn_prefetch_stop && in dsl_scan_prefetch_thread()
1664 (avl_numnodes(&scn->scn_prefetch_queue) == 0 || in dsl_scan_prefetch_thread()
1665 spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes)) { in dsl_scan_prefetch_thread()
1670 if (scn->scn_prefetch_stop) { in dsl_scan_prefetch_thread()
1676 spic = avl_first(&scn->scn_prefetch_queue); in dsl_scan_prefetch_thread()
1678 avl_remove(&scn->scn_prefetch_queue, spic); in dsl_scan_prefetch_thread()
1690 (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, in dsl_scan_prefetch_thread()
1697 ASSERT(scn->scn_prefetch_stop); in dsl_scan_prefetch_thread()
1701 while ((spic = avl_first(&scn->scn_prefetch_queue)) != NULL) { in dsl_scan_prefetch_thread()
1702 avl_remove(&scn->scn_prefetch_queue, spic); in dsl_scan_prefetch_thread()
1703 scan_prefetch_ctx_rele(spic->spic_spc, scn); in dsl_scan_prefetch_thread()
1706 ASSERT0(avl_numnodes(&scn->scn_prefetch_queue)); in dsl_scan_prefetch_thread()
1711 dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, in dsl_scan_check_resume() argument
1717 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && in dsl_scan_check_resume()
1724 &scn->scn_phys.scn_bookmark)) in dsl_scan_check_resume()
1733 if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || in dsl_scan_check_resume()
1734 zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { in dsl_scan_check_resume()
1740 bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); in dsl_scan_check_resume()
1747 dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
1758 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, in dsl_scan_recurse() argument
1762 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_recurse()
1776 scn->scn_phys.scn_errors++; in dsl_scan_recurse()
1786 ds, scn, ostype, tx); in dsl_scan_recurse()
1804 scn->scn_phys.scn_errors++; in dsl_scan_recurse()
1810 dsl_scan_visitdnode(scn, ds, ostype, in dsl_scan_recurse()
1823 scn->scn_phys.scn_errors++; in dsl_scan_recurse()
1829 dsl_scan_visitdnode(scn, ds, osp->os_type, in dsl_scan_recurse()
1840 dsl_scan_visitdnode(scn, ds, osp->os_type, in dsl_scan_recurse()
1843 dsl_scan_visitdnode(scn, ds, osp->os_type, in dsl_scan_recurse()
1846 dsl_scan_visitdnode(scn, ds, osp->os_type, in dsl_scan_recurse()
1857 dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, in dsl_scan_visitdnode() argument
1869 &czb, dnp, ds, scn, ostype, tx); in dsl_scan_visitdnode()
1877 &czb, dnp, ds, scn, ostype, tx); in dsl_scan_visitdnode()
1887 dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, in dsl_scan_visitbp() argument
1890 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_visitbp()
1893 if (dsl_scan_check_suspend(scn, zb)) in dsl_scan_visitbp()
1896 if (dsl_scan_check_resume(scn, dnp, zb)) in dsl_scan_visitbp()
1899 scn->scn_visited_this_txg++; in dsl_scan_visitbp()
1915 scn->scn_holes_this_txg++; in dsl_scan_visitbp()
1919 if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) { in dsl_scan_visitbp()
1920 scn->scn_lt_min_this_txg++; in dsl_scan_visitbp()
1927 if (dsl_scan_recurse(scn, ds, ostype, dnp, bp_toread, zb, tx) != 0) in dsl_scan_visitbp()
1936 scn->scn_phys.scn_ddt_class_max, bp)) { in dsl_scan_visitbp()
1937 scn->scn_ddt_contained_this_txg++; in dsl_scan_visitbp()
1948 if (BP_PHYSICAL_BIRTH(bp) > scn->scn_phys.scn_cur_max_txg) { in dsl_scan_visitbp()
1949 scn->scn_gt_max_this_txg++; in dsl_scan_visitbp()
1953 scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); in dsl_scan_visitbp()
1960 dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, in dsl_scan_visit_rootbp() argument
1969 if (ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) { in dsl_scan_visit_rootbp()
1970 SET_BOOKMARK(&scn->scn_prefetch_bookmark, in dsl_scan_visit_rootbp()
1973 scn->scn_prefetch_bookmark = scn->scn_phys.scn_bookmark; in dsl_scan_visit_rootbp()
1976 scn->scn_objsets_visited_this_txg++; in dsl_scan_visit_rootbp()
1978 spc = scan_prefetch_ctx_create(scn, NULL, FTAG); in dsl_scan_visit_rootbp()
1982 dsl_scan_visitbp(bp, &zb, NULL, ds, scn, DMU_OST_NONE, tx); in dsl_scan_visit_rootbp()
2035 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_ds_destroyed() local
2038 if (!dsl_scan_is_running(scn)) in dsl_scan_ds_destroyed()
2041 ds_destroyed_scn_phys(ds, &scn->scn_phys); in dsl_scan_ds_destroyed()
2042 ds_destroyed_scn_phys(ds, &scn->scn_phys_cached); in dsl_scan_ds_destroyed()
2044 if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { in dsl_scan_ds_destroyed()
2045 scan_ds_queue_remove(scn, ds->ds_object); in dsl_scan_ds_destroyed()
2047 scan_ds_queue_insert(scn, in dsl_scan_ds_destroyed()
2051 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, in dsl_scan_ds_destroyed()
2055 scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); in dsl_scan_ds_destroyed()
2063 scn->scn_phys.scn_queue_obj, in dsl_scan_ds_destroyed()
2081 dsl_scan_sync_state(scn, tx, SYNC_CACHED); in dsl_scan_ds_destroyed()
2107 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_ds_snapshotted() local
2110 if (!dsl_scan_is_running(scn)) in dsl_scan_ds_snapshotted()
2115 ds_snapshotted_bookmark(ds, &scn->scn_phys.scn_bookmark); in dsl_scan_ds_snapshotted()
2116 ds_snapshotted_bookmark(ds, &scn->scn_phys_cached.scn_bookmark); in dsl_scan_ds_snapshotted()
2118 if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { in dsl_scan_ds_snapshotted()
2119 scan_ds_queue_remove(scn, ds->ds_object); in dsl_scan_ds_snapshotted()
2120 scan_ds_queue_insert(scn, in dsl_scan_ds_snapshotted()
2124 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, in dsl_scan_ds_snapshotted()
2127 scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); in dsl_scan_ds_snapshotted()
2129 scn->scn_phys.scn_queue_obj, in dsl_scan_ds_snapshotted()
2137 dsl_scan_sync_state(scn, tx, SYNC_CACHED); in dsl_scan_ds_snapshotted()
2168 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_ds_clone_swapped() local
2171 if (!dsl_scan_is_running(scn)) in dsl_scan_ds_clone_swapped()
2174 ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys.scn_bookmark); in dsl_scan_ds_clone_swapped()
2175 ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys_cached.scn_bookmark); in dsl_scan_ds_clone_swapped()
2177 if (scan_ds_queue_contains(scn, ds1->ds_object, &mintxg)) { in dsl_scan_ds_clone_swapped()
2178 scan_ds_queue_remove(scn, ds1->ds_object); in dsl_scan_ds_clone_swapped()
2179 scan_ds_queue_insert(scn, ds2->ds_object, mintxg); in dsl_scan_ds_clone_swapped()
2181 if (scan_ds_queue_contains(scn, ds2->ds_object, &mintxg)) { in dsl_scan_ds_clone_swapped()
2182 scan_ds_queue_remove(scn, ds2->ds_object); in dsl_scan_ds_clone_swapped()
2183 scan_ds_queue_insert(scn, ds1->ds_object, mintxg); in dsl_scan_ds_clone_swapped()
2186 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, in dsl_scan_ds_clone_swapped()
2192 scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); in dsl_scan_ds_clone_swapped()
2194 scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); in dsl_scan_ds_clone_swapped()
2199 scn->scn_phys.scn_queue_obj, in dsl_scan_ds_clone_swapped()
2207 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, in dsl_scan_ds_clone_swapped()
2212 scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); in dsl_scan_ds_clone_swapped()
2214 scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); in dsl_scan_ds_clone_swapped()
2221 dsl_scan_sync_state(scn, tx, SYNC_CACHED); in dsl_scan_ds_clone_swapped()
2231 dsl_scan_t *scn = dp->dp_scan; in enqueue_clones_cb() local
2250 scan_ds_queue_insert(scn, ds->ds_object, in enqueue_clones_cb()
2257 dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) in dsl_scan_visitds() argument
2259 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_visitds()
2264 if (scn->scn_phys.scn_cur_min_txg >= in dsl_scan_visitds()
2265 scn->scn_phys.scn_max_txg) { in dsl_scan_visitds()
2297 (longlong_t)scn->scn_phys.scn_cur_min_txg, in dsl_scan_visitds()
2298 (longlong_t)scn->scn_phys.scn_max_txg); in dsl_scan_visitds()
2313 if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds) && in dsl_scan_visitds()
2328 dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx); in dsl_scan_visitds()
2336 (longlong_t)scn->scn_phys.scn_cur_min_txg, in dsl_scan_visitds()
2337 (longlong_t)scn->scn_phys.scn_cur_max_txg, in dsl_scan_visitds()
2338 (int)scn->scn_suspending); in dsl_scan_visitds()
2341 if (scn->scn_suspending) in dsl_scan_visitds()
2351 if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { in dsl_scan_visitds()
2353 scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; in dsl_scan_visitds()
2354 scan_ds_queue_insert(scn, ds->ds_object, in dsl_scan_visitds()
2355 scn->scn_phys.scn_cur_max_txg); in dsl_scan_visitds()
2363 scan_ds_queue_insert(scn, in dsl_scan_visitds()
2392 scan_ds_queue_insert(scn, in dsl_scan_visitds()
2414 dsl_scan_t *scn = dp->dp_scan; in enqueue_cb() local
2441 scan_ds_queue_insert(scn, ds->ds_object, in enqueue_cb()
2449 dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, in dsl_scan_ddt_entry() argument
2458 if (scn->scn_phys.scn_state != DSS_SCANNING) in dsl_scan_ddt_entry()
2472 if (scn->scn_done_txg != 0) in dsl_scan_ddt_entry()
2477 ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) in dsl_scan_ddt_entry()
2481 scn->scn_visited_this_txg++; in dsl_scan_ddt_entry()
2482 scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); in dsl_scan_ddt_entry()
2520 dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) in dsl_scan_ddt() argument
2522 ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; in dsl_scan_ddt()
2527 while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { in dsl_scan_ddt()
2530 if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) in dsl_scan_ddt()
2539 ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; in dsl_scan_ddt()
2542 dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); in dsl_scan_ddt()
2545 if (dsl_scan_check_suspend(scn, NULL)) in dsl_scan_ddt()
2551 (int)scn->scn_phys.scn_ddt_class_max, (int)scn->scn_suspending); in dsl_scan_ddt()
2555 ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); in dsl_scan_ddt()
2568 dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) in dsl_scan_visit() argument
2571 dsl_pool_t *dp = scn->scn_dp; in dsl_scan_visit()
2573 if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= in dsl_scan_visit()
2574 scn->scn_phys.scn_ddt_class_max) { in dsl_scan_visit()
2575 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; in dsl_scan_visit()
2576 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; in dsl_scan_visit()
2577 dsl_scan_ddt(scn, tx); in dsl_scan_visit()
2578 if (scn->scn_suspending) in dsl_scan_visit()
2582 if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { in dsl_scan_visit()
2585 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; in dsl_scan_visit()
2586 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; in dsl_scan_visit()
2587 dsl_scan_visit_rootbp(scn, NULL, in dsl_scan_visit()
2590 if (scn->scn_suspending) in dsl_scan_visit()
2597 dsl_scan_visitds(scn, in dsl_scan_visit()
2600 ASSERT(!scn->scn_suspending); in dsl_scan_visit()
2601 } else if (scn->scn_phys.scn_bookmark.zb_objset != in dsl_scan_visit()
2603 uint64_t dsobj = scn->scn_phys.scn_bookmark.zb_objset; in dsl_scan_visit()
2610 dsl_scan_visitds(scn, dsobj, tx); in dsl_scan_visit()
2611 if (scn->scn_suspending) in dsl_scan_visit()
2619 bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); in dsl_scan_visit()
2625 while ((sds = avl_first(&scn->scn_queue)) != NULL) { in dsl_scan_visit()
2631 scan_ds_queue_remove(scn, dsobj); in dsl_scan_visit()
2637 scn->scn_phys.scn_cur_min_txg = in dsl_scan_visit()
2638 MAX(scn->scn_phys.scn_min_txg, txg); in dsl_scan_visit()
2640 scn->scn_phys.scn_cur_min_txg = in dsl_scan_visit()
2641 MAX(scn->scn_phys.scn_min_txg, in dsl_scan_visit()
2644 scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); in dsl_scan_visit()
2647 dsl_scan_visitds(scn, dsobj, tx); in dsl_scan_visit()
2648 if (scn->scn_suspending) in dsl_scan_visit()
2652 scn->scn_phys.scn_bookmark.zb_objset = ZB_DESTROYED_OBJSET; in dsl_scan_visit()
2653 ASSERT0(scn->scn_suspending); in dsl_scan_visit()
2700 scan_io_queue_check_suspend(dsl_scan_t *scn) in scan_io_queue_check_suspend() argument
2704 uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; in scan_io_queue_check_suspend()
2706 scn->scn_dp->dp_spa->spa_sync_starttime; in scan_io_queue_check_suspend()
2707 int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; in scan_io_queue_check_suspend()
2708 int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? in scan_io_queue_check_suspend()
2713 txg_sync_waiting(scn->scn_dp) || in scan_io_queue_check_suspend()
2715 spa_shutting_down(scn->scn_dp->dp_spa)); in scan_io_queue_check_suspend()
2729 dsl_scan_t *scn = queue->q_scn; in scan_io_queue_issue() local
2737 if (scan_io_queue_check_suspend(scn)) { in scan_io_queue_issue()
2744 scan_exec_io(scn->scn_dp, &bp, sio->sio_flags, in scan_io_queue_issue()
2751 atomic_add_64(&scn->scn_bytes_pending, -bytes_issued); in scan_io_queue_issue()
2842 dsl_scan_t *scn = queue->q_scn; in scan_io_queue_fetch_ext() local
2846 ASSERT(scn->scn_is_sorted); in scan_io_queue_fetch_ext()
2849 if (scn->scn_checkpointing || scn->scn_clearing) { in scan_io_queue_fetch_ext()
2883 if (scn->scn_checkpointing) { in scan_io_queue_fetch_ext()
2885 } else if (scn->scn_clearing) { in scan_io_queue_fetch_ext()
3004 scan_io_queues_run(dsl_scan_t *scn) in scan_io_queues_run() argument
3006 spa_t *spa = scn->scn_dp->dp_spa; in scan_io_queues_run()
3008 ASSERT(scn->scn_is_sorted); in scan_io_queues_run()
3011 if (scn->scn_bytes_pending == 0) in scan_io_queues_run()
3014 if (scn->scn_taskq == NULL) { in scan_io_queues_run()
3028 scn->scn_taskq = taskq_create(tq_name, nthreads, minclsyspri, in scan_io_queues_run()
3038 VERIFY(taskq_dispatch(scn->scn_taskq, in scan_io_queues_run()
3050 taskq_wait(scn->scn_taskq); in scan_io_queues_run()
3054 dsl_scan_async_block_should_pause(dsl_scan_t *scn) in dsl_scan_async_block_should_pause() argument
3061 if (scn->scn_visited_this_txg >= zfs_async_block_max_blocks) in dsl_scan_async_block_should_pause()
3064 elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; in dsl_scan_async_block_should_pause()
3066 (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms && in dsl_scan_async_block_should_pause()
3067 txg_sync_waiting(scn->scn_dp)) || in dsl_scan_async_block_should_pause()
3068 spa_shutting_down(scn->scn_dp->dp_spa)); in dsl_scan_async_block_should_pause()
3074 dsl_scan_t *scn = arg; in dsl_scan_free_block_cb() local
3076 if (!scn->scn_is_bptree || in dsl_scan_free_block_cb()
3078 if (dsl_scan_async_block_should_pause(scn)) in dsl_scan_free_block_cb()
3082 zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, in dsl_scan_free_block_cb()
3085 -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), in dsl_scan_free_block_cb()
3087 scn->scn_visited_this_txg++; in dsl_scan_free_block_cb()
3092 dsl_scan_update_stats(dsl_scan_t *scn) in dsl_scan_update_stats() argument
3094 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_update_stats()
3113 scn->scn_avg_seg_size_this_txg = 0; in dsl_scan_update_stats()
3114 scn->scn_avg_zio_size_this_txg = 0; in dsl_scan_update_stats()
3115 scn->scn_segs_this_txg = 0; in dsl_scan_update_stats()
3116 scn->scn_zios_this_txg = 0; in dsl_scan_update_stats()
3120 scn->scn_avg_seg_size_this_txg = seg_size_total / seg_count_total; in dsl_scan_update_stats()
3121 scn->scn_avg_zio_size_this_txg = zio_size_total / zio_count_total; in dsl_scan_update_stats()
3122 scn->scn_segs_this_txg = seg_count_total; in dsl_scan_update_stats()
3123 scn->scn_zios_this_txg = zio_count_total; in dsl_scan_update_stats()
3129 dsl_scan_t *scn = arg; in dsl_scan_obsolete_block_cb() local
3132 if (dsl_scan_async_block_should_pause(scn)) in dsl_scan_obsolete_block_cb()
3135 spa_vdev_indirect_mark_obsolete(scn->scn_dp->dp_spa, in dsl_scan_obsolete_block_cb()
3138 scn->scn_visited_this_txg++; in dsl_scan_obsolete_block_cb()
3143 dsl_scan_active(dsl_scan_t *scn) in dsl_scan_active() argument
3145 spa_t *spa = scn->scn_dp->dp_spa; in dsl_scan_active()
3152 if ((dsl_scan_is_running(scn) && !dsl_scan_is_paused_scrub(scn)) || in dsl_scan_active()
3153 (scn->scn_async_destroying && !scn->scn_async_stalled)) in dsl_scan_active()
3156 if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { in dsl_scan_active()
3157 (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, in dsl_scan_active()
3244 dsl_scan_t *scn = dp->dp_scan; in dsl_process_async_destroys() local
3252 scn->scn_is_bptree = B_FALSE; in dsl_process_async_destroys()
3253 scn->scn_async_block_min_time_ms = zfs_free_min_time_ms; in dsl_process_async_destroys()
3254 scn->scn_zio_root = zio_root(spa, NULL, in dsl_process_async_destroys()
3257 dsl_scan_free_block_cb, scn, tx); in dsl_process_async_destroys()
3258 VERIFY0(zio_wait(scn->scn_zio_root)); in dsl_process_async_destroys()
3259 scn->scn_zio_root = NULL; in dsl_process_async_destroys()
3266 ASSERT(scn->scn_async_destroying); in dsl_process_async_destroys()
3267 scn->scn_is_bptree = B_TRUE; in dsl_process_async_destroys()
3268 scn->scn_zio_root = zio_root(spa, NULL, in dsl_process_async_destroys()
3271 dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); in dsl_process_async_destroys()
3272 VERIFY0(zio_wait(scn->scn_zio_root)); in dsl_process_async_destroys()
3273 scn->scn_zio_root = NULL; in dsl_process_async_destroys()
3293 scn->scn_async_destroying = B_FALSE; in dsl_process_async_destroys()
3294 scn->scn_async_stalled = B_FALSE; in dsl_process_async_destroys()
3304 scn->scn_async_stalled = in dsl_process_async_destroys()
3305 (scn->scn_visited_this_txg == 0); in dsl_process_async_destroys()
3308 if (scn->scn_visited_this_txg) { in dsl_process_async_destroys()
3311 (longlong_t)scn->scn_visited_this_txg, in dsl_process_async_destroys()
3313 NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), in dsl_process_async_destroys()
3315 scn->scn_visited_this_txg = 0; in dsl_process_async_destroys()
3326 if (dp->dp_free_dir != NULL && !scn->scn_async_destroying && in dsl_process_async_destroys()
3354 if (dp->dp_free_dir != NULL && !scn->scn_async_destroying) { in dsl_process_async_destroys()
3368 scn->scn_is_bptree = B_FALSE; in dsl_process_async_destroys()
3369 scn->scn_async_block_min_time_ms = zfs_obsolete_min_time_ms; in dsl_process_async_destroys()
3371 dsl_scan_obsolete_block_cb, scn, tx); in dsl_process_async_destroys()
3393 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_sync() local
3409 if (dsl_scan_restarting(scn, tx) || in dsl_scan_sync()
3412 dsl_scan_done(scn, B_FALSE, tx); in dsl_scan_sync()
3437 if (!scn->scn_async_stalled && !dsl_scan_active(scn)) in dsl_scan_sync()
3441 scn->scn_visited_this_txg = 0; in dsl_scan_sync()
3442 scn->scn_holes_this_txg = 0; in dsl_scan_sync()
3443 scn->scn_lt_min_this_txg = 0; in dsl_scan_sync()
3444 scn->scn_gt_max_this_txg = 0; in dsl_scan_sync()
3445 scn->scn_ddt_contained_this_txg = 0; in dsl_scan_sync()
3446 scn->scn_objsets_visited_this_txg = 0; in dsl_scan_sync()
3447 scn->scn_avg_seg_size_this_txg = 0; in dsl_scan_sync()
3448 scn->scn_segs_this_txg = 0; in dsl_scan_sync()
3449 scn->scn_avg_zio_size_this_txg = 0; in dsl_scan_sync()
3450 scn->scn_zios_this_txg = 0; in dsl_scan_sync()
3451 scn->scn_suspending = B_FALSE; in dsl_scan_sync()
3452 scn->scn_sync_start_time = gethrtime(); in dsl_scan_sync()
3466 if (!dsl_scan_is_running(scn) || dsl_scan_is_paused_scrub(scn)) in dsl_scan_sync()
3483 uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time; in dsl_scan_sync()
3484 int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? in dsl_scan_sync()
3488 !txg_sync_waiting(scn->scn_dp) && in dsl_scan_sync()
3489 !spa_shutting_down(scn->scn_dp->dp_spa) && in dsl_scan_sync()
3492 scan_time_ns = gethrtime() - scn->scn_sync_start_time; in dsl_scan_sync()
3503 scn->scn_is_sorted = B_TRUE; in dsl_scan_sync()
3504 if (scn->scn_last_checkpoint == 0) in dsl_scan_sync()
3505 scn->scn_last_checkpoint = ddi_get_lbolt(); in dsl_scan_sync()
3513 if (scn->scn_is_sorted) { in dsl_scan_sync()
3524 if (scn->scn_checkpointing || in dsl_scan_sync()
3525 ddi_get_lbolt() - scn->scn_last_checkpoint > in dsl_scan_sync()
3527 if (!scn->scn_checkpointing) in dsl_scan_sync()
3530 scn->scn_checkpointing = B_TRUE; in dsl_scan_sync()
3531 scn->scn_clearing = B_TRUE; in dsl_scan_sync()
3533 boolean_t should_clear = dsl_scan_should_clear(scn); in dsl_scan_sync()
3534 if (should_clear && !scn->scn_clearing) { in dsl_scan_sync()
3536 scn->scn_clearing = B_TRUE; in dsl_scan_sync()
3537 } else if (!should_clear && scn->scn_clearing) { in dsl_scan_sync()
3539 scn->scn_clearing = B_FALSE; in dsl_scan_sync()
3543 ASSERT0(scn->scn_checkpointing); in dsl_scan_sync()
3544 ASSERT0(scn->scn_clearing); in dsl_scan_sync()
3547 if (!scn->scn_clearing && scn->scn_done_txg == 0) { in dsl_scan_sync()
3549 dsl_scan_phys_t *scnp = &scn->scn_phys; in dsl_scan_sync()
3559 scn->scn_maxinflight_bytes = in dsl_scan_sync()
3582 scn->scn_zio_root = zio_root(dp->dp_spa, NULL, in dsl_scan_sync()
3585 scn->scn_prefetch_stop = B_FALSE; in dsl_scan_sync()
3587 dsl_scan_prefetch_thread, scn, TQ_SLEEP); in dsl_scan_sync()
3591 dsl_scan_visit(scn, tx); in dsl_scan_sync()
3595 scn->scn_prefetch_stop = B_TRUE; in dsl_scan_sync()
3600 (void) zio_wait(scn->scn_zio_root); in dsl_scan_sync()
3601 scn->scn_zio_root = NULL; in dsl_scan_sync()
3606 (longlong_t)scn->scn_visited_this_txg, in dsl_scan_sync()
3608 scn->scn_sync_start_time), in dsl_scan_sync()
3609 (longlong_t)scn->scn_objsets_visited_this_txg, in dsl_scan_sync()
3610 (longlong_t)scn->scn_holes_this_txg, in dsl_scan_sync()
3611 (longlong_t)scn->scn_lt_min_this_txg, in dsl_scan_sync()
3612 (longlong_t)scn->scn_ddt_contained_this_txg, in dsl_scan_sync()
3613 (longlong_t)scn->scn_gt_max_this_txg); in dsl_scan_sync()
3615 if (!scn->scn_suspending) { in dsl_scan_sync()
3616 ASSERT0(avl_numnodes(&scn->scn_queue)); in dsl_scan_sync()
3617 scn->scn_done_txg = tx->tx_txg + 1; in dsl_scan_sync()
3618 if (scn->scn_is_sorted) { in dsl_scan_sync()
3619 scn->scn_checkpointing = B_TRUE; in dsl_scan_sync()
3620 scn->scn_clearing = B_TRUE; in dsl_scan_sync()
3625 } else if (scn->scn_is_sorted && scn->scn_bytes_pending != 0) { in dsl_scan_sync()
3626 ASSERT(scn->scn_clearing); in dsl_scan_sync()
3629 scn->scn_zio_root = zio_root(dp->dp_spa, NULL, in dsl_scan_sync()
3631 scan_io_queues_run(scn); in dsl_scan_sync()
3632 (void) zio_wait(scn->scn_zio_root); in dsl_scan_sync()
3633 scn->scn_zio_root = NULL; in dsl_scan_sync()
3636 (void) dsl_scan_should_clear(scn); in dsl_scan_sync()
3637 dsl_scan_update_stats(scn); in dsl_scan_sync()
3641 (longlong_t)scn->scn_zios_this_txg, in dsl_scan_sync()
3642 (longlong_t)scn->scn_segs_this_txg, in dsl_scan_sync()
3644 scn->scn_sync_start_time), in dsl_scan_sync()
3645 (longlong_t)scn->scn_avg_zio_size_this_txg, in dsl_scan_sync()
3646 (longlong_t)scn->scn_avg_seg_size_this_txg); in dsl_scan_sync()
3647 } else if (scn->scn_done_txg != 0 && scn->scn_done_txg <= tx->tx_txg) { in dsl_scan_sync()
3651 ASSERT3U(scn->scn_done_txg, !=, 0); in dsl_scan_sync()
3653 ASSERT0(scn->scn_bytes_pending); in dsl_scan_sync()
3654 dsl_scan_done(scn, B_TRUE, tx); in dsl_scan_sync()
3658 dsl_scan_sync_state(scn, tx, sync_type); in dsl_scan_sync()
3662 count_block(dsl_scan_t *scn, zfs_all_blkstats_t *zab, const blkptr_t *bp) in count_block() argument
3680 if (scn->scn_is_sorted) { in count_block()
3681 atomic_add_64(&scn->scn_dp->dp_spa->spa_scan_pass_issued, in count_block()
3684 spa_t *spa = scn->scn_dp->dp_spa; in count_block()
3744 dsl_scan_t *scn = queue->q_scn; in scan_io_queue_insert_impl() local
3750 atomic_add_64(&scn->scn_bytes_pending, -asize); in scan_io_queue_insert_impl()
3769 dsl_scan_t *scn = queue->q_scn; in scan_io_queue_insert() local
3784 atomic_add_64(&scn->scn_bytes_pending, SIO_GET_ASIZE(sio)); in scan_io_queue_insert()
3832 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_scrub_cb() local
3840 if (phys_birth <= scn->scn_phys.scn_min_txg || in dsl_scan_scrub_cb()
3841 phys_birth >= scn->scn_phys.scn_max_txg) { in dsl_scan_scrub_cb()
3842 count_block(scn, dp->dp_blkstats, bp); in dsl_scan_scrub_cb()
3849 ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); in dsl_scan_scrub_cb()
3850 if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { in dsl_scan_scrub_cb()
3854 ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); in dsl_scan_scrub_cb()
3870 scn->scn_phys.scn_examined += DVA_GET_ASIZE(dva); in dsl_scan_scrub_cb()
3882 count_block(scn, dp->dp_blkstats, bp); in dsl_scan_scrub_cb()
3930 dsl_scan_t *scn = dp->dp_scan; in scan_exec_io() local
3936 while (spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes) in scan_exec_io()
4029 dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan; in scan_io_queue_create() local
4032 q->q_scn = scn; in scan_io_queue_create()
4052 dsl_scan_t *scn = queue->q_scn; in dsl_scan_io_queue_destroy() local
4069 atomic_add_64(&scn->scn_bytes_pending, -bytes_dequeued); in dsl_scan_io_queue_destroy()
4100 scan_io_queues_destroy(dsl_scan_t *scn) in scan_io_queues_destroy() argument
4102 vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; in scan_io_queues_destroy()
4119 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_freed_dva() local
4182 atomic_add_64(&scn->scn_bytes_pending, -asize); in dsl_scan_freed_dva()
4186 count_block(scn, dp->dp_blkstats, &tmpbp); in dsl_scan_freed_dva()
4203 dsl_scan_t *scn = dp->dp_scan; in dsl_scan_freed() local
4206 ASSERT(scn != NULL); in dsl_scan_freed()
4207 if (!dsl_scan_is_running(scn)) in dsl_scan_freed()