1b24ab676SJeff Bonwick /* 2b24ab676SJeff Bonwick * CDDL HEADER START 3b24ab676SJeff Bonwick * 4b24ab676SJeff Bonwick * The contents of this file are subject to the terms of the 5b24ab676SJeff Bonwick * Common Development and Distribution License (the "License"). 6b24ab676SJeff Bonwick * You may not use this file except in compliance with the License. 7b24ab676SJeff Bonwick * 8b24ab676SJeff Bonwick * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9b24ab676SJeff Bonwick * or http://www.opensolaris.org/os/licensing. 10b24ab676SJeff Bonwick * See the License for the specific language governing permissions 11b24ab676SJeff Bonwick * and limitations under the License. 12b24ab676SJeff Bonwick * 13b24ab676SJeff Bonwick * When distributing Covered Code, include this CDDL HEADER in each 14b24ab676SJeff Bonwick * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15b24ab676SJeff Bonwick * If applicable, add the following below this CDDL HEADER, with the 16b24ab676SJeff Bonwick * fields enclosed by brackets "[]" replaced with your own identifying 17b24ab676SJeff Bonwick * information: Portions Copyright [yyyy] [name of copyright owner] 18b24ab676SJeff Bonwick * 19b24ab676SJeff Bonwick * CDDL HEADER END 20b24ab676SJeff Bonwick */ 21b24ab676SJeff Bonwick 22b24ab676SJeff Bonwick /* 233f9d6ad7SLin Ling * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24b24ab676SJeff Bonwick */ 25b24ab676SJeff Bonwick 26b24ab676SJeff Bonwick #include <sys/zfs_context.h> 27b24ab676SJeff Bonwick #include <sys/spa.h> 28b24ab676SJeff Bonwick #include <sys/spa_impl.h> 29b24ab676SJeff Bonwick #include <sys/zio.h> 30b24ab676SJeff Bonwick #include <sys/ddt.h> 31b24ab676SJeff Bonwick #include <sys/zap.h> 32b24ab676SJeff Bonwick #include <sys/dmu_tx.h> 33b24ab676SJeff Bonwick #include <sys/arc.h> 34bbfd46c4SJeff Bonwick #include <sys/dsl_pool.h> 35b24ab676SJeff Bonwick #include <sys/zio_checksum.h> 36b24ab676SJeff Bonwick #include <sys/zio_compress.h> 373f9d6ad7SLin Ling #include <sys/dsl_scan.h> 38b24ab676SJeff Bonwick 39837b568bSGeorge Wilson /* 40837b568bSGeorge Wilson * Enable/disable prefetching of dedup-ed blocks which are going to be freed. 41837b568bSGeorge Wilson */ 42837b568bSGeorge Wilson int zfs_dedup_prefetch = 1; 43837b568bSGeorge Wilson 44b24ab676SJeff Bonwick static const ddt_ops_t *ddt_ops[DDT_TYPES] = { 45b24ab676SJeff Bonwick &ddt_zap_ops, 46b24ab676SJeff Bonwick }; 47b24ab676SJeff Bonwick 48b24ab676SJeff Bonwick static const char *ddt_class_name[DDT_CLASSES] = { 49b24ab676SJeff Bonwick "ditto", 50b24ab676SJeff Bonwick "duplicate", 51b24ab676SJeff Bonwick "unique", 52b24ab676SJeff Bonwick }; 53b24ab676SJeff Bonwick 54b24ab676SJeff Bonwick static void 55b24ab676SJeff Bonwick ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 56b24ab676SJeff Bonwick dmu_tx_t *tx) 57b24ab676SJeff Bonwick { 58b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 59b24ab676SJeff Bonwick objset_t *os = ddt->ddt_os; 60b24ab676SJeff Bonwick uint64_t *objectp = &ddt->ddt_object[type][class]; 61b24ab676SJeff Bonwick boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_dedup; 62b24ab676SJeff Bonwick char name[DDT_NAMELEN]; 63b24ab676SJeff Bonwick 64b24ab676SJeff Bonwick ddt_object_name(ddt, type, class, name); 65b24ab676SJeff Bonwick 66b24ab676SJeff Bonwick ASSERT(*objectp == 0); 67b24ab676SJeff Bonwick VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0); 68b24ab676SJeff Bonwick ASSERT(*objectp != 0); 69b24ab676SJeff Bonwick 70b24ab676SJeff Bonwick VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, 71b24ab676SJeff Bonwick sizeof (uint64_t), 1, objectp, tx) == 0); 72b24ab676SJeff Bonwick 73b24ab676SJeff Bonwick VERIFY(zap_add(os, spa->spa_ddt_stat_object, name, 74b24ab676SJeff Bonwick sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 75b24ab676SJeff Bonwick &ddt->ddt_histogram[type][class], tx) == 0); 76b24ab676SJeff Bonwick } 77b24ab676SJeff Bonwick 78b24ab676SJeff Bonwick static void 79b24ab676SJeff Bonwick ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 80b24ab676SJeff Bonwick dmu_tx_t *tx) 81b24ab676SJeff Bonwick { 82b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 83b24ab676SJeff Bonwick objset_t *os = ddt->ddt_os; 84b24ab676SJeff Bonwick uint64_t *objectp = &ddt->ddt_object[type][class]; 85b24ab676SJeff Bonwick char name[DDT_NAMELEN]; 86b24ab676SJeff Bonwick 87b24ab676SJeff Bonwick ddt_object_name(ddt, type, class, name); 88b24ab676SJeff Bonwick 89b24ab676SJeff Bonwick ASSERT(*objectp != 0); 90b24ab676SJeff Bonwick ASSERT(ddt_object_count(ddt, type, class) == 0); 91b24ab676SJeff Bonwick ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); 92b24ab676SJeff Bonwick VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0); 93b24ab676SJeff Bonwick VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0); 94b24ab676SJeff Bonwick VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0); 9530f608a4SGeorge Wilson bzero(&ddt->ddt_object_stats[type][class], sizeof (ddt_object_t)); 96b24ab676SJeff Bonwick 97b24ab676SJeff Bonwick *objectp = 0; 98b24ab676SJeff Bonwick } 99b24ab676SJeff Bonwick 100b24ab676SJeff Bonwick static int 101b24ab676SJeff Bonwick ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 102b24ab676SJeff Bonwick { 10330f608a4SGeorge Wilson ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 10430f608a4SGeorge Wilson dmu_object_info_t doi; 105b24ab676SJeff Bonwick char name[DDT_NAMELEN]; 106b24ab676SJeff Bonwick int error; 107b24ab676SJeff Bonwick 108b24ab676SJeff Bonwick ddt_object_name(ddt, type, class, name); 109b24ab676SJeff Bonwick 110b24ab676SJeff Bonwick error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, 111b24ab676SJeff Bonwick sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); 112b24ab676SJeff Bonwick 113b24ab676SJeff Bonwick if (error) 114b24ab676SJeff Bonwick return (error); 115b24ab676SJeff Bonwick 116b24ab676SJeff Bonwick error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 117b24ab676SJeff Bonwick sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 118b24ab676SJeff Bonwick &ddt->ddt_histogram[type][class]); 119b24ab676SJeff Bonwick 12030f608a4SGeorge Wilson /* 12130f608a4SGeorge Wilson * Seed the cached statistics. 12230f608a4SGeorge Wilson */ 12330f608a4SGeorge Wilson VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 12430f608a4SGeorge Wilson 12530f608a4SGeorge Wilson ddo->ddo_count = ddt_object_count(ddt, type, class); 12630f608a4SGeorge Wilson ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 12730f608a4SGeorge Wilson ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 12830f608a4SGeorge Wilson 129b24ab676SJeff Bonwick ASSERT(error == 0); 130b24ab676SJeff Bonwick return (error); 131b24ab676SJeff Bonwick } 132b24ab676SJeff Bonwick 133b24ab676SJeff Bonwick static void 134b24ab676SJeff Bonwick ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 135b24ab676SJeff Bonwick dmu_tx_t *tx) 136b24ab676SJeff Bonwick { 13730f608a4SGeorge Wilson ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; 13830f608a4SGeorge Wilson dmu_object_info_t doi; 139b24ab676SJeff Bonwick char name[DDT_NAMELEN]; 140b24ab676SJeff Bonwick 141b24ab676SJeff Bonwick ddt_object_name(ddt, type, class, name); 142b24ab676SJeff Bonwick 143b24ab676SJeff Bonwick VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, 144b24ab676SJeff Bonwick sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), 145b24ab676SJeff Bonwick &ddt->ddt_histogram[type][class], tx) == 0); 14630f608a4SGeorge Wilson 14730f608a4SGeorge Wilson /* 14830f608a4SGeorge Wilson * Cache DDT statistics; this is the only time they'll change. 14930f608a4SGeorge Wilson */ 15030f608a4SGeorge Wilson VERIFY(ddt_object_info(ddt, type, class, &doi) == 0); 15130f608a4SGeorge Wilson 15230f608a4SGeorge Wilson ddo->ddo_count = ddt_object_count(ddt, type, class); 15330f608a4SGeorge Wilson ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; 15430f608a4SGeorge Wilson ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; 155b24ab676SJeff Bonwick } 156b24ab676SJeff Bonwick 157b24ab676SJeff Bonwick static int 158b24ab676SJeff Bonwick ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 159b24ab676SJeff Bonwick ddt_entry_t *dde) 160b24ab676SJeff Bonwick { 161b24ab676SJeff Bonwick if (!ddt_object_exists(ddt, type, class)) 162b24ab676SJeff Bonwick return (ENOENT); 163b24ab676SJeff Bonwick 164b24ab676SJeff Bonwick return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, 165b24ab676SJeff Bonwick ddt->ddt_object[type][class], dde)); 166b24ab676SJeff Bonwick } 167b24ab676SJeff Bonwick 168c7cd2421SGeorge Wilson static void 169c7cd2421SGeorge Wilson ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 170c7cd2421SGeorge Wilson ddt_entry_t *dde) 171c7cd2421SGeorge Wilson { 172c7cd2421SGeorge Wilson if (!ddt_object_exists(ddt, type, class)) 173c7cd2421SGeorge Wilson return; 174c7cd2421SGeorge Wilson 175c7cd2421SGeorge Wilson ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os, 176c7cd2421SGeorge Wilson ddt->ddt_object[type][class], dde); 177c7cd2421SGeorge Wilson } 178c7cd2421SGeorge Wilson 1793f9d6ad7SLin Ling int 180b24ab676SJeff Bonwick ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 181b24ab676SJeff Bonwick ddt_entry_t *dde, dmu_tx_t *tx) 182b24ab676SJeff Bonwick { 183b24ab676SJeff Bonwick ASSERT(ddt_object_exists(ddt, type, class)); 184b24ab676SJeff Bonwick 185b24ab676SJeff Bonwick return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, 186b24ab676SJeff Bonwick ddt->ddt_object[type][class], dde, tx)); 187b24ab676SJeff Bonwick } 188b24ab676SJeff Bonwick 189b24ab676SJeff Bonwick static int 190b24ab676SJeff Bonwick ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 191b24ab676SJeff Bonwick ddt_entry_t *dde, dmu_tx_t *tx) 192b24ab676SJeff Bonwick { 193b24ab676SJeff Bonwick ASSERT(ddt_object_exists(ddt, type, class)); 194b24ab676SJeff Bonwick 195b24ab676SJeff Bonwick return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, 196b24ab676SJeff Bonwick ddt->ddt_object[type][class], dde, tx)); 197b24ab676SJeff Bonwick } 198b24ab676SJeff Bonwick 199b24ab676SJeff Bonwick int 200b24ab676SJeff Bonwick ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 201bbfd46c4SJeff Bonwick uint64_t *walk, ddt_entry_t *dde) 202b24ab676SJeff Bonwick { 203b24ab676SJeff Bonwick ASSERT(ddt_object_exists(ddt, type, class)); 204b24ab676SJeff Bonwick 205b24ab676SJeff Bonwick return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, 206b24ab676SJeff Bonwick ddt->ddt_object[type][class], dde, walk)); 207b24ab676SJeff Bonwick } 208b24ab676SJeff Bonwick 209b24ab676SJeff Bonwick uint64_t 210b24ab676SJeff Bonwick ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 211b24ab676SJeff Bonwick { 212b24ab676SJeff Bonwick ASSERT(ddt_object_exists(ddt, type, class)); 213b24ab676SJeff Bonwick 214b24ab676SJeff Bonwick return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, 215b24ab676SJeff Bonwick ddt->ddt_object[type][class])); 216b24ab676SJeff Bonwick } 217b24ab676SJeff Bonwick 218b24ab676SJeff Bonwick int 219b24ab676SJeff Bonwick ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 220b24ab676SJeff Bonwick dmu_object_info_t *doi) 221b24ab676SJeff Bonwick { 222b24ab676SJeff Bonwick if (!ddt_object_exists(ddt, type, class)) 223b24ab676SJeff Bonwick return (ENOENT); 224b24ab676SJeff Bonwick 225b24ab676SJeff Bonwick return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], 226b24ab676SJeff Bonwick doi)); 227b24ab676SJeff Bonwick } 228b24ab676SJeff Bonwick 229b24ab676SJeff Bonwick boolean_t 230b24ab676SJeff Bonwick ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) 231b24ab676SJeff Bonwick { 232b24ab676SJeff Bonwick return (!!ddt->ddt_object[type][class]); 233b24ab676SJeff Bonwick } 234b24ab676SJeff Bonwick 235b24ab676SJeff Bonwick void 236b24ab676SJeff Bonwick ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, 237b24ab676SJeff Bonwick char *name) 238b24ab676SJeff Bonwick { 239b24ab676SJeff Bonwick (void) sprintf(name, DMU_POOL_DDT, 240b24ab676SJeff Bonwick zio_checksum_table[ddt->ddt_checksum].ci_name, 241b24ab676SJeff Bonwick ddt_ops[type]->ddt_op_name, ddt_class_name[class]); 242b24ab676SJeff Bonwick } 243b24ab676SJeff Bonwick 244b24ab676SJeff Bonwick void 245b24ab676SJeff Bonwick ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) 246b24ab676SJeff Bonwick { 247b24ab676SJeff Bonwick ASSERT(txg != 0); 248b24ab676SJeff Bonwick 249b24ab676SJeff Bonwick for (int d = 0; d < SPA_DVAS_PER_BP; d++) 250b24ab676SJeff Bonwick bp->blk_dva[d] = ddp->ddp_dva[d]; 251b24ab676SJeff Bonwick BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); 252b24ab676SJeff Bonwick } 253b24ab676SJeff Bonwick 254b24ab676SJeff Bonwick void 255bbfd46c4SJeff Bonwick ddt_bp_create(enum zio_checksum checksum, 256bbfd46c4SJeff Bonwick const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) 257b24ab676SJeff Bonwick { 258b24ab676SJeff Bonwick BP_ZERO(bp); 259b24ab676SJeff Bonwick 260b24ab676SJeff Bonwick if (ddp != NULL) 261b24ab676SJeff Bonwick ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); 262b24ab676SJeff Bonwick 263b24ab676SJeff Bonwick bp->blk_cksum = ddk->ddk_cksum; 2643f9d6ad7SLin Ling bp->blk_fill = 1; 265b24ab676SJeff Bonwick 266b24ab676SJeff Bonwick BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); 267b24ab676SJeff Bonwick BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); 268b24ab676SJeff Bonwick BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); 269bbfd46c4SJeff Bonwick BP_SET_CHECKSUM(bp, checksum); 2703f9d6ad7SLin Ling BP_SET_TYPE(bp, DMU_OT_DEDUP); 271b24ab676SJeff Bonwick BP_SET_LEVEL(bp, 0); 272b24ab676SJeff Bonwick BP_SET_DEDUP(bp, 0); 273b24ab676SJeff Bonwick BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 274b24ab676SJeff Bonwick } 275b24ab676SJeff Bonwick 276b24ab676SJeff Bonwick void 277b24ab676SJeff Bonwick ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) 278b24ab676SJeff Bonwick { 279b24ab676SJeff Bonwick ddk->ddk_cksum = bp->blk_cksum; 280b24ab676SJeff Bonwick ddk->ddk_prop = 0; 281b24ab676SJeff Bonwick 282b24ab676SJeff Bonwick DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); 283b24ab676SJeff Bonwick DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); 284b24ab676SJeff Bonwick DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); 285b24ab676SJeff Bonwick } 286b24ab676SJeff Bonwick 287b24ab676SJeff Bonwick void 288b24ab676SJeff Bonwick ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) 289b24ab676SJeff Bonwick { 290b24ab676SJeff Bonwick ASSERT(ddp->ddp_phys_birth == 0); 291b24ab676SJeff Bonwick 292b24ab676SJeff Bonwick for (int d = 0; d < SPA_DVAS_PER_BP; d++) 293b24ab676SJeff Bonwick ddp->ddp_dva[d] = bp->blk_dva[d]; 294b24ab676SJeff Bonwick ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); 295b24ab676SJeff Bonwick } 296b24ab676SJeff Bonwick 297b24ab676SJeff Bonwick void 298b24ab676SJeff Bonwick ddt_phys_clear(ddt_phys_t *ddp) 299b24ab676SJeff Bonwick { 300b24ab676SJeff Bonwick bzero(ddp, sizeof (*ddp)); 301b24ab676SJeff Bonwick } 302b24ab676SJeff Bonwick 303b24ab676SJeff Bonwick void 304b24ab676SJeff Bonwick ddt_phys_addref(ddt_phys_t *ddp) 305b24ab676SJeff Bonwick { 306b24ab676SJeff Bonwick ddp->ddp_refcnt++; 307b24ab676SJeff Bonwick } 308b24ab676SJeff Bonwick 309b24ab676SJeff Bonwick void 310b24ab676SJeff Bonwick ddt_phys_decref(ddt_phys_t *ddp) 311b24ab676SJeff Bonwick { 312b24ab676SJeff Bonwick ASSERT((int64_t)ddp->ddp_refcnt > 0); 313b24ab676SJeff Bonwick ddp->ddp_refcnt--; 314b24ab676SJeff Bonwick } 315b24ab676SJeff Bonwick 316b24ab676SJeff Bonwick void 317b24ab676SJeff Bonwick ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) 318b24ab676SJeff Bonwick { 319b24ab676SJeff Bonwick blkptr_t blk; 320b24ab676SJeff Bonwick 321bbfd46c4SJeff Bonwick ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 322b24ab676SJeff Bonwick ddt_phys_clear(ddp); 323b24ab676SJeff Bonwick zio_free(ddt->ddt_spa, txg, &blk); 324b24ab676SJeff Bonwick } 325b24ab676SJeff Bonwick 326b24ab676SJeff Bonwick ddt_phys_t * 327b24ab676SJeff Bonwick ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) 328b24ab676SJeff Bonwick { 329b24ab676SJeff Bonwick ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; 330b24ab676SJeff Bonwick 331b24ab676SJeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 332b24ab676SJeff Bonwick if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && 333b24ab676SJeff Bonwick BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) 334b24ab676SJeff Bonwick return (ddp); 335b24ab676SJeff Bonwick } 336b24ab676SJeff Bonwick return (NULL); 337b24ab676SJeff Bonwick } 338b24ab676SJeff Bonwick 339b24ab676SJeff Bonwick uint64_t 340b24ab676SJeff Bonwick ddt_phys_total_refcnt(const ddt_entry_t *dde) 341b24ab676SJeff Bonwick { 342b24ab676SJeff Bonwick uint64_t refcnt = 0; 343b24ab676SJeff Bonwick 344b24ab676SJeff Bonwick for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) 345b24ab676SJeff Bonwick refcnt += dde->dde_phys[p].ddp_refcnt; 346b24ab676SJeff Bonwick 347b24ab676SJeff Bonwick return (refcnt); 348b24ab676SJeff Bonwick } 349b24ab676SJeff Bonwick 350b24ab676SJeff Bonwick static void 351b24ab676SJeff Bonwick ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) 352b24ab676SJeff Bonwick { 353b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 354b24ab676SJeff Bonwick ddt_phys_t *ddp = dde->dde_phys; 355b24ab676SJeff Bonwick ddt_key_t *ddk = &dde->dde_key; 356b24ab676SJeff Bonwick uint64_t lsize = DDK_GET_LSIZE(ddk); 357b24ab676SJeff Bonwick uint64_t psize = DDK_GET_PSIZE(ddk); 358b24ab676SJeff Bonwick 359b24ab676SJeff Bonwick bzero(dds, sizeof (*dds)); 360b24ab676SJeff Bonwick 361b24ab676SJeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 362b24ab676SJeff Bonwick uint64_t dsize = 0; 363b24ab676SJeff Bonwick uint64_t refcnt = ddp->ddp_refcnt; 364b24ab676SJeff Bonwick 365b24ab676SJeff Bonwick if (ddp->ddp_phys_birth == 0) 366b24ab676SJeff Bonwick continue; 367b24ab676SJeff Bonwick 368b24ab676SJeff Bonwick for (int d = 0; d < SPA_DVAS_PER_BP; d++) 369b24ab676SJeff Bonwick dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); 370b24ab676SJeff Bonwick 371b24ab676SJeff Bonwick dds->dds_blocks += 1; 372b24ab676SJeff Bonwick dds->dds_lsize += lsize; 373b24ab676SJeff Bonwick dds->dds_psize += psize; 374b24ab676SJeff Bonwick dds->dds_dsize += dsize; 375b24ab676SJeff Bonwick 376b24ab676SJeff Bonwick dds->dds_ref_blocks += refcnt; 377b24ab676SJeff Bonwick dds->dds_ref_lsize += lsize * refcnt; 378b24ab676SJeff Bonwick dds->dds_ref_psize += psize * refcnt; 379b24ab676SJeff Bonwick dds->dds_ref_dsize += dsize * refcnt; 380b24ab676SJeff Bonwick } 381b24ab676SJeff Bonwick } 382b24ab676SJeff Bonwick 383b24ab676SJeff Bonwick void 384b24ab676SJeff Bonwick ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) 385b24ab676SJeff Bonwick { 386b24ab676SJeff Bonwick const uint64_t *s = (const uint64_t *)src; 387b24ab676SJeff Bonwick uint64_t *d = (uint64_t *)dst; 388b24ab676SJeff Bonwick uint64_t *d_end = (uint64_t *)(dst + 1); 389b24ab676SJeff Bonwick 390b24ab676SJeff Bonwick ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ 391b24ab676SJeff Bonwick 392b24ab676SJeff Bonwick while (d < d_end) 393b24ab676SJeff Bonwick *d++ += (*s++ ^ neg) - neg; 394b24ab676SJeff Bonwick } 395b24ab676SJeff Bonwick 396b24ab676SJeff Bonwick static void 397b24ab676SJeff Bonwick ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) 398b24ab676SJeff Bonwick { 399b24ab676SJeff Bonwick ddt_stat_t dds; 400b24ab676SJeff Bonwick ddt_histogram_t *ddh; 401b24ab676SJeff Bonwick int bucket; 402b24ab676SJeff Bonwick 403b24ab676SJeff Bonwick ddt_stat_generate(ddt, dde, &dds); 404b24ab676SJeff Bonwick 405b24ab676SJeff Bonwick bucket = highbit(dds.dds_ref_blocks) - 1; 406b24ab676SJeff Bonwick ASSERT(bucket >= 0); 407b24ab676SJeff Bonwick 408b24ab676SJeff Bonwick ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; 409b24ab676SJeff Bonwick 410b24ab676SJeff Bonwick ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); 411b24ab676SJeff Bonwick } 412b24ab676SJeff Bonwick 413b24ab676SJeff Bonwick void 414b24ab676SJeff Bonwick ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) 415b24ab676SJeff Bonwick { 416b24ab676SJeff Bonwick for (int h = 0; h < 64; h++) 417b24ab676SJeff Bonwick ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); 418b24ab676SJeff Bonwick } 419b24ab676SJeff Bonwick 420b24ab676SJeff Bonwick void 421b24ab676SJeff Bonwick ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) 422b24ab676SJeff Bonwick { 423b24ab676SJeff Bonwick bzero(dds, sizeof (*dds)); 424b24ab676SJeff Bonwick 425b24ab676SJeff Bonwick for (int h = 0; h < 64; h++) 426b24ab676SJeff Bonwick ddt_stat_add(dds, &ddh->ddh_stat[h], 0); 427b24ab676SJeff Bonwick } 428b24ab676SJeff Bonwick 429b24ab676SJeff Bonwick boolean_t 430b24ab676SJeff Bonwick ddt_histogram_empty(const ddt_histogram_t *ddh) 431b24ab676SJeff Bonwick { 432b24ab676SJeff Bonwick const uint64_t *s = (const uint64_t *)ddh; 433b24ab676SJeff Bonwick const uint64_t *s_end = (const uint64_t *)(ddh + 1); 434b24ab676SJeff Bonwick 435b24ab676SJeff Bonwick while (s < s_end) 436b24ab676SJeff Bonwick if (*s++ != 0) 437b24ab676SJeff Bonwick return (B_FALSE); 438b24ab676SJeff Bonwick 439b24ab676SJeff Bonwick return (B_TRUE); 440b24ab676SJeff Bonwick } 441b24ab676SJeff Bonwick 4429eb19f4dSGeorge Wilson void 44330f608a4SGeorge Wilson ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) 444b24ab676SJeff Bonwick { 44530f608a4SGeorge Wilson /* Sum the statistics we cached in ddt_object_sync(). */ 446b24ab676SJeff Bonwick for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 447b24ab676SJeff Bonwick ddt_t *ddt = spa->spa_ddt[c]; 448b24ab676SJeff Bonwick for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 449b24ab676SJeff Bonwick for (enum ddt_class class = 0; class < DDT_CLASSES; 450b24ab676SJeff Bonwick class++) { 45130f608a4SGeorge Wilson ddt_object_t *ddo = 45230f608a4SGeorge Wilson &ddt->ddt_object_stats[type][class]; 45330f608a4SGeorge Wilson ddo_total->ddo_count += ddo->ddo_count; 45430f608a4SGeorge Wilson ddo_total->ddo_dspace += ddo->ddo_dspace; 45530f608a4SGeorge Wilson ddo_total->ddo_mspace += ddo->ddo_mspace; 4569eb19f4dSGeorge Wilson } 4579eb19f4dSGeorge Wilson } 4589eb19f4dSGeorge Wilson } 45930f608a4SGeorge Wilson 46030f608a4SGeorge Wilson /* ... and compute the averages. */ 46130f608a4SGeorge Wilson if (ddo_total->ddo_count != 0) { 46230f608a4SGeorge Wilson ddo_total->ddo_dspace /= ddo_total->ddo_count; 46330f608a4SGeorge Wilson ddo_total->ddo_mspace /= ddo_total->ddo_count; 46430f608a4SGeorge Wilson } else { 46530f608a4SGeorge Wilson ASSERT(ddo_total->ddo_dspace == 0); 46630f608a4SGeorge Wilson ASSERT(ddo_total->ddo_mspace == 0); 46730f608a4SGeorge Wilson } 4689eb19f4dSGeorge Wilson } 4699eb19f4dSGeorge Wilson 4709eb19f4dSGeorge Wilson void 4719eb19f4dSGeorge Wilson ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) 4729eb19f4dSGeorge Wilson { 4739eb19f4dSGeorge Wilson for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 4749eb19f4dSGeorge Wilson ddt_t *ddt = spa->spa_ddt[c]; 4759eb19f4dSGeorge Wilson for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 4769eb19f4dSGeorge Wilson for (enum ddt_class class = 0; class < DDT_CLASSES; 4779eb19f4dSGeorge Wilson class++) { 4789eb19f4dSGeorge Wilson ddt_histogram_add(ddh, 47930f608a4SGeorge Wilson &ddt->ddt_histogram_cache[type][class]); 480b24ab676SJeff Bonwick } 481b24ab676SJeff Bonwick } 482b24ab676SJeff Bonwick } 4839eb19f4dSGeorge Wilson } 4849eb19f4dSGeorge Wilson 4859eb19f4dSGeorge Wilson void 4869eb19f4dSGeorge Wilson ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) 4879eb19f4dSGeorge Wilson { 4889eb19f4dSGeorge Wilson ddt_histogram_t *ddh_total; 489b24ab676SJeff Bonwick 4909eb19f4dSGeorge Wilson ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); 4919eb19f4dSGeorge Wilson ddt_get_dedup_histogram(spa, ddh_total); 4929eb19f4dSGeorge Wilson ddt_histogram_stat(dds_total, ddh_total); 4939eb19f4dSGeorge Wilson kmem_free(ddh_total, sizeof (ddt_histogram_t)); 494485bbbf5SGeorge Wilson } 495485bbbf5SGeorge Wilson 496485bbbf5SGeorge Wilson uint64_t 497485bbbf5SGeorge Wilson ddt_get_dedup_dspace(spa_t *spa) 498485bbbf5SGeorge Wilson { 499485bbbf5SGeorge Wilson ddt_stat_t dds_total = { 0 }; 500485bbbf5SGeorge Wilson 501485bbbf5SGeorge Wilson ddt_get_dedup_stats(spa, &dds_total); 502485bbbf5SGeorge Wilson return (dds_total.dds_ref_dsize - dds_total.dds_dsize); 503485bbbf5SGeorge Wilson } 504485bbbf5SGeorge Wilson 505485bbbf5SGeorge Wilson uint64_t 506485bbbf5SGeorge Wilson ddt_get_pool_dedup_ratio(spa_t *spa) 507485bbbf5SGeorge Wilson { 508485bbbf5SGeorge Wilson ddt_stat_t dds_total = { 0 }; 509b24ab676SJeff Bonwick 510485bbbf5SGeorge Wilson ddt_get_dedup_stats(spa, &dds_total); 511b24ab676SJeff Bonwick if (dds_total.dds_dsize == 0) 512b24ab676SJeff Bonwick return (100); 513b24ab676SJeff Bonwick 514b24ab676SJeff Bonwick return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); 515b24ab676SJeff Bonwick } 516b24ab676SJeff Bonwick 517b24ab676SJeff Bonwick int 518b24ab676SJeff Bonwick ddt_ditto_copies_needed(ddt_t *ddt, ddt_entry_t *dde, ddt_phys_t *ddp_willref) 519b24ab676SJeff Bonwick { 520b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 521b24ab676SJeff Bonwick uint64_t total_refcnt = 0; 522b24ab676SJeff Bonwick uint64_t ditto = spa->spa_dedup_ditto; 523b24ab676SJeff Bonwick int total_copies = 0; 524b24ab676SJeff Bonwick int desired_copies = 0; 525b24ab676SJeff Bonwick 526b24ab676SJeff Bonwick for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 527b24ab676SJeff Bonwick ddt_phys_t *ddp = &dde->dde_phys[p]; 528b24ab676SJeff Bonwick zio_t *zio = dde->dde_lead_zio[p]; 529b24ab676SJeff Bonwick uint64_t refcnt = ddp->ddp_refcnt; /* committed refs */ 530b24ab676SJeff Bonwick if (zio != NULL) 531b24ab676SJeff Bonwick refcnt += zio->io_parent_count; /* pending refs */ 532b24ab676SJeff Bonwick if (ddp == ddp_willref) 533b24ab676SJeff Bonwick refcnt++; /* caller's ref */ 534b24ab676SJeff Bonwick if (refcnt != 0) { 535b24ab676SJeff Bonwick total_refcnt += refcnt; 536b24ab676SJeff Bonwick total_copies += p; 537b24ab676SJeff Bonwick } 538b24ab676SJeff Bonwick } 539b24ab676SJeff Bonwick 540b24ab676SJeff Bonwick if (ditto == 0 || ditto > UINT32_MAX) 541b24ab676SJeff Bonwick ditto = UINT32_MAX; 542b24ab676SJeff Bonwick 543b24ab676SJeff Bonwick if (total_refcnt >= 1) 544b24ab676SJeff Bonwick desired_copies++; 545b24ab676SJeff Bonwick if (total_refcnt >= ditto) 546b24ab676SJeff Bonwick desired_copies++; 547b24ab676SJeff Bonwick if (total_refcnt >= ditto * ditto) 548b24ab676SJeff Bonwick desired_copies++; 549b24ab676SJeff Bonwick 550b24ab676SJeff Bonwick return (MAX(desired_copies, total_copies) - total_copies); 551b24ab676SJeff Bonwick } 552b24ab676SJeff Bonwick 553b24ab676SJeff Bonwick int 554b24ab676SJeff Bonwick ddt_ditto_copies_present(ddt_entry_t *dde) 555b24ab676SJeff Bonwick { 556b24ab676SJeff Bonwick ddt_phys_t *ddp = &dde->dde_phys[DDT_PHYS_DITTO]; 557b24ab676SJeff Bonwick dva_t *dva = ddp->ddp_dva; 558b24ab676SJeff Bonwick int copies = 0 - DVA_GET_GANG(dva); 559b24ab676SJeff Bonwick 560b24ab676SJeff Bonwick for (int d = 0; d < SPA_DVAS_PER_BP; d++, dva++) 561b24ab676SJeff Bonwick if (DVA_IS_VALID(dva)) 562b24ab676SJeff Bonwick copies++; 563b24ab676SJeff Bonwick 564b24ab676SJeff Bonwick ASSERT(copies >= 0 && copies < SPA_DVAS_PER_BP); 565b24ab676SJeff Bonwick 566b24ab676SJeff Bonwick return (copies); 567b24ab676SJeff Bonwick } 568b24ab676SJeff Bonwick 569b24ab676SJeff Bonwick size_t 570b24ab676SJeff Bonwick ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len) 571b24ab676SJeff Bonwick { 572b24ab676SJeff Bonwick uchar_t *version = dst++; 573b24ab676SJeff Bonwick int cpfunc = ZIO_COMPRESS_ZLE; 574b24ab676SJeff Bonwick zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 575b24ab676SJeff Bonwick size_t c_len; 576b24ab676SJeff Bonwick 577b24ab676SJeff Bonwick ASSERT(d_len >= s_len + 1); /* no compression plus version byte */ 578b24ab676SJeff Bonwick 579b24ab676SJeff Bonwick c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level); 580b24ab676SJeff Bonwick 581b24ab676SJeff Bonwick if (c_len == s_len) { 582b24ab676SJeff Bonwick cpfunc = ZIO_COMPRESS_OFF; 583b24ab676SJeff Bonwick bcopy(src, dst, s_len); 584b24ab676SJeff Bonwick } 585b24ab676SJeff Bonwick 586b24ab676SJeff Bonwick *version = (ZFS_HOST_BYTEORDER & DDT_COMPRESS_BYTEORDER_MASK) | cpfunc; 587b24ab676SJeff Bonwick 588b24ab676SJeff Bonwick return (c_len + 1); 589b24ab676SJeff Bonwick } 590b24ab676SJeff Bonwick 591b24ab676SJeff Bonwick void 592b24ab676SJeff Bonwick ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len) 593b24ab676SJeff Bonwick { 594b24ab676SJeff Bonwick uchar_t version = *src++; 595b24ab676SJeff Bonwick int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK; 596b24ab676SJeff Bonwick zio_compress_info_t *ci = &zio_compress_table[cpfunc]; 597b24ab676SJeff Bonwick 598b24ab676SJeff Bonwick if (ci->ci_decompress != NULL) 599b24ab676SJeff Bonwick (void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level); 600b24ab676SJeff Bonwick else 601b24ab676SJeff Bonwick bcopy(src, dst, d_len); 602b24ab676SJeff Bonwick 603b24ab676SJeff Bonwick if ((version ^ ZFS_HOST_BYTEORDER) & DDT_COMPRESS_BYTEORDER_MASK) 604b24ab676SJeff Bonwick byteswap_uint64_array(dst, d_len); 605b24ab676SJeff Bonwick } 606b24ab676SJeff Bonwick 607b24ab676SJeff Bonwick ddt_t * 608b24ab676SJeff Bonwick ddt_select_by_checksum(spa_t *spa, enum zio_checksum c) 609b24ab676SJeff Bonwick { 610b24ab676SJeff Bonwick return (spa->spa_ddt[c]); 611b24ab676SJeff Bonwick } 612b24ab676SJeff Bonwick 613b24ab676SJeff Bonwick ddt_t * 614b24ab676SJeff Bonwick ddt_select(spa_t *spa, const blkptr_t *bp) 615b24ab676SJeff Bonwick { 616b24ab676SJeff Bonwick return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); 617b24ab676SJeff Bonwick } 618b24ab676SJeff Bonwick 619b24ab676SJeff Bonwick void 620b24ab676SJeff Bonwick ddt_enter(ddt_t *ddt) 621b24ab676SJeff Bonwick { 622b24ab676SJeff Bonwick mutex_enter(&ddt->ddt_lock); 623b24ab676SJeff Bonwick } 624b24ab676SJeff Bonwick 625b24ab676SJeff Bonwick void 626b24ab676SJeff Bonwick ddt_exit(ddt_t *ddt) 627b24ab676SJeff Bonwick { 628b24ab676SJeff Bonwick mutex_exit(&ddt->ddt_lock); 629b24ab676SJeff Bonwick } 630b24ab676SJeff Bonwick 631b24ab676SJeff Bonwick static ddt_entry_t * 632b24ab676SJeff Bonwick ddt_alloc(const ddt_key_t *ddk) 633b24ab676SJeff Bonwick { 634b24ab676SJeff Bonwick ddt_entry_t *dde; 635b24ab676SJeff Bonwick 636b24ab676SJeff Bonwick dde = kmem_zalloc(sizeof (ddt_entry_t), KM_SLEEP); 637b24ab676SJeff Bonwick cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); 638b24ab676SJeff Bonwick 639b24ab676SJeff Bonwick dde->dde_key = *ddk; 640b24ab676SJeff Bonwick 641b24ab676SJeff Bonwick return (dde); 642b24ab676SJeff Bonwick } 643b24ab676SJeff Bonwick 644b24ab676SJeff Bonwick static void 645b24ab676SJeff Bonwick ddt_free(ddt_entry_t *dde) 646b24ab676SJeff Bonwick { 647b24ab676SJeff Bonwick ASSERT(!dde->dde_loading); 648b24ab676SJeff Bonwick 649b24ab676SJeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++) 650b24ab676SJeff Bonwick ASSERT(dde->dde_lead_zio[p] == NULL); 651b24ab676SJeff Bonwick 652b24ab676SJeff Bonwick if (dde->dde_repair_data != NULL) 653b24ab676SJeff Bonwick zio_buf_free(dde->dde_repair_data, 654b24ab676SJeff Bonwick DDK_GET_PSIZE(&dde->dde_key)); 655b24ab676SJeff Bonwick 656b24ab676SJeff Bonwick cv_destroy(&dde->dde_cv); 657b24ab676SJeff Bonwick kmem_free(dde, sizeof (*dde)); 658b24ab676SJeff Bonwick } 659b24ab676SJeff Bonwick 660b24ab676SJeff Bonwick void 661b24ab676SJeff Bonwick ddt_remove(ddt_t *ddt, ddt_entry_t *dde) 662b24ab676SJeff Bonwick { 663b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 664b24ab676SJeff Bonwick 665b24ab676SJeff Bonwick avl_remove(&ddt->ddt_tree, dde); 666b24ab676SJeff Bonwick ddt_free(dde); 667b24ab676SJeff Bonwick } 668b24ab676SJeff Bonwick 669b24ab676SJeff Bonwick ddt_entry_t * 670b24ab676SJeff Bonwick ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) 671b24ab676SJeff Bonwick { 672b24ab676SJeff Bonwick ddt_entry_t *dde, dde_search; 673b24ab676SJeff Bonwick enum ddt_type type; 674b24ab676SJeff Bonwick enum ddt_class class; 675b24ab676SJeff Bonwick avl_index_t where; 676b24ab676SJeff Bonwick int error; 677b24ab676SJeff Bonwick 678b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&ddt->ddt_lock)); 679b24ab676SJeff Bonwick 680b24ab676SJeff Bonwick ddt_key_fill(&dde_search.dde_key, bp); 681b24ab676SJeff Bonwick 682b24ab676SJeff Bonwick dde = avl_find(&ddt->ddt_tree, &dde_search, &where); 683b24ab676SJeff Bonwick if (dde == NULL) { 684b24ab676SJeff Bonwick if (!add) 685b24ab676SJeff Bonwick return (NULL); 686b24ab676SJeff Bonwick dde = ddt_alloc(&dde_search.dde_key); 687b24ab676SJeff Bonwick avl_insert(&ddt->ddt_tree, dde, where); 688b24ab676SJeff Bonwick } 689b24ab676SJeff Bonwick 690b24ab676SJeff Bonwick while (dde->dde_loading) 691b24ab676SJeff Bonwick cv_wait(&dde->dde_cv, &ddt->ddt_lock); 692b24ab676SJeff Bonwick 693b24ab676SJeff Bonwick if (dde->dde_loaded) 694b24ab676SJeff Bonwick return (dde); 695b24ab676SJeff Bonwick 696b24ab676SJeff Bonwick dde->dde_loading = B_TRUE; 697b24ab676SJeff Bonwick 698b24ab676SJeff Bonwick ddt_exit(ddt); 699b24ab676SJeff Bonwick 700b24ab676SJeff Bonwick error = ENOENT; 701b24ab676SJeff Bonwick 702b24ab676SJeff Bonwick for (type = 0; type < DDT_TYPES; type++) { 703b24ab676SJeff Bonwick for (class = 0; class < DDT_CLASSES; class++) { 704b24ab676SJeff Bonwick error = ddt_object_lookup(ddt, type, class, dde); 705b24ab676SJeff Bonwick if (error != ENOENT) 706b24ab676SJeff Bonwick break; 707b24ab676SJeff Bonwick } 708b24ab676SJeff Bonwick if (error != ENOENT) 709b24ab676SJeff Bonwick break; 710b24ab676SJeff Bonwick } 711b24ab676SJeff Bonwick 712b24ab676SJeff Bonwick ASSERT(error == 0 || error == ENOENT); 713b24ab676SJeff Bonwick 714b24ab676SJeff Bonwick ddt_enter(ddt); 715b24ab676SJeff Bonwick 716b24ab676SJeff Bonwick ASSERT(dde->dde_loaded == B_FALSE); 717b24ab676SJeff Bonwick ASSERT(dde->dde_loading == B_TRUE); 718b24ab676SJeff Bonwick 719b24ab676SJeff Bonwick dde->dde_type = type; /* will be DDT_TYPES if no entry found */ 720b24ab676SJeff Bonwick dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ 721b24ab676SJeff Bonwick dde->dde_loaded = B_TRUE; 722b24ab676SJeff Bonwick dde->dde_loading = B_FALSE; 723b24ab676SJeff Bonwick 724b24ab676SJeff Bonwick if (error == 0) 725b24ab676SJeff Bonwick ddt_stat_update(ddt, dde, -1ULL); 726b24ab676SJeff Bonwick 727b24ab676SJeff Bonwick cv_broadcast(&dde->dde_cv); 728b24ab676SJeff Bonwick 729b24ab676SJeff Bonwick return (dde); 730b24ab676SJeff Bonwick } 731b24ab676SJeff Bonwick 732c7cd2421SGeorge Wilson void 733c7cd2421SGeorge Wilson ddt_prefetch(spa_t *spa, const blkptr_t *bp) 734c7cd2421SGeorge Wilson { 735c7cd2421SGeorge Wilson ddt_t *ddt; 736c7cd2421SGeorge Wilson ddt_entry_t dde; 737c7cd2421SGeorge Wilson 738837b568bSGeorge Wilson if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp)) 739c7cd2421SGeorge Wilson return; 740c7cd2421SGeorge Wilson 741c7cd2421SGeorge Wilson /* 742*ce053c9dSGeorge Wilson * We only remove the DDT once all tables are empty and only 743*ce053c9dSGeorge Wilson * prefetch dedup blocks when there are entries in the DDT. 744*ce053c9dSGeorge Wilson * Thus no locking is required as the DDT can't disappear on us. 745c7cd2421SGeorge Wilson */ 746c7cd2421SGeorge Wilson ddt = ddt_select(spa, bp); 747c7cd2421SGeorge Wilson ddt_key_fill(&dde.dde_key, bp); 748c7cd2421SGeorge Wilson 749c7cd2421SGeorge Wilson for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 750c7cd2421SGeorge Wilson for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 751c7cd2421SGeorge Wilson ddt_object_prefetch(ddt, type, class, &dde); 752c7cd2421SGeorge Wilson } 753c7cd2421SGeorge Wilson } 754c7cd2421SGeorge Wilson } 755c7cd2421SGeorge Wilson 756b24ab676SJeff Bonwick int 757b24ab676SJeff Bonwick ddt_entry_compare(const void *x1, const void *x2) 758b24ab676SJeff Bonwick { 759b24ab676SJeff Bonwick const ddt_entry_t *dde1 = x1; 760b24ab676SJeff Bonwick const ddt_entry_t *dde2 = x2; 761b24ab676SJeff Bonwick const uint64_t *u1 = (const uint64_t *)&dde1->dde_key; 762b24ab676SJeff Bonwick const uint64_t *u2 = (const uint64_t *)&dde2->dde_key; 763b24ab676SJeff Bonwick 764b24ab676SJeff Bonwick for (int i = 0; i < DDT_KEY_WORDS; i++) { 765b24ab676SJeff Bonwick if (u1[i] < u2[i]) 766b24ab676SJeff Bonwick return (-1); 767b24ab676SJeff Bonwick if (u1[i] > u2[i]) 768b24ab676SJeff Bonwick return (1); 769b24ab676SJeff Bonwick } 770b24ab676SJeff Bonwick 771b24ab676SJeff Bonwick return (0); 772b24ab676SJeff Bonwick } 773b24ab676SJeff Bonwick 774b24ab676SJeff Bonwick static ddt_t * 775b24ab676SJeff Bonwick ddt_table_alloc(spa_t *spa, enum zio_checksum c) 776b24ab676SJeff Bonwick { 777b24ab676SJeff Bonwick ddt_t *ddt; 778b24ab676SJeff Bonwick 779b24ab676SJeff Bonwick ddt = kmem_zalloc(sizeof (*ddt), KM_SLEEP); 780b24ab676SJeff Bonwick 781b24ab676SJeff Bonwick mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); 782b24ab676SJeff Bonwick avl_create(&ddt->ddt_tree, ddt_entry_compare, 783b24ab676SJeff Bonwick sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 784b24ab676SJeff Bonwick avl_create(&ddt->ddt_repair_tree, ddt_entry_compare, 785b24ab676SJeff Bonwick sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); 786b24ab676SJeff Bonwick ddt->ddt_checksum = c; 787b24ab676SJeff Bonwick ddt->ddt_spa = spa; 788b24ab676SJeff Bonwick ddt->ddt_os = spa->spa_meta_objset; 789b24ab676SJeff Bonwick 790b24ab676SJeff Bonwick return (ddt); 791b24ab676SJeff Bonwick } 792b24ab676SJeff Bonwick 793b24ab676SJeff Bonwick static void 794b24ab676SJeff Bonwick ddt_table_free(ddt_t *ddt) 795b24ab676SJeff Bonwick { 796b24ab676SJeff Bonwick ASSERT(avl_numnodes(&ddt->ddt_tree) == 0); 797b24ab676SJeff Bonwick ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0); 798b24ab676SJeff Bonwick avl_destroy(&ddt->ddt_tree); 799b24ab676SJeff Bonwick avl_destroy(&ddt->ddt_repair_tree); 800b24ab676SJeff Bonwick mutex_destroy(&ddt->ddt_lock); 801b24ab676SJeff Bonwick kmem_free(ddt, sizeof (*ddt)); 802b24ab676SJeff Bonwick } 803b24ab676SJeff Bonwick 804b24ab676SJeff Bonwick void 805b24ab676SJeff Bonwick ddt_create(spa_t *spa) 806b24ab676SJeff Bonwick { 807b24ab676SJeff Bonwick spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; 808b24ab676SJeff Bonwick 809b24ab676SJeff Bonwick for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) 810b24ab676SJeff Bonwick spa->spa_ddt[c] = ddt_table_alloc(spa, c); 811b24ab676SJeff Bonwick } 812b24ab676SJeff Bonwick 813b24ab676SJeff Bonwick int 814b24ab676SJeff Bonwick ddt_load(spa_t *spa) 815b24ab676SJeff Bonwick { 816b24ab676SJeff Bonwick int error; 817b24ab676SJeff Bonwick 818b24ab676SJeff Bonwick ddt_create(spa); 819b24ab676SJeff Bonwick 820b24ab676SJeff Bonwick error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 821b24ab676SJeff Bonwick DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 822b24ab676SJeff Bonwick &spa->spa_ddt_stat_object); 823b24ab676SJeff Bonwick 824b24ab676SJeff Bonwick if (error) 825b24ab676SJeff Bonwick return (error == ENOENT ? 0 : error); 826b24ab676SJeff Bonwick 827b24ab676SJeff Bonwick for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 82830f608a4SGeorge Wilson ddt_t *ddt = spa->spa_ddt[c]; 829b24ab676SJeff Bonwick for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 830b24ab676SJeff Bonwick for (enum ddt_class class = 0; class < DDT_CLASSES; 831b24ab676SJeff Bonwick class++) { 832b24ab676SJeff Bonwick error = ddt_object_load(ddt, type, class); 833b24ab676SJeff Bonwick if (error != 0 && error != ENOENT) 834b24ab676SJeff Bonwick return (error); 835b24ab676SJeff Bonwick } 836b24ab676SJeff Bonwick } 83730f608a4SGeorge Wilson 83830f608a4SGeorge Wilson /* 83930f608a4SGeorge Wilson * Seed the cached histograms. 84030f608a4SGeorge Wilson */ 84130f608a4SGeorge Wilson bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 84230f608a4SGeorge Wilson sizeof (ddt->ddt_histogram)); 843b24ab676SJeff Bonwick } 844b24ab676SJeff Bonwick 845b24ab676SJeff Bonwick return (0); 846b24ab676SJeff Bonwick } 847b24ab676SJeff Bonwick 848b24ab676SJeff Bonwick void 849b24ab676SJeff Bonwick ddt_unload(spa_t *spa) 850b24ab676SJeff Bonwick { 851b24ab676SJeff Bonwick for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 852b24ab676SJeff Bonwick if (spa->spa_ddt[c]) { 853b24ab676SJeff Bonwick ddt_table_free(spa->spa_ddt[c]); 854b24ab676SJeff Bonwick spa->spa_ddt[c] = NULL; 855b24ab676SJeff Bonwick } 856b24ab676SJeff Bonwick } 857b24ab676SJeff Bonwick } 858b24ab676SJeff Bonwick 859bbfd46c4SJeff Bonwick boolean_t 860bbfd46c4SJeff Bonwick ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) 861bbfd46c4SJeff Bonwick { 862bbfd46c4SJeff Bonwick ddt_t *ddt; 863bbfd46c4SJeff Bonwick ddt_entry_t dde; 864bbfd46c4SJeff Bonwick 865bbfd46c4SJeff Bonwick if (!BP_GET_DEDUP(bp)) 866bbfd46c4SJeff Bonwick return (B_FALSE); 867bbfd46c4SJeff Bonwick 868bbfd46c4SJeff Bonwick if (max_class == DDT_CLASS_UNIQUE) 869bbfd46c4SJeff Bonwick return (B_TRUE); 870bbfd46c4SJeff Bonwick 871bbfd46c4SJeff Bonwick ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; 872bbfd46c4SJeff Bonwick 873bbfd46c4SJeff Bonwick ddt_key_fill(&dde.dde_key, bp); 874bbfd46c4SJeff Bonwick 875bbfd46c4SJeff Bonwick for (enum ddt_type type = 0; type < DDT_TYPES; type++) 876bbfd46c4SJeff Bonwick for (enum ddt_class class = 0; class <= max_class; class++) 877bbfd46c4SJeff Bonwick if (ddt_object_lookup(ddt, type, class, &dde) == 0) 878bbfd46c4SJeff Bonwick return (B_TRUE); 879bbfd46c4SJeff Bonwick 880bbfd46c4SJeff Bonwick return (B_FALSE); 881bbfd46c4SJeff Bonwick } 882bbfd46c4SJeff Bonwick 883b24ab676SJeff Bonwick ddt_entry_t * 884b24ab676SJeff Bonwick ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) 885b24ab676SJeff Bonwick { 886b24ab676SJeff Bonwick ddt_key_t ddk; 887b24ab676SJeff Bonwick ddt_entry_t *dde; 888b24ab676SJeff Bonwick 889b24ab676SJeff Bonwick ddt_key_fill(&ddk, bp); 890b24ab676SJeff Bonwick 891b24ab676SJeff Bonwick dde = ddt_alloc(&ddk); 892b24ab676SJeff Bonwick 893b24ab676SJeff Bonwick for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 894b24ab676SJeff Bonwick for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 895b24ab676SJeff Bonwick /* 896b24ab676SJeff Bonwick * We can only do repair if there are multiple copies 897b24ab676SJeff Bonwick * of the block. For anything in the UNIQUE class, 898b24ab676SJeff Bonwick * there's definitely only one copy, so don't even try. 899b24ab676SJeff Bonwick */ 900b24ab676SJeff Bonwick if (class != DDT_CLASS_UNIQUE && 901b24ab676SJeff Bonwick ddt_object_lookup(ddt, type, class, dde) == 0) 902b24ab676SJeff Bonwick return (dde); 903b24ab676SJeff Bonwick } 904b24ab676SJeff Bonwick } 905b24ab676SJeff Bonwick 906b24ab676SJeff Bonwick bzero(dde->dde_phys, sizeof (dde->dde_phys)); 907b24ab676SJeff Bonwick 908b24ab676SJeff Bonwick return (dde); 909b24ab676SJeff Bonwick } 910b24ab676SJeff Bonwick 911b24ab676SJeff Bonwick void 912b24ab676SJeff Bonwick ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) 913b24ab676SJeff Bonwick { 914b24ab676SJeff Bonwick avl_index_t where; 915b24ab676SJeff Bonwick 916b24ab676SJeff Bonwick ddt_enter(ddt); 917b24ab676SJeff Bonwick 918b24ab676SJeff Bonwick if (dde->dde_repair_data != NULL && spa_writeable(ddt->ddt_spa) && 919b24ab676SJeff Bonwick avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) 920b24ab676SJeff Bonwick avl_insert(&ddt->ddt_repair_tree, dde, where); 921b24ab676SJeff Bonwick else 922b24ab676SJeff Bonwick ddt_free(dde); 923b24ab676SJeff Bonwick 924b24ab676SJeff Bonwick ddt_exit(ddt); 925b24ab676SJeff Bonwick } 926b24ab676SJeff Bonwick 927b24ab676SJeff Bonwick static void 928b24ab676SJeff Bonwick ddt_repair_entry_done(zio_t *zio) 929b24ab676SJeff Bonwick { 930b24ab676SJeff Bonwick ddt_entry_t *rdde = zio->io_private; 931b24ab676SJeff Bonwick 932b24ab676SJeff Bonwick ddt_free(rdde); 933b24ab676SJeff Bonwick } 934b24ab676SJeff Bonwick 935b24ab676SJeff Bonwick static void 936b24ab676SJeff Bonwick ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) 937b24ab676SJeff Bonwick { 938b24ab676SJeff Bonwick ddt_phys_t *ddp = dde->dde_phys; 939b24ab676SJeff Bonwick ddt_phys_t *rddp = rdde->dde_phys; 940b24ab676SJeff Bonwick ddt_key_t *ddk = &dde->dde_key; 941b24ab676SJeff Bonwick ddt_key_t *rddk = &rdde->dde_key; 942b24ab676SJeff Bonwick zio_t *zio; 943b24ab676SJeff Bonwick blkptr_t blk; 944b24ab676SJeff Bonwick 945b24ab676SJeff Bonwick zio = zio_null(rio, rio->io_spa, NULL, 946b24ab676SJeff Bonwick ddt_repair_entry_done, rdde, rio->io_flags); 947b24ab676SJeff Bonwick 948b24ab676SJeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { 949b24ab676SJeff Bonwick if (ddp->ddp_phys_birth == 0 || 950b24ab676SJeff Bonwick ddp->ddp_phys_birth != rddp->ddp_phys_birth || 951b24ab676SJeff Bonwick bcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) 952b24ab676SJeff Bonwick continue; 953bbfd46c4SJeff Bonwick ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); 954b24ab676SJeff Bonwick zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, 955b24ab676SJeff Bonwick rdde->dde_repair_data, DDK_GET_PSIZE(rddk), NULL, NULL, 956b24ab676SJeff Bonwick ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); 957b24ab676SJeff Bonwick } 958b24ab676SJeff Bonwick 959b24ab676SJeff Bonwick zio_nowait(zio); 960b24ab676SJeff Bonwick } 961b24ab676SJeff Bonwick 962b24ab676SJeff Bonwick static void 963b24ab676SJeff Bonwick ddt_repair_table(ddt_t *ddt, zio_t *rio) 964b24ab676SJeff Bonwick { 965b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 966b24ab676SJeff Bonwick ddt_entry_t *dde, *rdde_next, *rdde; 967b24ab676SJeff Bonwick avl_tree_t *t = &ddt->ddt_repair_tree; 968b24ab676SJeff Bonwick blkptr_t blk; 969b24ab676SJeff Bonwick 970b24ab676SJeff Bonwick if (spa_sync_pass(spa) > 1) 971b24ab676SJeff Bonwick return; 972b24ab676SJeff Bonwick 973b24ab676SJeff Bonwick ddt_enter(ddt); 974b24ab676SJeff Bonwick for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { 975b24ab676SJeff Bonwick rdde_next = AVL_NEXT(t, rdde); 976b24ab676SJeff Bonwick avl_remove(&ddt->ddt_repair_tree, rdde); 977b24ab676SJeff Bonwick ddt_exit(ddt); 978bbfd46c4SJeff Bonwick ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); 979b24ab676SJeff Bonwick dde = ddt_repair_start(ddt, &blk); 980b24ab676SJeff Bonwick ddt_repair_entry(ddt, dde, rdde, rio); 981b24ab676SJeff Bonwick ddt_repair_done(ddt, dde); 982b24ab676SJeff Bonwick ddt_enter(ddt); 983b24ab676SJeff Bonwick } 984b24ab676SJeff Bonwick ddt_exit(ddt); 985b24ab676SJeff Bonwick } 986b24ab676SJeff Bonwick 987b24ab676SJeff Bonwick static void 988b24ab676SJeff Bonwick ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) 989b24ab676SJeff Bonwick { 990bbfd46c4SJeff Bonwick dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; 991b24ab676SJeff Bonwick ddt_phys_t *ddp = dde->dde_phys; 992b24ab676SJeff Bonwick ddt_key_t *ddk = &dde->dde_key; 993b24ab676SJeff Bonwick enum ddt_type otype = dde->dde_type; 994b24ab676SJeff Bonwick enum ddt_type ntype = DDT_TYPE_CURRENT; 995b24ab676SJeff Bonwick enum ddt_class oclass = dde->dde_class; 996b24ab676SJeff Bonwick enum ddt_class nclass; 997b24ab676SJeff Bonwick uint64_t total_refcnt = 0; 998b24ab676SJeff Bonwick 999b24ab676SJeff Bonwick ASSERT(dde->dde_loaded); 1000b24ab676SJeff Bonwick ASSERT(!dde->dde_loading); 1001b24ab676SJeff Bonwick 1002b24ab676SJeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1003b24ab676SJeff Bonwick ASSERT(dde->dde_lead_zio[p] == NULL); 1004b24ab676SJeff Bonwick ASSERT((int64_t)ddp->ddp_refcnt >= 0); 1005b24ab676SJeff Bonwick if (ddp->ddp_phys_birth == 0) { 1006b24ab676SJeff Bonwick ASSERT(ddp->ddp_refcnt == 0); 1007b24ab676SJeff Bonwick continue; 1008b24ab676SJeff Bonwick } 1009b24ab676SJeff Bonwick if (p == DDT_PHYS_DITTO) { 1010b24ab676SJeff Bonwick if (ddt_ditto_copies_needed(ddt, dde, NULL) == 0) 1011b24ab676SJeff Bonwick ddt_phys_free(ddt, ddk, ddp, txg); 1012b24ab676SJeff Bonwick continue; 1013b24ab676SJeff Bonwick } 1014b24ab676SJeff Bonwick if (ddp->ddp_refcnt == 0) 1015b24ab676SJeff Bonwick ddt_phys_free(ddt, ddk, ddp, txg); 1016b24ab676SJeff Bonwick total_refcnt += ddp->ddp_refcnt; 1017b24ab676SJeff Bonwick } 1018b24ab676SJeff Bonwick 1019b24ab676SJeff Bonwick if (dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth != 0) 1020b24ab676SJeff Bonwick nclass = DDT_CLASS_DITTO; 1021b24ab676SJeff Bonwick else if (total_refcnt > 1) 1022b24ab676SJeff Bonwick nclass = DDT_CLASS_DUPLICATE; 1023b24ab676SJeff Bonwick else 1024b24ab676SJeff Bonwick nclass = DDT_CLASS_UNIQUE; 1025b24ab676SJeff Bonwick 1026b24ab676SJeff Bonwick if (otype != DDT_TYPES && 1027b24ab676SJeff Bonwick (otype != ntype || oclass != nclass || total_refcnt == 0)) { 1028b24ab676SJeff Bonwick VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0); 1029b24ab676SJeff Bonwick ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT); 1030b24ab676SJeff Bonwick } 1031b24ab676SJeff Bonwick 1032b24ab676SJeff Bonwick if (total_refcnt != 0) { 1033b24ab676SJeff Bonwick dde->dde_type = ntype; 1034b24ab676SJeff Bonwick dde->dde_class = nclass; 1035b24ab676SJeff Bonwick ddt_stat_update(ddt, dde, 0); 1036b24ab676SJeff Bonwick if (!ddt_object_exists(ddt, ntype, nclass)) 1037b24ab676SJeff Bonwick ddt_object_create(ddt, ntype, nclass, tx); 1038b24ab676SJeff Bonwick VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0); 1039bbfd46c4SJeff Bonwick 10403f9d6ad7SLin Ling /* 10413f9d6ad7SLin Ling * If the class changes, the order that we scan this bp 10423f9d6ad7SLin Ling * changes. If it decreases, we could miss it, so 10433f9d6ad7SLin Ling * scan it right now. (This covers both class changing 10443f9d6ad7SLin Ling * while we are doing ddt_walk(), and when we are 10453f9d6ad7SLin Ling * traversing.) 10463f9d6ad7SLin Ling */ 10473f9d6ad7SLin Ling if (nclass < oclass) { 10483f9d6ad7SLin Ling dsl_scan_ddt_entry(dp->dp_scan, 10493f9d6ad7SLin Ling ddt->ddt_checksum, dde, tx); 10503f9d6ad7SLin Ling } 1051b24ab676SJeff Bonwick } 1052b24ab676SJeff Bonwick } 1053b24ab676SJeff Bonwick 1054b24ab676SJeff Bonwick static void 1055b24ab676SJeff Bonwick ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) 1056b24ab676SJeff Bonwick { 1057b24ab676SJeff Bonwick spa_t *spa = ddt->ddt_spa; 1058b24ab676SJeff Bonwick ddt_entry_t *dde; 1059b24ab676SJeff Bonwick void *cookie = NULL; 1060b24ab676SJeff Bonwick 1061b24ab676SJeff Bonwick if (avl_numnodes(&ddt->ddt_tree) == 0) 1062b24ab676SJeff Bonwick return; 1063b24ab676SJeff Bonwick 1064b24ab676SJeff Bonwick ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP); 1065b24ab676SJeff Bonwick 1066b24ab676SJeff Bonwick if (spa->spa_ddt_stat_object == 0) { 1067b24ab676SJeff Bonwick spa->spa_ddt_stat_object = zap_create(ddt->ddt_os, 1068b24ab676SJeff Bonwick DMU_OT_DDT_STATS, DMU_OT_NONE, 0, tx); 1069b24ab676SJeff Bonwick VERIFY(zap_add(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, 1070b24ab676SJeff Bonwick DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, 1071b24ab676SJeff Bonwick &spa->spa_ddt_stat_object, tx) == 0); 1072b24ab676SJeff Bonwick } 1073b24ab676SJeff Bonwick 1074b24ab676SJeff Bonwick while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { 1075b24ab676SJeff Bonwick ddt_sync_entry(ddt, dde, tx, txg); 1076b24ab676SJeff Bonwick ddt_free(dde); 1077b24ab676SJeff Bonwick } 1078b24ab676SJeff Bonwick 1079b24ab676SJeff Bonwick for (enum ddt_type type = 0; type < DDT_TYPES; type++) { 1080*ce053c9dSGeorge Wilson uint64_t count = 0; 1081b24ab676SJeff Bonwick for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1082*ce053c9dSGeorge Wilson if (ddt_object_exists(ddt, type, class)) { 1083*ce053c9dSGeorge Wilson ddt_object_sync(ddt, type, class, tx); 1084*ce053c9dSGeorge Wilson count += ddt_object_count(ddt, type, class); 1085*ce053c9dSGeorge Wilson } 1086*ce053c9dSGeorge Wilson } 1087*ce053c9dSGeorge Wilson for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { 1088*ce053c9dSGeorge Wilson if (count == 0 && ddt_object_exists(ddt, type, class)) 1089b24ab676SJeff Bonwick ddt_object_destroy(ddt, type, class, tx); 1090b24ab676SJeff Bonwick } 1091b24ab676SJeff Bonwick } 109230f608a4SGeorge Wilson 109330f608a4SGeorge Wilson bcopy(ddt->ddt_histogram, &ddt->ddt_histogram_cache, 109430f608a4SGeorge Wilson sizeof (ddt->ddt_histogram)); 1095b24ab676SJeff Bonwick } 1096b24ab676SJeff Bonwick 1097b24ab676SJeff Bonwick void 1098b24ab676SJeff Bonwick ddt_sync(spa_t *spa, uint64_t txg) 1099b24ab676SJeff Bonwick { 1100b24ab676SJeff Bonwick dmu_tx_t *tx; 1101b24ab676SJeff Bonwick zio_t *rio = zio_root(spa, NULL, NULL, 1102b24ab676SJeff Bonwick ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 1103b24ab676SJeff Bonwick 1104b24ab676SJeff Bonwick ASSERT(spa_syncing_txg(spa) == txg); 1105b24ab676SJeff Bonwick 1106b24ab676SJeff Bonwick tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 1107b24ab676SJeff Bonwick 1108b24ab676SJeff Bonwick for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { 1109b24ab676SJeff Bonwick ddt_t *ddt = spa->spa_ddt[c]; 1110b24ab676SJeff Bonwick if (ddt == NULL) 1111b24ab676SJeff Bonwick continue; 1112b24ab676SJeff Bonwick ddt_sync_table(ddt, tx, txg); 1113b24ab676SJeff Bonwick ddt_repair_table(ddt, rio); 1114b24ab676SJeff Bonwick } 1115b24ab676SJeff Bonwick 1116b24ab676SJeff Bonwick (void) zio_wait(rio); 1117b24ab676SJeff Bonwick 1118b24ab676SJeff Bonwick dmu_tx_commit(tx); 1119b24ab676SJeff Bonwick } 1120bbfd46c4SJeff Bonwick 1121bbfd46c4SJeff Bonwick int 1122bbfd46c4SJeff Bonwick ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) 1123bbfd46c4SJeff Bonwick { 1124bbfd46c4SJeff Bonwick do { 1125bbfd46c4SJeff Bonwick do { 1126bbfd46c4SJeff Bonwick do { 1127bbfd46c4SJeff Bonwick ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; 1128bbfd46c4SJeff Bonwick int error = ENOENT; 1129bbfd46c4SJeff Bonwick if (ddt_object_exists(ddt, ddb->ddb_type, 1130bbfd46c4SJeff Bonwick ddb->ddb_class)) { 1131bbfd46c4SJeff Bonwick error = ddt_object_walk(ddt, 1132bbfd46c4SJeff Bonwick ddb->ddb_type, ddb->ddb_class, 1133bbfd46c4SJeff Bonwick &ddb->ddb_cursor, dde); 1134bbfd46c4SJeff Bonwick } 11353f9d6ad7SLin Ling dde->dde_type = ddb->ddb_type; 11363f9d6ad7SLin Ling dde->dde_class = ddb->ddb_class; 1137bbfd46c4SJeff Bonwick if (error == 0) 1138bbfd46c4SJeff Bonwick return (0); 1139bbfd46c4SJeff Bonwick if (error != ENOENT) 1140bbfd46c4SJeff Bonwick return (error); 1141bbfd46c4SJeff Bonwick ddb->ddb_cursor = 0; 1142bbfd46c4SJeff Bonwick } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); 1143bbfd46c4SJeff Bonwick ddb->ddb_checksum = 0; 1144bbfd46c4SJeff Bonwick } while (++ddb->ddb_type < DDT_TYPES); 1145bbfd46c4SJeff Bonwick ddb->ddb_type = 0; 1146bbfd46c4SJeff Bonwick } while (++ddb->ddb_class < DDT_CLASSES); 1147bbfd46c4SJeff Bonwick 1148bbfd46c4SJeff Bonwick return (ENOENT); 1149bbfd46c4SJeff Bonwick } 1150