1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 2394c2d0ebSMatthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 24bc9014e6SJustin Gibbs * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 25c3d26abcSMatthew Ahrens * Copyright (c) 2014 Integros [integros.com] 26f06dce2cSAndrew Stormont * Copyright 2017 RackTop Systems. 27fa9e4066Sahrens */ 28fa9e4066Sahrens 29fa9e4066Sahrens #include <sys/zfs_context.h> 30fa9e4066Sahrens #include <sys/dbuf.h> 31fa9e4066Sahrens #include <sys/dnode.h> 32fa9e4066Sahrens #include <sys/dmu.h> 33fa9e4066Sahrens #include <sys/dmu_impl.h> 34fa9e4066Sahrens #include <sys/dmu_tx.h> 35fa9e4066Sahrens #include <sys/dmu_objset.h> 36fa9e4066Sahrens #include <sys/dsl_dir.h> 37fa9e4066Sahrens #include <sys/dsl_dataset.h> 38fa9e4066Sahrens #include <sys/spa.h> 39fa9e4066Sahrens #include <sys/zio.h> 40fa9e4066Sahrens #include <sys/dmu_zfetch.h> 41bf16b11eSMatthew Ahrens #include <sys/range_tree.h> 42fa9e4066Sahrens 4354811da5SToomas Soome dnode_stats_t dnode_stats = { 4454811da5SToomas Soome { "dnode_hold_dbuf_hold", KSTAT_DATA_UINT64 }, 4554811da5SToomas Soome { "dnode_hold_dbuf_read", KSTAT_DATA_UINT64 }, 4654811da5SToomas Soome { "dnode_hold_alloc_hits", KSTAT_DATA_UINT64 }, 4754811da5SToomas Soome { "dnode_hold_alloc_misses", KSTAT_DATA_UINT64 }, 4854811da5SToomas Soome { "dnode_hold_alloc_interior", KSTAT_DATA_UINT64 }, 4954811da5SToomas Soome { "dnode_hold_alloc_lock_retry", KSTAT_DATA_UINT64 }, 5054811da5SToomas Soome { "dnode_hold_alloc_lock_misses", KSTAT_DATA_UINT64 }, 5154811da5SToomas Soome { "dnode_hold_alloc_type_none", KSTAT_DATA_UINT64 }, 5254811da5SToomas Soome { "dnode_hold_free_hits", KSTAT_DATA_UINT64 }, 5354811da5SToomas Soome { "dnode_hold_free_misses", KSTAT_DATA_UINT64 }, 5454811da5SToomas Soome { "dnode_hold_free_lock_misses", KSTAT_DATA_UINT64 }, 5554811da5SToomas Soome { "dnode_hold_free_lock_retry", KSTAT_DATA_UINT64 }, 5654811da5SToomas Soome { "dnode_hold_free_overflow", KSTAT_DATA_UINT64 }, 5754811da5SToomas Soome { "dnode_hold_free_refcount", KSTAT_DATA_UINT64 }, 5854811da5SToomas Soome { "dnode_hold_free_txg", KSTAT_DATA_UINT64 }, 5954811da5SToomas Soome { "dnode_free_interior_lock_retry", KSTAT_DATA_UINT64 }, 6054811da5SToomas Soome { "dnode_allocate", KSTAT_DATA_UINT64 }, 6154811da5SToomas Soome { "dnode_reallocate", KSTAT_DATA_UINT64 }, 6254811da5SToomas Soome { "dnode_buf_evict", KSTAT_DATA_UINT64 }, 6354811da5SToomas Soome { "dnode_alloc_next_chunk", KSTAT_DATA_UINT64 }, 6454811da5SToomas Soome { "dnode_alloc_race", KSTAT_DATA_UINT64 }, 6554811da5SToomas Soome { "dnode_alloc_next_block", KSTAT_DATA_UINT64 }, 6654811da5SToomas Soome { "dnode_move_invalid", KSTAT_DATA_UINT64 }, 6754811da5SToomas Soome { "dnode_move_recheck1", KSTAT_DATA_UINT64 }, 6854811da5SToomas Soome { "dnode_move_recheck2", KSTAT_DATA_UINT64 }, 6954811da5SToomas Soome { "dnode_move_special", KSTAT_DATA_UINT64 }, 7054811da5SToomas Soome { "dnode_move_handle", KSTAT_DATA_UINT64 }, 7154811da5SToomas Soome { "dnode_move_rwlock", KSTAT_DATA_UINT64 }, 7254811da5SToomas Soome { "dnode_move_active", KSTAT_DATA_UINT64 }, 7354811da5SToomas Soome }; 7454811da5SToomas Soome 7554811da5SToomas Soome static kstat_t *dnode_ksp; 76fa9e4066Sahrens static kmem_cache_t *dnode_cache; 77fa9e4066Sahrens 78fa9e4066Sahrens static dnode_phys_t dnode_phys_zero; 79fa9e4066Sahrens 80fa9e4066Sahrens int zfs_default_bs = SPA_MINBLOCKSHIFT; 81fa9e4066Sahrens int zfs_default_ibs = DN_MAX_INDBLKSHIFT; 82fa9e4066Sahrens 83f06dce2cSAndrew Stormont #ifdef _KERNEL 84744947dcSTom Erickson static kmem_cbrc_t dnode_move(void *, void *, size_t, void *); 85f06dce2cSAndrew Stormont #endif /* _KERNEL */ 86744947dcSTom Erickson 870f6d88adSAlex Reece static int 880f6d88adSAlex Reece dbuf_compare(const void *x1, const void *x2) 890f6d88adSAlex Reece { 900f6d88adSAlex Reece const dmu_buf_impl_t *d1 = x1; 910f6d88adSAlex Reece const dmu_buf_impl_t *d2 = x2; 920f6d88adSAlex Reece 93*c4ab0d3fSGvozden Neskovic int cmp = AVL_CMP(d1->db_level, d2->db_level); 94*c4ab0d3fSGvozden Neskovic if (likely(cmp)) 95*c4ab0d3fSGvozden Neskovic return (cmp); 960f6d88adSAlex Reece 97*c4ab0d3fSGvozden Neskovic cmp = AVL_CMP(d1->db_blkid, d2->db_blkid); 98*c4ab0d3fSGvozden Neskovic if (likely(cmp)) 99*c4ab0d3fSGvozden Neskovic return (cmp); 1000f6d88adSAlex Reece 101a846f19dSAlex Reece if (d1->db_state == DB_SEARCH) { 102a846f19dSAlex Reece ASSERT3S(d2->db_state, !=, DB_SEARCH); 1030f6d88adSAlex Reece return (-1); 104a846f19dSAlex Reece } else if (d2->db_state == DB_SEARCH) { 105a846f19dSAlex Reece ASSERT3S(d1->db_state, !=, DB_SEARCH); 10686bb58aeSAlex Reece return (1); 10786bb58aeSAlex Reece } 10886bb58aeSAlex Reece 109*c4ab0d3fSGvozden Neskovic return (AVL_PCMP(d1, d2)); 1100f6d88adSAlex Reece } 1110f6d88adSAlex Reece 112fa9e4066Sahrens /* ARGSUSED */ 113fa9e4066Sahrens static int 114fa9e4066Sahrens dnode_cons(void *arg, void *unused, int kmflag) 115fa9e4066Sahrens { 116fa9e4066Sahrens dnode_t *dn = arg; 117744947dcSTom Erickson int i; 118fa9e4066Sahrens 119fa9e4066Sahrens rw_init(&dn->dn_struct_rwlock, NULL, RW_DEFAULT, NULL); 120fa9e4066Sahrens mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); 121fa9e4066Sahrens mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL); 122b5e70f97SRicardo M. Correia cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL); 123b5e70f97SRicardo M. Correia 1243b2aab18SMatthew Ahrens /* 1253b2aab18SMatthew Ahrens * Every dbuf has a reference, and dropping a tracked reference is 1263b2aab18SMatthew Ahrens * O(number of references), so don't track dn_holds. 1273b2aab18SMatthew Ahrens */ 128e914ace2STim Schumacher zfs_refcount_create_untracked(&dn->dn_holds); 129e914ace2STim Schumacher zfs_refcount_create(&dn->dn_tx_holds); 130744947dcSTom Erickson list_link_init(&dn->dn_link); 131744947dcSTom Erickson 132744947dcSTom Erickson bzero(&dn->dn_next_nblkptr[0], sizeof (dn->dn_next_nblkptr)); 133744947dcSTom Erickson bzero(&dn->dn_next_nlevels[0], sizeof (dn->dn_next_nlevels)); 134744947dcSTom Erickson bzero(&dn->dn_next_indblkshift[0], sizeof (dn->dn_next_indblkshift)); 135744947dcSTom Erickson bzero(&dn->dn_next_bonustype[0], sizeof (dn->dn_next_bonustype)); 136744947dcSTom Erickson bzero(&dn->dn_rm_spillblk[0], sizeof (dn->dn_rm_spillblk)); 137744947dcSTom Erickson bzero(&dn->dn_next_bonuslen[0], sizeof (dn->dn_next_bonuslen)); 138744947dcSTom Erickson bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); 139fa9e4066Sahrens 140fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 141aa02ea01STom Caputi multilist_link_init(&dn->dn_dirty_link[i]); 142bf16b11eSMatthew Ahrens dn->dn_free_ranges[i] = NULL; 143c717a561Smaybee list_create(&dn->dn_dirty_records[i], 144c717a561Smaybee sizeof (dbuf_dirty_record_t), 145c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 146fa9e4066Sahrens } 147fa9e4066Sahrens 148744947dcSTom Erickson dn->dn_allocated_txg = 0; 149744947dcSTom Erickson dn->dn_free_txg = 0; 150744947dcSTom Erickson dn->dn_assigned_txg = 0; 151aa02ea01STom Caputi dn->dn_dirty_txg = 0; 152744947dcSTom Erickson dn->dn_dirtyctx = 0; 153744947dcSTom Erickson dn->dn_dirtyctx_firstset = NULL; 154744947dcSTom Erickson dn->dn_bonus = NULL; 155744947dcSTom Erickson dn->dn_have_spill = B_FALSE; 156744947dcSTom Erickson dn->dn_zio = NULL; 157744947dcSTom Erickson dn->dn_oldused = 0; 158744947dcSTom Erickson dn->dn_oldflags = 0; 159744947dcSTom Erickson dn->dn_olduid = 0; 160744947dcSTom Erickson dn->dn_oldgid = 0; 161744947dcSTom Erickson dn->dn_newuid = 0; 162744947dcSTom Erickson dn->dn_newgid = 0; 163744947dcSTom Erickson dn->dn_id_flags = 0; 164744947dcSTom Erickson 165744947dcSTom Erickson dn->dn_dbufs_count = 0; 1660f6d88adSAlex Reece avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), 167fa9e4066Sahrens offsetof(dmu_buf_impl_t, db_link)); 168fa9e4066Sahrens 169744947dcSTom Erickson dn->dn_moved = 0; 170fa9e4066Sahrens return (0); 171fa9e4066Sahrens } 172fa9e4066Sahrens 173fa9e4066Sahrens /* ARGSUSED */ 174fa9e4066Sahrens static void 175fa9e4066Sahrens dnode_dest(void *arg, void *unused) 176fa9e4066Sahrens { 177fa9e4066Sahrens int i; 178fa9e4066Sahrens dnode_t *dn = arg; 179fa9e4066Sahrens 180fa9e4066Sahrens rw_destroy(&dn->dn_struct_rwlock); 181fa9e4066Sahrens mutex_destroy(&dn->dn_mtx); 182fa9e4066Sahrens mutex_destroy(&dn->dn_dbufs_mtx); 183b5e70f97SRicardo M. Correia cv_destroy(&dn->dn_notxholds); 184e914ace2STim Schumacher zfs_refcount_destroy(&dn->dn_holds); 185e914ace2STim Schumacher zfs_refcount_destroy(&dn->dn_tx_holds); 186744947dcSTom Erickson ASSERT(!list_link_active(&dn->dn_link)); 187fa9e4066Sahrens 188fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 189aa02ea01STom Caputi ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); 190bf16b11eSMatthew Ahrens ASSERT3P(dn->dn_free_ranges[i], ==, NULL); 191c717a561Smaybee list_destroy(&dn->dn_dirty_records[i]); 192fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nblkptr[i]); 193fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nlevels[i]); 194fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_indblkshift[i]); 195fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[i]); 196fb09f5aaSMadhav Suresh ASSERT0(dn->dn_rm_spillblk[i]); 197fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[i]); 198fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[i]); 199fa9e4066Sahrens } 200fa9e4066Sahrens 201fb09f5aaSMadhav Suresh ASSERT0(dn->dn_allocated_txg); 202fb09f5aaSMadhav Suresh ASSERT0(dn->dn_free_txg); 203fb09f5aaSMadhav Suresh ASSERT0(dn->dn_assigned_txg); 204aa02ea01STom Caputi ASSERT0(dn->dn_dirty_txg); 205fb09f5aaSMadhav Suresh ASSERT0(dn->dn_dirtyctx); 206744947dcSTom Erickson ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); 207744947dcSTom Erickson ASSERT3P(dn->dn_bonus, ==, NULL); 208744947dcSTom Erickson ASSERT(!dn->dn_have_spill); 209744947dcSTom Erickson ASSERT3P(dn->dn_zio, ==, NULL); 210fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldused); 211fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldflags); 212fb09f5aaSMadhav Suresh ASSERT0(dn->dn_olduid); 213fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldgid); 214fb09f5aaSMadhav Suresh ASSERT0(dn->dn_newuid); 215fb09f5aaSMadhav Suresh ASSERT0(dn->dn_newgid); 216fb09f5aaSMadhav Suresh ASSERT0(dn->dn_id_flags); 217fb09f5aaSMadhav Suresh 218fb09f5aaSMadhav Suresh ASSERT0(dn->dn_dbufs_count); 2190f6d88adSAlex Reece avl_destroy(&dn->dn_dbufs); 220fa9e4066Sahrens } 221fa9e4066Sahrens 222fa9e4066Sahrens void 223fa9e4066Sahrens dnode_init(void) 224fa9e4066Sahrens { 225744947dcSTom Erickson ASSERT(dnode_cache == NULL); 226fa9e4066Sahrens dnode_cache = kmem_cache_create("dnode_t", 227fa9e4066Sahrens sizeof (dnode_t), 228fa9e4066Sahrens 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0); 229f06dce2cSAndrew Stormont #ifdef _KERNEL 230744947dcSTom Erickson kmem_cache_set_move(dnode_cache, dnode_move); 23154811da5SToomas Soome 23254811da5SToomas Soome dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc", 23354811da5SToomas Soome KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t), 23454811da5SToomas Soome KSTAT_FLAG_VIRTUAL); 23554811da5SToomas Soome if (dnode_ksp != NULL) { 23654811da5SToomas Soome dnode_ksp->ks_data = &dnode_stats; 23754811da5SToomas Soome kstat_install(dnode_ksp); 23854811da5SToomas Soome } 239f06dce2cSAndrew Stormont #endif /* _KERNEL */ 240fa9e4066Sahrens } 241fa9e4066Sahrens 242fa9e4066Sahrens void 243fa9e4066Sahrens dnode_fini(void) 244fa9e4066Sahrens { 24554811da5SToomas Soome if (dnode_ksp != NULL) { 24654811da5SToomas Soome kstat_delete(dnode_ksp); 24754811da5SToomas Soome dnode_ksp = NULL; 24854811da5SToomas Soome } 24954811da5SToomas Soome 250fa9e4066Sahrens kmem_cache_destroy(dnode_cache); 251744947dcSTom Erickson dnode_cache = NULL; 252fa9e4066Sahrens } 253fa9e4066Sahrens 254fa9e4066Sahrens 2559c9dc39aSek #ifdef ZFS_DEBUG 256fa9e4066Sahrens void 257fa9e4066Sahrens dnode_verify(dnode_t *dn) 258fa9e4066Sahrens { 259fa9e4066Sahrens int drop_struct_lock = FALSE; 260fa9e4066Sahrens 261fa9e4066Sahrens ASSERT(dn->dn_phys); 262fa9e4066Sahrens ASSERT(dn->dn_objset); 263744947dcSTom Erickson ASSERT(dn->dn_handle->dnh_dnode == dn); 264fa9e4066Sahrens 265ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); 266fa9e4066Sahrens 267fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY)) 268fa9e4066Sahrens return; 269fa9e4066Sahrens 270fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 271fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 272fa9e4066Sahrens drop_struct_lock = TRUE; 273fa9e4066Sahrens } 274fa9e4066Sahrens if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { 275fa9e4066Sahrens int i; 27654811da5SToomas Soome int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 277fa9e4066Sahrens ASSERT3U(dn->dn_indblkshift, >=, 0); 278fa9e4066Sahrens ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); 279fa9e4066Sahrens if (dn->dn_datablkshift) { 280fa9e4066Sahrens ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); 281fa9e4066Sahrens ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT); 282fa9e4066Sahrens ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz); 283fa9e4066Sahrens } 284fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, <=, 30); 285ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_type)); 286fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, >=, 1); 287fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); 28854811da5SToomas Soome ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen); 289fa9e4066Sahrens ASSERT3U(dn->dn_datablksz, ==, 290fa9e4066Sahrens dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); 291fa9e4066Sahrens ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); 292fa9e4066Sahrens ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + 29354811da5SToomas Soome dn->dn_bonuslen, <=, max_bonuslen); 294fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 295fa9e4066Sahrens ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); 296fa9e4066Sahrens } 297fa9e4066Sahrens } 298fa9e4066Sahrens if (dn->dn_phys->dn_type != DMU_OT_NONE) 299fa9e4066Sahrens ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels); 30014843421SMatthew Ahrens ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL); 301fa9e4066Sahrens if (dn->dn_dbuf != NULL) { 302fa9e4066Sahrens ASSERT3P(dn->dn_phys, ==, 303fa9e4066Sahrens (dnode_phys_t *)dn->dn_dbuf->db.db_data + 304fa9e4066Sahrens (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT))); 305fa9e4066Sahrens } 306fa9e4066Sahrens if (drop_struct_lock) 307fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 308fa9e4066Sahrens } 3099c9dc39aSek #endif 310fa9e4066Sahrens 311fa9e4066Sahrens void 312fa9e4066Sahrens dnode_byteswap(dnode_phys_t *dnp) 313fa9e4066Sahrens { 314fa9e4066Sahrens uint64_t *buf64 = (void*)&dnp->dn_blkptr; 315fa9e4066Sahrens int i; 316fa9e4066Sahrens 317fa9e4066Sahrens if (dnp->dn_type == DMU_OT_NONE) { 318fa9e4066Sahrens bzero(dnp, sizeof (dnode_phys_t)); 319fa9e4066Sahrens return; 320fa9e4066Sahrens } 321fa9e4066Sahrens 322fa9e4066Sahrens dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec); 323fa9e4066Sahrens dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen); 32454811da5SToomas Soome dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots); 325fa9e4066Sahrens dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid); 32699653d4eSeschrock dnp->dn_used = BSWAP_64(dnp->dn_used); 327fa9e4066Sahrens 328fa9e4066Sahrens /* 329fa9e4066Sahrens * dn_nblkptr is only one byte, so it's OK to read it in either 330fa9e4066Sahrens * byte order. We can't read dn_bouslen. 331fa9e4066Sahrens */ 332fa9e4066Sahrens ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT); 333fa9e4066Sahrens ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR); 334fa9e4066Sahrens for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++) 335fa9e4066Sahrens buf64[i] = BSWAP_64(buf64[i]); 336fa9e4066Sahrens 337fa9e4066Sahrens /* 338fa9e4066Sahrens * OK to check dn_bonuslen for zero, because it won't matter if 339fa9e4066Sahrens * we have the wrong byte order. This is necessary because the 340fa9e4066Sahrens * dnode dnode is smaller than a regular dnode. 341fa9e4066Sahrens */ 342fa9e4066Sahrens if (dnp->dn_bonuslen != 0) { 343fa9e4066Sahrens /* 344fa9e4066Sahrens * Note that the bonus length calculated here may be 345fa9e4066Sahrens * longer than the actual bonus buffer. This is because 346fa9e4066Sahrens * we always put the bonus buffer after the last block 347fa9e4066Sahrens * pointer (instead of packing it against the end of the 348fa9e4066Sahrens * dnode buffer). 349fa9e4066Sahrens */ 350fa9e4066Sahrens int off = (dnp->dn_nblkptr-1) * sizeof (blkptr_t); 35154811da5SToomas Soome int slots = dnp->dn_extra_slots + 1; 35254811da5SToomas Soome size_t len = DN_SLOTS_TO_BONUSLEN(slots) - off; 353ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype)); 354ad135b5dSChristopher Siden dmu_object_byteswap_t byteswap = 355ad135b5dSChristopher Siden DMU_OT_BYTESWAP(dnp->dn_bonustype); 356ad135b5dSChristopher Siden dmu_ot_byteswap[byteswap].ob_func(dnp->dn_bonus + off, len); 357fa9e4066Sahrens } 3580a586ceaSMark Shellenbaum 3590a586ceaSMark Shellenbaum /* Swap SPILL block if we have one */ 3600a586ceaSMark Shellenbaum if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) 36154811da5SToomas Soome byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t)); 3620a586ceaSMark Shellenbaum 363fa9e4066Sahrens } 364fa9e4066Sahrens 365fa9e4066Sahrens void 366fa9e4066Sahrens dnode_buf_byteswap(void *vbuf, size_t size) 367fa9e4066Sahrens { 36854811da5SToomas Soome int i = 0; 369fa9e4066Sahrens 370fa9e4066Sahrens ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT)); 371fa9e4066Sahrens ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0); 372fa9e4066Sahrens 37354811da5SToomas Soome while (i < size) { 37454811da5SToomas Soome dnode_phys_t *dnp = (void *)(((char *)vbuf) + i); 37554811da5SToomas Soome dnode_byteswap(dnp); 37654811da5SToomas Soome 37754811da5SToomas Soome i += DNODE_MIN_SIZE; 37854811da5SToomas Soome if (dnp->dn_type != DMU_OT_NONE) 37954811da5SToomas Soome i += dnp->dn_extra_slots * DNODE_MIN_SIZE; 380fa9e4066Sahrens } 381fa9e4066Sahrens } 382fa9e4066Sahrens 3831934e92fSmaybee void 3841934e92fSmaybee dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) 3851934e92fSmaybee { 386e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 3871934e92fSmaybee 3881934e92fSmaybee dnode_setdirty(dn, tx); 3891934e92fSmaybee rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 39054811da5SToomas Soome ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - 3911934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t)); 3921934e92fSmaybee dn->dn_bonuslen = newsize; 3931934e92fSmaybee if (newsize == 0) 3941934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN; 3951934e92fSmaybee else 3961934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; 3971934e92fSmaybee rw_exit(&dn->dn_struct_rwlock); 3981934e92fSmaybee } 3991934e92fSmaybee 4000a586ceaSMark Shellenbaum void 4010a586ceaSMark Shellenbaum dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx) 4020a586ceaSMark Shellenbaum { 403e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 4040a586ceaSMark Shellenbaum dnode_setdirty(dn, tx); 4050a586ceaSMark Shellenbaum rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 4060a586ceaSMark Shellenbaum dn->dn_bonustype = newtype; 4070a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; 4080a586ceaSMark Shellenbaum rw_exit(&dn->dn_struct_rwlock); 4090a586ceaSMark Shellenbaum } 4100a586ceaSMark Shellenbaum 4110a586ceaSMark Shellenbaum void 4120a586ceaSMark Shellenbaum dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx) 4130a586ceaSMark Shellenbaum { 414e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 41506e0070dSMark Shellenbaum ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 4160a586ceaSMark Shellenbaum dnode_setdirty(dn, tx); 4170a586ceaSMark Shellenbaum dn->dn_rm_spillblk[tx->tx_txg&TXG_MASK] = DN_KILL_SPILLBLK; 4180a586ceaSMark Shellenbaum dn->dn_have_spill = B_FALSE; 4190a586ceaSMark Shellenbaum } 4200a586ceaSMark Shellenbaum 421fa9e4066Sahrens static void 422fa9e4066Sahrens dnode_setdblksz(dnode_t *dn, int size) 423fa9e4066Sahrens { 424fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(size, SPA_MINBLOCKSIZE)); 425fa9e4066Sahrens ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 426fa9e4066Sahrens ASSERT3U(size, >=, SPA_MINBLOCKSIZE); 427fa9e4066Sahrens ASSERT3U(size >> SPA_MINBLOCKSHIFT, <, 428fa9e4066Sahrens 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8)); 429fa9e4066Sahrens dn->dn_datablksz = size; 430fa9e4066Sahrens dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT; 431bf16b11eSMatthew Ahrens dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0; 432fa9e4066Sahrens } 433fa9e4066Sahrens 434fa9e4066Sahrens static dnode_t * 435503ad85cSMatthew Ahrens dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, 436744947dcSTom Erickson uint64_t object, dnode_handle_t *dnh) 437fa9e4066Sahrens { 438bc9014e6SJustin Gibbs dnode_t *dn; 439fa9e4066Sahrens 440bc9014e6SJustin Gibbs dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); 441f06dce2cSAndrew Stormont #ifdef _KERNEL 442744947dcSTom Erickson ASSERT(!POINTER_IS_VALID(dn->dn_objset)); 443f06dce2cSAndrew Stormont #endif /* _KERNEL */ 444744947dcSTom Erickson dn->dn_moved = 0; 445744947dcSTom Erickson 446744947dcSTom Erickson /* 447744947dcSTom Erickson * Defer setting dn_objset until the dnode is ready to be a candidate 448744947dcSTom Erickson * for the dnode_move() callback. 449744947dcSTom Erickson */ 450fa9e4066Sahrens dn->dn_object = object; 451fa9e4066Sahrens dn->dn_dbuf = db; 452744947dcSTom Erickson dn->dn_handle = dnh; 453fa9e4066Sahrens dn->dn_phys = dnp; 454fa9e4066Sahrens 455744947dcSTom Erickson if (dnp->dn_datablkszsec) { 456fa9e4066Sahrens dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); 457744947dcSTom Erickson } else { 458744947dcSTom Erickson dn->dn_datablksz = 0; 459744947dcSTom Erickson dn->dn_datablkszsec = 0; 460744947dcSTom Erickson dn->dn_datablkshift = 0; 461744947dcSTom Erickson } 462fa9e4066Sahrens dn->dn_indblkshift = dnp->dn_indblkshift; 463fa9e4066Sahrens dn->dn_nlevels = dnp->dn_nlevels; 464fa9e4066Sahrens dn->dn_type = dnp->dn_type; 465fa9e4066Sahrens dn->dn_nblkptr = dnp->dn_nblkptr; 466fa9e4066Sahrens dn->dn_checksum = dnp->dn_checksum; 467fa9e4066Sahrens dn->dn_compress = dnp->dn_compress; 468fa9e4066Sahrens dn->dn_bonustype = dnp->dn_bonustype; 469fa9e4066Sahrens dn->dn_bonuslen = dnp->dn_bonuslen; 47054811da5SToomas Soome dn->dn_num_slots = dnp->dn_extra_slots + 1; 471fa9e4066Sahrens dn->dn_maxblkid = dnp->dn_maxblkid; 4720a586ceaSMark Shellenbaum dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); 47306e0070dSMark Shellenbaum dn->dn_id_flags = 0; 474fa9e4066Sahrens 475fa9e4066Sahrens dmu_zfetch_init(&dn->dn_zfetch, dn); 476fa9e4066Sahrens 477ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); 47854811da5SToomas Soome ASSERT(zrl_is_locked(&dnh->dnh_zrlock)); 47954811da5SToomas Soome ASSERT(!DN_SLOT_IS_PTR(dnh->dnh_dnode)); 480744947dcSTom Erickson 481fa9e4066Sahrens mutex_enter(&os->os_lock); 482bc9014e6SJustin Gibbs 483bc9014e6SJustin Gibbs /* 484bc9014e6SJustin Gibbs * Exclude special dnodes from os_dnodes so an empty os_dnodes 485bc9014e6SJustin Gibbs * signifies that the special dnodes have no references from 486bc9014e6SJustin Gibbs * their children (the entries in os_dnodes). This allows 487bc9014e6SJustin Gibbs * dnode_destroy() to easily determine if the last child has 488bc9014e6SJustin Gibbs * been removed and then complete eviction of the objset. 489bc9014e6SJustin Gibbs */ 490bc9014e6SJustin Gibbs if (!DMU_OBJECT_IS_SPECIAL(object)) 491bc9014e6SJustin Gibbs list_insert_head(&os->os_dnodes, dn); 492744947dcSTom Erickson membar_producer(); 493bc9014e6SJustin Gibbs 494744947dcSTom Erickson /* 495bc9014e6SJustin Gibbs * Everything else must be valid before assigning dn_objset 496bc9014e6SJustin Gibbs * makes the dnode eligible for dnode_move(). 497744947dcSTom Erickson */ 498744947dcSTom Erickson dn->dn_objset = os; 499bc9014e6SJustin Gibbs 500bc9014e6SJustin Gibbs dnh->dnh_dnode = dn; 501fa9e4066Sahrens mutex_exit(&os->os_lock); 502fa9e4066Sahrens 5035a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER); 50454811da5SToomas Soome 505fa9e4066Sahrens return (dn); 506fa9e4066Sahrens } 507fa9e4066Sahrens 508744947dcSTom Erickson /* 509744947dcSTom Erickson * Caller must be holding the dnode handle, which is released upon return. 510744947dcSTom Erickson */ 511fa9e4066Sahrens static void 512fa9e4066Sahrens dnode_destroy(dnode_t *dn) 513fa9e4066Sahrens { 514503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 515bc9014e6SJustin Gibbs boolean_t complete_os_eviction = B_FALSE; 516fa9e4066Sahrens 5170a586ceaSMark Shellenbaum ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0); 518a2eea2e1Sahrens 519fa9e4066Sahrens mutex_enter(&os->os_lock); 520744947dcSTom Erickson POINTER_INVALIDATE(&dn->dn_objset); 521bc9014e6SJustin Gibbs if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { 522bc9014e6SJustin Gibbs list_remove(&os->os_dnodes, dn); 523bc9014e6SJustin Gibbs complete_os_eviction = 524bc9014e6SJustin Gibbs list_is_empty(&os->os_dnodes) && 525bc9014e6SJustin Gibbs list_link_active(&os->os_evicting_node); 526bc9014e6SJustin Gibbs } 527fa9e4066Sahrens mutex_exit(&os->os_lock); 528fa9e4066Sahrens 529744947dcSTom Erickson /* the dnode can no longer move, so we can release the handle */ 53054811da5SToomas Soome if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock)) 53154811da5SToomas Soome zrl_remove(&dn->dn_handle->dnh_zrlock); 532744947dcSTom Erickson 533744947dcSTom Erickson dn->dn_allocated_txg = 0; 534744947dcSTom Erickson dn->dn_free_txg = 0; 535744947dcSTom Erickson dn->dn_assigned_txg = 0; 536aa02ea01STom Caputi dn->dn_dirty_txg = 0; 537744947dcSTom Erickson 538744947dcSTom Erickson dn->dn_dirtyctx = 0; 539744947dcSTom Erickson if (dn->dn_dirtyctx_firstset != NULL) { 540fa9e4066Sahrens kmem_free(dn->dn_dirtyctx_firstset, 1); 541fa9e4066Sahrens dn->dn_dirtyctx_firstset = NULL; 542fa9e4066Sahrens } 543744947dcSTom Erickson if (dn->dn_bonus != NULL) { 544ea8dc4b6Seschrock mutex_enter(&dn->dn_bonus->db_mtx); 545dcbf3bd6SGeorge Wilson dbuf_destroy(dn->dn_bonus); 546ea8dc4b6Seschrock dn->dn_bonus = NULL; 547ea8dc4b6Seschrock } 548744947dcSTom Erickson dn->dn_zio = NULL; 549744947dcSTom Erickson 550744947dcSTom Erickson dn->dn_have_spill = B_FALSE; 551744947dcSTom Erickson dn->dn_oldused = 0; 552744947dcSTom Erickson dn->dn_oldflags = 0; 553744947dcSTom Erickson dn->dn_olduid = 0; 554744947dcSTom Erickson dn->dn_oldgid = 0; 555744947dcSTom Erickson dn->dn_newuid = 0; 556744947dcSTom Erickson dn->dn_newgid = 0; 557744947dcSTom Erickson dn->dn_id_flags = 0; 558744947dcSTom Erickson 559cf6106c8SMatthew Ahrens dmu_zfetch_fini(&dn->dn_zfetch); 560fa9e4066Sahrens kmem_cache_free(dnode_cache, dn); 5615a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER); 562bc9014e6SJustin Gibbs 563bc9014e6SJustin Gibbs if (complete_os_eviction) 564bc9014e6SJustin Gibbs dmu_objset_evict_done(os); 565fa9e4066Sahrens } 566fa9e4066Sahrens 567fa9e4066Sahrens void 568fa9e4066Sahrens dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, 56954811da5SToomas Soome dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) 570fa9e4066Sahrens { 571fa9e4066Sahrens int i; 572fa9e4066Sahrens 57354811da5SToomas Soome ASSERT3U(dn_slots, >, 0); 57454811da5SToomas Soome ASSERT3U(dn_slots << DNODE_SHIFT, <=, 57554811da5SToomas Soome spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))); 576b5152584SMatthew Ahrens ASSERT3U(blocksize, <=, 577b5152584SMatthew Ahrens spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 578fa9e4066Sahrens if (blocksize == 0) 579fa9e4066Sahrens blocksize = 1 << zfs_default_bs; 5803b83abddSahrens else 5813b83abddSahrens blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE); 582fa9e4066Sahrens 583fa9e4066Sahrens if (ibs == 0) 584fa9e4066Sahrens ibs = zfs_default_ibs; 585fa9e4066Sahrens 586fa9e4066Sahrens ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT); 587fa9e4066Sahrens 58854811da5SToomas Soome dprintf("os=%p obj=%" PRIu64 " txg=%" PRIu64 58954811da5SToomas Soome " blocksize=%d ibs=%d dn_slots=%d\n", 59054811da5SToomas Soome dn->dn_objset, dn->dn_object, tx->tx_txg, blocksize, ibs, dn_slots); 59154811da5SToomas Soome DNODE_STAT_BUMP(dnode_allocate); 592fa9e4066Sahrens 593fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_NONE); 594fa9e4066Sahrens ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0); 595fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE); 596fa9e4066Sahrens ASSERT(ot != DMU_OT_NONE); 597ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(ot)); 598fa9e4066Sahrens ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) || 5990a586ceaSMark Shellenbaum (bonustype == DMU_OT_SA && bonuslen == 0) || 600fa9e4066Sahrens (bonustype != DMU_OT_NONE && bonuslen != 0)); 601ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(bonustype)); 60254811da5SToomas Soome ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots)); 603fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_NONE); 604fb09f5aaSMadhav Suresh ASSERT0(dn->dn_maxblkid); 605fb09f5aaSMadhav Suresh ASSERT0(dn->dn_allocated_txg); 606aa02ea01STom Caputi ASSERT0(dn->dn_dirty_txg); 607fb09f5aaSMadhav Suresh ASSERT0(dn->dn_assigned_txg); 608e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); 609e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1); 6100f6d88adSAlex Reece ASSERT(avl_is_empty(&dn->dn_dbufs)); 611fa9e4066Sahrens 612fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 613fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nblkptr[i]); 614fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nlevels[i]); 615fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_indblkshift[i]); 616fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[i]); 617fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[i]); 618fb09f5aaSMadhav Suresh ASSERT0(dn->dn_rm_spillblk[i]); 619fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[i]); 620aa02ea01STom Caputi ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); 621c717a561Smaybee ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); 622bf16b11eSMatthew Ahrens ASSERT3P(dn->dn_free_ranges[i], ==, NULL); 623fa9e4066Sahrens } 624fa9e4066Sahrens 625fa9e4066Sahrens dn->dn_type = ot; 626fa9e4066Sahrens dnode_setdblksz(dn, blocksize); 627fa9e4066Sahrens dn->dn_indblkshift = ibs; 628fa9e4066Sahrens dn->dn_nlevels = 1; 62954811da5SToomas Soome dn->dn_num_slots = dn_slots; 6300a586ceaSMark Shellenbaum if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ 6310a586ceaSMark Shellenbaum dn->dn_nblkptr = 1; 63254811da5SToomas Soome else { 63354811da5SToomas Soome dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR, 63454811da5SToomas Soome 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> 63554811da5SToomas Soome SPA_BLKPTRSHIFT)); 63654811da5SToomas Soome } 63754811da5SToomas Soome 638fa9e4066Sahrens dn->dn_bonustype = bonustype; 639fa9e4066Sahrens dn->dn_bonuslen = bonuslen; 640fa9e4066Sahrens dn->dn_checksum = ZIO_CHECKSUM_INHERIT; 641fa9e4066Sahrens dn->dn_compress = ZIO_COMPRESS_INHERIT; 642fa9e4066Sahrens dn->dn_dirtyctx = 0; 643fa9e4066Sahrens 644fa9e4066Sahrens dn->dn_free_txg = 0; 645fa9e4066Sahrens if (dn->dn_dirtyctx_firstset) { 646fa9e4066Sahrens kmem_free(dn->dn_dirtyctx_firstset, 1); 647fa9e4066Sahrens dn->dn_dirtyctx_firstset = NULL; 648fa9e4066Sahrens } 649fa9e4066Sahrens 650fa9e4066Sahrens dn->dn_allocated_txg = tx->tx_txg; 6510a586ceaSMark Shellenbaum dn->dn_id_flags = 0; 652f676ed34Sahrens 653fa9e4066Sahrens dnode_setdirty(dn, tx); 654f676ed34Sahrens dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; 6551934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; 6560a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; 657f676ed34Sahrens dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz; 658fa9e4066Sahrens } 659fa9e4066Sahrens 660fa9e4066Sahrens void 661fa9e4066Sahrens dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, 66254811da5SToomas Soome dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) 663fa9e4066Sahrens { 6642bf405a2SMark Maybee int nblkptr; 665c543ec06Sahrens 666fa9e4066Sahrens ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE); 667b5152584SMatthew Ahrens ASSERT3U(blocksize, <=, 668b5152584SMatthew Ahrens spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 669fb09f5aaSMadhav Suresh ASSERT0(blocksize % SPA_MINBLOCKSIZE); 670ea8dc4b6Seschrock ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); 671fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 672fa9e4066Sahrens ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) || 67306e0070dSMark Shellenbaum (bonustype != DMU_OT_NONE && bonuslen != 0) || 67406e0070dSMark Shellenbaum (bonustype == DMU_OT_SA && bonuslen == 0)); 675ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(bonustype)); 67654811da5SToomas Soome ASSERT3U(bonuslen, <=, 67754811da5SToomas Soome DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)))); 678946342a2SFabian Grünbichler ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT)); 67954811da5SToomas Soome 68054811da5SToomas Soome dnode_free_interior_slots(dn); 68154811da5SToomas Soome DNODE_STAT_BUMP(dnode_reallocate); 682c543ec06Sahrens 683ea8dc4b6Seschrock /* clean up any unreferenced dbufs */ 6841934e92fSmaybee dnode_evict_dbufs(dn); 685da03de99SMark Maybee 68628d97a71SMark Shellenbaum dn->dn_id_flags = 0; 68728d97a71SMark Shellenbaum 688fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 689fa9e4066Sahrens dnode_setdirty(dn, tx); 6902bf405a2SMark Maybee if (dn->dn_datablksz != blocksize) { 6912bf405a2SMark Maybee /* change blocksize */ 6922bf405a2SMark Maybee ASSERT(dn->dn_maxblkid == 0 && 6932bf405a2SMark Maybee (BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || 6942bf405a2SMark Maybee dnode_block_freed(dn, 0))); 6952bf405a2SMark Maybee dnode_setdblksz(dn, blocksize); 6962bf405a2SMark Maybee dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize; 6972bf405a2SMark Maybee } 6982bf405a2SMark Maybee if (dn->dn_bonuslen != bonuslen) 6992bf405a2SMark Maybee dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen; 70006e0070dSMark Shellenbaum 70106e0070dSMark Shellenbaum if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ 70206e0070dSMark Shellenbaum nblkptr = 1; 70306e0070dSMark Shellenbaum else 70454811da5SToomas Soome nblkptr = MIN(DN_MAX_NBLKPTR, 70554811da5SToomas Soome 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> 70654811da5SToomas Soome SPA_BLKPTRSHIFT)); 7070a586ceaSMark Shellenbaum if (dn->dn_bonustype != bonustype) 7080a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg&TXG_MASK] = bonustype; 709da03de99SMark Maybee if (dn->dn_nblkptr != nblkptr) 710da03de99SMark Maybee dn->dn_next_nblkptr[tx->tx_txg&TXG_MASK] = nblkptr; 7110a586ceaSMark Shellenbaum if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 71206e0070dSMark Shellenbaum dbuf_rm_spill(dn, tx); 71306e0070dSMark Shellenbaum dnode_rm_spill(dn, tx); 7140a586ceaSMark Shellenbaum } 715fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 716fa9e4066Sahrens 717fa9e4066Sahrens /* change type */ 718fa9e4066Sahrens dn->dn_type = ot; 719fa9e4066Sahrens 720fa9e4066Sahrens /* change bonus size and type */ 721fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 722fa9e4066Sahrens dn->dn_bonustype = bonustype; 723fa9e4066Sahrens dn->dn_bonuslen = bonuslen; 72454811da5SToomas Soome dn->dn_num_slots = dn_slots; 725da03de99SMark Maybee dn->dn_nblkptr = nblkptr; 726fa9e4066Sahrens dn->dn_checksum = ZIO_CHECKSUM_INHERIT; 727fa9e4066Sahrens dn->dn_compress = ZIO_COMPRESS_INHERIT; 728fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); 729fa9e4066Sahrens 730da03de99SMark Maybee /* fix up the bonus db_size */ 731da03de99SMark Maybee if (dn->dn_bonus) { 7321934e92fSmaybee dn->dn_bonus->db.db_size = 73354811da5SToomas Soome DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - 73454811da5SToomas Soome (dn->dn_nblkptr - 1) * sizeof (blkptr_t); 7351934e92fSmaybee ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); 7361934e92fSmaybee } 737432f72fdSahrens 738fa9e4066Sahrens dn->dn_allocated_txg = tx->tx_txg; 739fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 740fa9e4066Sahrens } 741fa9e4066Sahrens 742f06dce2cSAndrew Stormont #ifdef _KERNEL 743744947dcSTom Erickson static void 744744947dcSTom Erickson dnode_move_impl(dnode_t *odn, dnode_t *ndn) 745744947dcSTom Erickson { 746744947dcSTom Erickson int i; 747744947dcSTom Erickson 748744947dcSTom Erickson ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock)); 749744947dcSTom Erickson ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx)); 750744947dcSTom Erickson ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx)); 751744947dcSTom Erickson ASSERT(!RW_LOCK_HELD(&odn->dn_zfetch.zf_rwlock)); 752744947dcSTom Erickson 753744947dcSTom Erickson /* Copy fields. */ 754744947dcSTom Erickson ndn->dn_objset = odn->dn_objset; 755744947dcSTom Erickson ndn->dn_object = odn->dn_object; 756744947dcSTom Erickson ndn->dn_dbuf = odn->dn_dbuf; 757744947dcSTom Erickson ndn->dn_handle = odn->dn_handle; 758744947dcSTom Erickson ndn->dn_phys = odn->dn_phys; 759744947dcSTom Erickson ndn->dn_type = odn->dn_type; 760744947dcSTom Erickson ndn->dn_bonuslen = odn->dn_bonuslen; 761744947dcSTom Erickson ndn->dn_bonustype = odn->dn_bonustype; 762744947dcSTom Erickson ndn->dn_nblkptr = odn->dn_nblkptr; 763744947dcSTom Erickson ndn->dn_checksum = odn->dn_checksum; 764744947dcSTom Erickson ndn->dn_compress = odn->dn_compress; 765744947dcSTom Erickson ndn->dn_nlevels = odn->dn_nlevels; 766744947dcSTom Erickson ndn->dn_indblkshift = odn->dn_indblkshift; 767744947dcSTom Erickson ndn->dn_datablkshift = odn->dn_datablkshift; 768744947dcSTom Erickson ndn->dn_datablkszsec = odn->dn_datablkszsec; 769744947dcSTom Erickson ndn->dn_datablksz = odn->dn_datablksz; 770744947dcSTom Erickson ndn->dn_maxblkid = odn->dn_maxblkid; 77154811da5SToomas Soome ndn->dn_num_slots = odn->dn_num_slots; 772c7fbe46dSMatthew Ahrens bcopy(&odn->dn_next_type[0], &ndn->dn_next_type[0], 773c7fbe46dSMatthew Ahrens sizeof (odn->dn_next_type)); 774744947dcSTom Erickson bcopy(&odn->dn_next_nblkptr[0], &ndn->dn_next_nblkptr[0], 775744947dcSTom Erickson sizeof (odn->dn_next_nblkptr)); 776744947dcSTom Erickson bcopy(&odn->dn_next_nlevels[0], &ndn->dn_next_nlevels[0], 777744947dcSTom Erickson sizeof (odn->dn_next_nlevels)); 778744947dcSTom Erickson bcopy(&odn->dn_next_indblkshift[0], &ndn->dn_next_indblkshift[0], 779744947dcSTom Erickson sizeof (odn->dn_next_indblkshift)); 780744947dcSTom Erickson bcopy(&odn->dn_next_bonustype[0], &ndn->dn_next_bonustype[0], 781744947dcSTom Erickson sizeof (odn->dn_next_bonustype)); 782744947dcSTom Erickson bcopy(&odn->dn_rm_spillblk[0], &ndn->dn_rm_spillblk[0], 783744947dcSTom Erickson sizeof (odn->dn_rm_spillblk)); 784744947dcSTom Erickson bcopy(&odn->dn_next_bonuslen[0], &ndn->dn_next_bonuslen[0], 785744947dcSTom Erickson sizeof (odn->dn_next_bonuslen)); 786744947dcSTom Erickson bcopy(&odn->dn_next_blksz[0], &ndn->dn_next_blksz[0], 787744947dcSTom Erickson sizeof (odn->dn_next_blksz)); 788744947dcSTom Erickson for (i = 0; i < TXG_SIZE; i++) { 789744947dcSTom Erickson list_move_tail(&ndn->dn_dirty_records[i], 790744947dcSTom Erickson &odn->dn_dirty_records[i]); 791744947dcSTom Erickson } 792bf16b11eSMatthew Ahrens bcopy(&odn->dn_free_ranges[0], &ndn->dn_free_ranges[0], 793bf16b11eSMatthew Ahrens sizeof (odn->dn_free_ranges)); 794744947dcSTom Erickson ndn->dn_allocated_txg = odn->dn_allocated_txg; 795744947dcSTom Erickson ndn->dn_free_txg = odn->dn_free_txg; 796744947dcSTom Erickson ndn->dn_assigned_txg = odn->dn_assigned_txg; 797aa02ea01STom Caputi ndn->dn_dirty_txg = odn->dn_dirty_txg; 798744947dcSTom Erickson ndn->dn_dirtyctx = odn->dn_dirtyctx; 799744947dcSTom Erickson ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset; 800e914ace2STim Schumacher ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0); 801e914ace2STim Schumacher zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds); 8020f6d88adSAlex Reece ASSERT(avl_is_empty(&ndn->dn_dbufs)); 8030f6d88adSAlex Reece avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs); 804744947dcSTom Erickson ndn->dn_dbufs_count = odn->dn_dbufs_count; 805744947dcSTom Erickson ndn->dn_bonus = odn->dn_bonus; 806744947dcSTom Erickson ndn->dn_have_spill = odn->dn_have_spill; 807744947dcSTom Erickson ndn->dn_zio = odn->dn_zio; 808744947dcSTom Erickson ndn->dn_oldused = odn->dn_oldused; 809744947dcSTom Erickson ndn->dn_oldflags = odn->dn_oldflags; 810744947dcSTom Erickson ndn->dn_olduid = odn->dn_olduid; 811744947dcSTom Erickson ndn->dn_oldgid = odn->dn_oldgid; 812744947dcSTom Erickson ndn->dn_newuid = odn->dn_newuid; 813744947dcSTom Erickson ndn->dn_newgid = odn->dn_newgid; 814744947dcSTom Erickson ndn->dn_id_flags = odn->dn_id_flags; 815744947dcSTom Erickson dmu_zfetch_init(&ndn->dn_zfetch, NULL); 816744947dcSTom Erickson list_move_tail(&ndn->dn_zfetch.zf_stream, &odn->dn_zfetch.zf_stream); 817744947dcSTom Erickson ndn->dn_zfetch.zf_dnode = odn->dn_zfetch.zf_dnode; 818744947dcSTom Erickson 819744947dcSTom Erickson /* 820744947dcSTom Erickson * Update back pointers. Updating the handle fixes the back pointer of 821744947dcSTom Erickson * every descendant dbuf as well as the bonus dbuf. 822744947dcSTom Erickson */ 823744947dcSTom Erickson ASSERT(ndn->dn_handle->dnh_dnode == odn); 824744947dcSTom Erickson ndn->dn_handle->dnh_dnode = ndn; 825744947dcSTom Erickson if (ndn->dn_zfetch.zf_dnode == odn) { 826744947dcSTom Erickson ndn->dn_zfetch.zf_dnode = ndn; 827744947dcSTom Erickson } 828744947dcSTom Erickson 829744947dcSTom Erickson /* 830744947dcSTom Erickson * Invalidate the original dnode by clearing all of its back pointers. 831744947dcSTom Erickson */ 832744947dcSTom Erickson odn->dn_dbuf = NULL; 833744947dcSTom Erickson odn->dn_handle = NULL; 8340f6d88adSAlex Reece avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), 835744947dcSTom Erickson offsetof(dmu_buf_impl_t, db_link)); 836744947dcSTom Erickson odn->dn_dbufs_count = 0; 837744947dcSTom Erickson odn->dn_bonus = NULL; 838744947dcSTom Erickson odn->dn_zfetch.zf_dnode = NULL; 839744947dcSTom Erickson 840744947dcSTom Erickson /* 841744947dcSTom Erickson * Set the low bit of the objset pointer to ensure that dnode_move() 842744947dcSTom Erickson * recognizes the dnode as invalid in any subsequent callback. 843744947dcSTom Erickson */ 844744947dcSTom Erickson POINTER_INVALIDATE(&odn->dn_objset); 845744947dcSTom Erickson 846744947dcSTom Erickson /* 847744947dcSTom Erickson * Satisfy the destructor. 848744947dcSTom Erickson */ 849744947dcSTom Erickson for (i = 0; i < TXG_SIZE; i++) { 850744947dcSTom Erickson list_create(&odn->dn_dirty_records[i], 851744947dcSTom Erickson sizeof (dbuf_dirty_record_t), 852744947dcSTom Erickson offsetof(dbuf_dirty_record_t, dr_dirty_node)); 853bf16b11eSMatthew Ahrens odn->dn_free_ranges[i] = NULL; 854744947dcSTom Erickson odn->dn_next_nlevels[i] = 0; 855744947dcSTom Erickson odn->dn_next_indblkshift[i] = 0; 856744947dcSTom Erickson odn->dn_next_bonustype[i] = 0; 857744947dcSTom Erickson odn->dn_rm_spillblk[i] = 0; 858744947dcSTom Erickson odn->dn_next_bonuslen[i] = 0; 859744947dcSTom Erickson odn->dn_next_blksz[i] = 0; 860744947dcSTom Erickson } 861744947dcSTom Erickson odn->dn_allocated_txg = 0; 862744947dcSTom Erickson odn->dn_free_txg = 0; 863744947dcSTom Erickson odn->dn_assigned_txg = 0; 864aa02ea01STom Caputi odn->dn_dirty_txg = 0; 865744947dcSTom Erickson odn->dn_dirtyctx = 0; 866744947dcSTom Erickson odn->dn_dirtyctx_firstset = NULL; 867744947dcSTom Erickson odn->dn_have_spill = B_FALSE; 868744947dcSTom Erickson odn->dn_zio = NULL; 869744947dcSTom Erickson odn->dn_oldused = 0; 870744947dcSTom Erickson odn->dn_oldflags = 0; 871744947dcSTom Erickson odn->dn_olduid = 0; 872744947dcSTom Erickson odn->dn_oldgid = 0; 873744947dcSTom Erickson odn->dn_newuid = 0; 874744947dcSTom Erickson odn->dn_newgid = 0; 875744947dcSTom Erickson odn->dn_id_flags = 0; 876744947dcSTom Erickson 877744947dcSTom Erickson /* 878744947dcSTom Erickson * Mark the dnode. 879744947dcSTom Erickson */ 880744947dcSTom Erickson ndn->dn_moved = 1; 881744947dcSTom Erickson odn->dn_moved = (uint8_t)-1; 882744947dcSTom Erickson } 883744947dcSTom Erickson 884744947dcSTom Erickson /*ARGSUSED*/ 885744947dcSTom Erickson static kmem_cbrc_t 886744947dcSTom Erickson dnode_move(void *buf, void *newbuf, size_t size, void *arg) 887744947dcSTom Erickson { 888744947dcSTom Erickson dnode_t *odn = buf, *ndn = newbuf; 889744947dcSTom Erickson objset_t *os; 890744947dcSTom Erickson int64_t refcount; 891744947dcSTom Erickson uint32_t dbufs; 892744947dcSTom Erickson 893744947dcSTom Erickson /* 894744947dcSTom Erickson * The dnode is on the objset's list of known dnodes if the objset 895744947dcSTom Erickson * pointer is valid. We set the low bit of the objset pointer when 896744947dcSTom Erickson * freeing the dnode to invalidate it, and the memory patterns written 897744947dcSTom Erickson * by kmem (baddcafe and deadbeef) set at least one of the two low bits. 898744947dcSTom Erickson * A newly created dnode sets the objset pointer last of all to indicate 899744947dcSTom Erickson * that the dnode is known and in a valid state to be moved by this 900744947dcSTom Erickson * function. 901744947dcSTom Erickson */ 902744947dcSTom Erickson os = odn->dn_objset; 903744947dcSTom Erickson if (!POINTER_IS_VALID(os)) { 90454811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_invalid); 905744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 906744947dcSTom Erickson } 907744947dcSTom Erickson 908744947dcSTom Erickson /* 909744947dcSTom Erickson * Ensure that the objset does not go away during the move. 910744947dcSTom Erickson */ 911744947dcSTom Erickson rw_enter(&os_lock, RW_WRITER); 912744947dcSTom Erickson if (os != odn->dn_objset) { 913744947dcSTom Erickson rw_exit(&os_lock); 91454811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_recheck1); 915744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 916744947dcSTom Erickson } 917744947dcSTom Erickson 918744947dcSTom Erickson /* 919744947dcSTom Erickson * If the dnode is still valid, then so is the objset. We know that no 920744947dcSTom Erickson * valid objset can be freed while we hold os_lock, so we can safely 921744947dcSTom Erickson * ensure that the objset remains in use. 922744947dcSTom Erickson */ 923744947dcSTom Erickson mutex_enter(&os->os_lock); 924744947dcSTom Erickson 925744947dcSTom Erickson /* 926744947dcSTom Erickson * Recheck the objset pointer in case the dnode was removed just before 927744947dcSTom Erickson * acquiring the lock. 928744947dcSTom Erickson */ 929744947dcSTom Erickson if (os != odn->dn_objset) { 930744947dcSTom Erickson mutex_exit(&os->os_lock); 931744947dcSTom Erickson rw_exit(&os_lock); 93254811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_recheck2); 933744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 934744947dcSTom Erickson } 935744947dcSTom Erickson 936744947dcSTom Erickson /* 937744947dcSTom Erickson * At this point we know that as long as we hold os->os_lock, the dnode 938744947dcSTom Erickson * cannot be freed and fields within the dnode can be safely accessed. 939744947dcSTom Erickson * The objset listing this dnode cannot go away as long as this dnode is 940744947dcSTom Erickson * on its list. 941744947dcSTom Erickson */ 942744947dcSTom Erickson rw_exit(&os_lock); 943744947dcSTom Erickson if (DMU_OBJECT_IS_SPECIAL(odn->dn_object)) { 944744947dcSTom Erickson mutex_exit(&os->os_lock); 94554811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_special); 946744947dcSTom Erickson return (KMEM_CBRC_NO); 947744947dcSTom Erickson } 948744947dcSTom Erickson ASSERT(odn->dn_dbuf != NULL); /* only "special" dnodes have no parent */ 949744947dcSTom Erickson 950744947dcSTom Erickson /* 951744947dcSTom Erickson * Lock the dnode handle to prevent the dnode from obtaining any new 952744947dcSTom Erickson * holds. This also prevents the descendant dbufs and the bonus dbuf 953744947dcSTom Erickson * from accessing the dnode, so that we can discount their holds. The 954744947dcSTom Erickson * handle is safe to access because we know that while the dnode cannot 955744947dcSTom Erickson * go away, neither can its handle. Once we hold dnh_zrlock, we can 956744947dcSTom Erickson * safely move any dnode referenced only by dbufs. 957744947dcSTom Erickson */ 958744947dcSTom Erickson if (!zrl_tryenter(&odn->dn_handle->dnh_zrlock)) { 959744947dcSTom Erickson mutex_exit(&os->os_lock); 96054811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_handle); 961744947dcSTom Erickson return (KMEM_CBRC_LATER); 962744947dcSTom Erickson } 963744947dcSTom Erickson 964744947dcSTom Erickson /* 965744947dcSTom Erickson * Ensure a consistent view of the dnode's holds and the dnode's dbufs. 966744947dcSTom Erickson * We need to guarantee that there is a hold for every dbuf in order to 967744947dcSTom Erickson * determine whether the dnode is actively referenced. Falsely matching 968744947dcSTom Erickson * a dbuf to an active hold would lead to an unsafe move. It's possible 969744947dcSTom Erickson * that a thread already having an active dnode hold is about to add a 970744947dcSTom Erickson * dbuf, and we can't compare hold and dbuf counts while the add is in 971744947dcSTom Erickson * progress. 972744947dcSTom Erickson */ 973744947dcSTom Erickson if (!rw_tryenter(&odn->dn_struct_rwlock, RW_WRITER)) { 974744947dcSTom Erickson zrl_exit(&odn->dn_handle->dnh_zrlock); 975744947dcSTom Erickson mutex_exit(&os->os_lock); 97654811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_rwlock); 977744947dcSTom Erickson return (KMEM_CBRC_LATER); 978744947dcSTom Erickson } 979744947dcSTom Erickson 980744947dcSTom Erickson /* 981744947dcSTom Erickson * A dbuf may be removed (evicted) without an active dnode hold. In that 982744947dcSTom Erickson * case, the dbuf count is decremented under the handle lock before the 983744947dcSTom Erickson * dbuf's hold is released. This order ensures that if we count the hold 984744947dcSTom Erickson * after the dbuf is removed but before its hold is released, we will 985744947dcSTom Erickson * treat the unmatched hold as active and exit safely. If we count the 986744947dcSTom Erickson * hold before the dbuf is removed, the hold is discounted, and the 987744947dcSTom Erickson * removal is blocked until the move completes. 988744947dcSTom Erickson */ 989e914ace2STim Schumacher refcount = zfs_refcount_count(&odn->dn_holds); 990744947dcSTom Erickson ASSERT(refcount >= 0); 991744947dcSTom Erickson dbufs = odn->dn_dbufs_count; 992744947dcSTom Erickson 993744947dcSTom Erickson /* We can't have more dbufs than dnode holds. */ 994744947dcSTom Erickson ASSERT3U(dbufs, <=, refcount); 995744947dcSTom Erickson DTRACE_PROBE3(dnode__move, dnode_t *, odn, int64_t, refcount, 996744947dcSTom Erickson uint32_t, dbufs); 997744947dcSTom Erickson 998744947dcSTom Erickson if (refcount > dbufs) { 999744947dcSTom Erickson rw_exit(&odn->dn_struct_rwlock); 1000744947dcSTom Erickson zrl_exit(&odn->dn_handle->dnh_zrlock); 1001744947dcSTom Erickson mutex_exit(&os->os_lock); 100254811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_active); 1003744947dcSTom Erickson return (KMEM_CBRC_LATER); 1004744947dcSTom Erickson } 1005744947dcSTom Erickson 1006744947dcSTom Erickson rw_exit(&odn->dn_struct_rwlock); 1007744947dcSTom Erickson 1008744947dcSTom Erickson /* 1009744947dcSTom Erickson * At this point we know that anyone with a hold on the dnode is not 1010744947dcSTom Erickson * actively referencing it. The dnode is known and in a valid state to 1011744947dcSTom Erickson * move. We're holding the locks needed to execute the critical section. 1012744947dcSTom Erickson */ 1013744947dcSTom Erickson dnode_move_impl(odn, ndn); 1014744947dcSTom Erickson 1015744947dcSTom Erickson list_link_replace(&odn->dn_link, &ndn->dn_link); 1016744947dcSTom Erickson /* If the dnode was safe to move, the refcount cannot have changed. */ 1017e914ace2STim Schumacher ASSERT(refcount == zfs_refcount_count(&ndn->dn_holds)); 1018744947dcSTom Erickson ASSERT(dbufs == ndn->dn_dbufs_count); 1019744947dcSTom Erickson zrl_exit(&ndn->dn_handle->dnh_zrlock); /* handle has moved */ 1020744947dcSTom Erickson mutex_exit(&os->os_lock); 1021744947dcSTom Erickson 1022744947dcSTom Erickson return (KMEM_CBRC_YES); 1023744947dcSTom Erickson } 1024744947dcSTom Erickson #endif /* _KERNEL */ 1025744947dcSTom Erickson 102654811da5SToomas Soome static void 102754811da5SToomas Soome dnode_slots_hold(dnode_children_t *children, int idx, int slots) 102854811da5SToomas Soome { 102954811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 103054811da5SToomas Soome 103154811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 103254811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 103354811da5SToomas Soome zrl_add(&dnh->dnh_zrlock); 103454811da5SToomas Soome } 103554811da5SToomas Soome } 103654811da5SToomas Soome 103754811da5SToomas Soome static void 103854811da5SToomas Soome dnode_slots_rele(dnode_children_t *children, int idx, int slots) 103954811da5SToomas Soome { 104054811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 104154811da5SToomas Soome 104254811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 104354811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 104454811da5SToomas Soome 104554811da5SToomas Soome if (zrl_is_locked(&dnh->dnh_zrlock)) 104654811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 104754811da5SToomas Soome else 104854811da5SToomas Soome zrl_remove(&dnh->dnh_zrlock); 104954811da5SToomas Soome } 105054811da5SToomas Soome } 105154811da5SToomas Soome 105254811da5SToomas Soome static int 105354811da5SToomas Soome dnode_slots_tryenter(dnode_children_t *children, int idx, int slots) 105454811da5SToomas Soome { 105554811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 105654811da5SToomas Soome 105754811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 105854811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 105954811da5SToomas Soome 106054811da5SToomas Soome if (!zrl_tryenter(&dnh->dnh_zrlock)) { 106154811da5SToomas Soome for (int j = idx; j < i; j++) { 106254811da5SToomas Soome dnh = &children->dnc_children[j]; 106354811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 106454811da5SToomas Soome } 106554811da5SToomas Soome 106654811da5SToomas Soome return (0); 106754811da5SToomas Soome } 106854811da5SToomas Soome } 106954811da5SToomas Soome 107054811da5SToomas Soome return (1); 107154811da5SToomas Soome } 107254811da5SToomas Soome 107354811da5SToomas Soome static void 107454811da5SToomas Soome dnode_set_slots(dnode_children_t *children, int idx, int slots, void *ptr) 107554811da5SToomas Soome { 107654811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 107754811da5SToomas Soome 107854811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 107954811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 108054811da5SToomas Soome dnh->dnh_dnode = ptr; 108154811da5SToomas Soome } 108254811da5SToomas Soome } 108354811da5SToomas Soome 108454811da5SToomas Soome static boolean_t 108554811da5SToomas Soome dnode_check_slots_free(dnode_children_t *children, int idx, int slots) 108654811da5SToomas Soome { 108754811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 108854811da5SToomas Soome 1089aa02ea01STom Caputi /* 1090aa02ea01STom Caputi * If all dnode slots are either already free or 1091aa02ea01STom Caputi * evictable return B_TRUE. 1092aa02ea01STom Caputi */ 109354811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 109454811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 109554811da5SToomas Soome dnode_t *dn = dnh->dnh_dnode; 109654811da5SToomas Soome 109754811da5SToomas Soome if (dn == DN_SLOT_FREE) { 109854811da5SToomas Soome continue; 109954811da5SToomas Soome } else if (DN_SLOT_IS_PTR(dn)) { 110054811da5SToomas Soome mutex_enter(&dn->dn_mtx); 1101aa02ea01STom Caputi boolean_t can_free = (dn->dn_type == DMU_OT_NONE && 1102aa02ea01STom Caputi zfs_refcount_is_zero(&dn->dn_holds) && 1103aa02ea01STom Caputi !DNODE_IS_DIRTY(dn)); 110454811da5SToomas Soome mutex_exit(&dn->dn_mtx); 110554811da5SToomas Soome 1106aa02ea01STom Caputi if (!can_free) 110754811da5SToomas Soome return (B_FALSE); 1108aa02ea01STom Caputi else 1109aa02ea01STom Caputi continue; 111054811da5SToomas Soome } else { 111154811da5SToomas Soome return (B_FALSE); 111254811da5SToomas Soome } 111354811da5SToomas Soome } 111454811da5SToomas Soome 111554811da5SToomas Soome return (B_TRUE); 111654811da5SToomas Soome } 111754811da5SToomas Soome 111854811da5SToomas Soome static void 111954811da5SToomas Soome dnode_reclaim_slots(dnode_children_t *children, int idx, int slots) 112054811da5SToomas Soome { 112154811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 112254811da5SToomas Soome 112354811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 112454811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 112554811da5SToomas Soome 112654811da5SToomas Soome ASSERT(zrl_is_locked(&dnh->dnh_zrlock)); 112754811da5SToomas Soome 112854811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 112954811da5SToomas Soome ASSERT3S(dnh->dnh_dnode->dn_type, ==, DMU_OT_NONE); 113054811da5SToomas Soome dnode_destroy(dnh->dnh_dnode); 113154811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_FREE; 113254811da5SToomas Soome } 113354811da5SToomas Soome } 113454811da5SToomas Soome } 113554811da5SToomas Soome 113654811da5SToomas Soome void 113754811da5SToomas Soome dnode_free_interior_slots(dnode_t *dn) 113854811da5SToomas Soome { 113954811da5SToomas Soome dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db); 114054811da5SToomas Soome int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT; 114154811da5SToomas Soome int idx = (dn->dn_object & (epb - 1)) + 1; 114254811da5SToomas Soome int slots = dn->dn_num_slots - 1; 114354811da5SToomas Soome 114454811da5SToomas Soome if (slots == 0) 114554811da5SToomas Soome return; 114654811da5SToomas Soome 114754811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 114854811da5SToomas Soome 114954811da5SToomas Soome while (!dnode_slots_tryenter(children, idx, slots)) 115054811da5SToomas Soome DNODE_STAT_BUMP(dnode_free_interior_lock_retry); 115154811da5SToomas Soome 115254811da5SToomas Soome dnode_set_slots(children, idx, slots, DN_SLOT_FREE); 115354811da5SToomas Soome dnode_slots_rele(children, idx, slots); 115454811da5SToomas Soome } 115554811da5SToomas Soome 1156fa9e4066Sahrens void 1157744947dcSTom Erickson dnode_special_close(dnode_handle_t *dnh) 1158fa9e4066Sahrens { 1159744947dcSTom Erickson dnode_t *dn = dnh->dnh_dnode; 1160744947dcSTom Erickson 1161ea8dc4b6Seschrock /* 1162ea8dc4b6Seschrock * Wait for final references to the dnode to clear. This can 116354811da5SToomas Soome * only happen if the arc is asynchronously evicting state that 1164ea8dc4b6Seschrock * has a hold on this dnode while we are trying to evict this 1165ea8dc4b6Seschrock * dnode. 1166ea8dc4b6Seschrock */ 1167e914ace2STim Schumacher while (zfs_refcount_count(&dn->dn_holds) > 0) 1168ea8dc4b6Seschrock delay(1); 1169bc9014e6SJustin Gibbs ASSERT(dn->dn_dbuf == NULL || 1170bc9014e6SJustin Gibbs dmu_buf_get_user(&dn->dn_dbuf->db) == NULL); 1171744947dcSTom Erickson zrl_add(&dnh->dnh_zrlock); 1172744947dcSTom Erickson dnode_destroy(dn); /* implicit zrl_remove() */ 1173744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 1174744947dcSTom Erickson dnh->dnh_dnode = NULL; 1175fa9e4066Sahrens } 1176fa9e4066Sahrens 1177bc9014e6SJustin Gibbs void 1178744947dcSTom Erickson dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object, 1179744947dcSTom Erickson dnode_handle_t *dnh) 1180fa9e4066Sahrens { 1181bc9014e6SJustin Gibbs dnode_t *dn; 1182bc9014e6SJustin Gibbs 1183744947dcSTom Erickson zrl_init(&dnh->dnh_zrlock); 118454811da5SToomas Soome zrl_tryenter(&dnh->dnh_zrlock); 118554811da5SToomas Soome 118654811da5SToomas Soome dn = dnode_create(os, dnp, NULL, object, dnh); 11879c9dc39aSek DNODE_VERIFY(dn); 118854811da5SToomas Soome 118954811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 1190fa9e4066Sahrens } 1191fa9e4066Sahrens 1192fa9e4066Sahrens static void 119340510e8eSJosef 'Jeff' Sipek dnode_buf_evict_async(void *dbu) 1194fa9e4066Sahrens { 119554811da5SToomas Soome dnode_children_t *dnc = dbu; 119654811da5SToomas Soome 119754811da5SToomas Soome DNODE_STAT_BUMP(dnode_buf_evict); 1198fa9e4066Sahrens 119954811da5SToomas Soome for (int i = 0; i < dnc->dnc_count; i++) { 120054811da5SToomas Soome dnode_handle_t *dnh = &dnc->dnc_children[i]; 1201744947dcSTom Erickson dnode_t *dn; 1202fa9e4066Sahrens 1203744947dcSTom Erickson /* 1204744947dcSTom Erickson * The dnode handle lock guards against the dnode moving to 1205744947dcSTom Erickson * another valid address, so there is no need here to guard 1206744947dcSTom Erickson * against changes to or from NULL. 1207744947dcSTom Erickson */ 120854811da5SToomas Soome if (!DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 1209744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 121054811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_UNINIT; 1211fa9e4066Sahrens continue; 1212744947dcSTom Erickson } 1213744947dcSTom Erickson 1214744947dcSTom Erickson zrl_add(&dnh->dnh_zrlock); 1215744947dcSTom Erickson dn = dnh->dnh_dnode; 1216fa9e4066Sahrens /* 1217fa9e4066Sahrens * If there are holds on this dnode, then there should 1218fa9e4066Sahrens * be holds on the dnode's containing dbuf as well; thus 1219744947dcSTom Erickson * it wouldn't be eligible for eviction and this function 1220fa9e4066Sahrens * would not have been called. 1221fa9e4066Sahrens */ 1222e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_holds)); 1223e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); 1224fa9e4066Sahrens 122554811da5SToomas Soome dnode_destroy(dn); /* implicit zrl_remove() for first slot */ 1226744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 122754811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_UNINIT; 1228fa9e4066Sahrens } 122954811da5SToomas Soome kmem_free(dnc, sizeof (dnode_children_t) + 123054811da5SToomas Soome dnc->dnc_count * sizeof (dnode_handle_t)); 1231fa9e4066Sahrens } 1232fa9e4066Sahrens 1233fa9e4066Sahrens /* 123454811da5SToomas Soome * When the DNODE_MUST_BE_FREE flag is set, the "slots" parameter is used 123554811da5SToomas Soome * to ensure the hole at the specified object offset is large enough to 123654811da5SToomas Soome * hold the dnode being created. The slots parameter is also used to ensure 123754811da5SToomas Soome * a dnode does not span multiple dnode blocks. In both of these cases, if 123854811da5SToomas Soome * a failure occurs, ENOSPC is returned. Keep in mind, these failure cases 123954811da5SToomas Soome * are only possible when using DNODE_MUST_BE_FREE. 124054811da5SToomas Soome * 124154811da5SToomas Soome * If the DNODE_MUST_BE_ALLOCATED flag is set, "slots" must be 0. 124254811da5SToomas Soome * dnode_hold_impl() will check if the requested dnode is already consumed 124354811da5SToomas Soome * as an extra dnode slot by an large dnode, in which case it returns 124454811da5SToomas Soome * ENOENT. 124554811da5SToomas Soome * 1246ea8dc4b6Seschrock * errors: 124754811da5SToomas Soome * EINVAL - invalid object number or flags. 124854811da5SToomas Soome * ENOSPC - hole too small to fulfill "slots" request (DNODE_MUST_BE_FREE) 124954811da5SToomas Soome * EEXIST - Refers to an allocated dnode (DNODE_MUST_BE_FREE) 125054811da5SToomas Soome * - Refers to a freeing dnode (DNODE_MUST_BE_FREE) 125154811da5SToomas Soome * - Refers to an interior dnode slot (DNODE_MUST_BE_ALLOCATED) 125254811da5SToomas Soome * ENOENT - The requested dnode is not allocated (DNODE_MUST_BE_ALLOCATED) 125354811da5SToomas Soome * - The requested dnode is being freed (DNODE_MUST_BE_ALLOCATED) 125454811da5SToomas Soome * EIO - i/o error error when reading the meta dnode dbuf. 1255ea8dc4b6Seschrock * succeeds even for free dnodes. 1256fa9e4066Sahrens */ 1257ea8dc4b6Seschrock int 125854811da5SToomas Soome dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots, 1259ea8dc4b6Seschrock void *tag, dnode_t **dnp) 1260fa9e4066Sahrens { 1261ea8dc4b6Seschrock int epb, idx, err; 1262fa9e4066Sahrens int drop_struct_lock = FALSE; 1263ea8dc4b6Seschrock int type; 1264fa9e4066Sahrens uint64_t blk; 1265fa9e4066Sahrens dnode_t *mdn, *dn; 1266fa9e4066Sahrens dmu_buf_impl_t *db; 126754811da5SToomas Soome dnode_children_t *dnc; 126854811da5SToomas Soome dnode_phys_t *dn_block; 1269744947dcSTom Erickson dnode_handle_t *dnh; 1270fa9e4066Sahrens 127154811da5SToomas Soome ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0)); 127254811da5SToomas Soome ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0)); 127354811da5SToomas Soome 1274e14bb325SJeff Bonwick /* 1275e14bb325SJeff Bonwick * If you are holding the spa config lock as writer, you shouldn't 1276dcba9f3fSGeorge Wilson * be asking the DMU to do *anything* unless it's the root pool 1277dcba9f3fSGeorge Wilson * which may require us to read from the root filesystem while 1278dcba9f3fSGeorge Wilson * holding some (not all) of the locks as writer. 1279e14bb325SJeff Bonwick */ 1280dcba9f3fSGeorge Wilson ASSERT(spa_config_held(os->os_spa, SCL_ALL, RW_WRITER) == 0 || 1281dcba9f3fSGeorge Wilson (spa_is_root(os->os_spa) && 128244ecc532SGeorge Wilson spa_config_held(os->os_spa, SCL_STATE, RW_WRITER))); 1283e14bb325SJeff Bonwick 128486714001SSerapheim Dimitropoulos ASSERT((flag & DNODE_MUST_BE_ALLOCATED) || (flag & DNODE_MUST_BE_FREE)); 128586714001SSerapheim Dimitropoulos 128614843421SMatthew Ahrens if (object == DMU_USERUSED_OBJECT || object == DMU_GROUPUSED_OBJECT) { 128714843421SMatthew Ahrens dn = (object == DMU_USERUSED_OBJECT) ? 1288744947dcSTom Erickson DMU_USERUSED_DNODE(os) : DMU_GROUPUSED_DNODE(os); 128914843421SMatthew Ahrens if (dn == NULL) 1290be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 129114843421SMatthew Ahrens type = dn->dn_type; 129214843421SMatthew Ahrens if ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) 1293be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 129414843421SMatthew Ahrens if ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE) 1295be6fd75aSMatthew Ahrens return (SET_ERROR(EEXIST)); 129614843421SMatthew Ahrens DNODE_VERIFY(dn); 1297e914ace2STim Schumacher (void) zfs_refcount_add(&dn->dn_holds, tag); 129814843421SMatthew Ahrens *dnp = dn; 129914843421SMatthew Ahrens return (0); 130014843421SMatthew Ahrens } 130114843421SMatthew Ahrens 1302fa9e4066Sahrens if (object == 0 || object >= DN_MAX_OBJECT) 1303be6fd75aSMatthew Ahrens return (SET_ERROR(EINVAL)); 1304fa9e4066Sahrens 1305744947dcSTom Erickson mdn = DMU_META_DNODE(os); 1306744947dcSTom Erickson ASSERT(mdn->dn_object == DMU_META_DNODE_OBJECT); 1307fa9e4066Sahrens 13089c9dc39aSek DNODE_VERIFY(mdn); 1309fa9e4066Sahrens 1310fa9e4066Sahrens if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) { 1311fa9e4066Sahrens rw_enter(&mdn->dn_struct_rwlock, RW_READER); 1312fa9e4066Sahrens drop_struct_lock = TRUE; 1313fa9e4066Sahrens } 1314fa9e4066Sahrens 1315a2cdcdd2SPaul Dagnelie blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t)); 1316fa9e4066Sahrens 1317ea8dc4b6Seschrock db = dbuf_hold(mdn, blk, FTAG); 1318fa9e4066Sahrens if (drop_struct_lock) 1319fa9e4066Sahrens rw_exit(&mdn->dn_struct_rwlock); 132054811da5SToomas Soome if (db == NULL) { 132154811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_dbuf_hold); 1322be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 132354811da5SToomas Soome } 1324ea8dc4b6Seschrock err = dbuf_read(db, NULL, DB_RF_CANFAIL); 1325ea8dc4b6Seschrock if (err) { 132654811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_dbuf_read); 1327ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1328ea8dc4b6Seschrock return (err); 1329ea8dc4b6Seschrock } 1330fa9e4066Sahrens 1331fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT); 1332fa9e4066Sahrens epb = db->db.db_size >> DNODE_SHIFT; 1333fa9e4066Sahrens 133454811da5SToomas Soome idx = object & (epb - 1); 133554811da5SToomas Soome dn_block = (dnode_phys_t *)db->db.db_data; 1336fa9e4066Sahrens 1337744947dcSTom Erickson ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE); 133854811da5SToomas Soome dnc = dmu_buf_get_user(&db->db); 133954811da5SToomas Soome dnh = NULL; 134054811da5SToomas Soome if (dnc == NULL) { 1341744947dcSTom Erickson dnode_children_t *winner; 134254811da5SToomas Soome int skip = 0; 134354811da5SToomas Soome 134454811da5SToomas Soome dnc = kmem_zalloc(sizeof (dnode_children_t) + 13457f18da4cSJustin T. Gibbs epb * sizeof (dnode_handle_t), KM_SLEEP); 134654811da5SToomas Soome dnc->dnc_count = epb; 134754811da5SToomas Soome dnh = &dnc->dnc_children[0]; 134854811da5SToomas Soome 134954811da5SToomas Soome /* Initialize dnode slot status from dnode_phys_t */ 135054811da5SToomas Soome for (int i = 0; i < epb; i++) { 1351744947dcSTom Erickson zrl_init(&dnh[i].dnh_zrlock); 135254811da5SToomas Soome 135354811da5SToomas Soome if (skip) { 135454811da5SToomas Soome skip--; 135554811da5SToomas Soome continue; 135654811da5SToomas Soome } 135754811da5SToomas Soome 135854811da5SToomas Soome if (dn_block[i].dn_type != DMU_OT_NONE) { 135954811da5SToomas Soome int interior = dn_block[i].dn_extra_slots; 136054811da5SToomas Soome 136154811da5SToomas Soome dnode_set_slots(dnc, i, 1, DN_SLOT_ALLOCATED); 136254811da5SToomas Soome dnode_set_slots(dnc, i + 1, interior, 136354811da5SToomas Soome DN_SLOT_INTERIOR); 136454811da5SToomas Soome skip = interior; 136554811da5SToomas Soome } else { 136654811da5SToomas Soome dnh[i].dnh_dnode = DN_SLOT_FREE; 136754811da5SToomas Soome skip = 0; 136854811da5SToomas Soome } 1369744947dcSTom Erickson } 137054811da5SToomas Soome 137154811da5SToomas Soome dmu_buf_init_user(&dnc->dnc_dbu, NULL, 137240510e8eSJosef 'Jeff' Sipek dnode_buf_evict_async, NULL); 137354811da5SToomas Soome winner = dmu_buf_set_user(&db->db, &dnc->dnc_dbu); 1374bc9014e6SJustin Gibbs if (winner != NULL) { 1375d2b3cbbdSJorgen Lundman 137654811da5SToomas Soome for (int i = 0; i < epb; i++) 1377d2b3cbbdSJorgen Lundman zrl_destroy(&dnh[i].dnh_zrlock); 1378d2b3cbbdSJorgen Lundman 137954811da5SToomas Soome kmem_free(dnc, sizeof (dnode_children_t) + 13807f18da4cSJustin T. Gibbs epb * sizeof (dnode_handle_t)); 138154811da5SToomas Soome dnc = winner; 1382fa9e4066Sahrens } 1383fa9e4066Sahrens } 1384fa9e4066Sahrens 138554811da5SToomas Soome ASSERT(dnc->dnc_count == epb); 138654811da5SToomas Soome dn = DN_SLOT_UNINIT; 138754811da5SToomas Soome 138854811da5SToomas Soome if (flag & DNODE_MUST_BE_ALLOCATED) { 138954811da5SToomas Soome slots = 1; 139054811da5SToomas Soome 139154811da5SToomas Soome while (dn == DN_SLOT_UNINIT) { 139254811da5SToomas Soome dnode_slots_hold(dnc, idx, slots); 139354811da5SToomas Soome dnh = &dnc->dnc_children[idx]; 139454811da5SToomas Soome 139554811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 139654811da5SToomas Soome dn = dnh->dnh_dnode; 139754811da5SToomas Soome break; 139854811da5SToomas Soome } else if (dnh->dnh_dnode == DN_SLOT_INTERIOR) { 139954811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_interior); 140054811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 140154811da5SToomas Soome dbuf_rele(db, FTAG); 140254811da5SToomas Soome return (SET_ERROR(EEXIST)); 140354811da5SToomas Soome } else if (dnh->dnh_dnode != DN_SLOT_ALLOCATED) { 140454811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_misses); 140554811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 140654811da5SToomas Soome dbuf_rele(db, FTAG); 140754811da5SToomas Soome return (SET_ERROR(ENOENT)); 140854811da5SToomas Soome } 140954811da5SToomas Soome 141054811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 141154811da5SToomas Soome if (!dnode_slots_tryenter(dnc, idx, slots)) { 141254811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_lock_retry); 141354811da5SToomas Soome continue; 141454811da5SToomas Soome } 141554811da5SToomas Soome 141654811da5SToomas Soome /* 141754811da5SToomas Soome * Someone else won the race and called dnode_create() 141854811da5SToomas Soome * after we checked DN_SLOT_IS_PTR() above but before 141954811da5SToomas Soome * we acquired the lock. 142054811da5SToomas Soome */ 142154811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 142254811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_lock_misses); 142354811da5SToomas Soome dn = dnh->dnh_dnode; 142454811da5SToomas Soome } else { 142554811da5SToomas Soome dn = dnode_create(os, dn_block + idx, db, 142654811da5SToomas Soome object, dnh); 142754811da5SToomas Soome } 142854811da5SToomas Soome } 142954811da5SToomas Soome 143054811da5SToomas Soome mutex_enter(&dn->dn_mtx); 143154811da5SToomas Soome if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) { 143254811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_type_none); 143354811da5SToomas Soome mutex_exit(&dn->dn_mtx); 143454811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 143554811da5SToomas Soome dbuf_rele(db, FTAG); 143654811da5SToomas Soome return (SET_ERROR(ENOENT)); 143754811da5SToomas Soome } 143854811da5SToomas Soome 143954811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_hits); 144054811da5SToomas Soome } else if (flag & DNODE_MUST_BE_FREE) { 144154811da5SToomas Soome 144254811da5SToomas Soome if (idx + slots - 1 >= DNODES_PER_BLOCK) { 144354811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_overflow); 144454811da5SToomas Soome dbuf_rele(db, FTAG); 144554811da5SToomas Soome return (SET_ERROR(ENOSPC)); 144654811da5SToomas Soome } 144754811da5SToomas Soome 144854811da5SToomas Soome while (dn == DN_SLOT_UNINIT) { 144954811da5SToomas Soome dnode_slots_hold(dnc, idx, slots); 145054811da5SToomas Soome 145154811da5SToomas Soome if (!dnode_check_slots_free(dnc, idx, slots)) { 145254811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_misses); 145354811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 145454811da5SToomas Soome dbuf_rele(db, FTAG); 145554811da5SToomas Soome return (SET_ERROR(ENOSPC)); 145654811da5SToomas Soome } 145754811da5SToomas Soome 145854811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 145954811da5SToomas Soome if (!dnode_slots_tryenter(dnc, idx, slots)) { 146054811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_lock_retry); 146154811da5SToomas Soome continue; 146254811da5SToomas Soome } 146354811da5SToomas Soome 146454811da5SToomas Soome if (!dnode_check_slots_free(dnc, idx, slots)) { 146554811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_lock_misses); 146654811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 146754811da5SToomas Soome dbuf_rele(db, FTAG); 146854811da5SToomas Soome return (SET_ERROR(ENOSPC)); 146954811da5SToomas Soome } 147054811da5SToomas Soome 147154811da5SToomas Soome /* 147254811da5SToomas Soome * Allocated but otherwise free dnodes which would 147354811da5SToomas Soome * be in the interior of a multi-slot dnodes need 147454811da5SToomas Soome * to be freed. Single slot dnodes can be safely 147554811da5SToomas Soome * re-purposed as a performance optimization. 147654811da5SToomas Soome */ 147754811da5SToomas Soome if (slots > 1) 147854811da5SToomas Soome dnode_reclaim_slots(dnc, idx + 1, slots - 1); 147954811da5SToomas Soome 148054811da5SToomas Soome dnh = &dnc->dnc_children[idx]; 148154811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 148254811da5SToomas Soome dn = dnh->dnh_dnode; 148354811da5SToomas Soome } else { 148454811da5SToomas Soome dn = dnode_create(os, dn_block + idx, db, 148554811da5SToomas Soome object, dnh); 148654811da5SToomas Soome } 148754811da5SToomas Soome } 148854811da5SToomas Soome 148954811da5SToomas Soome mutex_enter(&dn->dn_mtx); 1490e914ace2STim Schumacher if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) { 149154811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_refcount); 149254811da5SToomas Soome mutex_exit(&dn->dn_mtx); 149354811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 149454811da5SToomas Soome dbuf_rele(db, FTAG); 149554811da5SToomas Soome return (SET_ERROR(EEXIST)); 149654811da5SToomas Soome } 14970e8c6158Smaybee 149854811da5SToomas Soome dnode_set_slots(dnc, idx + 1, slots - 1, DN_SLOT_INTERIOR); 149954811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_hits); 150054811da5SToomas Soome } else { 150154811da5SToomas Soome dbuf_rele(db, FTAG); 150254811da5SToomas Soome return (SET_ERROR(EINVAL)); 1503fa9e4066Sahrens } 1504fa9e4066Sahrens 150554811da5SToomas Soome if (dn->dn_free_txg) { 150654811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_txg); 150754811da5SToomas Soome type = dn->dn_type; 1508fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 150954811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 1510ea8dc4b6Seschrock dbuf_rele(db, FTAG); 151154811da5SToomas Soome return (SET_ERROR((flag & DNODE_MUST_BE_ALLOCATED) ? 151254811da5SToomas Soome ENOENT : EEXIST)); 1513fa9e4066Sahrens } 151454811da5SToomas Soome 1515e914ace2STim Schumacher if (zfs_refcount_add(&dn->dn_holds, tag) == 1) 1516744947dcSTom Erickson dbuf_add_ref(db, dnh); 151754811da5SToomas Soome 1518bc9014e6SJustin Gibbs mutex_exit(&dn->dn_mtx); 1519bc9014e6SJustin Gibbs 1520744947dcSTom Erickson /* Now we can rely on the hold to prevent the dnode from moving. */ 152154811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 1522fa9e4066Sahrens 15239c9dc39aSek DNODE_VERIFY(dn); 1524fa9e4066Sahrens ASSERT3P(dn->dn_dbuf, ==, db); 1525fa9e4066Sahrens ASSERT3U(dn->dn_object, ==, object); 1526ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1527fa9e4066Sahrens 1528ea8dc4b6Seschrock *dnp = dn; 1529ea8dc4b6Seschrock return (0); 1530fa9e4066Sahrens } 1531fa9e4066Sahrens 1532fa9e4066Sahrens /* 1533fa9e4066Sahrens * Return held dnode if the object is allocated, NULL if not. 1534fa9e4066Sahrens */ 1535ea8dc4b6Seschrock int 1536503ad85cSMatthew Ahrens dnode_hold(objset_t *os, uint64_t object, void *tag, dnode_t **dnp) 1537fa9e4066Sahrens { 153854811da5SToomas Soome return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag, 153954811da5SToomas Soome dnp)); 1540fa9e4066Sahrens } 1541fa9e4066Sahrens 15421934e92fSmaybee /* 15431934e92fSmaybee * Can only add a reference if there is already at least one 15441934e92fSmaybee * reference on the dnode. Returns FALSE if unable to add a 15451934e92fSmaybee * new reference. 15461934e92fSmaybee */ 15471934e92fSmaybee boolean_t 1548ea8dc4b6Seschrock dnode_add_ref(dnode_t *dn, void *tag) 1549fa9e4066Sahrens { 15501934e92fSmaybee mutex_enter(&dn->dn_mtx); 1551e914ace2STim Schumacher if (zfs_refcount_is_zero(&dn->dn_holds)) { 15521934e92fSmaybee mutex_exit(&dn->dn_mtx); 15531934e92fSmaybee return (FALSE); 15541934e92fSmaybee } 1555e914ace2STim Schumacher VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag)); 15561934e92fSmaybee mutex_exit(&dn->dn_mtx); 15571934e92fSmaybee return (TRUE); 1558fa9e4066Sahrens } 1559fa9e4066Sahrens 1560fa9e4066Sahrens void 1561ea8dc4b6Seschrock dnode_rele(dnode_t *dn, void *tag) 1562cd485b49SJustin T. Gibbs { 1563cd485b49SJustin T. Gibbs mutex_enter(&dn->dn_mtx); 1564c2919acbSMatthew Ahrens dnode_rele_and_unlock(dn, tag, B_FALSE); 1565cd485b49SJustin T. Gibbs } 1566cd485b49SJustin T. Gibbs 1567cd485b49SJustin T. Gibbs void 1568c2919acbSMatthew Ahrens dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting) 1569fa9e4066Sahrens { 1570fa9e4066Sahrens uint64_t refs; 1571744947dcSTom Erickson /* Get while the hold prevents the dnode from moving. */ 1572744947dcSTom Erickson dmu_buf_impl_t *db = dn->dn_dbuf; 1573744947dcSTom Erickson dnode_handle_t *dnh = dn->dn_handle; 1574fa9e4066Sahrens 1575e914ace2STim Schumacher refs = zfs_refcount_remove(&dn->dn_holds, tag); 15761934e92fSmaybee mutex_exit(&dn->dn_mtx); 1577744947dcSTom Erickson 1578744947dcSTom Erickson /* 1579744947dcSTom Erickson * It's unsafe to release the last hold on a dnode by dnode_rele() or 1580744947dcSTom Erickson * indirectly by dbuf_rele() while relying on the dnode handle to 1581744947dcSTom Erickson * prevent the dnode from moving, since releasing the last hold could 1582744947dcSTom Erickson * result in the dnode's parent dbuf evicting its dnode handles. For 1583744947dcSTom Erickson * that reason anyone calling dnode_rele() or dbuf_rele() without some 1584744947dcSTom Erickson * other direct or indirect hold on the dnode must first drop the dnode 1585744947dcSTom Erickson * handle. 1586744947dcSTom Erickson */ 1587744947dcSTom Erickson ASSERT(refs > 0 || dnh->dnh_zrlock.zr_owner != curthread); 1588744947dcSTom Erickson 1589fa9e4066Sahrens /* NOTE: the DNODE_DNODE does not have a dn_dbuf */ 1590744947dcSTom Erickson if (refs == 0 && db != NULL) { 1591744947dcSTom Erickson /* 1592744947dcSTom Erickson * Another thread could add a hold to the dnode handle in 1593744947dcSTom Erickson * dnode_hold_impl() while holding the parent dbuf. Since the 1594744947dcSTom Erickson * hold on the parent dbuf prevents the handle from being 1595744947dcSTom Erickson * destroyed, the hold on the handle is OK. We can't yet assert 1596744947dcSTom Erickson * that the handle has zero references, but that will be 1597744947dcSTom Erickson * asserted anyway when the handle gets destroyed. 1598744947dcSTom Erickson */ 1599c2919acbSMatthew Ahrens mutex_enter(&db->db_mtx); 1600c2919acbSMatthew Ahrens dbuf_rele_and_unlock(db, dnh, evicting); 1601744947dcSTom Erickson } 1602fa9e4066Sahrens } 1603fa9e4066Sahrens 1604fa9e4066Sahrens void 1605fa9e4066Sahrens dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) 1606fa9e4066Sahrens { 1607503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1608fa9e4066Sahrens uint64_t txg = tx->tx_txg; 1609fa9e4066Sahrens 161014843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { 161114843421SMatthew Ahrens dsl_dataset_dirty(os->os_dsl_dataset, tx); 1612fa9e4066Sahrens return; 161314843421SMatthew Ahrens } 1614fa9e4066Sahrens 16159c9dc39aSek DNODE_VERIFY(dn); 1616fa9e4066Sahrens 1617fa9e4066Sahrens #ifdef ZFS_DEBUG 1618fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1619fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg); 1620744947dcSTom Erickson ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); 1621fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1622fa9e4066Sahrens #endif 1623fa9e4066Sahrens 16240a586ceaSMark Shellenbaum /* 16250a586ceaSMark Shellenbaum * Determine old uid/gid when necessary 16260a586ceaSMark Shellenbaum */ 162706e0070dSMark Shellenbaum dmu_objset_userquota_get_ids(dn, B_TRUE, tx); 16280a586ceaSMark Shellenbaum 162994c2d0ebSMatthew Ahrens multilist_t *dirtylist = os->os_dirty_dnodes[txg & TXG_MASK]; 163094c2d0ebSMatthew Ahrens multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn); 1631fa9e4066Sahrens 1632fa9e4066Sahrens /* 1633fa9e4066Sahrens * If we are already marked dirty, we're done. 1634fa9e4066Sahrens */ 1635aa02ea01STom Caputi if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { 163694c2d0ebSMatthew Ahrens multilist_sublist_unlock(mls); 1637fa9e4066Sahrens return; 1638fa9e4066Sahrens } 1639fa9e4066Sahrens 1640e914ace2STim Schumacher ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) || 16410f6d88adSAlex Reece !avl_is_empty(&dn->dn_dbufs)); 1642fa9e4066Sahrens ASSERT(dn->dn_datablksz != 0); 1643fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[txg&TXG_MASK]); 1644fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[txg&TXG_MASK]); 1645fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[txg&TXG_MASK]); 1646fa9e4066Sahrens 1647fa9e4066Sahrens dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n", 1648fa9e4066Sahrens dn->dn_object, txg); 1649fa9e4066Sahrens 165094c2d0ebSMatthew Ahrens multilist_sublist_insert_head(mls, dn); 1651fa9e4066Sahrens 165294c2d0ebSMatthew Ahrens multilist_sublist_unlock(mls); 1653fa9e4066Sahrens 1654fa9e4066Sahrens /* 1655fa9e4066Sahrens * The dnode maintains a hold on its containing dbuf as 1656fa9e4066Sahrens * long as there are holds on it. Each instantiated child 1657744947dcSTom Erickson * dbuf maintains a hold on the dnode. When the last child 1658fa9e4066Sahrens * drops its hold, the dnode will drop its hold on the 1659fa9e4066Sahrens * containing dbuf. We add a "dirty hold" here so that the 1660fa9e4066Sahrens * dnode will hang around after we finish processing its 1661fa9e4066Sahrens * children. 1662fa9e4066Sahrens */ 16631934e92fSmaybee VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg)); 1664fa9e4066Sahrens 1665c717a561Smaybee (void) dbuf_dirty(dn->dn_dbuf, tx); 1666fa9e4066Sahrens 1667fa9e4066Sahrens dsl_dataset_dirty(os->os_dsl_dataset, tx); 1668fa9e4066Sahrens } 1669fa9e4066Sahrens 1670fa9e4066Sahrens void 1671fa9e4066Sahrens dnode_free(dnode_t *dn, dmu_tx_t *tx) 1672fa9e4066Sahrens { 1673fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1674fa9e4066Sahrens if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { 1675fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1676fa9e4066Sahrens return; 1677fa9e4066Sahrens } 1678fa9e4066Sahrens dn->dn_free_txg = tx->tx_txg; 1679fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1680fa9e4066Sahrens 168194c2d0ebSMatthew Ahrens dnode_setdirty(dn, tx); 1682fa9e4066Sahrens } 1683fa9e4066Sahrens 1684fa9e4066Sahrens /* 1685fa9e4066Sahrens * Try to change the block size for the indicated dnode. This can only 1686fa9e4066Sahrens * succeed if there are no blocks allocated or dirty beyond first block 1687fa9e4066Sahrens */ 1688fa9e4066Sahrens int 1689fa9e4066Sahrens dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx) 1690fa9e4066Sahrens { 16910f6d88adSAlex Reece dmu_buf_impl_t *db; 1692cdb0ab79Smaybee int err; 1693fa9e4066Sahrens 1694b5152584SMatthew Ahrens ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 1695fa9e4066Sahrens if (size == 0) 1696fa9e4066Sahrens size = SPA_MINBLOCKSIZE; 1697fa9e4066Sahrens else 1698fa9e4066Sahrens size = P2ROUNDUP(size, SPA_MINBLOCKSIZE); 1699fa9e4066Sahrens 1700b143e04bSahrens if (ibs == dn->dn_indblkshift) 1701b143e04bSahrens ibs = 0; 1702fa9e4066Sahrens 1703b143e04bSahrens if (size >> SPA_MINBLOCKSHIFT == dn->dn_datablkszsec && ibs == 0) 1704fa9e4066Sahrens return (0); 1705fa9e4066Sahrens 1706fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1707fa9e4066Sahrens 1708fa9e4066Sahrens /* Check for any allocated blocks beyond the first */ 17090713e232SGeorge Wilson if (dn->dn_maxblkid != 0) 1710b143e04bSahrens goto fail; 1711fa9e4066Sahrens 1712fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 17130f6d88adSAlex Reece for (db = avl_first(&dn->dn_dbufs); db != NULL; 17140f6d88adSAlex Reece db = AVL_NEXT(&dn->dn_dbufs, db)) { 17150a586ceaSMark Shellenbaum if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID && 17160a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1717fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1718b143e04bSahrens goto fail; 1719fa9e4066Sahrens } 1720fa9e4066Sahrens } 1721fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1722fa9e4066Sahrens 1723b143e04bSahrens if (ibs && dn->dn_nlevels != 1) 1724b143e04bSahrens goto fail; 1725b143e04bSahrens 1726cdb0ab79Smaybee /* resize the old block */ 1727a2cdcdd2SPaul Dagnelie err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db); 1728cdb0ab79Smaybee if (err == 0) 1729c543ec06Sahrens dbuf_new_size(db, size, tx); 1730cdb0ab79Smaybee else if (err != ENOENT) 1731cdb0ab79Smaybee goto fail; 1732fa9e4066Sahrens 1733fa9e4066Sahrens dnode_setdblksz(dn, size); 1734c543ec06Sahrens dnode_setdirty(dn, tx); 1735c543ec06Sahrens dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = size; 1736b143e04bSahrens if (ibs) { 1737b143e04bSahrens dn->dn_indblkshift = ibs; 1738b143e04bSahrens dn->dn_next_indblkshift[tx->tx_txg&TXG_MASK] = ibs; 1739b143e04bSahrens } 1740cdb0ab79Smaybee /* rele after we have fixed the blocksize in the dnode */ 1741c543ec06Sahrens if (db) 1742c543ec06Sahrens dbuf_rele(db, FTAG); 1743fa9e4066Sahrens 1744fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1745b143e04bSahrens return (0); 1746b143e04bSahrens 1747b143e04bSahrens fail: 1748b143e04bSahrens rw_exit(&dn->dn_struct_rwlock); 1749be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 1750fa9e4066Sahrens } 1751fa9e4066Sahrens 17528346f03fSJonathan W Adams /* read-holding callers must not rely on the lock being continuously held */ 1753fa9e4066Sahrens void 17548346f03fSJonathan W Adams dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read) 1755fa9e4066Sahrens { 1756fa9e4066Sahrens uint64_t txgoff = tx->tx_txg & TXG_MASK; 1757c543ec06Sahrens int epbs, new_nlevels; 1758fa9e4066Sahrens uint64_t sz; 1759fa9e4066Sahrens 17600a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1761fa9e4066Sahrens 17628346f03fSJonathan W Adams ASSERT(have_read ? 17638346f03fSJonathan W Adams RW_READ_HELD(&dn->dn_struct_rwlock) : 17648346f03fSJonathan W Adams RW_WRITE_HELD(&dn->dn_struct_rwlock)); 17658346f03fSJonathan W Adams 17668346f03fSJonathan W Adams /* 17678346f03fSJonathan W Adams * if we have a read-lock, check to see if we need to do any work 17688346f03fSJonathan W Adams * before upgrading to a write-lock. 17698346f03fSJonathan W Adams */ 17708346f03fSJonathan W Adams if (have_read) { 17718346f03fSJonathan W Adams if (blkid <= dn->dn_maxblkid) 17728346f03fSJonathan W Adams return; 17738346f03fSJonathan W Adams 17748346f03fSJonathan W Adams if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { 17758346f03fSJonathan W Adams rw_exit(&dn->dn_struct_rwlock); 17768346f03fSJonathan W Adams rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 17778346f03fSJonathan W Adams } 1778fa9e4066Sahrens } 1779fa9e4066Sahrens 1780c543ec06Sahrens if (blkid <= dn->dn_maxblkid) 1781c543ec06Sahrens goto out; 1782c543ec06Sahrens 1783c543ec06Sahrens dn->dn_maxblkid = blkid; 1784fa9e4066Sahrens 1785fa9e4066Sahrens /* 1786c543ec06Sahrens * Compute the number of levels necessary to support the new maxblkid. 1787fa9e4066Sahrens */ 1788fa9e4066Sahrens new_nlevels = 1; 1789fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1790c543ec06Sahrens for (sz = dn->dn_nblkptr; 1791c543ec06Sahrens sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs) 1792fa9e4066Sahrens new_nlevels++; 1793fa9e4066Sahrens 1794c543ec06Sahrens if (new_nlevels > dn->dn_nlevels) { 1795c543ec06Sahrens int old_nlevels = dn->dn_nlevels; 1796c543ec06Sahrens dmu_buf_impl_t *db; 1797c717a561Smaybee list_t *list; 1798c717a561Smaybee dbuf_dirty_record_t *new, *dr, *dr_next; 1799fa9e4066Sahrens 1800fa9e4066Sahrens dn->dn_nlevels = new_nlevels; 1801fa9e4066Sahrens 1802c543ec06Sahrens ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]); 1803c543ec06Sahrens dn->dn_next_nlevels[txgoff] = new_nlevels; 1804c543ec06Sahrens 1805c717a561Smaybee /* dirty the left indirects */ 1806c543ec06Sahrens db = dbuf_hold_level(dn, old_nlevels, 0, FTAG); 180701025c89SJohn Harres ASSERT(db != NULL); 1808c717a561Smaybee new = dbuf_dirty(db, tx); 1809ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1810fa9e4066Sahrens 1811c717a561Smaybee /* transfer the dirty records to the new indirect */ 1812c717a561Smaybee mutex_enter(&dn->dn_mtx); 1813c717a561Smaybee mutex_enter(&new->dt.di.dr_mtx); 1814c717a561Smaybee list = &dn->dn_dirty_records[txgoff]; 1815c717a561Smaybee for (dr = list_head(list); dr; dr = dr_next) { 1816c717a561Smaybee dr_next = list_next(&dn->dn_dirty_records[txgoff], dr); 1817c717a561Smaybee if (dr->dr_dbuf->db_level != new_nlevels-1 && 18180a586ceaSMark Shellenbaum dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 18190a586ceaSMark Shellenbaum dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 1820c717a561Smaybee ASSERT(dr->dr_dbuf->db_level == old_nlevels-1); 1821c717a561Smaybee list_remove(&dn->dn_dirty_records[txgoff], dr); 1822c717a561Smaybee list_insert_tail(&new->dt.di.dr_children, dr); 1823c717a561Smaybee dr->dr_parent = new; 1824c717a561Smaybee } 1825c717a561Smaybee } 1826c717a561Smaybee mutex_exit(&new->dt.di.dr_mtx); 1827c717a561Smaybee mutex_exit(&dn->dn_mtx); 1828fa9e4066Sahrens } 1829fa9e4066Sahrens 1830fa9e4066Sahrens out: 18318346f03fSJonathan W Adams if (have_read) 18328346f03fSJonathan W Adams rw_downgrade(&dn->dn_struct_rwlock); 1833fa9e4066Sahrens } 1834fa9e4066Sahrens 183546e1baa6SMatthew Ahrens static void 183646e1baa6SMatthew Ahrens dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx) 183746e1baa6SMatthew Ahrens { 183846e1baa6SMatthew Ahrens dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG); 183946e1baa6SMatthew Ahrens if (db != NULL) { 184046e1baa6SMatthew Ahrens dmu_buf_will_dirty(&db->db, tx); 184146e1baa6SMatthew Ahrens dbuf_rele(db, FTAG); 184246e1baa6SMatthew Ahrens } 184346e1baa6SMatthew Ahrens } 184446e1baa6SMatthew Ahrens 1845738e2a3cSPaul Dagnelie /* 1846738e2a3cSPaul Dagnelie * Dirty all the in-core level-1 dbufs in the range specified by start_blkid 1847738e2a3cSPaul Dagnelie * and end_blkid. 1848738e2a3cSPaul Dagnelie */ 1849738e2a3cSPaul Dagnelie static void 1850738e2a3cSPaul Dagnelie dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 1851738e2a3cSPaul Dagnelie dmu_tx_t *tx) 1852738e2a3cSPaul Dagnelie { 1853738e2a3cSPaul Dagnelie dmu_buf_impl_t db_search; 1854738e2a3cSPaul Dagnelie dmu_buf_impl_t *db; 1855738e2a3cSPaul Dagnelie avl_index_t where; 1856738e2a3cSPaul Dagnelie 1857738e2a3cSPaul Dagnelie mutex_enter(&dn->dn_dbufs_mtx); 1858738e2a3cSPaul Dagnelie 1859738e2a3cSPaul Dagnelie db_search.db_level = 1; 1860738e2a3cSPaul Dagnelie db_search.db_blkid = start_blkid + 1; 1861738e2a3cSPaul Dagnelie db_search.db_state = DB_SEARCH; 1862738e2a3cSPaul Dagnelie for (;;) { 1863738e2a3cSPaul Dagnelie 1864738e2a3cSPaul Dagnelie db = avl_find(&dn->dn_dbufs, &db_search, &where); 1865738e2a3cSPaul Dagnelie if (db == NULL) 1866738e2a3cSPaul Dagnelie db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1867738e2a3cSPaul Dagnelie 1868738e2a3cSPaul Dagnelie if (db == NULL || db->db_level != 1 || 1869738e2a3cSPaul Dagnelie db->db_blkid >= end_blkid) { 1870738e2a3cSPaul Dagnelie break; 1871738e2a3cSPaul Dagnelie } 1872738e2a3cSPaul Dagnelie 1873738e2a3cSPaul Dagnelie /* 1874738e2a3cSPaul Dagnelie * Setup the next blkid we want to search for. 1875738e2a3cSPaul Dagnelie */ 1876738e2a3cSPaul Dagnelie db_search.db_blkid = db->db_blkid + 1; 1877738e2a3cSPaul Dagnelie ASSERT3U(db->db_blkid, >=, start_blkid); 1878738e2a3cSPaul Dagnelie 1879738e2a3cSPaul Dagnelie /* 1880738e2a3cSPaul Dagnelie * If the dbuf transitions to DB_EVICTING while we're trying 1881738e2a3cSPaul Dagnelie * to dirty it, then we will be unable to discover it in 1882738e2a3cSPaul Dagnelie * the dbuf hash table. This will result in a call to 1883738e2a3cSPaul Dagnelie * dbuf_create() which needs to acquire the dn_dbufs_mtx 1884738e2a3cSPaul Dagnelie * lock. To avoid a deadlock, we drop the lock before 1885738e2a3cSPaul Dagnelie * dirtying the level-1 dbuf. 1886738e2a3cSPaul Dagnelie */ 1887738e2a3cSPaul Dagnelie mutex_exit(&dn->dn_dbufs_mtx); 1888738e2a3cSPaul Dagnelie dnode_dirty_l1(dn, db->db_blkid, tx); 1889738e2a3cSPaul Dagnelie mutex_enter(&dn->dn_dbufs_mtx); 1890738e2a3cSPaul Dagnelie } 1891738e2a3cSPaul Dagnelie 1892738e2a3cSPaul Dagnelie #ifdef ZFS_DEBUG 1893738e2a3cSPaul Dagnelie /* 1894738e2a3cSPaul Dagnelie * Walk all the in-core level-1 dbufs and verify they have been dirtied. 1895738e2a3cSPaul Dagnelie */ 1896738e2a3cSPaul Dagnelie db_search.db_level = 1; 1897738e2a3cSPaul Dagnelie db_search.db_blkid = start_blkid + 1; 1898738e2a3cSPaul Dagnelie db_search.db_state = DB_SEARCH; 1899738e2a3cSPaul Dagnelie db = avl_find(&dn->dn_dbufs, &db_search, &where); 1900738e2a3cSPaul Dagnelie if (db == NULL) 1901738e2a3cSPaul Dagnelie db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1902738e2a3cSPaul Dagnelie for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) { 1903738e2a3cSPaul Dagnelie if (db->db_level != 1 || db->db_blkid >= end_blkid) 1904738e2a3cSPaul Dagnelie break; 1905738e2a3cSPaul Dagnelie ASSERT(db->db_dirtycnt > 0); 1906738e2a3cSPaul Dagnelie } 1907738e2a3cSPaul Dagnelie #endif 1908738e2a3cSPaul Dagnelie mutex_exit(&dn->dn_dbufs_mtx); 1909738e2a3cSPaul Dagnelie } 1910738e2a3cSPaul Dagnelie 1911fa9e4066Sahrens void 1912fa9e4066Sahrens dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) 1913fa9e4066Sahrens { 1914fa9e4066Sahrens dmu_buf_impl_t *db; 1915b143e04bSahrens uint64_t blkoff, blkid, nblks; 1916cdb0ab79Smaybee int blksz, blkshift, head, tail; 1917fa9e4066Sahrens int trunc = FALSE; 1918cdb0ab79Smaybee int epbs; 1919fa9e4066Sahrens 1920fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1921fa9e4066Sahrens blksz = dn->dn_datablksz; 1922cdb0ab79Smaybee blkshift = dn->dn_datablkshift; 1923cdb0ab79Smaybee epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1924fa9e4066Sahrens 1925713d6c20SMatthew Ahrens if (len == DMU_OBJECT_END) { 1926fa9e4066Sahrens len = UINT64_MAX - off; 1927fa9e4066Sahrens trunc = TRUE; 1928fa9e4066Sahrens } 1929fa9e4066Sahrens 1930fa9e4066Sahrens /* 1931fa9e4066Sahrens * First, block align the region to free: 1932fa9e4066Sahrens */ 1933b143e04bSahrens if (ISP2(blksz)) { 1934b143e04bSahrens head = P2NPHASE(off, blksz); 1935b143e04bSahrens blkoff = P2PHASE(off, blksz); 1936cdb0ab79Smaybee if ((off >> blkshift) > dn->dn_maxblkid) 1937cdb0ab79Smaybee goto out; 1938b143e04bSahrens } else { 1939b143e04bSahrens ASSERT(dn->dn_maxblkid == 0); 1940b143e04bSahrens if (off == 0 && len >= blksz) { 194143466aaeSMax Grossman /* 194243466aaeSMax Grossman * Freeing the whole block; fast-track this request. 194343466aaeSMax Grossman */ 1944cdb0ab79Smaybee blkid = 0; 1945cdb0ab79Smaybee nblks = 1; 194699a19144SMatthew Ahrens if (dn->dn_nlevels > 1) 194799a19144SMatthew Ahrens dnode_dirty_l1(dn, 0, tx); 1948cdb0ab79Smaybee goto done; 19491c8564a7SMark Maybee } else if (off >= blksz) { 1950cdb0ab79Smaybee /* Freeing past end-of-data */ 1951cdb0ab79Smaybee goto out; 1952fa9e4066Sahrens } else { 1953b143e04bSahrens /* Freeing part of the block. */ 1954fa9e4066Sahrens head = blksz - off; 1955fa9e4066Sahrens ASSERT3U(head, >, 0); 1956fa9e4066Sahrens } 1957b143e04bSahrens blkoff = off; 1958fa9e4066Sahrens } 1959fa9e4066Sahrens /* zero out any partial block data at the start of the range */ 1960fa9e4066Sahrens if (head) { 1961b143e04bSahrens ASSERT3U(blkoff + head, ==, blksz); 1962fa9e4066Sahrens if (len < head) 1963fa9e4066Sahrens head = len; 1964a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), 1965a2cdcdd2SPaul Dagnelie TRUE, FALSE, FTAG, &db) == 0) { 1966fa9e4066Sahrens caddr_t data; 1967fa9e4066Sahrens 1968fa9e4066Sahrens /* don't dirty if it isn't on disk and isn't dirty */ 1969c717a561Smaybee if (db->db_last_dirty || 1970fa9e4066Sahrens (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr))) { 1971fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 197243466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 1973fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1974fa9e4066Sahrens data = db->db.db_data; 1975b143e04bSahrens bzero(data + blkoff, head); 1976fa9e4066Sahrens } 1977ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1978fa9e4066Sahrens } 1979fa9e4066Sahrens off += head; 1980fa9e4066Sahrens len -= head; 1981fa9e4066Sahrens } 1982fa9e4066Sahrens 1983b143e04bSahrens /* If the range was less than one block, we're done */ 1984cdb0ab79Smaybee if (len == 0) 1985fa9e4066Sahrens goto out; 1986fa9e4066Sahrens 1987cdb0ab79Smaybee /* If the remaining range is past end of file, we're done */ 1988cdb0ab79Smaybee if ((off >> blkshift) > dn->dn_maxblkid) 1989cdb0ab79Smaybee goto out; 1990b143e04bSahrens 19911c8564a7SMark Maybee ASSERT(ISP2(blksz)); 1992cdb0ab79Smaybee if (trunc) 1993cdb0ab79Smaybee tail = 0; 1994cdb0ab79Smaybee else 1995cdb0ab79Smaybee tail = P2PHASE(len, blksz); 1996cdb0ab79Smaybee 1997fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(off, blksz)); 1998cdb0ab79Smaybee /* zero out any partial block data at the end of the range */ 1999cdb0ab79Smaybee if (tail) { 2000cdb0ab79Smaybee if (len < tail) 2001cdb0ab79Smaybee tail = len; 2002a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off+len), 2003a2cdcdd2SPaul Dagnelie TRUE, FALSE, FTAG, &db) == 0) { 2004cdb0ab79Smaybee /* don't dirty if not on disk and not dirty */ 2005cdb0ab79Smaybee if (db->db_last_dirty || 2006cdb0ab79Smaybee (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr))) { 2007cdb0ab79Smaybee rw_exit(&dn->dn_struct_rwlock); 200843466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 2009cdb0ab79Smaybee rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2010cdb0ab79Smaybee bzero(db->db.db_data, tail); 2011fa9e4066Sahrens } 2012ea8dc4b6Seschrock dbuf_rele(db, FTAG); 2013fa9e4066Sahrens } 2014cdb0ab79Smaybee len -= tail; 2015cdb0ab79Smaybee } 2016fa9e4066Sahrens 2017cdb0ab79Smaybee /* If the range did not include a full block, we are done */ 2018cdb0ab79Smaybee if (len == 0) 2019cdb0ab79Smaybee goto out; 2020fa9e4066Sahrens 2021cdb0ab79Smaybee ASSERT(IS_P2ALIGNED(off, blksz)); 2022cdb0ab79Smaybee ASSERT(trunc || IS_P2ALIGNED(len, blksz)); 2023cdb0ab79Smaybee blkid = off >> blkshift; 2024cdb0ab79Smaybee nblks = len >> blkshift; 2025cdb0ab79Smaybee if (trunc) 2026cdb0ab79Smaybee nblks += 1; 2027cdb0ab79Smaybee 2028cdb0ab79Smaybee /* 202946e1baa6SMatthew Ahrens * Dirty all the indirect blocks in this range. Note that only 203046e1baa6SMatthew Ahrens * the first and last indirect blocks can actually be written 203146e1baa6SMatthew Ahrens * (if they were partially freed) -- they must be dirtied, even if 203246e1baa6SMatthew Ahrens * they do not exist on disk yet. The interior blocks will 203346e1baa6SMatthew Ahrens * be freed by free_children(), so they will not actually be written. 203446e1baa6SMatthew Ahrens * Even though these interior blocks will not be written, we 203546e1baa6SMatthew Ahrens * dirty them for two reasons: 203646e1baa6SMatthew Ahrens * 203746e1baa6SMatthew Ahrens * - It ensures that the indirect blocks remain in memory until 203846e1baa6SMatthew Ahrens * syncing context. (They have already been prefetched by 203946e1baa6SMatthew Ahrens * dmu_tx_hold_free(), so we don't have to worry about reading 204046e1baa6SMatthew Ahrens * them serially here.) 204146e1baa6SMatthew Ahrens * 204246e1baa6SMatthew Ahrens * - The dirty space accounting will put pressure on the txg sync 204346e1baa6SMatthew Ahrens * mechanism to begin syncing, and to delay transactions if there 204446e1baa6SMatthew Ahrens * is a large amount of freeing. Even though these indirect 204546e1baa6SMatthew Ahrens * blocks will not be written, we could need to write the same 204646e1baa6SMatthew Ahrens * amount of space if we copy the freed BPs into deadlists. 2047cdb0ab79Smaybee */ 2048cdb0ab79Smaybee if (dn->dn_nlevels > 1) { 204943466aaeSMax Grossman uint64_t first, last; 2050b143e04bSahrens 2051cdb0ab79Smaybee first = blkid >> epbs; 205246e1baa6SMatthew Ahrens dnode_dirty_l1(dn, first, tx); 2053b143e04bSahrens if (trunc) 2054cdb0ab79Smaybee last = dn->dn_maxblkid >> epbs; 2055cdb0ab79Smaybee else 2056cdb0ab79Smaybee last = (blkid + nblks - 1) >> epbs; 205746e1baa6SMatthew Ahrens if (last != first) 205846e1baa6SMatthew Ahrens dnode_dirty_l1(dn, last, tx); 205946e1baa6SMatthew Ahrens 2060738e2a3cSPaul Dagnelie dnode_dirty_l1range(dn, first, last, tx); 2061738e2a3cSPaul Dagnelie 206246e1baa6SMatthew Ahrens int shift = dn->dn_datablkshift + dn->dn_indblkshift - 206346e1baa6SMatthew Ahrens SPA_BLKPTRSHIFT; 206446e1baa6SMatthew Ahrens for (uint64_t i = first + 1; i < last; i++) { 206546e1baa6SMatthew Ahrens /* 206646e1baa6SMatthew Ahrens * Set i to the blockid of the next non-hole 206746e1baa6SMatthew Ahrens * level-1 indirect block at or after i. Note 206846e1baa6SMatthew Ahrens * that dnode_next_offset() operates in terms of 206946e1baa6SMatthew Ahrens * level-0-equivalent bytes. 207046e1baa6SMatthew Ahrens */ 207146e1baa6SMatthew Ahrens uint64_t ibyte = i << shift; 207246e1baa6SMatthew Ahrens int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK, 207346e1baa6SMatthew Ahrens &ibyte, 2, 1, 0); 207446e1baa6SMatthew Ahrens i = ibyte >> shift; 207546e1baa6SMatthew Ahrens if (i >= last) 207646e1baa6SMatthew Ahrens break; 207746e1baa6SMatthew Ahrens 207846e1baa6SMatthew Ahrens /* 207946e1baa6SMatthew Ahrens * Normally we should not see an error, either 208046e1baa6SMatthew Ahrens * from dnode_next_offset() or dbuf_hold_level() 208146e1baa6SMatthew Ahrens * (except for ESRCH from dnode_next_offset). 208246e1baa6SMatthew Ahrens * If there is an i/o error, then when we read 208346e1baa6SMatthew Ahrens * this block in syncing context, it will use 208446e1baa6SMatthew Ahrens * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according 208546e1baa6SMatthew Ahrens * to the "failmode" property. dnode_next_offset() 208646e1baa6SMatthew Ahrens * doesn't have a flag to indicate MUSTSUCCEED. 208746e1baa6SMatthew Ahrens */ 208846e1baa6SMatthew Ahrens if (err != 0) 208946e1baa6SMatthew Ahrens break; 209046e1baa6SMatthew Ahrens 209146e1baa6SMatthew Ahrens dnode_dirty_l1(dn, i, tx); 209256d55a53Smaybee } 2093fa9e4066Sahrens } 209443466aaeSMax Grossman 2095cdb0ab79Smaybee done: 2096cdb0ab79Smaybee /* 2097cdb0ab79Smaybee * Add this range to the dnode range list. 2098cdb0ab79Smaybee * We will finish up this free operation in the syncing phase. 2099cdb0ab79Smaybee */ 2100fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2101bf16b11eSMatthew Ahrens int txgoff = tx->tx_txg & TXG_MASK; 2102bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] == NULL) { 21035cabbc6bSPrashanth Sreenivasa dn->dn_free_ranges[txgoff] = range_tree_create(NULL, NULL); 2104fa9e4066Sahrens } 2105bf16b11eSMatthew Ahrens range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks); 2106bf16b11eSMatthew Ahrens range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks); 2107bf16b11eSMatthew Ahrens dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", 2108bf16b11eSMatthew Ahrens blkid, nblks, tx->tx_txg); 2109fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2110fa9e4066Sahrens 2111cdb0ab79Smaybee dbuf_free_range(dn, blkid, blkid + nblks - 1, tx); 2112fa9e4066Sahrens dnode_setdirty(dn, tx); 2113fa9e4066Sahrens out: 2114cdb0ab79Smaybee 2115fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 2116fa9e4066Sahrens } 2117fa9e4066Sahrens 21180a586ceaSMark Shellenbaum static boolean_t 21190a586ceaSMark Shellenbaum dnode_spill_freed(dnode_t *dn) 21200a586ceaSMark Shellenbaum { 21210a586ceaSMark Shellenbaum int i; 21220a586ceaSMark Shellenbaum 21230a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 21240a586ceaSMark Shellenbaum for (i = 0; i < TXG_SIZE; i++) { 21250a586ceaSMark Shellenbaum if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK) 21260a586ceaSMark Shellenbaum break; 21270a586ceaSMark Shellenbaum } 21280a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 21290a586ceaSMark Shellenbaum return (i < TXG_SIZE); 21300a586ceaSMark Shellenbaum } 21310a586ceaSMark Shellenbaum 2132fa9e4066Sahrens /* return TRUE if this blkid was freed in a recent txg, or FALSE if it wasn't */ 2133fa9e4066Sahrens uint64_t 2134fa9e4066Sahrens dnode_block_freed(dnode_t *dn, uint64_t blkid) 2135fa9e4066Sahrens { 2136fa9e4066Sahrens void *dp = spa_get_dsl(dn->dn_objset->os_spa); 2137fa9e4066Sahrens int i; 2138fa9e4066Sahrens 21390a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) 2140fa9e4066Sahrens return (FALSE); 2141fa9e4066Sahrens 2142fa9e4066Sahrens /* 2143fa9e4066Sahrens * If we're in the process of opening the pool, dp will not be 2144fa9e4066Sahrens * set yet, but there shouldn't be anything dirty. 2145fa9e4066Sahrens */ 2146fa9e4066Sahrens if (dp == NULL) 2147fa9e4066Sahrens return (FALSE); 2148fa9e4066Sahrens 2149fa9e4066Sahrens if (dn->dn_free_txg) 2150fa9e4066Sahrens return (TRUE); 2151fa9e4066Sahrens 21520a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) 21530a586ceaSMark Shellenbaum return (dnode_spill_freed(dn)); 21540a586ceaSMark Shellenbaum 2155fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2156fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 2157bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[i] != NULL && 2158bf16b11eSMatthew Ahrens range_tree_contains(dn->dn_free_ranges[i], blkid, 1)) 2159fa9e4066Sahrens break; 2160fa9e4066Sahrens } 2161fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2162fa9e4066Sahrens return (i < TXG_SIZE); 2163fa9e4066Sahrens } 2164fa9e4066Sahrens 2165fa9e4066Sahrens /* call from syncing context when we actually write/free space for this dnode */ 2166fa9e4066Sahrens void 216799653d4eSeschrock dnode_diduse_space(dnode_t *dn, int64_t delta) 2168fa9e4066Sahrens { 216999653d4eSeschrock uint64_t space; 217099653d4eSeschrock dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n", 2171fa9e4066Sahrens dn, dn->dn_phys, 217299653d4eSeschrock (u_longlong_t)dn->dn_phys->dn_used, 217399653d4eSeschrock (longlong_t)delta); 2174fa9e4066Sahrens 2175fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 217699653d4eSeschrock space = DN_USED_BYTES(dn->dn_phys); 217799653d4eSeschrock if (delta > 0) { 217899653d4eSeschrock ASSERT3U(space + delta, >=, space); /* no overflow */ 217999653d4eSeschrock } else { 218099653d4eSeschrock ASSERT3U(space, >=, -delta); /* no underflow */ 218199653d4eSeschrock } 218299653d4eSeschrock space += delta; 2183e7437265Sahrens if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) { 218499653d4eSeschrock ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0); 2185fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT)); 218699653d4eSeschrock dn->dn_phys->dn_used = space >> DEV_BSHIFT; 2187fa9e4066Sahrens } else { 218899653d4eSeschrock dn->dn_phys->dn_used = space; 218999653d4eSeschrock dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES; 2190fa9e4066Sahrens } 2191fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2192fa9e4066Sahrens } 2193fa9e4066Sahrens 219476256205SMark Maybee /* 2195f7170741SWill Andrews * Scans a block at the indicated "level" looking for a hole or data, 2196f7170741SWill Andrews * depending on 'flags'. 2197f7170741SWill Andrews * 2198f7170741SWill Andrews * If level > 0, then we are scanning an indirect block looking at its 2199f7170741SWill Andrews * pointers. If level == 0, then we are looking at a block of dnodes. 2200f7170741SWill Andrews * 2201f7170741SWill Andrews * If we don't find what we are looking for in the block, we return ESRCH. 2202f7170741SWill Andrews * Otherwise, return with *offset pointing to the beginning (if searching 2203f7170741SWill Andrews * forwards) or end (if searching backwards) of the range covered by the 2204f7170741SWill Andrews * block pointer we matched on (or dnode). 220576256205SMark Maybee * 220676256205SMark Maybee * The basic search algorithm used below by dnode_next_offset() is to 220776256205SMark Maybee * use this function to search up the block tree (widen the search) until 220876256205SMark Maybee * we find something (i.e., we don't return ESRCH) and then search back 220976256205SMark Maybee * down the tree (narrow the search) until we reach our original search 221076256205SMark Maybee * level. 221176256205SMark Maybee */ 2212fa9e4066Sahrens static int 2213cdb0ab79Smaybee dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, 2214a2cdcdd2SPaul Dagnelie int lvl, uint64_t blkfill, uint64_t txg) 2215fa9e4066Sahrens { 2216fa9e4066Sahrens dmu_buf_impl_t *db = NULL; 2217fa9e4066Sahrens void *data = NULL; 2218fa9e4066Sahrens uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2219fa9e4066Sahrens uint64_t epb = 1ULL << epbs; 2220fa9e4066Sahrens uint64_t minfill, maxfill; 2221cdb0ab79Smaybee boolean_t hole; 2222cdb0ab79Smaybee int i, inc, error, span; 2223fa9e4066Sahrens 2224fa9e4066Sahrens dprintf("probing object %llu offset %llx level %d of %u\n", 2225fa9e4066Sahrens dn->dn_object, *offset, lvl, dn->dn_phys->dn_nlevels); 2226fa9e4066Sahrens 222714843421SMatthew Ahrens hole = ((flags & DNODE_FIND_HOLE) != 0); 2228cdb0ab79Smaybee inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1; 22291c8564a7SMark Maybee ASSERT(txg == 0 || !hole); 2230cdb0ab79Smaybee 2231fa9e4066Sahrens if (lvl == dn->dn_phys->dn_nlevels) { 2232fa9e4066Sahrens error = 0; 2233fa9e4066Sahrens epb = dn->dn_phys->dn_nblkptr; 2234fa9e4066Sahrens data = dn->dn_phys->dn_blkptr; 2235fa9e4066Sahrens } else { 2236a2cdcdd2SPaul Dagnelie uint64_t blkid = dbuf_whichblock(dn, lvl, *offset); 2237a2cdcdd2SPaul Dagnelie error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db); 2238fa9e4066Sahrens if (error) { 22391c8564a7SMark Maybee if (error != ENOENT) 22401c8564a7SMark Maybee return (error); 22411c8564a7SMark Maybee if (hole) 22421c8564a7SMark Maybee return (0); 22431c8564a7SMark Maybee /* 22441c8564a7SMark Maybee * This can only happen when we are searching up 22451c8564a7SMark Maybee * the block tree for data. We don't really need to 22461c8564a7SMark Maybee * adjust the offset, as we will just end up looking 22471c8564a7SMark Maybee * at the pointer to this block in its parent, and its 22481c8564a7SMark Maybee * going to be unallocated, so we will skip over it. 22491c8564a7SMark Maybee */ 2250be6fd75aSMatthew Ahrens return (SET_ERROR(ESRCH)); 2251fa9e4066Sahrens } 225298572ac1Sahrens error = dbuf_read(db, NULL, DB_RF_CANFAIL | DB_RF_HAVESTRUCT); 225398572ac1Sahrens if (error) { 225498572ac1Sahrens dbuf_rele(db, FTAG); 225598572ac1Sahrens return (error); 225698572ac1Sahrens } 2257fa9e4066Sahrens data = db->db.db_data; 2258fa9e4066Sahrens } 2259fa9e4066Sahrens 226043466aaeSMax Grossman 226143466aaeSMax Grossman if (db != NULL && txg != 0 && (db->db_blkptr == NULL || 226243466aaeSMax Grossman db->db_blkptr->blk_birth <= txg || 226343466aaeSMax Grossman BP_IS_HOLE(db->db_blkptr))) { 22641c8564a7SMark Maybee /* 22651c8564a7SMark Maybee * This can only happen when we are searching up the tree 22661c8564a7SMark Maybee * and these conditions mean that we need to keep climbing. 22671c8564a7SMark Maybee */ 2268be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 22696754306eSahrens } else if (lvl == 0) { 2270fa9e4066Sahrens dnode_phys_t *dnp = data; 227154811da5SToomas Soome 2272fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_DNODE); 227354811da5SToomas Soome ASSERT(!(flags & DNODE_FIND_BACKWARDS)); 2274fa9e4066Sahrens 227554811da5SToomas Soome for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1); 227654811da5SToomas Soome i < blkfill; i += dnp[i].dn_extra_slots + 1) { 227708f3f137SJonathan W Adams if ((dnp[i].dn_type == DMU_OT_NONE) == hole) 2278fa9e4066Sahrens break; 2279fa9e4066Sahrens } 228054811da5SToomas Soome 228154811da5SToomas Soome if (i == blkfill) 2282be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 228354811da5SToomas Soome 228454811da5SToomas Soome *offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) + 228554811da5SToomas Soome (i << DNODE_SHIFT); 2286fa9e4066Sahrens } else { 2287fa9e4066Sahrens blkptr_t *bp = data; 228876256205SMark Maybee uint64_t start = *offset; 2289fa9e4066Sahrens span = (lvl - 1) * epbs + dn->dn_datablkshift; 2290fa9e4066Sahrens minfill = 0; 2291fa9e4066Sahrens maxfill = blkfill << ((lvl - 1) * epbs); 2292fa9e4066Sahrens 2293fa9e4066Sahrens if (hole) 2294fa9e4066Sahrens maxfill--; 2295fa9e4066Sahrens else 2296fa9e4066Sahrens minfill++; 2297fa9e4066Sahrens 229876256205SMark Maybee *offset = *offset >> span; 229976256205SMark Maybee for (i = BF64_GET(*offset, 0, epbs); 2300cdb0ab79Smaybee i >= 0 && i < epb; i += inc) { 23015d7b4d43SMatthew Ahrens if (BP_GET_FILL(&bp[i]) >= minfill && 23025d7b4d43SMatthew Ahrens BP_GET_FILL(&bp[i]) <= maxfill && 23031c8564a7SMark Maybee (hole || bp[i].blk_birth > txg)) 2304fa9e4066Sahrens break; 230576256205SMark Maybee if (inc > 0 || *offset > 0) 230676256205SMark Maybee *offset += inc; 230776256205SMark Maybee } 230876256205SMark Maybee *offset = *offset << span; 230976256205SMark Maybee if (inc < 0) { 231076256205SMark Maybee /* traversing backwards; position offset at the end */ 231176256205SMark Maybee ASSERT3U(*offset, <=, start); 231276256205SMark Maybee *offset = MIN(*offset + (1ULL << span) - 1, start); 231376256205SMark Maybee } else if (*offset < start) { 231476256205SMark Maybee *offset = start; 2315fa9e4066Sahrens } 231676256205SMark Maybee if (i < 0 || i >= epb) 2317be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2318fa9e4066Sahrens } 2319fa9e4066Sahrens 2320fa9e4066Sahrens if (db) 2321ea8dc4b6Seschrock dbuf_rele(db, FTAG); 2322fa9e4066Sahrens 2323fa9e4066Sahrens return (error); 2324fa9e4066Sahrens } 2325fa9e4066Sahrens 2326fa9e4066Sahrens /* 2327fa9e4066Sahrens * Find the next hole, data, or sparse region at or after *offset. 2328fa9e4066Sahrens * The value 'blkfill' tells us how many items we expect to find 2329fa9e4066Sahrens * in an L0 data block; this value is 1 for normal objects, 2330fa9e4066Sahrens * DNODES_PER_BLOCK for the meta dnode, and some fraction of 2331fa9e4066Sahrens * DNODES_PER_BLOCK when searching for sparse regions thereof. 23326754306eSahrens * 2333fa9e4066Sahrens * Examples: 2334fa9e4066Sahrens * 2335cdb0ab79Smaybee * dnode_next_offset(dn, flags, offset, 1, 1, 0); 2336cdb0ab79Smaybee * Finds the next/previous hole/data in a file. 2337fa9e4066Sahrens * Used in dmu_offset_next(). 2338fa9e4066Sahrens * 2339cdb0ab79Smaybee * dnode_next_offset(mdn, flags, offset, 0, DNODES_PER_BLOCK, txg); 2340fa9e4066Sahrens * Finds the next free/allocated dnode an objset's meta-dnode. 23416754306eSahrens * Only finds objects that have new contents since txg (ie. 23426754306eSahrens * bonus buffer changes and content removal are ignored). 2343fa9e4066Sahrens * Used in dmu_object_next(). 2344fa9e4066Sahrens * 2345cdb0ab79Smaybee * dnode_next_offset(mdn, DNODE_FIND_HOLE, offset, 2, DNODES_PER_BLOCK >> 2, 0); 2346fa9e4066Sahrens * Finds the next L2 meta-dnode bp that's at most 1/4 full. 2347fa9e4066Sahrens * Used in dmu_object_alloc(). 2348fa9e4066Sahrens */ 2349fa9e4066Sahrens int 2350cdb0ab79Smaybee dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, 23516754306eSahrens int minlvl, uint64_t blkfill, uint64_t txg) 2352fa9e4066Sahrens { 2353cdb0ab79Smaybee uint64_t initial_offset = *offset; 2354fa9e4066Sahrens int lvl, maxlvl; 2355fa9e4066Sahrens int error = 0; 2356fa9e4066Sahrens 2357cdb0ab79Smaybee if (!(flags & DNODE_FIND_HAVELOCK)) 2358cdb0ab79Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2359fa9e4066Sahrens 2360fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) { 2361be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2362cdb0ab79Smaybee goto out; 2363fa9e4066Sahrens } 2364fa9e4066Sahrens 2365fa9e4066Sahrens if (dn->dn_datablkshift == 0) { 2366fa9e4066Sahrens if (*offset < dn->dn_datablksz) { 2367cdb0ab79Smaybee if (flags & DNODE_FIND_HOLE) 2368fa9e4066Sahrens *offset = dn->dn_datablksz; 2369fa9e4066Sahrens } else { 2370be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2371fa9e4066Sahrens } 2372cdb0ab79Smaybee goto out; 2373fa9e4066Sahrens } 2374fa9e4066Sahrens 2375fa9e4066Sahrens maxlvl = dn->dn_phys->dn_nlevels; 2376fa9e4066Sahrens 2377fa9e4066Sahrens for (lvl = minlvl; lvl <= maxlvl; lvl++) { 23786754306eSahrens error = dnode_next_offset_level(dn, 2379cdb0ab79Smaybee flags, offset, lvl, blkfill, txg); 238098572ac1Sahrens if (error != ESRCH) 2381fa9e4066Sahrens break; 2382fa9e4066Sahrens } 2383fa9e4066Sahrens 2384cdb0ab79Smaybee while (error == 0 && --lvl >= minlvl) { 23856754306eSahrens error = dnode_next_offset_level(dn, 2386cdb0ab79Smaybee flags, offset, lvl, blkfill, txg); 23876754306eSahrens } 2388fa9e4066Sahrens 23890fbc0cd0SMatthew Ahrens /* 23900fbc0cd0SMatthew Ahrens * There's always a "virtual hole" at the end of the object, even 23910fbc0cd0SMatthew Ahrens * if all BP's which physically exist are non-holes. 23920fbc0cd0SMatthew Ahrens */ 23930fbc0cd0SMatthew Ahrens if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 && 23940fbc0cd0SMatthew Ahrens minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) { 23950fbc0cd0SMatthew Ahrens error = 0; 23960fbc0cd0SMatthew Ahrens } 23970fbc0cd0SMatthew Ahrens 2398cdb0ab79Smaybee if (error == 0 && (flags & DNODE_FIND_BACKWARDS ? 2399cdb0ab79Smaybee initial_offset < *offset : initial_offset > *offset)) 2400be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2401cdb0ab79Smaybee out: 2402cdb0ab79Smaybee if (!(flags & DNODE_FIND_HAVELOCK)) 2403cdb0ab79Smaybee rw_exit(&dn->dn_struct_rwlock); 2404fa9e4066Sahrens 2405fa9e4066Sahrens return (error); 2406fa9e4066Sahrens } 2407