1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 2394c2d0ebSMatthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 24bc9014e6SJustin Gibbs * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 25c3d26abcSMatthew Ahrens * Copyright (c) 2014 Integros [integros.com] 26f06dce2cSAndrew Stormont * Copyright 2017 RackTop Systems. 27fa9e4066Sahrens */ 28fa9e4066Sahrens 29fa9e4066Sahrens #include <sys/zfs_context.h> 30fa9e4066Sahrens #include <sys/dbuf.h> 31fa9e4066Sahrens #include <sys/dnode.h> 32fa9e4066Sahrens #include <sys/dmu.h> 33fa9e4066Sahrens #include <sys/dmu_impl.h> 34fa9e4066Sahrens #include <sys/dmu_tx.h> 35fa9e4066Sahrens #include <sys/dmu_objset.h> 36fa9e4066Sahrens #include <sys/dsl_dir.h> 37fa9e4066Sahrens #include <sys/dsl_dataset.h> 38fa9e4066Sahrens #include <sys/spa.h> 39fa9e4066Sahrens #include <sys/zio.h> 40fa9e4066Sahrens #include <sys/dmu_zfetch.h> 41bf16b11eSMatthew Ahrens #include <sys/range_tree.h> 42fa9e4066Sahrens 4354811da5SToomas Soome dnode_stats_t dnode_stats = { 4454811da5SToomas Soome { "dnode_hold_dbuf_hold", KSTAT_DATA_UINT64 }, 4554811da5SToomas Soome { "dnode_hold_dbuf_read", KSTAT_DATA_UINT64 }, 4654811da5SToomas Soome { "dnode_hold_alloc_hits", KSTAT_DATA_UINT64 }, 4754811da5SToomas Soome { "dnode_hold_alloc_misses", KSTAT_DATA_UINT64 }, 4854811da5SToomas Soome { "dnode_hold_alloc_interior", KSTAT_DATA_UINT64 }, 4954811da5SToomas Soome { "dnode_hold_alloc_lock_retry", KSTAT_DATA_UINT64 }, 5054811da5SToomas Soome { "dnode_hold_alloc_lock_misses", KSTAT_DATA_UINT64 }, 5154811da5SToomas Soome { "dnode_hold_alloc_type_none", KSTAT_DATA_UINT64 }, 5254811da5SToomas Soome { "dnode_hold_free_hits", KSTAT_DATA_UINT64 }, 5354811da5SToomas Soome { "dnode_hold_free_misses", KSTAT_DATA_UINT64 }, 5454811da5SToomas Soome { "dnode_hold_free_lock_misses", KSTAT_DATA_UINT64 }, 5554811da5SToomas Soome { "dnode_hold_free_lock_retry", KSTAT_DATA_UINT64 }, 5654811da5SToomas Soome { "dnode_hold_free_overflow", KSTAT_DATA_UINT64 }, 5754811da5SToomas Soome { "dnode_hold_free_refcount", KSTAT_DATA_UINT64 }, 5854811da5SToomas Soome { "dnode_hold_free_txg", KSTAT_DATA_UINT64 }, 5954811da5SToomas Soome { "dnode_free_interior_lock_retry", KSTAT_DATA_UINT64 }, 6054811da5SToomas Soome { "dnode_allocate", KSTAT_DATA_UINT64 }, 6154811da5SToomas Soome { "dnode_reallocate", KSTAT_DATA_UINT64 }, 6254811da5SToomas Soome { "dnode_buf_evict", KSTAT_DATA_UINT64 }, 6354811da5SToomas Soome { "dnode_alloc_next_chunk", KSTAT_DATA_UINT64 }, 6454811da5SToomas Soome { "dnode_alloc_race", KSTAT_DATA_UINT64 }, 6554811da5SToomas Soome { "dnode_alloc_next_block", KSTAT_DATA_UINT64 }, 6654811da5SToomas Soome { "dnode_move_invalid", KSTAT_DATA_UINT64 }, 6754811da5SToomas Soome { "dnode_move_recheck1", KSTAT_DATA_UINT64 }, 6854811da5SToomas Soome { "dnode_move_recheck2", KSTAT_DATA_UINT64 }, 6954811da5SToomas Soome { "dnode_move_special", KSTAT_DATA_UINT64 }, 7054811da5SToomas Soome { "dnode_move_handle", KSTAT_DATA_UINT64 }, 7154811da5SToomas Soome { "dnode_move_rwlock", KSTAT_DATA_UINT64 }, 7254811da5SToomas Soome { "dnode_move_active", KSTAT_DATA_UINT64 }, 7354811da5SToomas Soome }; 7454811da5SToomas Soome 7554811da5SToomas Soome static kstat_t *dnode_ksp; 76fa9e4066Sahrens static kmem_cache_t *dnode_cache; 77fa9e4066Sahrens 78fa9e4066Sahrens static dnode_phys_t dnode_phys_zero; 79fa9e4066Sahrens 80fa9e4066Sahrens int zfs_default_bs = SPA_MINBLOCKSHIFT; 81fa9e4066Sahrens int zfs_default_ibs = DN_MAX_INDBLKSHIFT; 82fa9e4066Sahrens 83f06dce2cSAndrew Stormont #ifdef _KERNEL 84744947dcSTom Erickson static kmem_cbrc_t dnode_move(void *, void *, size_t, void *); 85f06dce2cSAndrew Stormont #endif /* _KERNEL */ 86744947dcSTom Erickson 870f6d88adSAlex Reece static int 880f6d88adSAlex Reece dbuf_compare(const void *x1, const void *x2) 890f6d88adSAlex Reece { 900f6d88adSAlex Reece const dmu_buf_impl_t *d1 = x1; 910f6d88adSAlex Reece const dmu_buf_impl_t *d2 = x2; 920f6d88adSAlex Reece 930f6d88adSAlex Reece if (d1->db_level < d2->db_level) { 940f6d88adSAlex Reece return (-1); 9586bb58aeSAlex Reece } 9686bb58aeSAlex Reece if (d1->db_level > d2->db_level) { 970f6d88adSAlex Reece return (1); 980f6d88adSAlex Reece } 990f6d88adSAlex Reece 1000f6d88adSAlex Reece if (d1->db_blkid < d2->db_blkid) { 1010f6d88adSAlex Reece return (-1); 10286bb58aeSAlex Reece } 10386bb58aeSAlex Reece if (d1->db_blkid > d2->db_blkid) { 1040f6d88adSAlex Reece return (1); 1050f6d88adSAlex Reece } 1060f6d88adSAlex Reece 107a846f19dSAlex Reece if (d1->db_state == DB_SEARCH) { 108a846f19dSAlex Reece ASSERT3S(d2->db_state, !=, DB_SEARCH); 1090f6d88adSAlex Reece return (-1); 110a846f19dSAlex Reece } else if (d2->db_state == DB_SEARCH) { 111a846f19dSAlex Reece ASSERT3S(d1->db_state, !=, DB_SEARCH); 11286bb58aeSAlex Reece return (1); 11386bb58aeSAlex Reece } 11486bb58aeSAlex Reece 11586bb58aeSAlex Reece if ((uintptr_t)d1 < (uintptr_t)d2) { 11686bb58aeSAlex Reece return (-1); 11786bb58aeSAlex Reece } 11886bb58aeSAlex Reece if ((uintptr_t)d1 > (uintptr_t)d2) { 1190f6d88adSAlex Reece return (1); 1200f6d88adSAlex Reece } 12186bb58aeSAlex Reece return (0); 1220f6d88adSAlex Reece } 1230f6d88adSAlex Reece 124fa9e4066Sahrens /* ARGSUSED */ 125fa9e4066Sahrens static int 126fa9e4066Sahrens dnode_cons(void *arg, void *unused, int kmflag) 127fa9e4066Sahrens { 128fa9e4066Sahrens dnode_t *dn = arg; 129744947dcSTom Erickson int i; 130fa9e4066Sahrens 131fa9e4066Sahrens rw_init(&dn->dn_struct_rwlock, NULL, RW_DEFAULT, NULL); 132fa9e4066Sahrens mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); 133fa9e4066Sahrens mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL); 134b5e70f97SRicardo M. Correia cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL); 135b5e70f97SRicardo M. Correia 1363b2aab18SMatthew Ahrens /* 1373b2aab18SMatthew Ahrens * Every dbuf has a reference, and dropping a tracked reference is 1383b2aab18SMatthew Ahrens * O(number of references), so don't track dn_holds. 1393b2aab18SMatthew Ahrens */ 140e914ace2STim Schumacher zfs_refcount_create_untracked(&dn->dn_holds); 141e914ace2STim Schumacher zfs_refcount_create(&dn->dn_tx_holds); 142744947dcSTom Erickson list_link_init(&dn->dn_link); 143744947dcSTom Erickson 144744947dcSTom Erickson bzero(&dn->dn_next_nblkptr[0], sizeof (dn->dn_next_nblkptr)); 145744947dcSTom Erickson bzero(&dn->dn_next_nlevels[0], sizeof (dn->dn_next_nlevels)); 146744947dcSTom Erickson bzero(&dn->dn_next_indblkshift[0], sizeof (dn->dn_next_indblkshift)); 147744947dcSTom Erickson bzero(&dn->dn_next_bonustype[0], sizeof (dn->dn_next_bonustype)); 148744947dcSTom Erickson bzero(&dn->dn_rm_spillblk[0], sizeof (dn->dn_rm_spillblk)); 149744947dcSTom Erickson bzero(&dn->dn_next_bonuslen[0], sizeof (dn->dn_next_bonuslen)); 150744947dcSTom Erickson bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); 151fa9e4066Sahrens 152fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 153744947dcSTom Erickson list_link_init(&dn->dn_dirty_link[i]); 154bf16b11eSMatthew Ahrens dn->dn_free_ranges[i] = NULL; 155c717a561Smaybee list_create(&dn->dn_dirty_records[i], 156c717a561Smaybee sizeof (dbuf_dirty_record_t), 157c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 158fa9e4066Sahrens } 159fa9e4066Sahrens 160744947dcSTom Erickson dn->dn_allocated_txg = 0; 161744947dcSTom Erickson dn->dn_free_txg = 0; 162744947dcSTom Erickson dn->dn_assigned_txg = 0; 163744947dcSTom Erickson dn->dn_dirtyctx = 0; 164744947dcSTom Erickson dn->dn_dirtyctx_firstset = NULL; 165744947dcSTom Erickson dn->dn_bonus = NULL; 166744947dcSTom Erickson dn->dn_have_spill = B_FALSE; 167744947dcSTom Erickson dn->dn_zio = NULL; 168744947dcSTom Erickson dn->dn_oldused = 0; 169744947dcSTom Erickson dn->dn_oldflags = 0; 170744947dcSTom Erickson dn->dn_olduid = 0; 171744947dcSTom Erickson dn->dn_oldgid = 0; 172744947dcSTom Erickson dn->dn_newuid = 0; 173744947dcSTom Erickson dn->dn_newgid = 0; 174744947dcSTom Erickson dn->dn_id_flags = 0; 175744947dcSTom Erickson 176744947dcSTom Erickson dn->dn_dbufs_count = 0; 1770f6d88adSAlex Reece avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), 178fa9e4066Sahrens offsetof(dmu_buf_impl_t, db_link)); 179fa9e4066Sahrens 180744947dcSTom Erickson dn->dn_moved = 0; 181fa9e4066Sahrens return (0); 182fa9e4066Sahrens } 183fa9e4066Sahrens 184fa9e4066Sahrens /* ARGSUSED */ 185fa9e4066Sahrens static void 186fa9e4066Sahrens dnode_dest(void *arg, void *unused) 187fa9e4066Sahrens { 188fa9e4066Sahrens int i; 189fa9e4066Sahrens dnode_t *dn = arg; 190fa9e4066Sahrens 191fa9e4066Sahrens rw_destroy(&dn->dn_struct_rwlock); 192fa9e4066Sahrens mutex_destroy(&dn->dn_mtx); 193fa9e4066Sahrens mutex_destroy(&dn->dn_dbufs_mtx); 194b5e70f97SRicardo M. Correia cv_destroy(&dn->dn_notxholds); 195e914ace2STim Schumacher zfs_refcount_destroy(&dn->dn_holds); 196e914ace2STim Schumacher zfs_refcount_destroy(&dn->dn_tx_holds); 197744947dcSTom Erickson ASSERT(!list_link_active(&dn->dn_link)); 198fa9e4066Sahrens 199fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 200744947dcSTom Erickson ASSERT(!list_link_active(&dn->dn_dirty_link[i])); 201bf16b11eSMatthew Ahrens ASSERT3P(dn->dn_free_ranges[i], ==, NULL); 202c717a561Smaybee list_destroy(&dn->dn_dirty_records[i]); 203fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nblkptr[i]); 204fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nlevels[i]); 205fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_indblkshift[i]); 206fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[i]); 207fb09f5aaSMadhav Suresh ASSERT0(dn->dn_rm_spillblk[i]); 208fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[i]); 209fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[i]); 210fa9e4066Sahrens } 211fa9e4066Sahrens 212fb09f5aaSMadhav Suresh ASSERT0(dn->dn_allocated_txg); 213fb09f5aaSMadhav Suresh ASSERT0(dn->dn_free_txg); 214fb09f5aaSMadhav Suresh ASSERT0(dn->dn_assigned_txg); 215fb09f5aaSMadhav Suresh ASSERT0(dn->dn_dirtyctx); 216744947dcSTom Erickson ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); 217744947dcSTom Erickson ASSERT3P(dn->dn_bonus, ==, NULL); 218744947dcSTom Erickson ASSERT(!dn->dn_have_spill); 219744947dcSTom Erickson ASSERT3P(dn->dn_zio, ==, NULL); 220fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldused); 221fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldflags); 222fb09f5aaSMadhav Suresh ASSERT0(dn->dn_olduid); 223fb09f5aaSMadhav Suresh ASSERT0(dn->dn_oldgid); 224fb09f5aaSMadhav Suresh ASSERT0(dn->dn_newuid); 225fb09f5aaSMadhav Suresh ASSERT0(dn->dn_newgid); 226fb09f5aaSMadhav Suresh ASSERT0(dn->dn_id_flags); 227fb09f5aaSMadhav Suresh 228fb09f5aaSMadhav Suresh ASSERT0(dn->dn_dbufs_count); 2290f6d88adSAlex Reece avl_destroy(&dn->dn_dbufs); 230fa9e4066Sahrens } 231fa9e4066Sahrens 232fa9e4066Sahrens void 233fa9e4066Sahrens dnode_init(void) 234fa9e4066Sahrens { 235744947dcSTom Erickson ASSERT(dnode_cache == NULL); 236fa9e4066Sahrens dnode_cache = kmem_cache_create("dnode_t", 237fa9e4066Sahrens sizeof (dnode_t), 238fa9e4066Sahrens 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0); 239f06dce2cSAndrew Stormont #ifdef _KERNEL 240744947dcSTom Erickson kmem_cache_set_move(dnode_cache, dnode_move); 24154811da5SToomas Soome 24254811da5SToomas Soome dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc", 24354811da5SToomas Soome KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t), 24454811da5SToomas Soome KSTAT_FLAG_VIRTUAL); 24554811da5SToomas Soome if (dnode_ksp != NULL) { 24654811da5SToomas Soome dnode_ksp->ks_data = &dnode_stats; 24754811da5SToomas Soome kstat_install(dnode_ksp); 24854811da5SToomas Soome } 249f06dce2cSAndrew Stormont #endif /* _KERNEL */ 250fa9e4066Sahrens } 251fa9e4066Sahrens 252fa9e4066Sahrens void 253fa9e4066Sahrens dnode_fini(void) 254fa9e4066Sahrens { 25554811da5SToomas Soome if (dnode_ksp != NULL) { 25654811da5SToomas Soome kstat_delete(dnode_ksp); 25754811da5SToomas Soome dnode_ksp = NULL; 25854811da5SToomas Soome } 25954811da5SToomas Soome 260fa9e4066Sahrens kmem_cache_destroy(dnode_cache); 261744947dcSTom Erickson dnode_cache = NULL; 262fa9e4066Sahrens } 263fa9e4066Sahrens 264fa9e4066Sahrens 2659c9dc39aSek #ifdef ZFS_DEBUG 266fa9e4066Sahrens void 267fa9e4066Sahrens dnode_verify(dnode_t *dn) 268fa9e4066Sahrens { 269fa9e4066Sahrens int drop_struct_lock = FALSE; 270fa9e4066Sahrens 271fa9e4066Sahrens ASSERT(dn->dn_phys); 272fa9e4066Sahrens ASSERT(dn->dn_objset); 273744947dcSTom Erickson ASSERT(dn->dn_handle->dnh_dnode == dn); 274fa9e4066Sahrens 275ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); 276fa9e4066Sahrens 277fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY)) 278fa9e4066Sahrens return; 279fa9e4066Sahrens 280fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 281fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 282fa9e4066Sahrens drop_struct_lock = TRUE; 283fa9e4066Sahrens } 284fa9e4066Sahrens if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { 285fa9e4066Sahrens int i; 28654811da5SToomas Soome int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); 287fa9e4066Sahrens ASSERT3U(dn->dn_indblkshift, >=, 0); 288fa9e4066Sahrens ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); 289fa9e4066Sahrens if (dn->dn_datablkshift) { 290fa9e4066Sahrens ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); 291fa9e4066Sahrens ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT); 292fa9e4066Sahrens ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz); 293fa9e4066Sahrens } 294fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, <=, 30); 295ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_type)); 296fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, >=, 1); 297fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); 29854811da5SToomas Soome ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen); 299fa9e4066Sahrens ASSERT3U(dn->dn_datablksz, ==, 300fa9e4066Sahrens dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); 301fa9e4066Sahrens ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); 302fa9e4066Sahrens ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + 30354811da5SToomas Soome dn->dn_bonuslen, <=, max_bonuslen); 304fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 305fa9e4066Sahrens ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); 306fa9e4066Sahrens } 307fa9e4066Sahrens } 308fa9e4066Sahrens if (dn->dn_phys->dn_type != DMU_OT_NONE) 309fa9e4066Sahrens ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels); 31014843421SMatthew Ahrens ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL); 311fa9e4066Sahrens if (dn->dn_dbuf != NULL) { 312fa9e4066Sahrens ASSERT3P(dn->dn_phys, ==, 313fa9e4066Sahrens (dnode_phys_t *)dn->dn_dbuf->db.db_data + 314fa9e4066Sahrens (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT))); 315fa9e4066Sahrens } 316fa9e4066Sahrens if (drop_struct_lock) 317fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 318fa9e4066Sahrens } 3199c9dc39aSek #endif 320fa9e4066Sahrens 321fa9e4066Sahrens void 322fa9e4066Sahrens dnode_byteswap(dnode_phys_t *dnp) 323fa9e4066Sahrens { 324fa9e4066Sahrens uint64_t *buf64 = (void*)&dnp->dn_blkptr; 325fa9e4066Sahrens int i; 326fa9e4066Sahrens 327fa9e4066Sahrens if (dnp->dn_type == DMU_OT_NONE) { 328fa9e4066Sahrens bzero(dnp, sizeof (dnode_phys_t)); 329fa9e4066Sahrens return; 330fa9e4066Sahrens } 331fa9e4066Sahrens 332fa9e4066Sahrens dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec); 333fa9e4066Sahrens dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen); 33454811da5SToomas Soome dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots); 335fa9e4066Sahrens dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid); 33699653d4eSeschrock dnp->dn_used = BSWAP_64(dnp->dn_used); 337fa9e4066Sahrens 338fa9e4066Sahrens /* 339fa9e4066Sahrens * dn_nblkptr is only one byte, so it's OK to read it in either 340fa9e4066Sahrens * byte order. We can't read dn_bouslen. 341fa9e4066Sahrens */ 342fa9e4066Sahrens ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT); 343fa9e4066Sahrens ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR); 344fa9e4066Sahrens for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++) 345fa9e4066Sahrens buf64[i] = BSWAP_64(buf64[i]); 346fa9e4066Sahrens 347fa9e4066Sahrens /* 348fa9e4066Sahrens * OK to check dn_bonuslen for zero, because it won't matter if 349fa9e4066Sahrens * we have the wrong byte order. This is necessary because the 350fa9e4066Sahrens * dnode dnode is smaller than a regular dnode. 351fa9e4066Sahrens */ 352fa9e4066Sahrens if (dnp->dn_bonuslen != 0) { 353fa9e4066Sahrens /* 354fa9e4066Sahrens * Note that the bonus length calculated here may be 355fa9e4066Sahrens * longer than the actual bonus buffer. This is because 356fa9e4066Sahrens * we always put the bonus buffer after the last block 357fa9e4066Sahrens * pointer (instead of packing it against the end of the 358fa9e4066Sahrens * dnode buffer). 359fa9e4066Sahrens */ 360fa9e4066Sahrens int off = (dnp->dn_nblkptr-1) * sizeof (blkptr_t); 36154811da5SToomas Soome int slots = dnp->dn_extra_slots + 1; 36254811da5SToomas Soome size_t len = DN_SLOTS_TO_BONUSLEN(slots) - off; 363ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype)); 364ad135b5dSChristopher Siden dmu_object_byteswap_t byteswap = 365ad135b5dSChristopher Siden DMU_OT_BYTESWAP(dnp->dn_bonustype); 366ad135b5dSChristopher Siden dmu_ot_byteswap[byteswap].ob_func(dnp->dn_bonus + off, len); 367fa9e4066Sahrens } 3680a586ceaSMark Shellenbaum 3690a586ceaSMark Shellenbaum /* Swap SPILL block if we have one */ 3700a586ceaSMark Shellenbaum if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) 37154811da5SToomas Soome byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t)); 3720a586ceaSMark Shellenbaum 373fa9e4066Sahrens } 374fa9e4066Sahrens 375fa9e4066Sahrens void 376fa9e4066Sahrens dnode_buf_byteswap(void *vbuf, size_t size) 377fa9e4066Sahrens { 37854811da5SToomas Soome int i = 0; 379fa9e4066Sahrens 380fa9e4066Sahrens ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT)); 381fa9e4066Sahrens ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0); 382fa9e4066Sahrens 38354811da5SToomas Soome while (i < size) { 38454811da5SToomas Soome dnode_phys_t *dnp = (void *)(((char *)vbuf) + i); 38554811da5SToomas Soome dnode_byteswap(dnp); 38654811da5SToomas Soome 38754811da5SToomas Soome i += DNODE_MIN_SIZE; 38854811da5SToomas Soome if (dnp->dn_type != DMU_OT_NONE) 38954811da5SToomas Soome i += dnp->dn_extra_slots * DNODE_MIN_SIZE; 390fa9e4066Sahrens } 391fa9e4066Sahrens } 392fa9e4066Sahrens 3931934e92fSmaybee void 3941934e92fSmaybee dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) 3951934e92fSmaybee { 396e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 3971934e92fSmaybee 3981934e92fSmaybee dnode_setdirty(dn, tx); 3991934e92fSmaybee rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 40054811da5SToomas Soome ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - 4011934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t)); 4021934e92fSmaybee dn->dn_bonuslen = newsize; 4031934e92fSmaybee if (newsize == 0) 4041934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN; 4051934e92fSmaybee else 4061934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; 4071934e92fSmaybee rw_exit(&dn->dn_struct_rwlock); 4081934e92fSmaybee } 4091934e92fSmaybee 4100a586ceaSMark Shellenbaum void 4110a586ceaSMark Shellenbaum dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx) 4120a586ceaSMark Shellenbaum { 413e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 4140a586ceaSMark Shellenbaum dnode_setdirty(dn, tx); 4150a586ceaSMark Shellenbaum rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 4160a586ceaSMark Shellenbaum dn->dn_bonustype = newtype; 4170a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; 4180a586ceaSMark Shellenbaum rw_exit(&dn->dn_struct_rwlock); 4190a586ceaSMark Shellenbaum } 4200a586ceaSMark Shellenbaum 4210a586ceaSMark Shellenbaum void 4220a586ceaSMark Shellenbaum dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx) 4230a586ceaSMark Shellenbaum { 424e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1); 42506e0070dSMark Shellenbaum ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 4260a586ceaSMark Shellenbaum dnode_setdirty(dn, tx); 4270a586ceaSMark Shellenbaum dn->dn_rm_spillblk[tx->tx_txg&TXG_MASK] = DN_KILL_SPILLBLK; 4280a586ceaSMark Shellenbaum dn->dn_have_spill = B_FALSE; 4290a586ceaSMark Shellenbaum } 4300a586ceaSMark Shellenbaum 431fa9e4066Sahrens static void 432fa9e4066Sahrens dnode_setdblksz(dnode_t *dn, int size) 433fa9e4066Sahrens { 434fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(size, SPA_MINBLOCKSIZE)); 435fa9e4066Sahrens ASSERT3U(size, <=, SPA_MAXBLOCKSIZE); 436fa9e4066Sahrens ASSERT3U(size, >=, SPA_MINBLOCKSIZE); 437fa9e4066Sahrens ASSERT3U(size >> SPA_MINBLOCKSHIFT, <, 438fa9e4066Sahrens 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8)); 439fa9e4066Sahrens dn->dn_datablksz = size; 440fa9e4066Sahrens dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT; 441bf16b11eSMatthew Ahrens dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0; 442fa9e4066Sahrens } 443fa9e4066Sahrens 444fa9e4066Sahrens static dnode_t * 445503ad85cSMatthew Ahrens dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, 446744947dcSTom Erickson uint64_t object, dnode_handle_t *dnh) 447fa9e4066Sahrens { 448bc9014e6SJustin Gibbs dnode_t *dn; 449fa9e4066Sahrens 450bc9014e6SJustin Gibbs dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); 451f06dce2cSAndrew Stormont #ifdef _KERNEL 452744947dcSTom Erickson ASSERT(!POINTER_IS_VALID(dn->dn_objset)); 453f06dce2cSAndrew Stormont #endif /* _KERNEL */ 454744947dcSTom Erickson dn->dn_moved = 0; 455744947dcSTom Erickson 456744947dcSTom Erickson /* 457744947dcSTom Erickson * Defer setting dn_objset until the dnode is ready to be a candidate 458744947dcSTom Erickson * for the dnode_move() callback. 459744947dcSTom Erickson */ 460fa9e4066Sahrens dn->dn_object = object; 461fa9e4066Sahrens dn->dn_dbuf = db; 462744947dcSTom Erickson dn->dn_handle = dnh; 463fa9e4066Sahrens dn->dn_phys = dnp; 464fa9e4066Sahrens 465744947dcSTom Erickson if (dnp->dn_datablkszsec) { 466fa9e4066Sahrens dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); 467744947dcSTom Erickson } else { 468744947dcSTom Erickson dn->dn_datablksz = 0; 469744947dcSTom Erickson dn->dn_datablkszsec = 0; 470744947dcSTom Erickson dn->dn_datablkshift = 0; 471744947dcSTom Erickson } 472fa9e4066Sahrens dn->dn_indblkshift = dnp->dn_indblkshift; 473fa9e4066Sahrens dn->dn_nlevels = dnp->dn_nlevels; 474fa9e4066Sahrens dn->dn_type = dnp->dn_type; 475fa9e4066Sahrens dn->dn_nblkptr = dnp->dn_nblkptr; 476fa9e4066Sahrens dn->dn_checksum = dnp->dn_checksum; 477fa9e4066Sahrens dn->dn_compress = dnp->dn_compress; 478fa9e4066Sahrens dn->dn_bonustype = dnp->dn_bonustype; 479fa9e4066Sahrens dn->dn_bonuslen = dnp->dn_bonuslen; 48054811da5SToomas Soome dn->dn_num_slots = dnp->dn_extra_slots + 1; 481fa9e4066Sahrens dn->dn_maxblkid = dnp->dn_maxblkid; 4820a586ceaSMark Shellenbaum dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); 48306e0070dSMark Shellenbaum dn->dn_id_flags = 0; 484fa9e4066Sahrens 485fa9e4066Sahrens dmu_zfetch_init(&dn->dn_zfetch, dn); 486fa9e4066Sahrens 487ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type)); 48854811da5SToomas Soome ASSERT(zrl_is_locked(&dnh->dnh_zrlock)); 48954811da5SToomas Soome ASSERT(!DN_SLOT_IS_PTR(dnh->dnh_dnode)); 490744947dcSTom Erickson 491fa9e4066Sahrens mutex_enter(&os->os_lock); 492bc9014e6SJustin Gibbs 493bc9014e6SJustin Gibbs /* 494bc9014e6SJustin Gibbs * Exclude special dnodes from os_dnodes so an empty os_dnodes 495bc9014e6SJustin Gibbs * signifies that the special dnodes have no references from 496bc9014e6SJustin Gibbs * their children (the entries in os_dnodes). This allows 497bc9014e6SJustin Gibbs * dnode_destroy() to easily determine if the last child has 498bc9014e6SJustin Gibbs * been removed and then complete eviction of the objset. 499bc9014e6SJustin Gibbs */ 500bc9014e6SJustin Gibbs if (!DMU_OBJECT_IS_SPECIAL(object)) 501bc9014e6SJustin Gibbs list_insert_head(&os->os_dnodes, dn); 502744947dcSTom Erickson membar_producer(); 503bc9014e6SJustin Gibbs 504744947dcSTom Erickson /* 505bc9014e6SJustin Gibbs * Everything else must be valid before assigning dn_objset 506bc9014e6SJustin Gibbs * makes the dnode eligible for dnode_move(). 507744947dcSTom Erickson */ 508744947dcSTom Erickson dn->dn_objset = os; 509bc9014e6SJustin Gibbs 510bc9014e6SJustin Gibbs dnh->dnh_dnode = dn; 511fa9e4066Sahrens mutex_exit(&os->os_lock); 512fa9e4066Sahrens 5135a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER); 51454811da5SToomas Soome 515fa9e4066Sahrens return (dn); 516fa9e4066Sahrens } 517fa9e4066Sahrens 518744947dcSTom Erickson /* 519744947dcSTom Erickson * Caller must be holding the dnode handle, which is released upon return. 520744947dcSTom Erickson */ 521fa9e4066Sahrens static void 522fa9e4066Sahrens dnode_destroy(dnode_t *dn) 523fa9e4066Sahrens { 524503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 525bc9014e6SJustin Gibbs boolean_t complete_os_eviction = B_FALSE; 526fa9e4066Sahrens 5270a586ceaSMark Shellenbaum ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0); 528a2eea2e1Sahrens 529fa9e4066Sahrens mutex_enter(&os->os_lock); 530744947dcSTom Erickson POINTER_INVALIDATE(&dn->dn_objset); 531bc9014e6SJustin Gibbs if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { 532bc9014e6SJustin Gibbs list_remove(&os->os_dnodes, dn); 533bc9014e6SJustin Gibbs complete_os_eviction = 534bc9014e6SJustin Gibbs list_is_empty(&os->os_dnodes) && 535bc9014e6SJustin Gibbs list_link_active(&os->os_evicting_node); 536bc9014e6SJustin Gibbs } 537fa9e4066Sahrens mutex_exit(&os->os_lock); 538fa9e4066Sahrens 539744947dcSTom Erickson /* the dnode can no longer move, so we can release the handle */ 54054811da5SToomas Soome if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock)) 54154811da5SToomas Soome zrl_remove(&dn->dn_handle->dnh_zrlock); 542744947dcSTom Erickson 543744947dcSTom Erickson dn->dn_allocated_txg = 0; 544744947dcSTom Erickson dn->dn_free_txg = 0; 545744947dcSTom Erickson dn->dn_assigned_txg = 0; 546744947dcSTom Erickson 547744947dcSTom Erickson dn->dn_dirtyctx = 0; 548744947dcSTom Erickson if (dn->dn_dirtyctx_firstset != NULL) { 549fa9e4066Sahrens kmem_free(dn->dn_dirtyctx_firstset, 1); 550fa9e4066Sahrens dn->dn_dirtyctx_firstset = NULL; 551fa9e4066Sahrens } 552744947dcSTom Erickson if (dn->dn_bonus != NULL) { 553ea8dc4b6Seschrock mutex_enter(&dn->dn_bonus->db_mtx); 554dcbf3bd6SGeorge Wilson dbuf_destroy(dn->dn_bonus); 555ea8dc4b6Seschrock dn->dn_bonus = NULL; 556ea8dc4b6Seschrock } 557744947dcSTom Erickson dn->dn_zio = NULL; 558744947dcSTom Erickson 559744947dcSTom Erickson dn->dn_have_spill = B_FALSE; 560744947dcSTom Erickson dn->dn_oldused = 0; 561744947dcSTom Erickson dn->dn_oldflags = 0; 562744947dcSTom Erickson dn->dn_olduid = 0; 563744947dcSTom Erickson dn->dn_oldgid = 0; 564744947dcSTom Erickson dn->dn_newuid = 0; 565744947dcSTom Erickson dn->dn_newgid = 0; 566744947dcSTom Erickson dn->dn_id_flags = 0; 567744947dcSTom Erickson 568cf6106c8SMatthew Ahrens dmu_zfetch_fini(&dn->dn_zfetch); 569fa9e4066Sahrens kmem_cache_free(dnode_cache, dn); 5705a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER); 571bc9014e6SJustin Gibbs 572bc9014e6SJustin Gibbs if (complete_os_eviction) 573bc9014e6SJustin Gibbs dmu_objset_evict_done(os); 574fa9e4066Sahrens } 575fa9e4066Sahrens 576fa9e4066Sahrens void 577fa9e4066Sahrens dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, 57854811da5SToomas Soome dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) 579fa9e4066Sahrens { 580fa9e4066Sahrens int i; 581fa9e4066Sahrens 58254811da5SToomas Soome ASSERT3U(dn_slots, >, 0); 58354811da5SToomas Soome ASSERT3U(dn_slots << DNODE_SHIFT, <=, 58454811da5SToomas Soome spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))); 585b5152584SMatthew Ahrens ASSERT3U(blocksize, <=, 586b5152584SMatthew Ahrens spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 587fa9e4066Sahrens if (blocksize == 0) 588fa9e4066Sahrens blocksize = 1 << zfs_default_bs; 5893b83abddSahrens else 5903b83abddSahrens blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE); 591fa9e4066Sahrens 592fa9e4066Sahrens if (ibs == 0) 593fa9e4066Sahrens ibs = zfs_default_ibs; 594fa9e4066Sahrens 595fa9e4066Sahrens ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT); 596fa9e4066Sahrens 59754811da5SToomas Soome dprintf("os=%p obj=%" PRIu64 " txg=%" PRIu64 59854811da5SToomas Soome " blocksize=%d ibs=%d dn_slots=%d\n", 59954811da5SToomas Soome dn->dn_objset, dn->dn_object, tx->tx_txg, blocksize, ibs, dn_slots); 60054811da5SToomas Soome DNODE_STAT_BUMP(dnode_allocate); 601fa9e4066Sahrens 602fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_NONE); 603fa9e4066Sahrens ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0); 604fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE); 605fa9e4066Sahrens ASSERT(ot != DMU_OT_NONE); 606ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(ot)); 607fa9e4066Sahrens ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) || 6080a586ceaSMark Shellenbaum (bonustype == DMU_OT_SA && bonuslen == 0) || 609fa9e4066Sahrens (bonustype != DMU_OT_NONE && bonuslen != 0)); 610ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(bonustype)); 61154811da5SToomas Soome ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots)); 612fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_NONE); 613fb09f5aaSMadhav Suresh ASSERT0(dn->dn_maxblkid); 614fb09f5aaSMadhav Suresh ASSERT0(dn->dn_allocated_txg); 615fb09f5aaSMadhav Suresh ASSERT0(dn->dn_assigned_txg); 616e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); 617e914ace2STim Schumacher ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1); 6180f6d88adSAlex Reece ASSERT(avl_is_empty(&dn->dn_dbufs)); 619fa9e4066Sahrens 620fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 621fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nblkptr[i]); 622fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_nlevels[i]); 623fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_indblkshift[i]); 624fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[i]); 625fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[i]); 626fb09f5aaSMadhav Suresh ASSERT0(dn->dn_rm_spillblk[i]); 627fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[i]); 628c543ec06Sahrens ASSERT(!list_link_active(&dn->dn_dirty_link[i])); 629c717a561Smaybee ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); 630bf16b11eSMatthew Ahrens ASSERT3P(dn->dn_free_ranges[i], ==, NULL); 631fa9e4066Sahrens } 632fa9e4066Sahrens 633fa9e4066Sahrens dn->dn_type = ot; 634fa9e4066Sahrens dnode_setdblksz(dn, blocksize); 635fa9e4066Sahrens dn->dn_indblkshift = ibs; 636fa9e4066Sahrens dn->dn_nlevels = 1; 63754811da5SToomas Soome dn->dn_num_slots = dn_slots; 6380a586ceaSMark Shellenbaum if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ 6390a586ceaSMark Shellenbaum dn->dn_nblkptr = 1; 64054811da5SToomas Soome else { 64154811da5SToomas Soome dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR, 64254811da5SToomas Soome 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> 64354811da5SToomas Soome SPA_BLKPTRSHIFT)); 64454811da5SToomas Soome } 64554811da5SToomas Soome 646fa9e4066Sahrens dn->dn_bonustype = bonustype; 647fa9e4066Sahrens dn->dn_bonuslen = bonuslen; 648fa9e4066Sahrens dn->dn_checksum = ZIO_CHECKSUM_INHERIT; 649fa9e4066Sahrens dn->dn_compress = ZIO_COMPRESS_INHERIT; 650fa9e4066Sahrens dn->dn_dirtyctx = 0; 651fa9e4066Sahrens 652fa9e4066Sahrens dn->dn_free_txg = 0; 653fa9e4066Sahrens if (dn->dn_dirtyctx_firstset) { 654fa9e4066Sahrens kmem_free(dn->dn_dirtyctx_firstset, 1); 655fa9e4066Sahrens dn->dn_dirtyctx_firstset = NULL; 656fa9e4066Sahrens } 657fa9e4066Sahrens 658fa9e4066Sahrens dn->dn_allocated_txg = tx->tx_txg; 6590a586ceaSMark Shellenbaum dn->dn_id_flags = 0; 660f676ed34Sahrens 661fa9e4066Sahrens dnode_setdirty(dn, tx); 662f676ed34Sahrens dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; 6631934e92fSmaybee dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; 6640a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; 665f676ed34Sahrens dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz; 666fa9e4066Sahrens } 667fa9e4066Sahrens 668fa9e4066Sahrens void 669fa9e4066Sahrens dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, 67054811da5SToomas Soome dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) 671fa9e4066Sahrens { 6722bf405a2SMark Maybee int nblkptr; 673c543ec06Sahrens 674fa9e4066Sahrens ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE); 675b5152584SMatthew Ahrens ASSERT3U(blocksize, <=, 676b5152584SMatthew Ahrens spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 677fb09f5aaSMadhav Suresh ASSERT0(blocksize % SPA_MINBLOCKSIZE); 678ea8dc4b6Seschrock ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); 679fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 680fa9e4066Sahrens ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) || 68106e0070dSMark Shellenbaum (bonustype != DMU_OT_NONE && bonuslen != 0) || 68206e0070dSMark Shellenbaum (bonustype == DMU_OT_SA && bonuslen == 0)); 683ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(bonustype)); 68454811da5SToomas Soome ASSERT3U(bonuslen, <=, 68554811da5SToomas Soome DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)))); 686*946342a2SFabian Grünbichler ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT)); 68754811da5SToomas Soome 68854811da5SToomas Soome dnode_free_interior_slots(dn); 68954811da5SToomas Soome DNODE_STAT_BUMP(dnode_reallocate); 690c543ec06Sahrens 691ea8dc4b6Seschrock /* clean up any unreferenced dbufs */ 6921934e92fSmaybee dnode_evict_dbufs(dn); 693da03de99SMark Maybee 69428d97a71SMark Shellenbaum dn->dn_id_flags = 0; 69528d97a71SMark Shellenbaum 696fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 697fa9e4066Sahrens dnode_setdirty(dn, tx); 6982bf405a2SMark Maybee if (dn->dn_datablksz != blocksize) { 6992bf405a2SMark Maybee /* change blocksize */ 7002bf405a2SMark Maybee ASSERT(dn->dn_maxblkid == 0 && 7012bf405a2SMark Maybee (BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || 7022bf405a2SMark Maybee dnode_block_freed(dn, 0))); 7032bf405a2SMark Maybee dnode_setdblksz(dn, blocksize); 7042bf405a2SMark Maybee dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize; 7052bf405a2SMark Maybee } 7062bf405a2SMark Maybee if (dn->dn_bonuslen != bonuslen) 7072bf405a2SMark Maybee dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen; 70806e0070dSMark Shellenbaum 70906e0070dSMark Shellenbaum if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ 71006e0070dSMark Shellenbaum nblkptr = 1; 71106e0070dSMark Shellenbaum else 71254811da5SToomas Soome nblkptr = MIN(DN_MAX_NBLKPTR, 71354811da5SToomas Soome 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> 71454811da5SToomas Soome SPA_BLKPTRSHIFT)); 7150a586ceaSMark Shellenbaum if (dn->dn_bonustype != bonustype) 7160a586ceaSMark Shellenbaum dn->dn_next_bonustype[tx->tx_txg&TXG_MASK] = bonustype; 717da03de99SMark Maybee if (dn->dn_nblkptr != nblkptr) 718da03de99SMark Maybee dn->dn_next_nblkptr[tx->tx_txg&TXG_MASK] = nblkptr; 7190a586ceaSMark Shellenbaum if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 72006e0070dSMark Shellenbaum dbuf_rm_spill(dn, tx); 72106e0070dSMark Shellenbaum dnode_rm_spill(dn, tx); 7220a586ceaSMark Shellenbaum } 723fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 724fa9e4066Sahrens 725fa9e4066Sahrens /* change type */ 726fa9e4066Sahrens dn->dn_type = ot; 727fa9e4066Sahrens 728fa9e4066Sahrens /* change bonus size and type */ 729fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 730fa9e4066Sahrens dn->dn_bonustype = bonustype; 731fa9e4066Sahrens dn->dn_bonuslen = bonuslen; 73254811da5SToomas Soome dn->dn_num_slots = dn_slots; 733da03de99SMark Maybee dn->dn_nblkptr = nblkptr; 734fa9e4066Sahrens dn->dn_checksum = ZIO_CHECKSUM_INHERIT; 735fa9e4066Sahrens dn->dn_compress = ZIO_COMPRESS_INHERIT; 736fa9e4066Sahrens ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); 737fa9e4066Sahrens 738da03de99SMark Maybee /* fix up the bonus db_size */ 739da03de99SMark Maybee if (dn->dn_bonus) { 7401934e92fSmaybee dn->dn_bonus->db.db_size = 74154811da5SToomas Soome DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - 74254811da5SToomas Soome (dn->dn_nblkptr - 1) * sizeof (blkptr_t); 7431934e92fSmaybee ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); 7441934e92fSmaybee } 745432f72fdSahrens 746fa9e4066Sahrens dn->dn_allocated_txg = tx->tx_txg; 747fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 748fa9e4066Sahrens } 749fa9e4066Sahrens 750f06dce2cSAndrew Stormont #ifdef _KERNEL 751744947dcSTom Erickson static void 752744947dcSTom Erickson dnode_move_impl(dnode_t *odn, dnode_t *ndn) 753744947dcSTom Erickson { 754744947dcSTom Erickson int i; 755744947dcSTom Erickson 756744947dcSTom Erickson ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock)); 757744947dcSTom Erickson ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx)); 758744947dcSTom Erickson ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx)); 759744947dcSTom Erickson ASSERT(!RW_LOCK_HELD(&odn->dn_zfetch.zf_rwlock)); 760744947dcSTom Erickson 761744947dcSTom Erickson /* Copy fields. */ 762744947dcSTom Erickson ndn->dn_objset = odn->dn_objset; 763744947dcSTom Erickson ndn->dn_object = odn->dn_object; 764744947dcSTom Erickson ndn->dn_dbuf = odn->dn_dbuf; 765744947dcSTom Erickson ndn->dn_handle = odn->dn_handle; 766744947dcSTom Erickson ndn->dn_phys = odn->dn_phys; 767744947dcSTom Erickson ndn->dn_type = odn->dn_type; 768744947dcSTom Erickson ndn->dn_bonuslen = odn->dn_bonuslen; 769744947dcSTom Erickson ndn->dn_bonustype = odn->dn_bonustype; 770744947dcSTom Erickson ndn->dn_nblkptr = odn->dn_nblkptr; 771744947dcSTom Erickson ndn->dn_checksum = odn->dn_checksum; 772744947dcSTom Erickson ndn->dn_compress = odn->dn_compress; 773744947dcSTom Erickson ndn->dn_nlevels = odn->dn_nlevels; 774744947dcSTom Erickson ndn->dn_indblkshift = odn->dn_indblkshift; 775744947dcSTom Erickson ndn->dn_datablkshift = odn->dn_datablkshift; 776744947dcSTom Erickson ndn->dn_datablkszsec = odn->dn_datablkszsec; 777744947dcSTom Erickson ndn->dn_datablksz = odn->dn_datablksz; 778744947dcSTom Erickson ndn->dn_maxblkid = odn->dn_maxblkid; 77954811da5SToomas Soome ndn->dn_num_slots = odn->dn_num_slots; 780c7fbe46dSMatthew Ahrens bcopy(&odn->dn_next_type[0], &ndn->dn_next_type[0], 781c7fbe46dSMatthew Ahrens sizeof (odn->dn_next_type)); 782744947dcSTom Erickson bcopy(&odn->dn_next_nblkptr[0], &ndn->dn_next_nblkptr[0], 783744947dcSTom Erickson sizeof (odn->dn_next_nblkptr)); 784744947dcSTom Erickson bcopy(&odn->dn_next_nlevels[0], &ndn->dn_next_nlevels[0], 785744947dcSTom Erickson sizeof (odn->dn_next_nlevels)); 786744947dcSTom Erickson bcopy(&odn->dn_next_indblkshift[0], &ndn->dn_next_indblkshift[0], 787744947dcSTom Erickson sizeof (odn->dn_next_indblkshift)); 788744947dcSTom Erickson bcopy(&odn->dn_next_bonustype[0], &ndn->dn_next_bonustype[0], 789744947dcSTom Erickson sizeof (odn->dn_next_bonustype)); 790744947dcSTom Erickson bcopy(&odn->dn_rm_spillblk[0], &ndn->dn_rm_spillblk[0], 791744947dcSTom Erickson sizeof (odn->dn_rm_spillblk)); 792744947dcSTom Erickson bcopy(&odn->dn_next_bonuslen[0], &ndn->dn_next_bonuslen[0], 793744947dcSTom Erickson sizeof (odn->dn_next_bonuslen)); 794744947dcSTom Erickson bcopy(&odn->dn_next_blksz[0], &ndn->dn_next_blksz[0], 795744947dcSTom Erickson sizeof (odn->dn_next_blksz)); 796744947dcSTom Erickson for (i = 0; i < TXG_SIZE; i++) { 797744947dcSTom Erickson list_move_tail(&ndn->dn_dirty_records[i], 798744947dcSTom Erickson &odn->dn_dirty_records[i]); 799744947dcSTom Erickson } 800bf16b11eSMatthew Ahrens bcopy(&odn->dn_free_ranges[0], &ndn->dn_free_ranges[0], 801bf16b11eSMatthew Ahrens sizeof (odn->dn_free_ranges)); 802744947dcSTom Erickson ndn->dn_allocated_txg = odn->dn_allocated_txg; 803744947dcSTom Erickson ndn->dn_free_txg = odn->dn_free_txg; 804744947dcSTom Erickson ndn->dn_assigned_txg = odn->dn_assigned_txg; 805744947dcSTom Erickson ndn->dn_dirtyctx = odn->dn_dirtyctx; 806744947dcSTom Erickson ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset; 807e914ace2STim Schumacher ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0); 808e914ace2STim Schumacher zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds); 8090f6d88adSAlex Reece ASSERT(avl_is_empty(&ndn->dn_dbufs)); 8100f6d88adSAlex Reece avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs); 811744947dcSTom Erickson ndn->dn_dbufs_count = odn->dn_dbufs_count; 812744947dcSTom Erickson ndn->dn_bonus = odn->dn_bonus; 813744947dcSTom Erickson ndn->dn_have_spill = odn->dn_have_spill; 814744947dcSTom Erickson ndn->dn_zio = odn->dn_zio; 815744947dcSTom Erickson ndn->dn_oldused = odn->dn_oldused; 816744947dcSTom Erickson ndn->dn_oldflags = odn->dn_oldflags; 817744947dcSTom Erickson ndn->dn_olduid = odn->dn_olduid; 818744947dcSTom Erickson ndn->dn_oldgid = odn->dn_oldgid; 819744947dcSTom Erickson ndn->dn_newuid = odn->dn_newuid; 820744947dcSTom Erickson ndn->dn_newgid = odn->dn_newgid; 821744947dcSTom Erickson ndn->dn_id_flags = odn->dn_id_flags; 822744947dcSTom Erickson dmu_zfetch_init(&ndn->dn_zfetch, NULL); 823744947dcSTom Erickson list_move_tail(&ndn->dn_zfetch.zf_stream, &odn->dn_zfetch.zf_stream); 824744947dcSTom Erickson ndn->dn_zfetch.zf_dnode = odn->dn_zfetch.zf_dnode; 825744947dcSTom Erickson 826744947dcSTom Erickson /* 827744947dcSTom Erickson * Update back pointers. Updating the handle fixes the back pointer of 828744947dcSTom Erickson * every descendant dbuf as well as the bonus dbuf. 829744947dcSTom Erickson */ 830744947dcSTom Erickson ASSERT(ndn->dn_handle->dnh_dnode == odn); 831744947dcSTom Erickson ndn->dn_handle->dnh_dnode = ndn; 832744947dcSTom Erickson if (ndn->dn_zfetch.zf_dnode == odn) { 833744947dcSTom Erickson ndn->dn_zfetch.zf_dnode = ndn; 834744947dcSTom Erickson } 835744947dcSTom Erickson 836744947dcSTom Erickson /* 837744947dcSTom Erickson * Invalidate the original dnode by clearing all of its back pointers. 838744947dcSTom Erickson */ 839744947dcSTom Erickson odn->dn_dbuf = NULL; 840744947dcSTom Erickson odn->dn_handle = NULL; 8410f6d88adSAlex Reece avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t), 842744947dcSTom Erickson offsetof(dmu_buf_impl_t, db_link)); 843744947dcSTom Erickson odn->dn_dbufs_count = 0; 844744947dcSTom Erickson odn->dn_bonus = NULL; 845744947dcSTom Erickson odn->dn_zfetch.zf_dnode = NULL; 846744947dcSTom Erickson 847744947dcSTom Erickson /* 848744947dcSTom Erickson * Set the low bit of the objset pointer to ensure that dnode_move() 849744947dcSTom Erickson * recognizes the dnode as invalid in any subsequent callback. 850744947dcSTom Erickson */ 851744947dcSTom Erickson POINTER_INVALIDATE(&odn->dn_objset); 852744947dcSTom Erickson 853744947dcSTom Erickson /* 854744947dcSTom Erickson * Satisfy the destructor. 855744947dcSTom Erickson */ 856744947dcSTom Erickson for (i = 0; i < TXG_SIZE; i++) { 857744947dcSTom Erickson list_create(&odn->dn_dirty_records[i], 858744947dcSTom Erickson sizeof (dbuf_dirty_record_t), 859744947dcSTom Erickson offsetof(dbuf_dirty_record_t, dr_dirty_node)); 860bf16b11eSMatthew Ahrens odn->dn_free_ranges[i] = NULL; 861744947dcSTom Erickson odn->dn_next_nlevels[i] = 0; 862744947dcSTom Erickson odn->dn_next_indblkshift[i] = 0; 863744947dcSTom Erickson odn->dn_next_bonustype[i] = 0; 864744947dcSTom Erickson odn->dn_rm_spillblk[i] = 0; 865744947dcSTom Erickson odn->dn_next_bonuslen[i] = 0; 866744947dcSTom Erickson odn->dn_next_blksz[i] = 0; 867744947dcSTom Erickson } 868744947dcSTom Erickson odn->dn_allocated_txg = 0; 869744947dcSTom Erickson odn->dn_free_txg = 0; 870744947dcSTom Erickson odn->dn_assigned_txg = 0; 871744947dcSTom Erickson odn->dn_dirtyctx = 0; 872744947dcSTom Erickson odn->dn_dirtyctx_firstset = NULL; 873744947dcSTom Erickson odn->dn_have_spill = B_FALSE; 874744947dcSTom Erickson odn->dn_zio = NULL; 875744947dcSTom Erickson odn->dn_oldused = 0; 876744947dcSTom Erickson odn->dn_oldflags = 0; 877744947dcSTom Erickson odn->dn_olduid = 0; 878744947dcSTom Erickson odn->dn_oldgid = 0; 879744947dcSTom Erickson odn->dn_newuid = 0; 880744947dcSTom Erickson odn->dn_newgid = 0; 881744947dcSTom Erickson odn->dn_id_flags = 0; 882744947dcSTom Erickson 883744947dcSTom Erickson /* 884744947dcSTom Erickson * Mark the dnode. 885744947dcSTom Erickson */ 886744947dcSTom Erickson ndn->dn_moved = 1; 887744947dcSTom Erickson odn->dn_moved = (uint8_t)-1; 888744947dcSTom Erickson } 889744947dcSTom Erickson 890744947dcSTom Erickson /*ARGSUSED*/ 891744947dcSTom Erickson static kmem_cbrc_t 892744947dcSTom Erickson dnode_move(void *buf, void *newbuf, size_t size, void *arg) 893744947dcSTom Erickson { 894744947dcSTom Erickson dnode_t *odn = buf, *ndn = newbuf; 895744947dcSTom Erickson objset_t *os; 896744947dcSTom Erickson int64_t refcount; 897744947dcSTom Erickson uint32_t dbufs; 898744947dcSTom Erickson 899744947dcSTom Erickson /* 900744947dcSTom Erickson * The dnode is on the objset's list of known dnodes if the objset 901744947dcSTom Erickson * pointer is valid. We set the low bit of the objset pointer when 902744947dcSTom Erickson * freeing the dnode to invalidate it, and the memory patterns written 903744947dcSTom Erickson * by kmem (baddcafe and deadbeef) set at least one of the two low bits. 904744947dcSTom Erickson * A newly created dnode sets the objset pointer last of all to indicate 905744947dcSTom Erickson * that the dnode is known and in a valid state to be moved by this 906744947dcSTom Erickson * function. 907744947dcSTom Erickson */ 908744947dcSTom Erickson os = odn->dn_objset; 909744947dcSTom Erickson if (!POINTER_IS_VALID(os)) { 91054811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_invalid); 911744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 912744947dcSTom Erickson } 913744947dcSTom Erickson 914744947dcSTom Erickson /* 915744947dcSTom Erickson * Ensure that the objset does not go away during the move. 916744947dcSTom Erickson */ 917744947dcSTom Erickson rw_enter(&os_lock, RW_WRITER); 918744947dcSTom Erickson if (os != odn->dn_objset) { 919744947dcSTom Erickson rw_exit(&os_lock); 92054811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_recheck1); 921744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 922744947dcSTom Erickson } 923744947dcSTom Erickson 924744947dcSTom Erickson /* 925744947dcSTom Erickson * If the dnode is still valid, then so is the objset. We know that no 926744947dcSTom Erickson * valid objset can be freed while we hold os_lock, so we can safely 927744947dcSTom Erickson * ensure that the objset remains in use. 928744947dcSTom Erickson */ 929744947dcSTom Erickson mutex_enter(&os->os_lock); 930744947dcSTom Erickson 931744947dcSTom Erickson /* 932744947dcSTom Erickson * Recheck the objset pointer in case the dnode was removed just before 933744947dcSTom Erickson * acquiring the lock. 934744947dcSTom Erickson */ 935744947dcSTom Erickson if (os != odn->dn_objset) { 936744947dcSTom Erickson mutex_exit(&os->os_lock); 937744947dcSTom Erickson rw_exit(&os_lock); 93854811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_recheck2); 939744947dcSTom Erickson return (KMEM_CBRC_DONT_KNOW); 940744947dcSTom Erickson } 941744947dcSTom Erickson 942744947dcSTom Erickson /* 943744947dcSTom Erickson * At this point we know that as long as we hold os->os_lock, the dnode 944744947dcSTom Erickson * cannot be freed and fields within the dnode can be safely accessed. 945744947dcSTom Erickson * The objset listing this dnode cannot go away as long as this dnode is 946744947dcSTom Erickson * on its list. 947744947dcSTom Erickson */ 948744947dcSTom Erickson rw_exit(&os_lock); 949744947dcSTom Erickson if (DMU_OBJECT_IS_SPECIAL(odn->dn_object)) { 950744947dcSTom Erickson mutex_exit(&os->os_lock); 95154811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_special); 952744947dcSTom Erickson return (KMEM_CBRC_NO); 953744947dcSTom Erickson } 954744947dcSTom Erickson ASSERT(odn->dn_dbuf != NULL); /* only "special" dnodes have no parent */ 955744947dcSTom Erickson 956744947dcSTom Erickson /* 957744947dcSTom Erickson * Lock the dnode handle to prevent the dnode from obtaining any new 958744947dcSTom Erickson * holds. This also prevents the descendant dbufs and the bonus dbuf 959744947dcSTom Erickson * from accessing the dnode, so that we can discount their holds. The 960744947dcSTom Erickson * handle is safe to access because we know that while the dnode cannot 961744947dcSTom Erickson * go away, neither can its handle. Once we hold dnh_zrlock, we can 962744947dcSTom Erickson * safely move any dnode referenced only by dbufs. 963744947dcSTom Erickson */ 964744947dcSTom Erickson if (!zrl_tryenter(&odn->dn_handle->dnh_zrlock)) { 965744947dcSTom Erickson mutex_exit(&os->os_lock); 96654811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_handle); 967744947dcSTom Erickson return (KMEM_CBRC_LATER); 968744947dcSTom Erickson } 969744947dcSTom Erickson 970744947dcSTom Erickson /* 971744947dcSTom Erickson * Ensure a consistent view of the dnode's holds and the dnode's dbufs. 972744947dcSTom Erickson * We need to guarantee that there is a hold for every dbuf in order to 973744947dcSTom Erickson * determine whether the dnode is actively referenced. Falsely matching 974744947dcSTom Erickson * a dbuf to an active hold would lead to an unsafe move. It's possible 975744947dcSTom Erickson * that a thread already having an active dnode hold is about to add a 976744947dcSTom Erickson * dbuf, and we can't compare hold and dbuf counts while the add is in 977744947dcSTom Erickson * progress. 978744947dcSTom Erickson */ 979744947dcSTom Erickson if (!rw_tryenter(&odn->dn_struct_rwlock, RW_WRITER)) { 980744947dcSTom Erickson zrl_exit(&odn->dn_handle->dnh_zrlock); 981744947dcSTom Erickson mutex_exit(&os->os_lock); 98254811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_rwlock); 983744947dcSTom Erickson return (KMEM_CBRC_LATER); 984744947dcSTom Erickson } 985744947dcSTom Erickson 986744947dcSTom Erickson /* 987744947dcSTom Erickson * A dbuf may be removed (evicted) without an active dnode hold. In that 988744947dcSTom Erickson * case, the dbuf count is decremented under the handle lock before the 989744947dcSTom Erickson * dbuf's hold is released. This order ensures that if we count the hold 990744947dcSTom Erickson * after the dbuf is removed but before its hold is released, we will 991744947dcSTom Erickson * treat the unmatched hold as active and exit safely. If we count the 992744947dcSTom Erickson * hold before the dbuf is removed, the hold is discounted, and the 993744947dcSTom Erickson * removal is blocked until the move completes. 994744947dcSTom Erickson */ 995e914ace2STim Schumacher refcount = zfs_refcount_count(&odn->dn_holds); 996744947dcSTom Erickson ASSERT(refcount >= 0); 997744947dcSTom Erickson dbufs = odn->dn_dbufs_count; 998744947dcSTom Erickson 999744947dcSTom Erickson /* We can't have more dbufs than dnode holds. */ 1000744947dcSTom Erickson ASSERT3U(dbufs, <=, refcount); 1001744947dcSTom Erickson DTRACE_PROBE3(dnode__move, dnode_t *, odn, int64_t, refcount, 1002744947dcSTom Erickson uint32_t, dbufs); 1003744947dcSTom Erickson 1004744947dcSTom Erickson if (refcount > dbufs) { 1005744947dcSTom Erickson rw_exit(&odn->dn_struct_rwlock); 1006744947dcSTom Erickson zrl_exit(&odn->dn_handle->dnh_zrlock); 1007744947dcSTom Erickson mutex_exit(&os->os_lock); 100854811da5SToomas Soome DNODE_STAT_BUMP(dnode_move_active); 1009744947dcSTom Erickson return (KMEM_CBRC_LATER); 1010744947dcSTom Erickson } 1011744947dcSTom Erickson 1012744947dcSTom Erickson rw_exit(&odn->dn_struct_rwlock); 1013744947dcSTom Erickson 1014744947dcSTom Erickson /* 1015744947dcSTom Erickson * At this point we know that anyone with a hold on the dnode is not 1016744947dcSTom Erickson * actively referencing it. The dnode is known and in a valid state to 1017744947dcSTom Erickson * move. We're holding the locks needed to execute the critical section. 1018744947dcSTom Erickson */ 1019744947dcSTom Erickson dnode_move_impl(odn, ndn); 1020744947dcSTom Erickson 1021744947dcSTom Erickson list_link_replace(&odn->dn_link, &ndn->dn_link); 1022744947dcSTom Erickson /* If the dnode was safe to move, the refcount cannot have changed. */ 1023e914ace2STim Schumacher ASSERT(refcount == zfs_refcount_count(&ndn->dn_holds)); 1024744947dcSTom Erickson ASSERT(dbufs == ndn->dn_dbufs_count); 1025744947dcSTom Erickson zrl_exit(&ndn->dn_handle->dnh_zrlock); /* handle has moved */ 1026744947dcSTom Erickson mutex_exit(&os->os_lock); 1027744947dcSTom Erickson 1028744947dcSTom Erickson return (KMEM_CBRC_YES); 1029744947dcSTom Erickson } 1030744947dcSTom Erickson #endif /* _KERNEL */ 1031744947dcSTom Erickson 103254811da5SToomas Soome static void 103354811da5SToomas Soome dnode_slots_hold(dnode_children_t *children, int idx, int slots) 103454811da5SToomas Soome { 103554811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 103654811da5SToomas Soome 103754811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 103854811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 103954811da5SToomas Soome zrl_add(&dnh->dnh_zrlock); 104054811da5SToomas Soome } 104154811da5SToomas Soome } 104254811da5SToomas Soome 104354811da5SToomas Soome static void 104454811da5SToomas Soome dnode_slots_rele(dnode_children_t *children, int idx, int slots) 104554811da5SToomas Soome { 104654811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 104754811da5SToomas Soome 104854811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 104954811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 105054811da5SToomas Soome 105154811da5SToomas Soome if (zrl_is_locked(&dnh->dnh_zrlock)) 105254811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 105354811da5SToomas Soome else 105454811da5SToomas Soome zrl_remove(&dnh->dnh_zrlock); 105554811da5SToomas Soome } 105654811da5SToomas Soome } 105754811da5SToomas Soome 105854811da5SToomas Soome static int 105954811da5SToomas Soome dnode_slots_tryenter(dnode_children_t *children, int idx, int slots) 106054811da5SToomas Soome { 106154811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 106254811da5SToomas Soome 106354811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 106454811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 106554811da5SToomas Soome 106654811da5SToomas Soome if (!zrl_tryenter(&dnh->dnh_zrlock)) { 106754811da5SToomas Soome for (int j = idx; j < i; j++) { 106854811da5SToomas Soome dnh = &children->dnc_children[j]; 106954811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 107054811da5SToomas Soome } 107154811da5SToomas Soome 107254811da5SToomas Soome return (0); 107354811da5SToomas Soome } 107454811da5SToomas Soome } 107554811da5SToomas Soome 107654811da5SToomas Soome return (1); 107754811da5SToomas Soome } 107854811da5SToomas Soome 107954811da5SToomas Soome static void 108054811da5SToomas Soome dnode_set_slots(dnode_children_t *children, int idx, int slots, void *ptr) 108154811da5SToomas Soome { 108254811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 108354811da5SToomas Soome 108454811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 108554811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 108654811da5SToomas Soome dnh->dnh_dnode = ptr; 108754811da5SToomas Soome } 108854811da5SToomas Soome } 108954811da5SToomas Soome 109054811da5SToomas Soome static boolean_t 109154811da5SToomas Soome dnode_check_slots_free(dnode_children_t *children, int idx, int slots) 109254811da5SToomas Soome { 109354811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 109454811da5SToomas Soome 109554811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 109654811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 109754811da5SToomas Soome dnode_t *dn = dnh->dnh_dnode; 109854811da5SToomas Soome 109954811da5SToomas Soome if (dn == DN_SLOT_FREE) { 110054811da5SToomas Soome continue; 110154811da5SToomas Soome } else if (DN_SLOT_IS_PTR(dn)) { 110254811da5SToomas Soome mutex_enter(&dn->dn_mtx); 110354811da5SToomas Soome dmu_object_type_t type = dn->dn_type; 110454811da5SToomas Soome mutex_exit(&dn->dn_mtx); 110554811da5SToomas Soome 110654811da5SToomas Soome if (type != DMU_OT_NONE) 110754811da5SToomas Soome return (B_FALSE); 110854811da5SToomas Soome 110954811da5SToomas Soome continue; 111054811da5SToomas Soome } else { 111154811da5SToomas Soome return (B_FALSE); 111254811da5SToomas Soome } 111354811da5SToomas Soome 111454811da5SToomas Soome return (B_FALSE); 111554811da5SToomas Soome } 111654811da5SToomas Soome 111754811da5SToomas Soome return (B_TRUE); 111854811da5SToomas Soome } 111954811da5SToomas Soome 112054811da5SToomas Soome static void 112154811da5SToomas Soome dnode_reclaim_slots(dnode_children_t *children, int idx, int slots) 112254811da5SToomas Soome { 112354811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 112454811da5SToomas Soome 112554811da5SToomas Soome for (int i = idx; i < idx + slots; i++) { 112654811da5SToomas Soome dnode_handle_t *dnh = &children->dnc_children[i]; 112754811da5SToomas Soome 112854811da5SToomas Soome ASSERT(zrl_is_locked(&dnh->dnh_zrlock)); 112954811da5SToomas Soome 113054811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 113154811da5SToomas Soome ASSERT3S(dnh->dnh_dnode->dn_type, ==, DMU_OT_NONE); 113254811da5SToomas Soome dnode_destroy(dnh->dnh_dnode); 113354811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_FREE; 113454811da5SToomas Soome } 113554811da5SToomas Soome } 113654811da5SToomas Soome } 113754811da5SToomas Soome 113854811da5SToomas Soome void 113954811da5SToomas Soome dnode_free_interior_slots(dnode_t *dn) 114054811da5SToomas Soome { 114154811da5SToomas Soome dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db); 114254811da5SToomas Soome int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT; 114354811da5SToomas Soome int idx = (dn->dn_object & (epb - 1)) + 1; 114454811da5SToomas Soome int slots = dn->dn_num_slots - 1; 114554811da5SToomas Soome 114654811da5SToomas Soome if (slots == 0) 114754811da5SToomas Soome return; 114854811da5SToomas Soome 114954811da5SToomas Soome ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); 115054811da5SToomas Soome 115154811da5SToomas Soome while (!dnode_slots_tryenter(children, idx, slots)) 115254811da5SToomas Soome DNODE_STAT_BUMP(dnode_free_interior_lock_retry); 115354811da5SToomas Soome 115454811da5SToomas Soome dnode_set_slots(children, idx, slots, DN_SLOT_FREE); 115554811da5SToomas Soome dnode_slots_rele(children, idx, slots); 115654811da5SToomas Soome } 115754811da5SToomas Soome 1158fa9e4066Sahrens void 1159744947dcSTom Erickson dnode_special_close(dnode_handle_t *dnh) 1160fa9e4066Sahrens { 1161744947dcSTom Erickson dnode_t *dn = dnh->dnh_dnode; 1162744947dcSTom Erickson 1163ea8dc4b6Seschrock /* 1164ea8dc4b6Seschrock * Wait for final references to the dnode to clear. This can 116554811da5SToomas Soome * only happen if the arc is asynchronously evicting state that 1166ea8dc4b6Seschrock * has a hold on this dnode while we are trying to evict this 1167ea8dc4b6Seschrock * dnode. 1168ea8dc4b6Seschrock */ 1169e914ace2STim Schumacher while (zfs_refcount_count(&dn->dn_holds) > 0) 1170ea8dc4b6Seschrock delay(1); 1171bc9014e6SJustin Gibbs ASSERT(dn->dn_dbuf == NULL || 1172bc9014e6SJustin Gibbs dmu_buf_get_user(&dn->dn_dbuf->db) == NULL); 1173744947dcSTom Erickson zrl_add(&dnh->dnh_zrlock); 1174744947dcSTom Erickson dnode_destroy(dn); /* implicit zrl_remove() */ 1175744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 1176744947dcSTom Erickson dnh->dnh_dnode = NULL; 1177fa9e4066Sahrens } 1178fa9e4066Sahrens 1179bc9014e6SJustin Gibbs void 1180744947dcSTom Erickson dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object, 1181744947dcSTom Erickson dnode_handle_t *dnh) 1182fa9e4066Sahrens { 1183bc9014e6SJustin Gibbs dnode_t *dn; 1184bc9014e6SJustin Gibbs 1185744947dcSTom Erickson zrl_init(&dnh->dnh_zrlock); 118654811da5SToomas Soome zrl_tryenter(&dnh->dnh_zrlock); 118754811da5SToomas Soome 118854811da5SToomas Soome dn = dnode_create(os, dnp, NULL, object, dnh); 11899c9dc39aSek DNODE_VERIFY(dn); 119054811da5SToomas Soome 119154811da5SToomas Soome zrl_exit(&dnh->dnh_zrlock); 1192fa9e4066Sahrens } 1193fa9e4066Sahrens 1194fa9e4066Sahrens static void 119540510e8eSJosef 'Jeff' Sipek dnode_buf_evict_async(void *dbu) 1196fa9e4066Sahrens { 119754811da5SToomas Soome dnode_children_t *dnc = dbu; 119854811da5SToomas Soome 119954811da5SToomas Soome DNODE_STAT_BUMP(dnode_buf_evict); 1200fa9e4066Sahrens 120154811da5SToomas Soome for (int i = 0; i < dnc->dnc_count; i++) { 120254811da5SToomas Soome dnode_handle_t *dnh = &dnc->dnc_children[i]; 1203744947dcSTom Erickson dnode_t *dn; 1204fa9e4066Sahrens 1205744947dcSTom Erickson /* 1206744947dcSTom Erickson * The dnode handle lock guards against the dnode moving to 1207744947dcSTom Erickson * another valid address, so there is no need here to guard 1208744947dcSTom Erickson * against changes to or from NULL. 1209744947dcSTom Erickson */ 121054811da5SToomas Soome if (!DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 1211744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 121254811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_UNINIT; 1213fa9e4066Sahrens continue; 1214744947dcSTom Erickson } 1215744947dcSTom Erickson 1216744947dcSTom Erickson zrl_add(&dnh->dnh_zrlock); 1217744947dcSTom Erickson dn = dnh->dnh_dnode; 1218fa9e4066Sahrens /* 1219fa9e4066Sahrens * If there are holds on this dnode, then there should 1220fa9e4066Sahrens * be holds on the dnode's containing dbuf as well; thus 1221744947dcSTom Erickson * it wouldn't be eligible for eviction and this function 1222fa9e4066Sahrens * would not have been called. 1223fa9e4066Sahrens */ 1224e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_holds)); 1225e914ace2STim Schumacher ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds)); 1226fa9e4066Sahrens 122754811da5SToomas Soome dnode_destroy(dn); /* implicit zrl_remove() for first slot */ 1228744947dcSTom Erickson zrl_destroy(&dnh->dnh_zrlock); 122954811da5SToomas Soome dnh->dnh_dnode = DN_SLOT_UNINIT; 1230fa9e4066Sahrens } 123154811da5SToomas Soome kmem_free(dnc, sizeof (dnode_children_t) + 123254811da5SToomas Soome dnc->dnc_count * sizeof (dnode_handle_t)); 1233fa9e4066Sahrens } 1234fa9e4066Sahrens 1235fa9e4066Sahrens /* 123654811da5SToomas Soome * When the DNODE_MUST_BE_FREE flag is set, the "slots" parameter is used 123754811da5SToomas Soome * to ensure the hole at the specified object offset is large enough to 123854811da5SToomas Soome * hold the dnode being created. The slots parameter is also used to ensure 123954811da5SToomas Soome * a dnode does not span multiple dnode blocks. In both of these cases, if 124054811da5SToomas Soome * a failure occurs, ENOSPC is returned. Keep in mind, these failure cases 124154811da5SToomas Soome * are only possible when using DNODE_MUST_BE_FREE. 124254811da5SToomas Soome * 124354811da5SToomas Soome * If the DNODE_MUST_BE_ALLOCATED flag is set, "slots" must be 0. 124454811da5SToomas Soome * dnode_hold_impl() will check if the requested dnode is already consumed 124554811da5SToomas Soome * as an extra dnode slot by an large dnode, in which case it returns 124654811da5SToomas Soome * ENOENT. 124754811da5SToomas Soome * 1248ea8dc4b6Seschrock * errors: 124954811da5SToomas Soome * EINVAL - invalid object number or flags. 125054811da5SToomas Soome * ENOSPC - hole too small to fulfill "slots" request (DNODE_MUST_BE_FREE) 125154811da5SToomas Soome * EEXIST - Refers to an allocated dnode (DNODE_MUST_BE_FREE) 125254811da5SToomas Soome * - Refers to a freeing dnode (DNODE_MUST_BE_FREE) 125354811da5SToomas Soome * - Refers to an interior dnode slot (DNODE_MUST_BE_ALLOCATED) 125454811da5SToomas Soome * ENOENT - The requested dnode is not allocated (DNODE_MUST_BE_ALLOCATED) 125554811da5SToomas Soome * - The requested dnode is being freed (DNODE_MUST_BE_ALLOCATED) 125654811da5SToomas Soome * EIO - i/o error error when reading the meta dnode dbuf. 1257ea8dc4b6Seschrock * succeeds even for free dnodes. 1258fa9e4066Sahrens */ 1259ea8dc4b6Seschrock int 126054811da5SToomas Soome dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots, 1261ea8dc4b6Seschrock void *tag, dnode_t **dnp) 1262fa9e4066Sahrens { 1263ea8dc4b6Seschrock int epb, idx, err; 1264fa9e4066Sahrens int drop_struct_lock = FALSE; 1265ea8dc4b6Seschrock int type; 1266fa9e4066Sahrens uint64_t blk; 1267fa9e4066Sahrens dnode_t *mdn, *dn; 1268fa9e4066Sahrens dmu_buf_impl_t *db; 126954811da5SToomas Soome dnode_children_t *dnc; 127054811da5SToomas Soome dnode_phys_t *dn_block; 1271744947dcSTom Erickson dnode_handle_t *dnh; 1272fa9e4066Sahrens 127354811da5SToomas Soome ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0)); 127454811da5SToomas Soome ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0)); 127554811da5SToomas Soome 1276e14bb325SJeff Bonwick /* 1277e14bb325SJeff Bonwick * If you are holding the spa config lock as writer, you shouldn't 1278dcba9f3fSGeorge Wilson * be asking the DMU to do *anything* unless it's the root pool 1279dcba9f3fSGeorge Wilson * which may require us to read from the root filesystem while 1280dcba9f3fSGeorge Wilson * holding some (not all) of the locks as writer. 1281e14bb325SJeff Bonwick */ 1282dcba9f3fSGeorge Wilson ASSERT(spa_config_held(os->os_spa, SCL_ALL, RW_WRITER) == 0 || 1283dcba9f3fSGeorge Wilson (spa_is_root(os->os_spa) && 128444ecc532SGeorge Wilson spa_config_held(os->os_spa, SCL_STATE, RW_WRITER))); 1285e14bb325SJeff Bonwick 128686714001SSerapheim Dimitropoulos ASSERT((flag & DNODE_MUST_BE_ALLOCATED) || (flag & DNODE_MUST_BE_FREE)); 128786714001SSerapheim Dimitropoulos 128814843421SMatthew Ahrens if (object == DMU_USERUSED_OBJECT || object == DMU_GROUPUSED_OBJECT) { 128914843421SMatthew Ahrens dn = (object == DMU_USERUSED_OBJECT) ? 1290744947dcSTom Erickson DMU_USERUSED_DNODE(os) : DMU_GROUPUSED_DNODE(os); 129114843421SMatthew Ahrens if (dn == NULL) 1292be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 129314843421SMatthew Ahrens type = dn->dn_type; 129414843421SMatthew Ahrens if ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) 1295be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 129614843421SMatthew Ahrens if ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE) 1297be6fd75aSMatthew Ahrens return (SET_ERROR(EEXIST)); 129814843421SMatthew Ahrens DNODE_VERIFY(dn); 1299e914ace2STim Schumacher (void) zfs_refcount_add(&dn->dn_holds, tag); 130014843421SMatthew Ahrens *dnp = dn; 130114843421SMatthew Ahrens return (0); 130214843421SMatthew Ahrens } 130314843421SMatthew Ahrens 1304fa9e4066Sahrens if (object == 0 || object >= DN_MAX_OBJECT) 1305be6fd75aSMatthew Ahrens return (SET_ERROR(EINVAL)); 1306fa9e4066Sahrens 1307744947dcSTom Erickson mdn = DMU_META_DNODE(os); 1308744947dcSTom Erickson ASSERT(mdn->dn_object == DMU_META_DNODE_OBJECT); 1309fa9e4066Sahrens 13109c9dc39aSek DNODE_VERIFY(mdn); 1311fa9e4066Sahrens 1312fa9e4066Sahrens if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) { 1313fa9e4066Sahrens rw_enter(&mdn->dn_struct_rwlock, RW_READER); 1314fa9e4066Sahrens drop_struct_lock = TRUE; 1315fa9e4066Sahrens } 1316fa9e4066Sahrens 1317a2cdcdd2SPaul Dagnelie blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t)); 1318fa9e4066Sahrens 1319ea8dc4b6Seschrock db = dbuf_hold(mdn, blk, FTAG); 1320fa9e4066Sahrens if (drop_struct_lock) 1321fa9e4066Sahrens rw_exit(&mdn->dn_struct_rwlock); 132254811da5SToomas Soome if (db == NULL) { 132354811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_dbuf_hold); 1324be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 132554811da5SToomas Soome } 1326ea8dc4b6Seschrock err = dbuf_read(db, NULL, DB_RF_CANFAIL); 1327ea8dc4b6Seschrock if (err) { 132854811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_dbuf_read); 1329ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1330ea8dc4b6Seschrock return (err); 1331ea8dc4b6Seschrock } 1332fa9e4066Sahrens 1333fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT); 1334fa9e4066Sahrens epb = db->db.db_size >> DNODE_SHIFT; 1335fa9e4066Sahrens 133654811da5SToomas Soome idx = object & (epb - 1); 133754811da5SToomas Soome dn_block = (dnode_phys_t *)db->db.db_data; 1338fa9e4066Sahrens 1339744947dcSTom Erickson ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE); 134054811da5SToomas Soome dnc = dmu_buf_get_user(&db->db); 134154811da5SToomas Soome dnh = NULL; 134254811da5SToomas Soome if (dnc == NULL) { 1343744947dcSTom Erickson dnode_children_t *winner; 134454811da5SToomas Soome int skip = 0; 134554811da5SToomas Soome 134654811da5SToomas Soome dnc = kmem_zalloc(sizeof (dnode_children_t) + 13477f18da4cSJustin T. Gibbs epb * sizeof (dnode_handle_t), KM_SLEEP); 134854811da5SToomas Soome dnc->dnc_count = epb; 134954811da5SToomas Soome dnh = &dnc->dnc_children[0]; 135054811da5SToomas Soome 135154811da5SToomas Soome /* Initialize dnode slot status from dnode_phys_t */ 135254811da5SToomas Soome for (int i = 0; i < epb; i++) { 1353744947dcSTom Erickson zrl_init(&dnh[i].dnh_zrlock); 135454811da5SToomas Soome 135554811da5SToomas Soome if (skip) { 135654811da5SToomas Soome skip--; 135754811da5SToomas Soome continue; 135854811da5SToomas Soome } 135954811da5SToomas Soome 136054811da5SToomas Soome if (dn_block[i].dn_type != DMU_OT_NONE) { 136154811da5SToomas Soome int interior = dn_block[i].dn_extra_slots; 136254811da5SToomas Soome 136354811da5SToomas Soome dnode_set_slots(dnc, i, 1, DN_SLOT_ALLOCATED); 136454811da5SToomas Soome dnode_set_slots(dnc, i + 1, interior, 136554811da5SToomas Soome DN_SLOT_INTERIOR); 136654811da5SToomas Soome skip = interior; 136754811da5SToomas Soome } else { 136854811da5SToomas Soome dnh[i].dnh_dnode = DN_SLOT_FREE; 136954811da5SToomas Soome skip = 0; 137054811da5SToomas Soome } 1371744947dcSTom Erickson } 137254811da5SToomas Soome 137354811da5SToomas Soome dmu_buf_init_user(&dnc->dnc_dbu, NULL, 137440510e8eSJosef 'Jeff' Sipek dnode_buf_evict_async, NULL); 137554811da5SToomas Soome winner = dmu_buf_set_user(&db->db, &dnc->dnc_dbu); 1376bc9014e6SJustin Gibbs if (winner != NULL) { 1377d2b3cbbdSJorgen Lundman 137854811da5SToomas Soome for (int i = 0; i < epb; i++) 1379d2b3cbbdSJorgen Lundman zrl_destroy(&dnh[i].dnh_zrlock); 1380d2b3cbbdSJorgen Lundman 138154811da5SToomas Soome kmem_free(dnc, sizeof (dnode_children_t) + 13827f18da4cSJustin T. Gibbs epb * sizeof (dnode_handle_t)); 138354811da5SToomas Soome dnc = winner; 1384fa9e4066Sahrens } 1385fa9e4066Sahrens } 1386fa9e4066Sahrens 138754811da5SToomas Soome ASSERT(dnc->dnc_count == epb); 138854811da5SToomas Soome dn = DN_SLOT_UNINIT; 138954811da5SToomas Soome 139054811da5SToomas Soome if (flag & DNODE_MUST_BE_ALLOCATED) { 139154811da5SToomas Soome slots = 1; 139254811da5SToomas Soome 139354811da5SToomas Soome while (dn == DN_SLOT_UNINIT) { 139454811da5SToomas Soome dnode_slots_hold(dnc, idx, slots); 139554811da5SToomas Soome dnh = &dnc->dnc_children[idx]; 139654811da5SToomas Soome 139754811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 139854811da5SToomas Soome dn = dnh->dnh_dnode; 139954811da5SToomas Soome break; 140054811da5SToomas Soome } else if (dnh->dnh_dnode == DN_SLOT_INTERIOR) { 140154811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_interior); 140254811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 140354811da5SToomas Soome dbuf_rele(db, FTAG); 140454811da5SToomas Soome return (SET_ERROR(EEXIST)); 140554811da5SToomas Soome } else if (dnh->dnh_dnode != DN_SLOT_ALLOCATED) { 140654811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_misses); 140754811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 140854811da5SToomas Soome dbuf_rele(db, FTAG); 140954811da5SToomas Soome return (SET_ERROR(ENOENT)); 141054811da5SToomas Soome } 141154811da5SToomas Soome 141254811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 141354811da5SToomas Soome if (!dnode_slots_tryenter(dnc, idx, slots)) { 141454811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_lock_retry); 141554811da5SToomas Soome continue; 141654811da5SToomas Soome } 141754811da5SToomas Soome 141854811da5SToomas Soome /* 141954811da5SToomas Soome * Someone else won the race and called dnode_create() 142054811da5SToomas Soome * after we checked DN_SLOT_IS_PTR() above but before 142154811da5SToomas Soome * we acquired the lock. 142254811da5SToomas Soome */ 142354811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 142454811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_lock_misses); 142554811da5SToomas Soome dn = dnh->dnh_dnode; 142654811da5SToomas Soome } else { 142754811da5SToomas Soome dn = dnode_create(os, dn_block + idx, db, 142854811da5SToomas Soome object, dnh); 142954811da5SToomas Soome } 143054811da5SToomas Soome } 143154811da5SToomas Soome 143254811da5SToomas Soome mutex_enter(&dn->dn_mtx); 143354811da5SToomas Soome if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) { 143454811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_type_none); 143554811da5SToomas Soome mutex_exit(&dn->dn_mtx); 143654811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 143754811da5SToomas Soome dbuf_rele(db, FTAG); 143854811da5SToomas Soome return (SET_ERROR(ENOENT)); 143954811da5SToomas Soome } 144054811da5SToomas Soome 144154811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_alloc_hits); 144254811da5SToomas Soome } else if (flag & DNODE_MUST_BE_FREE) { 144354811da5SToomas Soome 144454811da5SToomas Soome if (idx + slots - 1 >= DNODES_PER_BLOCK) { 144554811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_overflow); 144654811da5SToomas Soome dbuf_rele(db, FTAG); 144754811da5SToomas Soome return (SET_ERROR(ENOSPC)); 144854811da5SToomas Soome } 144954811da5SToomas Soome 145054811da5SToomas Soome while (dn == DN_SLOT_UNINIT) { 145154811da5SToomas Soome dnode_slots_hold(dnc, idx, slots); 145254811da5SToomas Soome 145354811da5SToomas Soome if (!dnode_check_slots_free(dnc, idx, slots)) { 145454811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_misses); 145554811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 145654811da5SToomas Soome dbuf_rele(db, FTAG); 145754811da5SToomas Soome return (SET_ERROR(ENOSPC)); 145854811da5SToomas Soome } 145954811da5SToomas Soome 146054811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 146154811da5SToomas Soome if (!dnode_slots_tryenter(dnc, idx, slots)) { 146254811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_lock_retry); 146354811da5SToomas Soome continue; 146454811da5SToomas Soome } 146554811da5SToomas Soome 146654811da5SToomas Soome if (!dnode_check_slots_free(dnc, idx, slots)) { 146754811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_lock_misses); 146854811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 146954811da5SToomas Soome dbuf_rele(db, FTAG); 147054811da5SToomas Soome return (SET_ERROR(ENOSPC)); 147154811da5SToomas Soome } 147254811da5SToomas Soome 147354811da5SToomas Soome /* 147454811da5SToomas Soome * Allocated but otherwise free dnodes which would 147554811da5SToomas Soome * be in the interior of a multi-slot dnodes need 147654811da5SToomas Soome * to be freed. Single slot dnodes can be safely 147754811da5SToomas Soome * re-purposed as a performance optimization. 147854811da5SToomas Soome */ 147954811da5SToomas Soome if (slots > 1) 148054811da5SToomas Soome dnode_reclaim_slots(dnc, idx + 1, slots - 1); 148154811da5SToomas Soome 148254811da5SToomas Soome dnh = &dnc->dnc_children[idx]; 148354811da5SToomas Soome if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) { 148454811da5SToomas Soome dn = dnh->dnh_dnode; 148554811da5SToomas Soome } else { 148654811da5SToomas Soome dn = dnode_create(os, dn_block + idx, db, 148754811da5SToomas Soome object, dnh); 148854811da5SToomas Soome } 148954811da5SToomas Soome } 149054811da5SToomas Soome 149154811da5SToomas Soome mutex_enter(&dn->dn_mtx); 1492e914ace2STim Schumacher if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) { 149354811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_refcount); 149454811da5SToomas Soome mutex_exit(&dn->dn_mtx); 149554811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 149654811da5SToomas Soome dbuf_rele(db, FTAG); 149754811da5SToomas Soome return (SET_ERROR(EEXIST)); 149854811da5SToomas Soome } 14990e8c6158Smaybee 150054811da5SToomas Soome dnode_set_slots(dnc, idx + 1, slots - 1, DN_SLOT_INTERIOR); 150154811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_hits); 150254811da5SToomas Soome } else { 150354811da5SToomas Soome dbuf_rele(db, FTAG); 150454811da5SToomas Soome return (SET_ERROR(EINVAL)); 1505fa9e4066Sahrens } 1506fa9e4066Sahrens 150754811da5SToomas Soome if (dn->dn_free_txg) { 150854811da5SToomas Soome DNODE_STAT_BUMP(dnode_hold_free_txg); 150954811da5SToomas Soome type = dn->dn_type; 1510fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 151154811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 1512ea8dc4b6Seschrock dbuf_rele(db, FTAG); 151354811da5SToomas Soome return (SET_ERROR((flag & DNODE_MUST_BE_ALLOCATED) ? 151454811da5SToomas Soome ENOENT : EEXIST)); 1515fa9e4066Sahrens } 151654811da5SToomas Soome 1517e914ace2STim Schumacher if (zfs_refcount_add(&dn->dn_holds, tag) == 1) 1518744947dcSTom Erickson dbuf_add_ref(db, dnh); 151954811da5SToomas Soome 1520bc9014e6SJustin Gibbs mutex_exit(&dn->dn_mtx); 1521bc9014e6SJustin Gibbs 1522744947dcSTom Erickson /* Now we can rely on the hold to prevent the dnode from moving. */ 152354811da5SToomas Soome dnode_slots_rele(dnc, idx, slots); 1524fa9e4066Sahrens 15259c9dc39aSek DNODE_VERIFY(dn); 1526fa9e4066Sahrens ASSERT3P(dn->dn_dbuf, ==, db); 1527fa9e4066Sahrens ASSERT3U(dn->dn_object, ==, object); 1528ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1529fa9e4066Sahrens 1530ea8dc4b6Seschrock *dnp = dn; 1531ea8dc4b6Seschrock return (0); 1532fa9e4066Sahrens } 1533fa9e4066Sahrens 1534fa9e4066Sahrens /* 1535fa9e4066Sahrens * Return held dnode if the object is allocated, NULL if not. 1536fa9e4066Sahrens */ 1537ea8dc4b6Seschrock int 1538503ad85cSMatthew Ahrens dnode_hold(objset_t *os, uint64_t object, void *tag, dnode_t **dnp) 1539fa9e4066Sahrens { 154054811da5SToomas Soome return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag, 154154811da5SToomas Soome dnp)); 1542fa9e4066Sahrens } 1543fa9e4066Sahrens 15441934e92fSmaybee /* 15451934e92fSmaybee * Can only add a reference if there is already at least one 15461934e92fSmaybee * reference on the dnode. Returns FALSE if unable to add a 15471934e92fSmaybee * new reference. 15481934e92fSmaybee */ 15491934e92fSmaybee boolean_t 1550ea8dc4b6Seschrock dnode_add_ref(dnode_t *dn, void *tag) 1551fa9e4066Sahrens { 15521934e92fSmaybee mutex_enter(&dn->dn_mtx); 1553e914ace2STim Schumacher if (zfs_refcount_is_zero(&dn->dn_holds)) { 15541934e92fSmaybee mutex_exit(&dn->dn_mtx); 15551934e92fSmaybee return (FALSE); 15561934e92fSmaybee } 1557e914ace2STim Schumacher VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag)); 15581934e92fSmaybee mutex_exit(&dn->dn_mtx); 15591934e92fSmaybee return (TRUE); 1560fa9e4066Sahrens } 1561fa9e4066Sahrens 1562fa9e4066Sahrens void 1563ea8dc4b6Seschrock dnode_rele(dnode_t *dn, void *tag) 1564cd485b49SJustin T. Gibbs { 1565cd485b49SJustin T. Gibbs mutex_enter(&dn->dn_mtx); 1566c2919acbSMatthew Ahrens dnode_rele_and_unlock(dn, tag, B_FALSE); 1567cd485b49SJustin T. Gibbs } 1568cd485b49SJustin T. Gibbs 1569cd485b49SJustin T. Gibbs void 1570c2919acbSMatthew Ahrens dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting) 1571fa9e4066Sahrens { 1572fa9e4066Sahrens uint64_t refs; 1573744947dcSTom Erickson /* Get while the hold prevents the dnode from moving. */ 1574744947dcSTom Erickson dmu_buf_impl_t *db = dn->dn_dbuf; 1575744947dcSTom Erickson dnode_handle_t *dnh = dn->dn_handle; 1576fa9e4066Sahrens 1577e914ace2STim Schumacher refs = zfs_refcount_remove(&dn->dn_holds, tag); 15781934e92fSmaybee mutex_exit(&dn->dn_mtx); 1579744947dcSTom Erickson 1580744947dcSTom Erickson /* 1581744947dcSTom Erickson * It's unsafe to release the last hold on a dnode by dnode_rele() or 1582744947dcSTom Erickson * indirectly by dbuf_rele() while relying on the dnode handle to 1583744947dcSTom Erickson * prevent the dnode from moving, since releasing the last hold could 1584744947dcSTom Erickson * result in the dnode's parent dbuf evicting its dnode handles. For 1585744947dcSTom Erickson * that reason anyone calling dnode_rele() or dbuf_rele() without some 1586744947dcSTom Erickson * other direct or indirect hold on the dnode must first drop the dnode 1587744947dcSTom Erickson * handle. 1588744947dcSTom Erickson */ 1589744947dcSTom Erickson ASSERT(refs > 0 || dnh->dnh_zrlock.zr_owner != curthread); 1590744947dcSTom Erickson 1591fa9e4066Sahrens /* NOTE: the DNODE_DNODE does not have a dn_dbuf */ 1592744947dcSTom Erickson if (refs == 0 && db != NULL) { 1593744947dcSTom Erickson /* 1594744947dcSTom Erickson * Another thread could add a hold to the dnode handle in 1595744947dcSTom Erickson * dnode_hold_impl() while holding the parent dbuf. Since the 1596744947dcSTom Erickson * hold on the parent dbuf prevents the handle from being 1597744947dcSTom Erickson * destroyed, the hold on the handle is OK. We can't yet assert 1598744947dcSTom Erickson * that the handle has zero references, but that will be 1599744947dcSTom Erickson * asserted anyway when the handle gets destroyed. 1600744947dcSTom Erickson */ 1601c2919acbSMatthew Ahrens mutex_enter(&db->db_mtx); 1602c2919acbSMatthew Ahrens dbuf_rele_and_unlock(db, dnh, evicting); 1603744947dcSTom Erickson } 1604fa9e4066Sahrens } 1605fa9e4066Sahrens 1606fa9e4066Sahrens void 1607fa9e4066Sahrens dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) 1608fa9e4066Sahrens { 1609503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1610fa9e4066Sahrens uint64_t txg = tx->tx_txg; 1611fa9e4066Sahrens 161214843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { 161314843421SMatthew Ahrens dsl_dataset_dirty(os->os_dsl_dataset, tx); 1614fa9e4066Sahrens return; 161514843421SMatthew Ahrens } 1616fa9e4066Sahrens 16179c9dc39aSek DNODE_VERIFY(dn); 1618fa9e4066Sahrens 1619fa9e4066Sahrens #ifdef ZFS_DEBUG 1620fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1621fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg); 1622744947dcSTom Erickson ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); 1623fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1624fa9e4066Sahrens #endif 1625fa9e4066Sahrens 16260a586ceaSMark Shellenbaum /* 16270a586ceaSMark Shellenbaum * Determine old uid/gid when necessary 16280a586ceaSMark Shellenbaum */ 162906e0070dSMark Shellenbaum dmu_objset_userquota_get_ids(dn, B_TRUE, tx); 16300a586ceaSMark Shellenbaum 163194c2d0ebSMatthew Ahrens multilist_t *dirtylist = os->os_dirty_dnodes[txg & TXG_MASK]; 163294c2d0ebSMatthew Ahrens multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn); 1633fa9e4066Sahrens 1634fa9e4066Sahrens /* 1635fa9e4066Sahrens * If we are already marked dirty, we're done. 1636fa9e4066Sahrens */ 1637c543ec06Sahrens if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { 163894c2d0ebSMatthew Ahrens multilist_sublist_unlock(mls); 1639fa9e4066Sahrens return; 1640fa9e4066Sahrens } 1641fa9e4066Sahrens 1642e914ace2STim Schumacher ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) || 16430f6d88adSAlex Reece !avl_is_empty(&dn->dn_dbufs)); 1644fa9e4066Sahrens ASSERT(dn->dn_datablksz != 0); 1645fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonuslen[txg&TXG_MASK]); 1646fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_blksz[txg&TXG_MASK]); 1647fb09f5aaSMadhav Suresh ASSERT0(dn->dn_next_bonustype[txg&TXG_MASK]); 1648fa9e4066Sahrens 1649fa9e4066Sahrens dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n", 1650fa9e4066Sahrens dn->dn_object, txg); 1651fa9e4066Sahrens 165294c2d0ebSMatthew Ahrens multilist_sublist_insert_head(mls, dn); 1653fa9e4066Sahrens 165494c2d0ebSMatthew Ahrens multilist_sublist_unlock(mls); 1655fa9e4066Sahrens 1656fa9e4066Sahrens /* 1657fa9e4066Sahrens * The dnode maintains a hold on its containing dbuf as 1658fa9e4066Sahrens * long as there are holds on it. Each instantiated child 1659744947dcSTom Erickson * dbuf maintains a hold on the dnode. When the last child 1660fa9e4066Sahrens * drops its hold, the dnode will drop its hold on the 1661fa9e4066Sahrens * containing dbuf. We add a "dirty hold" here so that the 1662fa9e4066Sahrens * dnode will hang around after we finish processing its 1663fa9e4066Sahrens * children. 1664fa9e4066Sahrens */ 16651934e92fSmaybee VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg)); 1666fa9e4066Sahrens 1667c717a561Smaybee (void) dbuf_dirty(dn->dn_dbuf, tx); 1668fa9e4066Sahrens 1669fa9e4066Sahrens dsl_dataset_dirty(os->os_dsl_dataset, tx); 1670fa9e4066Sahrens } 1671fa9e4066Sahrens 1672fa9e4066Sahrens void 1673fa9e4066Sahrens dnode_free(dnode_t *dn, dmu_tx_t *tx) 1674fa9e4066Sahrens { 1675fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1676fa9e4066Sahrens if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { 1677fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1678fa9e4066Sahrens return; 1679fa9e4066Sahrens } 1680fa9e4066Sahrens dn->dn_free_txg = tx->tx_txg; 1681fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1682fa9e4066Sahrens 168394c2d0ebSMatthew Ahrens dnode_setdirty(dn, tx); 1684fa9e4066Sahrens } 1685fa9e4066Sahrens 1686fa9e4066Sahrens /* 1687fa9e4066Sahrens * Try to change the block size for the indicated dnode. This can only 1688fa9e4066Sahrens * succeed if there are no blocks allocated or dirty beyond first block 1689fa9e4066Sahrens */ 1690fa9e4066Sahrens int 1691fa9e4066Sahrens dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx) 1692fa9e4066Sahrens { 16930f6d88adSAlex Reece dmu_buf_impl_t *db; 1694cdb0ab79Smaybee int err; 1695fa9e4066Sahrens 1696b5152584SMatthew Ahrens ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); 1697fa9e4066Sahrens if (size == 0) 1698fa9e4066Sahrens size = SPA_MINBLOCKSIZE; 1699fa9e4066Sahrens else 1700fa9e4066Sahrens size = P2ROUNDUP(size, SPA_MINBLOCKSIZE); 1701fa9e4066Sahrens 1702b143e04bSahrens if (ibs == dn->dn_indblkshift) 1703b143e04bSahrens ibs = 0; 1704fa9e4066Sahrens 1705b143e04bSahrens if (size >> SPA_MINBLOCKSHIFT == dn->dn_datablkszsec && ibs == 0) 1706fa9e4066Sahrens return (0); 1707fa9e4066Sahrens 1708fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1709fa9e4066Sahrens 1710fa9e4066Sahrens /* Check for any allocated blocks beyond the first */ 17110713e232SGeorge Wilson if (dn->dn_maxblkid != 0) 1712b143e04bSahrens goto fail; 1713fa9e4066Sahrens 1714fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 17150f6d88adSAlex Reece for (db = avl_first(&dn->dn_dbufs); db != NULL; 17160f6d88adSAlex Reece db = AVL_NEXT(&dn->dn_dbufs, db)) { 17170a586ceaSMark Shellenbaum if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID && 17180a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1719fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1720b143e04bSahrens goto fail; 1721fa9e4066Sahrens } 1722fa9e4066Sahrens } 1723fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1724fa9e4066Sahrens 1725b143e04bSahrens if (ibs && dn->dn_nlevels != 1) 1726b143e04bSahrens goto fail; 1727b143e04bSahrens 1728cdb0ab79Smaybee /* resize the old block */ 1729a2cdcdd2SPaul Dagnelie err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db); 1730cdb0ab79Smaybee if (err == 0) 1731c543ec06Sahrens dbuf_new_size(db, size, tx); 1732cdb0ab79Smaybee else if (err != ENOENT) 1733cdb0ab79Smaybee goto fail; 1734fa9e4066Sahrens 1735fa9e4066Sahrens dnode_setdblksz(dn, size); 1736c543ec06Sahrens dnode_setdirty(dn, tx); 1737c543ec06Sahrens dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = size; 1738b143e04bSahrens if (ibs) { 1739b143e04bSahrens dn->dn_indblkshift = ibs; 1740b143e04bSahrens dn->dn_next_indblkshift[tx->tx_txg&TXG_MASK] = ibs; 1741b143e04bSahrens } 1742cdb0ab79Smaybee /* rele after we have fixed the blocksize in the dnode */ 1743c543ec06Sahrens if (db) 1744c543ec06Sahrens dbuf_rele(db, FTAG); 1745fa9e4066Sahrens 1746fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1747b143e04bSahrens return (0); 1748b143e04bSahrens 1749b143e04bSahrens fail: 1750b143e04bSahrens rw_exit(&dn->dn_struct_rwlock); 1751be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 1752fa9e4066Sahrens } 1753fa9e4066Sahrens 17548346f03fSJonathan W Adams /* read-holding callers must not rely on the lock being continuously held */ 1755fa9e4066Sahrens void 17568346f03fSJonathan W Adams dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read) 1757fa9e4066Sahrens { 1758fa9e4066Sahrens uint64_t txgoff = tx->tx_txg & TXG_MASK; 1759c543ec06Sahrens int epbs, new_nlevels; 1760fa9e4066Sahrens uint64_t sz; 1761fa9e4066Sahrens 17620a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1763fa9e4066Sahrens 17648346f03fSJonathan W Adams ASSERT(have_read ? 17658346f03fSJonathan W Adams RW_READ_HELD(&dn->dn_struct_rwlock) : 17668346f03fSJonathan W Adams RW_WRITE_HELD(&dn->dn_struct_rwlock)); 17678346f03fSJonathan W Adams 17688346f03fSJonathan W Adams /* 17698346f03fSJonathan W Adams * if we have a read-lock, check to see if we need to do any work 17708346f03fSJonathan W Adams * before upgrading to a write-lock. 17718346f03fSJonathan W Adams */ 17728346f03fSJonathan W Adams if (have_read) { 17738346f03fSJonathan W Adams if (blkid <= dn->dn_maxblkid) 17748346f03fSJonathan W Adams return; 17758346f03fSJonathan W Adams 17768346f03fSJonathan W Adams if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { 17778346f03fSJonathan W Adams rw_exit(&dn->dn_struct_rwlock); 17788346f03fSJonathan W Adams rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 17798346f03fSJonathan W Adams } 1780fa9e4066Sahrens } 1781fa9e4066Sahrens 1782c543ec06Sahrens if (blkid <= dn->dn_maxblkid) 1783c543ec06Sahrens goto out; 1784c543ec06Sahrens 1785c543ec06Sahrens dn->dn_maxblkid = blkid; 1786fa9e4066Sahrens 1787fa9e4066Sahrens /* 1788c543ec06Sahrens * Compute the number of levels necessary to support the new maxblkid. 1789fa9e4066Sahrens */ 1790fa9e4066Sahrens new_nlevels = 1; 1791fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1792c543ec06Sahrens for (sz = dn->dn_nblkptr; 1793c543ec06Sahrens sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs) 1794fa9e4066Sahrens new_nlevels++; 1795fa9e4066Sahrens 1796c543ec06Sahrens if (new_nlevels > dn->dn_nlevels) { 1797c543ec06Sahrens int old_nlevels = dn->dn_nlevels; 1798c543ec06Sahrens dmu_buf_impl_t *db; 1799c717a561Smaybee list_t *list; 1800c717a561Smaybee dbuf_dirty_record_t *new, *dr, *dr_next; 1801fa9e4066Sahrens 1802fa9e4066Sahrens dn->dn_nlevels = new_nlevels; 1803fa9e4066Sahrens 1804c543ec06Sahrens ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]); 1805c543ec06Sahrens dn->dn_next_nlevels[txgoff] = new_nlevels; 1806c543ec06Sahrens 1807c717a561Smaybee /* dirty the left indirects */ 1808c543ec06Sahrens db = dbuf_hold_level(dn, old_nlevels, 0, FTAG); 180901025c89SJohn Harres ASSERT(db != NULL); 1810c717a561Smaybee new = dbuf_dirty(db, tx); 1811ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1812fa9e4066Sahrens 1813c717a561Smaybee /* transfer the dirty records to the new indirect */ 1814c717a561Smaybee mutex_enter(&dn->dn_mtx); 1815c717a561Smaybee mutex_enter(&new->dt.di.dr_mtx); 1816c717a561Smaybee list = &dn->dn_dirty_records[txgoff]; 1817c717a561Smaybee for (dr = list_head(list); dr; dr = dr_next) { 1818c717a561Smaybee dr_next = list_next(&dn->dn_dirty_records[txgoff], dr); 1819c717a561Smaybee if (dr->dr_dbuf->db_level != new_nlevels-1 && 18200a586ceaSMark Shellenbaum dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && 18210a586ceaSMark Shellenbaum dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID) { 1822c717a561Smaybee ASSERT(dr->dr_dbuf->db_level == old_nlevels-1); 1823c717a561Smaybee list_remove(&dn->dn_dirty_records[txgoff], dr); 1824c717a561Smaybee list_insert_tail(&new->dt.di.dr_children, dr); 1825c717a561Smaybee dr->dr_parent = new; 1826c717a561Smaybee } 1827c717a561Smaybee } 1828c717a561Smaybee mutex_exit(&new->dt.di.dr_mtx); 1829c717a561Smaybee mutex_exit(&dn->dn_mtx); 1830fa9e4066Sahrens } 1831fa9e4066Sahrens 1832fa9e4066Sahrens out: 18338346f03fSJonathan W Adams if (have_read) 18348346f03fSJonathan W Adams rw_downgrade(&dn->dn_struct_rwlock); 1835fa9e4066Sahrens } 1836fa9e4066Sahrens 183746e1baa6SMatthew Ahrens static void 183846e1baa6SMatthew Ahrens dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx) 183946e1baa6SMatthew Ahrens { 184046e1baa6SMatthew Ahrens dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG); 184146e1baa6SMatthew Ahrens if (db != NULL) { 184246e1baa6SMatthew Ahrens dmu_buf_will_dirty(&db->db, tx); 184346e1baa6SMatthew Ahrens dbuf_rele(db, FTAG); 184446e1baa6SMatthew Ahrens } 184546e1baa6SMatthew Ahrens } 184646e1baa6SMatthew Ahrens 1847738e2a3cSPaul Dagnelie /* 1848738e2a3cSPaul Dagnelie * Dirty all the in-core level-1 dbufs in the range specified by start_blkid 1849738e2a3cSPaul Dagnelie * and end_blkid. 1850738e2a3cSPaul Dagnelie */ 1851738e2a3cSPaul Dagnelie static void 1852738e2a3cSPaul Dagnelie dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 1853738e2a3cSPaul Dagnelie dmu_tx_t *tx) 1854738e2a3cSPaul Dagnelie { 1855738e2a3cSPaul Dagnelie dmu_buf_impl_t db_search; 1856738e2a3cSPaul Dagnelie dmu_buf_impl_t *db; 1857738e2a3cSPaul Dagnelie avl_index_t where; 1858738e2a3cSPaul Dagnelie 1859738e2a3cSPaul Dagnelie mutex_enter(&dn->dn_dbufs_mtx); 1860738e2a3cSPaul Dagnelie 1861738e2a3cSPaul Dagnelie db_search.db_level = 1; 1862738e2a3cSPaul Dagnelie db_search.db_blkid = start_blkid + 1; 1863738e2a3cSPaul Dagnelie db_search.db_state = DB_SEARCH; 1864738e2a3cSPaul Dagnelie for (;;) { 1865738e2a3cSPaul Dagnelie 1866738e2a3cSPaul Dagnelie db = avl_find(&dn->dn_dbufs, &db_search, &where); 1867738e2a3cSPaul Dagnelie if (db == NULL) 1868738e2a3cSPaul Dagnelie db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1869738e2a3cSPaul Dagnelie 1870738e2a3cSPaul Dagnelie if (db == NULL || db->db_level != 1 || 1871738e2a3cSPaul Dagnelie db->db_blkid >= end_blkid) { 1872738e2a3cSPaul Dagnelie break; 1873738e2a3cSPaul Dagnelie } 1874738e2a3cSPaul Dagnelie 1875738e2a3cSPaul Dagnelie /* 1876738e2a3cSPaul Dagnelie * Setup the next blkid we want to search for. 1877738e2a3cSPaul Dagnelie */ 1878738e2a3cSPaul Dagnelie db_search.db_blkid = db->db_blkid + 1; 1879738e2a3cSPaul Dagnelie ASSERT3U(db->db_blkid, >=, start_blkid); 1880738e2a3cSPaul Dagnelie 1881738e2a3cSPaul Dagnelie /* 1882738e2a3cSPaul Dagnelie * If the dbuf transitions to DB_EVICTING while we're trying 1883738e2a3cSPaul Dagnelie * to dirty it, then we will be unable to discover it in 1884738e2a3cSPaul Dagnelie * the dbuf hash table. This will result in a call to 1885738e2a3cSPaul Dagnelie * dbuf_create() which needs to acquire the dn_dbufs_mtx 1886738e2a3cSPaul Dagnelie * lock. To avoid a deadlock, we drop the lock before 1887738e2a3cSPaul Dagnelie * dirtying the level-1 dbuf. 1888738e2a3cSPaul Dagnelie */ 1889738e2a3cSPaul Dagnelie mutex_exit(&dn->dn_dbufs_mtx); 1890738e2a3cSPaul Dagnelie dnode_dirty_l1(dn, db->db_blkid, tx); 1891738e2a3cSPaul Dagnelie mutex_enter(&dn->dn_dbufs_mtx); 1892738e2a3cSPaul Dagnelie } 1893738e2a3cSPaul Dagnelie 1894738e2a3cSPaul Dagnelie #ifdef ZFS_DEBUG 1895738e2a3cSPaul Dagnelie /* 1896738e2a3cSPaul Dagnelie * Walk all the in-core level-1 dbufs and verify they have been dirtied. 1897738e2a3cSPaul Dagnelie */ 1898738e2a3cSPaul Dagnelie db_search.db_level = 1; 1899738e2a3cSPaul Dagnelie db_search.db_blkid = start_blkid + 1; 1900738e2a3cSPaul Dagnelie db_search.db_state = DB_SEARCH; 1901738e2a3cSPaul Dagnelie db = avl_find(&dn->dn_dbufs, &db_search, &where); 1902738e2a3cSPaul Dagnelie if (db == NULL) 1903738e2a3cSPaul Dagnelie db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 1904738e2a3cSPaul Dagnelie for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) { 1905738e2a3cSPaul Dagnelie if (db->db_level != 1 || db->db_blkid >= end_blkid) 1906738e2a3cSPaul Dagnelie break; 1907738e2a3cSPaul Dagnelie ASSERT(db->db_dirtycnt > 0); 1908738e2a3cSPaul Dagnelie } 1909738e2a3cSPaul Dagnelie #endif 1910738e2a3cSPaul Dagnelie mutex_exit(&dn->dn_dbufs_mtx); 1911738e2a3cSPaul Dagnelie } 1912738e2a3cSPaul Dagnelie 1913fa9e4066Sahrens void 1914fa9e4066Sahrens dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) 1915fa9e4066Sahrens { 1916fa9e4066Sahrens dmu_buf_impl_t *db; 1917b143e04bSahrens uint64_t blkoff, blkid, nblks; 1918cdb0ab79Smaybee int blksz, blkshift, head, tail; 1919fa9e4066Sahrens int trunc = FALSE; 1920cdb0ab79Smaybee int epbs; 1921fa9e4066Sahrens 1922fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1923fa9e4066Sahrens blksz = dn->dn_datablksz; 1924cdb0ab79Smaybee blkshift = dn->dn_datablkshift; 1925cdb0ab79Smaybee epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1926fa9e4066Sahrens 1927713d6c20SMatthew Ahrens if (len == DMU_OBJECT_END) { 1928fa9e4066Sahrens len = UINT64_MAX - off; 1929fa9e4066Sahrens trunc = TRUE; 1930fa9e4066Sahrens } 1931fa9e4066Sahrens 1932fa9e4066Sahrens /* 1933fa9e4066Sahrens * First, block align the region to free: 1934fa9e4066Sahrens */ 1935b143e04bSahrens if (ISP2(blksz)) { 1936b143e04bSahrens head = P2NPHASE(off, blksz); 1937b143e04bSahrens blkoff = P2PHASE(off, blksz); 1938cdb0ab79Smaybee if ((off >> blkshift) > dn->dn_maxblkid) 1939cdb0ab79Smaybee goto out; 1940b143e04bSahrens } else { 1941b143e04bSahrens ASSERT(dn->dn_maxblkid == 0); 1942b143e04bSahrens if (off == 0 && len >= blksz) { 194343466aaeSMax Grossman /* 194443466aaeSMax Grossman * Freeing the whole block; fast-track this request. 194543466aaeSMax Grossman */ 1946cdb0ab79Smaybee blkid = 0; 1947cdb0ab79Smaybee nblks = 1; 194899a19144SMatthew Ahrens if (dn->dn_nlevels > 1) 194999a19144SMatthew Ahrens dnode_dirty_l1(dn, 0, tx); 1950cdb0ab79Smaybee goto done; 19511c8564a7SMark Maybee } else if (off >= blksz) { 1952cdb0ab79Smaybee /* Freeing past end-of-data */ 1953cdb0ab79Smaybee goto out; 1954fa9e4066Sahrens } else { 1955b143e04bSahrens /* Freeing part of the block. */ 1956fa9e4066Sahrens head = blksz - off; 1957fa9e4066Sahrens ASSERT3U(head, >, 0); 1958fa9e4066Sahrens } 1959b143e04bSahrens blkoff = off; 1960fa9e4066Sahrens } 1961fa9e4066Sahrens /* zero out any partial block data at the start of the range */ 1962fa9e4066Sahrens if (head) { 1963b143e04bSahrens ASSERT3U(blkoff + head, ==, blksz); 1964fa9e4066Sahrens if (len < head) 1965fa9e4066Sahrens head = len; 1966a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), 1967a2cdcdd2SPaul Dagnelie TRUE, FALSE, FTAG, &db) == 0) { 1968fa9e4066Sahrens caddr_t data; 1969fa9e4066Sahrens 1970fa9e4066Sahrens /* don't dirty if it isn't on disk and isn't dirty */ 1971c717a561Smaybee if (db->db_last_dirty || 1972fa9e4066Sahrens (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr))) { 1973fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 197443466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 1975fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 1976fa9e4066Sahrens data = db->db.db_data; 1977b143e04bSahrens bzero(data + blkoff, head); 1978fa9e4066Sahrens } 1979ea8dc4b6Seschrock dbuf_rele(db, FTAG); 1980fa9e4066Sahrens } 1981fa9e4066Sahrens off += head; 1982fa9e4066Sahrens len -= head; 1983fa9e4066Sahrens } 1984fa9e4066Sahrens 1985b143e04bSahrens /* If the range was less than one block, we're done */ 1986cdb0ab79Smaybee if (len == 0) 1987fa9e4066Sahrens goto out; 1988fa9e4066Sahrens 1989cdb0ab79Smaybee /* If the remaining range is past end of file, we're done */ 1990cdb0ab79Smaybee if ((off >> blkshift) > dn->dn_maxblkid) 1991cdb0ab79Smaybee goto out; 1992b143e04bSahrens 19931c8564a7SMark Maybee ASSERT(ISP2(blksz)); 1994cdb0ab79Smaybee if (trunc) 1995cdb0ab79Smaybee tail = 0; 1996cdb0ab79Smaybee else 1997cdb0ab79Smaybee tail = P2PHASE(len, blksz); 1998cdb0ab79Smaybee 1999fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(off, blksz)); 2000cdb0ab79Smaybee /* zero out any partial block data at the end of the range */ 2001cdb0ab79Smaybee if (tail) { 2002cdb0ab79Smaybee if (len < tail) 2003cdb0ab79Smaybee tail = len; 2004a2cdcdd2SPaul Dagnelie if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off+len), 2005a2cdcdd2SPaul Dagnelie TRUE, FALSE, FTAG, &db) == 0) { 2006cdb0ab79Smaybee /* don't dirty if not on disk and not dirty */ 2007cdb0ab79Smaybee if (db->db_last_dirty || 2008cdb0ab79Smaybee (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr))) { 2009cdb0ab79Smaybee rw_exit(&dn->dn_struct_rwlock); 201043466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 2011cdb0ab79Smaybee rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2012cdb0ab79Smaybee bzero(db->db.db_data, tail); 2013fa9e4066Sahrens } 2014ea8dc4b6Seschrock dbuf_rele(db, FTAG); 2015fa9e4066Sahrens } 2016cdb0ab79Smaybee len -= tail; 2017cdb0ab79Smaybee } 2018fa9e4066Sahrens 2019cdb0ab79Smaybee /* If the range did not include a full block, we are done */ 2020cdb0ab79Smaybee if (len == 0) 2021cdb0ab79Smaybee goto out; 2022fa9e4066Sahrens 2023cdb0ab79Smaybee ASSERT(IS_P2ALIGNED(off, blksz)); 2024cdb0ab79Smaybee ASSERT(trunc || IS_P2ALIGNED(len, blksz)); 2025cdb0ab79Smaybee blkid = off >> blkshift; 2026cdb0ab79Smaybee nblks = len >> blkshift; 2027cdb0ab79Smaybee if (trunc) 2028cdb0ab79Smaybee nblks += 1; 2029cdb0ab79Smaybee 2030cdb0ab79Smaybee /* 203146e1baa6SMatthew Ahrens * Dirty all the indirect blocks in this range. Note that only 203246e1baa6SMatthew Ahrens * the first and last indirect blocks can actually be written 203346e1baa6SMatthew Ahrens * (if they were partially freed) -- they must be dirtied, even if 203446e1baa6SMatthew Ahrens * they do not exist on disk yet. The interior blocks will 203546e1baa6SMatthew Ahrens * be freed by free_children(), so they will not actually be written. 203646e1baa6SMatthew Ahrens * Even though these interior blocks will not be written, we 203746e1baa6SMatthew Ahrens * dirty them for two reasons: 203846e1baa6SMatthew Ahrens * 203946e1baa6SMatthew Ahrens * - It ensures that the indirect blocks remain in memory until 204046e1baa6SMatthew Ahrens * syncing context. (They have already been prefetched by 204146e1baa6SMatthew Ahrens * dmu_tx_hold_free(), so we don't have to worry about reading 204246e1baa6SMatthew Ahrens * them serially here.) 204346e1baa6SMatthew Ahrens * 204446e1baa6SMatthew Ahrens * - The dirty space accounting will put pressure on the txg sync 204546e1baa6SMatthew Ahrens * mechanism to begin syncing, and to delay transactions if there 204646e1baa6SMatthew Ahrens * is a large amount of freeing. Even though these indirect 204746e1baa6SMatthew Ahrens * blocks will not be written, we could need to write the same 204846e1baa6SMatthew Ahrens * amount of space if we copy the freed BPs into deadlists. 2049cdb0ab79Smaybee */ 2050cdb0ab79Smaybee if (dn->dn_nlevels > 1) { 205143466aaeSMax Grossman uint64_t first, last; 2052b143e04bSahrens 2053cdb0ab79Smaybee first = blkid >> epbs; 205446e1baa6SMatthew Ahrens dnode_dirty_l1(dn, first, tx); 2055b143e04bSahrens if (trunc) 2056cdb0ab79Smaybee last = dn->dn_maxblkid >> epbs; 2057cdb0ab79Smaybee else 2058cdb0ab79Smaybee last = (blkid + nblks - 1) >> epbs; 205946e1baa6SMatthew Ahrens if (last != first) 206046e1baa6SMatthew Ahrens dnode_dirty_l1(dn, last, tx); 206146e1baa6SMatthew Ahrens 2062738e2a3cSPaul Dagnelie dnode_dirty_l1range(dn, first, last, tx); 2063738e2a3cSPaul Dagnelie 206446e1baa6SMatthew Ahrens int shift = dn->dn_datablkshift + dn->dn_indblkshift - 206546e1baa6SMatthew Ahrens SPA_BLKPTRSHIFT; 206646e1baa6SMatthew Ahrens for (uint64_t i = first + 1; i < last; i++) { 206746e1baa6SMatthew Ahrens /* 206846e1baa6SMatthew Ahrens * Set i to the blockid of the next non-hole 206946e1baa6SMatthew Ahrens * level-1 indirect block at or after i. Note 207046e1baa6SMatthew Ahrens * that dnode_next_offset() operates in terms of 207146e1baa6SMatthew Ahrens * level-0-equivalent bytes. 207246e1baa6SMatthew Ahrens */ 207346e1baa6SMatthew Ahrens uint64_t ibyte = i << shift; 207446e1baa6SMatthew Ahrens int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK, 207546e1baa6SMatthew Ahrens &ibyte, 2, 1, 0); 207646e1baa6SMatthew Ahrens i = ibyte >> shift; 207746e1baa6SMatthew Ahrens if (i >= last) 207846e1baa6SMatthew Ahrens break; 207946e1baa6SMatthew Ahrens 208046e1baa6SMatthew Ahrens /* 208146e1baa6SMatthew Ahrens * Normally we should not see an error, either 208246e1baa6SMatthew Ahrens * from dnode_next_offset() or dbuf_hold_level() 208346e1baa6SMatthew Ahrens * (except for ESRCH from dnode_next_offset). 208446e1baa6SMatthew Ahrens * If there is an i/o error, then when we read 208546e1baa6SMatthew Ahrens * this block in syncing context, it will use 208646e1baa6SMatthew Ahrens * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according 208746e1baa6SMatthew Ahrens * to the "failmode" property. dnode_next_offset() 208846e1baa6SMatthew Ahrens * doesn't have a flag to indicate MUSTSUCCEED. 208946e1baa6SMatthew Ahrens */ 209046e1baa6SMatthew Ahrens if (err != 0) 209146e1baa6SMatthew Ahrens break; 209246e1baa6SMatthew Ahrens 209346e1baa6SMatthew Ahrens dnode_dirty_l1(dn, i, tx); 209456d55a53Smaybee } 2095fa9e4066Sahrens } 209643466aaeSMax Grossman 2097cdb0ab79Smaybee done: 2098cdb0ab79Smaybee /* 2099cdb0ab79Smaybee * Add this range to the dnode range list. 2100cdb0ab79Smaybee * We will finish up this free operation in the syncing phase. 2101cdb0ab79Smaybee */ 2102fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2103bf16b11eSMatthew Ahrens int txgoff = tx->tx_txg & TXG_MASK; 2104bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] == NULL) { 21055cabbc6bSPrashanth Sreenivasa dn->dn_free_ranges[txgoff] = range_tree_create(NULL, NULL); 2106fa9e4066Sahrens } 2107bf16b11eSMatthew Ahrens range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks); 2108bf16b11eSMatthew Ahrens range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks); 2109bf16b11eSMatthew Ahrens dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", 2110bf16b11eSMatthew Ahrens blkid, nblks, tx->tx_txg); 2111fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2112fa9e4066Sahrens 2113cdb0ab79Smaybee dbuf_free_range(dn, blkid, blkid + nblks - 1, tx); 2114fa9e4066Sahrens dnode_setdirty(dn, tx); 2115fa9e4066Sahrens out: 2116cdb0ab79Smaybee 2117fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 2118fa9e4066Sahrens } 2119fa9e4066Sahrens 21200a586ceaSMark Shellenbaum static boolean_t 21210a586ceaSMark Shellenbaum dnode_spill_freed(dnode_t *dn) 21220a586ceaSMark Shellenbaum { 21230a586ceaSMark Shellenbaum int i; 21240a586ceaSMark Shellenbaum 21250a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 21260a586ceaSMark Shellenbaum for (i = 0; i < TXG_SIZE; i++) { 21270a586ceaSMark Shellenbaum if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK) 21280a586ceaSMark Shellenbaum break; 21290a586ceaSMark Shellenbaum } 21300a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 21310a586ceaSMark Shellenbaum return (i < TXG_SIZE); 21320a586ceaSMark Shellenbaum } 21330a586ceaSMark Shellenbaum 2134fa9e4066Sahrens /* return TRUE if this blkid was freed in a recent txg, or FALSE if it wasn't */ 2135fa9e4066Sahrens uint64_t 2136fa9e4066Sahrens dnode_block_freed(dnode_t *dn, uint64_t blkid) 2137fa9e4066Sahrens { 2138fa9e4066Sahrens void *dp = spa_get_dsl(dn->dn_objset->os_spa); 2139fa9e4066Sahrens int i; 2140fa9e4066Sahrens 21410a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) 2142fa9e4066Sahrens return (FALSE); 2143fa9e4066Sahrens 2144fa9e4066Sahrens /* 2145fa9e4066Sahrens * If we're in the process of opening the pool, dp will not be 2146fa9e4066Sahrens * set yet, but there shouldn't be anything dirty. 2147fa9e4066Sahrens */ 2148fa9e4066Sahrens if (dp == NULL) 2149fa9e4066Sahrens return (FALSE); 2150fa9e4066Sahrens 2151fa9e4066Sahrens if (dn->dn_free_txg) 2152fa9e4066Sahrens return (TRUE); 2153fa9e4066Sahrens 21540a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) 21550a586ceaSMark Shellenbaum return (dnode_spill_freed(dn)); 21560a586ceaSMark Shellenbaum 2157fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2158fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 2159bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[i] != NULL && 2160bf16b11eSMatthew Ahrens range_tree_contains(dn->dn_free_ranges[i], blkid, 1)) 2161fa9e4066Sahrens break; 2162fa9e4066Sahrens } 2163fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2164fa9e4066Sahrens return (i < TXG_SIZE); 2165fa9e4066Sahrens } 2166fa9e4066Sahrens 2167fa9e4066Sahrens /* call from syncing context when we actually write/free space for this dnode */ 2168fa9e4066Sahrens void 216999653d4eSeschrock dnode_diduse_space(dnode_t *dn, int64_t delta) 2170fa9e4066Sahrens { 217199653d4eSeschrock uint64_t space; 217299653d4eSeschrock dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n", 2173fa9e4066Sahrens dn, dn->dn_phys, 217499653d4eSeschrock (u_longlong_t)dn->dn_phys->dn_used, 217599653d4eSeschrock (longlong_t)delta); 2176fa9e4066Sahrens 2177fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 217899653d4eSeschrock space = DN_USED_BYTES(dn->dn_phys); 217999653d4eSeschrock if (delta > 0) { 218099653d4eSeschrock ASSERT3U(space + delta, >=, space); /* no overflow */ 218199653d4eSeschrock } else { 218299653d4eSeschrock ASSERT3U(space, >=, -delta); /* no underflow */ 218399653d4eSeschrock } 218499653d4eSeschrock space += delta; 2185e7437265Sahrens if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) { 218699653d4eSeschrock ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0); 2187fb09f5aaSMadhav Suresh ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT)); 218899653d4eSeschrock dn->dn_phys->dn_used = space >> DEV_BSHIFT; 2189fa9e4066Sahrens } else { 219099653d4eSeschrock dn->dn_phys->dn_used = space; 219199653d4eSeschrock dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES; 2192fa9e4066Sahrens } 2193fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2194fa9e4066Sahrens } 2195fa9e4066Sahrens 219676256205SMark Maybee /* 2197f7170741SWill Andrews * Scans a block at the indicated "level" looking for a hole or data, 2198f7170741SWill Andrews * depending on 'flags'. 2199f7170741SWill Andrews * 2200f7170741SWill Andrews * If level > 0, then we are scanning an indirect block looking at its 2201f7170741SWill Andrews * pointers. If level == 0, then we are looking at a block of dnodes. 2202f7170741SWill Andrews * 2203f7170741SWill Andrews * If we don't find what we are looking for in the block, we return ESRCH. 2204f7170741SWill Andrews * Otherwise, return with *offset pointing to the beginning (if searching 2205f7170741SWill Andrews * forwards) or end (if searching backwards) of the range covered by the 2206f7170741SWill Andrews * block pointer we matched on (or dnode). 220776256205SMark Maybee * 220876256205SMark Maybee * The basic search algorithm used below by dnode_next_offset() is to 220976256205SMark Maybee * use this function to search up the block tree (widen the search) until 221076256205SMark Maybee * we find something (i.e., we don't return ESRCH) and then search back 221176256205SMark Maybee * down the tree (narrow the search) until we reach our original search 221276256205SMark Maybee * level. 221376256205SMark Maybee */ 2214fa9e4066Sahrens static int 2215cdb0ab79Smaybee dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, 2216a2cdcdd2SPaul Dagnelie int lvl, uint64_t blkfill, uint64_t txg) 2217fa9e4066Sahrens { 2218fa9e4066Sahrens dmu_buf_impl_t *db = NULL; 2219fa9e4066Sahrens void *data = NULL; 2220fa9e4066Sahrens uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2221fa9e4066Sahrens uint64_t epb = 1ULL << epbs; 2222fa9e4066Sahrens uint64_t minfill, maxfill; 2223cdb0ab79Smaybee boolean_t hole; 2224cdb0ab79Smaybee int i, inc, error, span; 2225fa9e4066Sahrens 2226fa9e4066Sahrens dprintf("probing object %llu offset %llx level %d of %u\n", 2227fa9e4066Sahrens dn->dn_object, *offset, lvl, dn->dn_phys->dn_nlevels); 2228fa9e4066Sahrens 222914843421SMatthew Ahrens hole = ((flags & DNODE_FIND_HOLE) != 0); 2230cdb0ab79Smaybee inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1; 22311c8564a7SMark Maybee ASSERT(txg == 0 || !hole); 2232cdb0ab79Smaybee 2233fa9e4066Sahrens if (lvl == dn->dn_phys->dn_nlevels) { 2234fa9e4066Sahrens error = 0; 2235fa9e4066Sahrens epb = dn->dn_phys->dn_nblkptr; 2236fa9e4066Sahrens data = dn->dn_phys->dn_blkptr; 2237fa9e4066Sahrens } else { 2238a2cdcdd2SPaul Dagnelie uint64_t blkid = dbuf_whichblock(dn, lvl, *offset); 2239a2cdcdd2SPaul Dagnelie error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db); 2240fa9e4066Sahrens if (error) { 22411c8564a7SMark Maybee if (error != ENOENT) 22421c8564a7SMark Maybee return (error); 22431c8564a7SMark Maybee if (hole) 22441c8564a7SMark Maybee return (0); 22451c8564a7SMark Maybee /* 22461c8564a7SMark Maybee * This can only happen when we are searching up 22471c8564a7SMark Maybee * the block tree for data. We don't really need to 22481c8564a7SMark Maybee * adjust the offset, as we will just end up looking 22491c8564a7SMark Maybee * at the pointer to this block in its parent, and its 22501c8564a7SMark Maybee * going to be unallocated, so we will skip over it. 22511c8564a7SMark Maybee */ 2252be6fd75aSMatthew Ahrens return (SET_ERROR(ESRCH)); 2253fa9e4066Sahrens } 225498572ac1Sahrens error = dbuf_read(db, NULL, DB_RF_CANFAIL | DB_RF_HAVESTRUCT); 225598572ac1Sahrens if (error) { 225698572ac1Sahrens dbuf_rele(db, FTAG); 225798572ac1Sahrens return (error); 225898572ac1Sahrens } 2259fa9e4066Sahrens data = db->db.db_data; 2260fa9e4066Sahrens } 2261fa9e4066Sahrens 226243466aaeSMax Grossman 226343466aaeSMax Grossman if (db != NULL && txg != 0 && (db->db_blkptr == NULL || 226443466aaeSMax Grossman db->db_blkptr->blk_birth <= txg || 226543466aaeSMax Grossman BP_IS_HOLE(db->db_blkptr))) { 22661c8564a7SMark Maybee /* 22671c8564a7SMark Maybee * This can only happen when we are searching up the tree 22681c8564a7SMark Maybee * and these conditions mean that we need to keep climbing. 22691c8564a7SMark Maybee */ 2270be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 22716754306eSahrens } else if (lvl == 0) { 2272fa9e4066Sahrens dnode_phys_t *dnp = data; 227354811da5SToomas Soome 2274fa9e4066Sahrens ASSERT(dn->dn_type == DMU_OT_DNODE); 227554811da5SToomas Soome ASSERT(!(flags & DNODE_FIND_BACKWARDS)); 2276fa9e4066Sahrens 227754811da5SToomas Soome for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1); 227854811da5SToomas Soome i < blkfill; i += dnp[i].dn_extra_slots + 1) { 227908f3f137SJonathan W Adams if ((dnp[i].dn_type == DMU_OT_NONE) == hole) 2280fa9e4066Sahrens break; 2281fa9e4066Sahrens } 228254811da5SToomas Soome 228354811da5SToomas Soome if (i == blkfill) 2284be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 228554811da5SToomas Soome 228654811da5SToomas Soome *offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) + 228754811da5SToomas Soome (i << DNODE_SHIFT); 2288fa9e4066Sahrens } else { 2289fa9e4066Sahrens blkptr_t *bp = data; 229076256205SMark Maybee uint64_t start = *offset; 2291fa9e4066Sahrens span = (lvl - 1) * epbs + dn->dn_datablkshift; 2292fa9e4066Sahrens minfill = 0; 2293fa9e4066Sahrens maxfill = blkfill << ((lvl - 1) * epbs); 2294fa9e4066Sahrens 2295fa9e4066Sahrens if (hole) 2296fa9e4066Sahrens maxfill--; 2297fa9e4066Sahrens else 2298fa9e4066Sahrens minfill++; 2299fa9e4066Sahrens 230076256205SMark Maybee *offset = *offset >> span; 230176256205SMark Maybee for (i = BF64_GET(*offset, 0, epbs); 2302cdb0ab79Smaybee i >= 0 && i < epb; i += inc) { 23035d7b4d43SMatthew Ahrens if (BP_GET_FILL(&bp[i]) >= minfill && 23045d7b4d43SMatthew Ahrens BP_GET_FILL(&bp[i]) <= maxfill && 23051c8564a7SMark Maybee (hole || bp[i].blk_birth > txg)) 2306fa9e4066Sahrens break; 230776256205SMark Maybee if (inc > 0 || *offset > 0) 230876256205SMark Maybee *offset += inc; 230976256205SMark Maybee } 231076256205SMark Maybee *offset = *offset << span; 231176256205SMark Maybee if (inc < 0) { 231276256205SMark Maybee /* traversing backwards; position offset at the end */ 231376256205SMark Maybee ASSERT3U(*offset, <=, start); 231476256205SMark Maybee *offset = MIN(*offset + (1ULL << span) - 1, start); 231576256205SMark Maybee } else if (*offset < start) { 231676256205SMark Maybee *offset = start; 2317fa9e4066Sahrens } 231876256205SMark Maybee if (i < 0 || i >= epb) 2319be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2320fa9e4066Sahrens } 2321fa9e4066Sahrens 2322fa9e4066Sahrens if (db) 2323ea8dc4b6Seschrock dbuf_rele(db, FTAG); 2324fa9e4066Sahrens 2325fa9e4066Sahrens return (error); 2326fa9e4066Sahrens } 2327fa9e4066Sahrens 2328fa9e4066Sahrens /* 2329fa9e4066Sahrens * Find the next hole, data, or sparse region at or after *offset. 2330fa9e4066Sahrens * The value 'blkfill' tells us how many items we expect to find 2331fa9e4066Sahrens * in an L0 data block; this value is 1 for normal objects, 2332fa9e4066Sahrens * DNODES_PER_BLOCK for the meta dnode, and some fraction of 2333fa9e4066Sahrens * DNODES_PER_BLOCK when searching for sparse regions thereof. 23346754306eSahrens * 2335fa9e4066Sahrens * Examples: 2336fa9e4066Sahrens * 2337cdb0ab79Smaybee * dnode_next_offset(dn, flags, offset, 1, 1, 0); 2338cdb0ab79Smaybee * Finds the next/previous hole/data in a file. 2339fa9e4066Sahrens * Used in dmu_offset_next(). 2340fa9e4066Sahrens * 2341cdb0ab79Smaybee * dnode_next_offset(mdn, flags, offset, 0, DNODES_PER_BLOCK, txg); 2342fa9e4066Sahrens * Finds the next free/allocated dnode an objset's meta-dnode. 23436754306eSahrens * Only finds objects that have new contents since txg (ie. 23446754306eSahrens * bonus buffer changes and content removal are ignored). 2345fa9e4066Sahrens * Used in dmu_object_next(). 2346fa9e4066Sahrens * 2347cdb0ab79Smaybee * dnode_next_offset(mdn, DNODE_FIND_HOLE, offset, 2, DNODES_PER_BLOCK >> 2, 0); 2348fa9e4066Sahrens * Finds the next L2 meta-dnode bp that's at most 1/4 full. 2349fa9e4066Sahrens * Used in dmu_object_alloc(). 2350fa9e4066Sahrens */ 2351fa9e4066Sahrens int 2352cdb0ab79Smaybee dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, 23536754306eSahrens int minlvl, uint64_t blkfill, uint64_t txg) 2354fa9e4066Sahrens { 2355cdb0ab79Smaybee uint64_t initial_offset = *offset; 2356fa9e4066Sahrens int lvl, maxlvl; 2357fa9e4066Sahrens int error = 0; 2358fa9e4066Sahrens 2359cdb0ab79Smaybee if (!(flags & DNODE_FIND_HAVELOCK)) 2360cdb0ab79Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2361fa9e4066Sahrens 2362fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) { 2363be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2364cdb0ab79Smaybee goto out; 2365fa9e4066Sahrens } 2366fa9e4066Sahrens 2367fa9e4066Sahrens if (dn->dn_datablkshift == 0) { 2368fa9e4066Sahrens if (*offset < dn->dn_datablksz) { 2369cdb0ab79Smaybee if (flags & DNODE_FIND_HOLE) 2370fa9e4066Sahrens *offset = dn->dn_datablksz; 2371fa9e4066Sahrens } else { 2372be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2373fa9e4066Sahrens } 2374cdb0ab79Smaybee goto out; 2375fa9e4066Sahrens } 2376fa9e4066Sahrens 2377fa9e4066Sahrens maxlvl = dn->dn_phys->dn_nlevels; 2378fa9e4066Sahrens 2379fa9e4066Sahrens for (lvl = minlvl; lvl <= maxlvl; lvl++) { 23806754306eSahrens error = dnode_next_offset_level(dn, 2381cdb0ab79Smaybee flags, offset, lvl, blkfill, txg); 238298572ac1Sahrens if (error != ESRCH) 2383fa9e4066Sahrens break; 2384fa9e4066Sahrens } 2385fa9e4066Sahrens 2386cdb0ab79Smaybee while (error == 0 && --lvl >= minlvl) { 23876754306eSahrens error = dnode_next_offset_level(dn, 2388cdb0ab79Smaybee flags, offset, lvl, blkfill, txg); 23896754306eSahrens } 2390fa9e4066Sahrens 23910fbc0cd0SMatthew Ahrens /* 23920fbc0cd0SMatthew Ahrens * There's always a "virtual hole" at the end of the object, even 23930fbc0cd0SMatthew Ahrens * if all BP's which physically exist are non-holes. 23940fbc0cd0SMatthew Ahrens */ 23950fbc0cd0SMatthew Ahrens if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 && 23960fbc0cd0SMatthew Ahrens minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) { 23970fbc0cd0SMatthew Ahrens error = 0; 23980fbc0cd0SMatthew Ahrens } 23990fbc0cd0SMatthew Ahrens 2400cdb0ab79Smaybee if (error == 0 && (flags & DNODE_FIND_BACKWARDS ? 2401cdb0ab79Smaybee initial_offset < *offset : initial_offset > *offset)) 2402be6fd75aSMatthew Ahrens error = SET_ERROR(ESRCH); 2403cdb0ab79Smaybee out: 2404cdb0ab79Smaybee if (!(flags & DNODE_FIND_HAVELOCK)) 2405cdb0ab79Smaybee rw_exit(&dn->dn_struct_rwlock); 2406fa9e4066Sahrens 2407fa9e4066Sahrens return (error); 2408fa9e4066Sahrens } 2409