1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 221ab7f2deSmaybee * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #include <sys/zfs_context.h> 27fa9e4066Sahrens #include <sys/dmu.h> 28fa9e4066Sahrens #include <sys/dmu_impl.h> 29fa9e4066Sahrens #include <sys/dbuf.h> 30fa9e4066Sahrens #include <sys/dmu_objset.h> 31fa9e4066Sahrens #include <sys/dsl_dataset.h> 32fa9e4066Sahrens #include <sys/dsl_dir.h> 33fa9e4066Sahrens #include <sys/dmu_tx.h> 34fa9e4066Sahrens #include <sys/spa.h> 35fa9e4066Sahrens #include <sys/zio.h> 36fa9e4066Sahrens #include <sys/dmu_zfetch.h> 37fa9e4066Sahrens 38fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 39fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 40088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 41c717a561Smaybee static arc_done_func_t dbuf_write_ready; 42fa9e4066Sahrens static arc_done_func_t dbuf_write_done; 43fa9e4066Sahrens 44fa9e4066Sahrens /* 45fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 46fa9e4066Sahrens */ 47fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 48fa9e4066Sahrens 49fa9e4066Sahrens /* ARGSUSED */ 50fa9e4066Sahrens static int 51fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 52fa9e4066Sahrens { 53fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 54fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 55fa9e4066Sahrens 56fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 57fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 58fa9e4066Sahrens refcount_create(&db->db_holds); 59fa9e4066Sahrens return (0); 60fa9e4066Sahrens } 61fa9e4066Sahrens 62fa9e4066Sahrens /* ARGSUSED */ 63fa9e4066Sahrens static void 64fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 65fa9e4066Sahrens { 66fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 67fa9e4066Sahrens mutex_destroy(&db->db_mtx); 68fa9e4066Sahrens cv_destroy(&db->db_changed); 69fa9e4066Sahrens refcount_destroy(&db->db_holds); 70fa9e4066Sahrens } 71fa9e4066Sahrens 72fa9e4066Sahrens /* 73fa9e4066Sahrens * dbuf hash table routines 74fa9e4066Sahrens */ 75fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 76fa9e4066Sahrens 77fa9e4066Sahrens static uint64_t dbuf_hash_count; 78fa9e4066Sahrens 79fa9e4066Sahrens static uint64_t 80fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 81fa9e4066Sahrens { 82fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 83fa9e4066Sahrens uint64_t crc = -1ULL; 84fa9e4066Sahrens 85fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 86fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 87fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 91fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 92fa9e4066Sahrens 93fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 94fa9e4066Sahrens 95fa9e4066Sahrens return (crc); 96fa9e4066Sahrens } 97fa9e4066Sahrens 98fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 99fa9e4066Sahrens 100fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 101fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 102fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 103fa9e4066Sahrens (dbuf)->db_level == (level) && \ 104fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 105fa9e4066Sahrens 106fa9e4066Sahrens dmu_buf_impl_t * 107fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 108fa9e4066Sahrens { 109fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 110fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 111fa9e4066Sahrens uint64_t obj = dn->dn_object; 112fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 113fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 114fa9e4066Sahrens dmu_buf_impl_t *db; 115fa9e4066Sahrens 116fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 117fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 118fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 119fa9e4066Sahrens mutex_enter(&db->db_mtx); 120ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 121fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 122fa9e4066Sahrens return (db); 123fa9e4066Sahrens } 124fa9e4066Sahrens mutex_exit(&db->db_mtx); 125fa9e4066Sahrens } 126fa9e4066Sahrens } 127fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 128fa9e4066Sahrens return (NULL); 129fa9e4066Sahrens } 130fa9e4066Sahrens 131fa9e4066Sahrens /* 132fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 133fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 134fa9e4066Sahrens * will be returned and the new element will not be inserted. 135fa9e4066Sahrens * Otherwise returns NULL. 136fa9e4066Sahrens */ 137fa9e4066Sahrens static dmu_buf_impl_t * 138fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 139fa9e4066Sahrens { 140fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 141fa9e4066Sahrens objset_impl_t *os = db->db_objset; 142fa9e4066Sahrens uint64_t obj = db->db.db_object; 143fa9e4066Sahrens int level = db->db_level; 144fa9e4066Sahrens uint64_t blkid = db->db_blkid; 145fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 146fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 147fa9e4066Sahrens dmu_buf_impl_t *dbf; 148fa9e4066Sahrens 149fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 150fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 151fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 152fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 153ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 154fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 155fa9e4066Sahrens return (dbf); 156fa9e4066Sahrens } 157fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 158fa9e4066Sahrens } 159fa9e4066Sahrens } 160fa9e4066Sahrens 161fa9e4066Sahrens mutex_enter(&db->db_mtx); 162fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 163fa9e4066Sahrens h->hash_table[idx] = db; 164fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 165fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 166fa9e4066Sahrens 167fa9e4066Sahrens return (NULL); 168fa9e4066Sahrens } 169fa9e4066Sahrens 170fa9e4066Sahrens /* 171fa9e4066Sahrens * Remove an entry from the hash table. This operation will 172fa9e4066Sahrens * fail if there are any existing holds on the db. 173fa9e4066Sahrens */ 174fa9e4066Sahrens static void 175fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 176fa9e4066Sahrens { 177fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 178fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 179fa9e4066Sahrens db->db_level, db->db_blkid); 180fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 181fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 182fa9e4066Sahrens 183fa9e4066Sahrens /* 184fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 185fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 186fa9e4066Sahrens */ 187fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 188ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 189fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 190fa9e4066Sahrens 191fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 192fa9e4066Sahrens dbp = &h->hash_table[idx]; 193fa9e4066Sahrens while ((dbf = *dbp) != db) { 194fa9e4066Sahrens dbp = &dbf->db_hash_next; 195fa9e4066Sahrens ASSERT(dbf != NULL); 196fa9e4066Sahrens } 197fa9e4066Sahrens *dbp = db->db_hash_next; 198fa9e4066Sahrens db->db_hash_next = NULL; 199fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 200fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 201fa9e4066Sahrens } 202fa9e4066Sahrens 203ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 204fa9e4066Sahrens 205fa9e4066Sahrens static void 206fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 207fa9e4066Sahrens { 208fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 209fa9e4066Sahrens 210c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 211fa9e4066Sahrens return; 212fa9e4066Sahrens 213c717a561Smaybee if (db->db_user_data_ptr_ptr) 214c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 215c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 216c717a561Smaybee db->db_user_ptr = NULL; 217c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 218c717a561Smaybee db->db_evict_func = NULL; 219fa9e4066Sahrens } 220fa9e4066Sahrens 221ea8dc4b6Seschrock void 222ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 223ea8dc4b6Seschrock { 224ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 225ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 226c717a561Smaybee ASSERT(db->db_data_pending == NULL); 227ea8dc4b6Seschrock 228ea8dc4b6Seschrock dbuf_clear(db); 229ea8dc4b6Seschrock dbuf_destroy(db); 230ea8dc4b6Seschrock } 231ea8dc4b6Seschrock 232fa9e4066Sahrens void 233fa9e4066Sahrens dbuf_init(void) 234fa9e4066Sahrens { 235ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 236fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 237fa9e4066Sahrens int i; 238fa9e4066Sahrens 239fa9e4066Sahrens /* 240fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 241ea8dc4b6Seschrock * with an average 4K block size. The table will take up 242ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 243fa9e4066Sahrens */ 244ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 245fa9e4066Sahrens hsize <<= 1; 246fa9e4066Sahrens 247ea8dc4b6Seschrock retry: 248fa9e4066Sahrens h->hash_table_mask = hsize - 1; 249ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 250ea8dc4b6Seschrock if (h->hash_table == NULL) { 251ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 252ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 253ea8dc4b6Seschrock hsize >>= 1; 254ea8dc4b6Seschrock goto retry; 255ea8dc4b6Seschrock } 256fa9e4066Sahrens 257fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 258fa9e4066Sahrens sizeof (dmu_buf_impl_t), 259fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 260fa9e4066Sahrens 261fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 262fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 263fa9e4066Sahrens } 264fa9e4066Sahrens 265fa9e4066Sahrens void 266fa9e4066Sahrens dbuf_fini(void) 267fa9e4066Sahrens { 268fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 269fa9e4066Sahrens int i; 270fa9e4066Sahrens 271fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 272fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 273fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 274fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 275fa9e4066Sahrens } 276fa9e4066Sahrens 277fa9e4066Sahrens /* 278fa9e4066Sahrens * Other stuff. 279fa9e4066Sahrens */ 280fa9e4066Sahrens 2819c9dc39aSek #ifdef ZFS_DEBUG 282fa9e4066Sahrens static void 283fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 284fa9e4066Sahrens { 285fa9e4066Sahrens dnode_t *dn = db->db_dnode; 286fa9e4066Sahrens 287fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 288fa9e4066Sahrens 289fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 290fa9e4066Sahrens return; 291fa9e4066Sahrens 292fa9e4066Sahrens ASSERT(db->db_objset != NULL); 293fa9e4066Sahrens if (dn == NULL) { 294fa9e4066Sahrens ASSERT(db->db_parent == NULL); 295fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 296fa9e4066Sahrens } else { 297fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 298fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 299fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 300ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID || 301ea8dc4b6Seschrock list_head(&dn->dn_dbufs)); 302fa9e4066Sahrens } 303fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 304fa9e4066Sahrens ASSERT(dn != NULL); 3051934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 306fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 307fa9e4066Sahrens } else { 308fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 309fa9e4066Sahrens } 310fa9e4066Sahrens 311fa9e4066Sahrens if (db->db_level == 0) { 312fa9e4066Sahrens /* we can be momentarily larger in dnode_set_blksz() */ 313fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID && dn) { 314fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 315fa9e4066Sahrens } 316ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) { 317c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 318c717a561Smaybee /* 319c717a561Smaybee * it should only be modified in syncing 320c717a561Smaybee * context, so make sure we only have 321c717a561Smaybee * one copy of the data. 322c717a561Smaybee */ 323c717a561Smaybee ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 324fa9e4066Sahrens } 325fa9e4066Sahrens } 326fa9e4066Sahrens 327fa9e4066Sahrens /* verify db->db_blkptr */ 328fa9e4066Sahrens if (db->db_blkptr) { 329fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 330fa9e4066Sahrens /* db is pointed to by the dnode */ 331fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 332ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) 333fa9e4066Sahrens ASSERT(db->db_parent == NULL); 334fa9e4066Sahrens else 335fa9e4066Sahrens ASSERT(db->db_parent != NULL); 336fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 337fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 338fa9e4066Sahrens } else { 339fa9e4066Sahrens /* db is pointed to by an indirect block */ 340fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 341fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 342fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 343fa9e4066Sahrens db->db.db_object); 344fa9e4066Sahrens /* 345fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 346fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 347fa9e4066Sahrens * grows. safe to do this now? 348fa9e4066Sahrens */ 349fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 350fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 351fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 352fa9e4066Sahrens db->db_blkid % epb)); 353fa9e4066Sahrens } 354fa9e4066Sahrens } 355fa9e4066Sahrens } 356fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 357fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 358fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 359fa9e4066Sahrens /* 360fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 361fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 362fa9e4066Sahrens * data when we evict this buffer. 363fa9e4066Sahrens */ 364fa9e4066Sahrens if (db->db_dirtycnt == 0) { 365fa9e4066Sahrens uint64_t *buf = db->db.db_data; 366fa9e4066Sahrens int i; 367fa9e4066Sahrens 368fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 369fa9e4066Sahrens ASSERT(buf[i] == 0); 370fa9e4066Sahrens } 371fa9e4066Sahrens } 372fa9e4066Sahrens } 373fa9e4066Sahrens } 3749c9dc39aSek #endif 375fa9e4066Sahrens 376fa9e4066Sahrens static void 377fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 378fa9e4066Sahrens { 379fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 380c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 381fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 382c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 383fa9e4066Sahrens } 384fa9e4066Sahrens } 385fa9e4066Sahrens 386fa9e4066Sahrens static void 387fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 388fa9e4066Sahrens { 389fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 390ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 391fa9e4066Sahrens db->db_buf = buf; 392ea8dc4b6Seschrock if (buf != NULL) { 393ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 394ea8dc4b6Seschrock db->db.db_data = buf->b_data; 395ea8dc4b6Seschrock if (!arc_released(buf)) 396ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 397ea8dc4b6Seschrock dbuf_update_data(db); 398ea8dc4b6Seschrock } else { 399ea8dc4b6Seschrock dbuf_evict_user(db); 400ea8dc4b6Seschrock db->db.db_data = NULL; 401ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 402ea8dc4b6Seschrock } 403fa9e4066Sahrens } 404fa9e4066Sahrens 405fa9e4066Sahrens uint64_t 406fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 407fa9e4066Sahrens { 408fa9e4066Sahrens if (dn->dn_datablkshift) { 409fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 410fa9e4066Sahrens } else { 411fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 412fa9e4066Sahrens return (0); 413fa9e4066Sahrens } 414fa9e4066Sahrens } 415fa9e4066Sahrens 416fa9e4066Sahrens static void 417fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 418fa9e4066Sahrens { 419fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 420fa9e4066Sahrens 421fa9e4066Sahrens mutex_enter(&db->db_mtx); 422fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 423fa9e4066Sahrens /* 424fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 425fa9e4066Sahrens */ 426fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 427ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 428fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 429c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 430fa9e4066Sahrens /* we were freed in flight; disregard any error */ 431fa9e4066Sahrens arc_release(buf, db); 432fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4336b4acc8bSahrens arc_buf_freeze(buf); 434c717a561Smaybee db->db_freed_in_flight = FALSE; 435fa9e4066Sahrens dbuf_set_data(db, buf); 436fa9e4066Sahrens db->db_state = DB_CACHED; 437fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 438fa9e4066Sahrens dbuf_set_data(db, buf); 439fa9e4066Sahrens db->db_state = DB_CACHED; 440fa9e4066Sahrens } else { 441fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 442fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 443ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 444ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 445fa9e4066Sahrens } 446fa9e4066Sahrens cv_broadcast(&db->db_changed); 447fa9e4066Sahrens mutex_exit(&db->db_mtx); 448ea8dc4b6Seschrock dbuf_rele(db, NULL); 449fa9e4066Sahrens } 450fa9e4066Sahrens 451ea8dc4b6Seschrock static void 45213506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 453fa9e4066Sahrens { 454088f3894Sahrens dnode_t *dn = db->db_dnode; 455ea8dc4b6Seschrock zbookmark_t zb; 45613506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 457088f3894Sahrens arc_buf_t *pbuf; 458fa9e4066Sahrens 459fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 460fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 461088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 462ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 463ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 464ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 465fa9e4066Sahrens 466fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 467088f3894Sahrens int bonuslen = dn->dn_bonuslen; 4681934e92fSmaybee 4691934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 470ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 47125c2057bSmaybee arc_space_consume(DN_MAX_BONUSLEN); 4721934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 473ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 474088f3894Sahrens bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, 4751934e92fSmaybee bonuslen); 476ea8dc4b6Seschrock dbuf_update_data(db); 477fa9e4066Sahrens db->db_state = DB_CACHED; 478fa9e4066Sahrens mutex_exit(&db->db_mtx); 479fa9e4066Sahrens return; 480fa9e4066Sahrens } 481fa9e4066Sahrens 482088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 483088f3894Sahrens (db->db_level == 0 && dnode_block_freed(dn, db->db_blkid))) { 484ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 485ad23a2dbSjohansen 486088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 487ad23a2dbSjohansen db->db.db_size, db, type)); 488fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 489fa9e4066Sahrens db->db_state = DB_CACHED; 49013506d1eSmaybee *flags |= DB_RF_CACHED; 491fa9e4066Sahrens mutex_exit(&db->db_mtx); 492fa9e4066Sahrens return; 493fa9e4066Sahrens } 494fa9e4066Sahrens 495fa9e4066Sahrens db->db_state = DB_READ; 496fa9e4066Sahrens mutex_exit(&db->db_mtx); 497fa9e4066Sahrens 4983baa08fcSek if (DBUF_IS_L2CACHEABLE(db)) 4993baa08fcSek aflags |= ARC_L2CACHE; 5003baa08fcSek 501ea8dc4b6Seschrock zb.zb_objset = db->db_objset->os_dsl_dataset ? 502ea8dc4b6Seschrock db->db_objset->os_dsl_dataset->ds_object : 0; 503ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 504ea8dc4b6Seschrock zb.zb_level = db->db_level; 505ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 506ea8dc4b6Seschrock 507ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 508fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 509088f3894Sahrens 510088f3894Sahrens if (db->db_parent) 511088f3894Sahrens pbuf = db->db_parent->db_buf; 512088f3894Sahrens else 513088f3894Sahrens pbuf = db->db_objset->os_phys_buf; 514088f3894Sahrens 515088f3894Sahrens (void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf, 516fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 51713506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 51813506d1eSmaybee &aflags, &zb); 51913506d1eSmaybee if (aflags & ARC_CACHED) 52013506d1eSmaybee *flags |= DB_RF_CACHED; 521fa9e4066Sahrens } 522fa9e4066Sahrens 523ea8dc4b6Seschrock int 524ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 525fa9e4066Sahrens { 526ea8dc4b6Seschrock int err = 0; 527ea8dc4b6Seschrock int havepzio = (zio != NULL); 52813506d1eSmaybee int prefetch; 529fa9e4066Sahrens 530fa9e4066Sahrens /* 531fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 532fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 533fa9e4066Sahrens */ 534fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 535fa9e4066Sahrens 536ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 537ea8dc4b6Seschrock rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 538ea8dc4b6Seschrock 53913506d1eSmaybee prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 5403baa08fcSek (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL && 5413baa08fcSek DBUF_IS_CACHEABLE(db); 54213506d1eSmaybee 543ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 544ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 545ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 54613506d1eSmaybee if (prefetch) 54713506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 54813506d1eSmaybee db->db.db_size, TRUE); 549fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 550ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 551ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 552ea8dc4b6Seschrock if (zio == NULL) { 553ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 554ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 555ea8dc4b6Seschrock } 55613506d1eSmaybee dbuf_read_impl(db, zio, &flags); 55713506d1eSmaybee 558ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 559ea8dc4b6Seschrock 56013506d1eSmaybee if (prefetch) 561ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 56213506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 563ea8dc4b6Seschrock 564fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 565fa9e4066Sahrens rw_exit(&db->db_dnode->dn_struct_rwlock); 566fa9e4066Sahrens 567ea8dc4b6Seschrock if (!havepzio) 568ea8dc4b6Seschrock err = zio_wait(zio); 569ea8dc4b6Seschrock } else { 57013506d1eSmaybee mutex_exit(&db->db_mtx); 57113506d1eSmaybee if (prefetch) 57213506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 57313506d1eSmaybee db->db.db_size, TRUE); 574ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 575ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 57613506d1eSmaybee 57713506d1eSmaybee mutex_enter(&db->db_mtx); 578ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 579ea8dc4b6Seschrock while (db->db_state == DB_READ || 580ea8dc4b6Seschrock db->db_state == DB_FILL) { 581ea8dc4b6Seschrock ASSERT(db->db_state == DB_READ || 582ea8dc4b6Seschrock (flags & DB_RF_HAVESTRUCT) == 0); 583ea8dc4b6Seschrock cv_wait(&db->db_changed, &db->db_mtx); 584ea8dc4b6Seschrock } 585ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 586ea8dc4b6Seschrock err = EIO; 587ea8dc4b6Seschrock } 588ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 589fa9e4066Sahrens } 590fa9e4066Sahrens 591ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 592ea8dc4b6Seschrock return (err); 593fa9e4066Sahrens } 594fa9e4066Sahrens 595fa9e4066Sahrens static void 596fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 597fa9e4066Sahrens { 598fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 599ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 600fa9e4066Sahrens mutex_enter(&db->db_mtx); 601fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 602fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 603fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 604ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 605ad23a2dbSjohansen 606ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 607fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 608fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 609ad23a2dbSjohansen db->db.db_size, db, type)); 610fa9e4066Sahrens db->db_state = DB_FILL; 611fa9e4066Sahrens } else { 612fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 613fa9e4066Sahrens } 614fa9e4066Sahrens mutex_exit(&db->db_mtx); 615fa9e4066Sahrens } 616fa9e4066Sahrens 617fa9e4066Sahrens /* 618fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 619fa9e4066Sahrens * buffers, that have been modified in a previous transaction 620fa9e4066Sahrens * group, before we modify them in the current active group. 621fa9e4066Sahrens * 622fa9e4066Sahrens * This function is used in two places: when we are dirtying a 623fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 624fa9e4066Sahrens * a range in a dnode that includes this buffer. 625fa9e4066Sahrens * 626fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 627fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 628fa9e4066Sahrens * dbuf list for the dnode. 629fa9e4066Sahrens */ 630fa9e4066Sahrens static void 631fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 632fa9e4066Sahrens { 633c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 634fa9e4066Sahrens 635fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 636fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 637c717a561Smaybee ASSERT(db->db_level == 0); 638c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 639fa9e4066Sahrens 6404d31c452Smaybee if (dr == NULL || 6414d31c452Smaybee (dr->dt.dl.dr_data != 6424d31c452Smaybee ((db->db_blkid == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 643fa9e4066Sahrens return; 644fa9e4066Sahrens 645fa9e4066Sahrens /* 646c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 647c717a561Smaybee * and its referencing the dbuf data, either: 648c717a561Smaybee * reset the reference to point to a new copy, 649c717a561Smaybee * or (if there a no active holders) 650c717a561Smaybee * just null out the current db_data pointer. 651fa9e4066Sahrens */ 652c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 653c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 654c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 655c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 6560e8c6158Smaybee arc_space_consume(DN_MAX_BONUSLEN); 657c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 658c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 659c717a561Smaybee int size = db->db.db_size; 660c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 661c717a561Smaybee dr->dt.dl.dr_data = arc_buf_alloc( 662c717a561Smaybee db->db_dnode->dn_objset->os_spa, size, db, type); 663c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 664c717a561Smaybee } else { 665c717a561Smaybee dbuf_set_data(db, NULL); 666fa9e4066Sahrens } 667fa9e4066Sahrens } 668fa9e4066Sahrens 669c717a561Smaybee void 670c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 671ea8dc4b6Seschrock { 672c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 673c717a561Smaybee uint64_t txg = dr->dr_txg; 674ea8dc4b6Seschrock 675ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 676c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 677c717a561Smaybee ASSERT(db->db_level == 0); 678ea8dc4b6Seschrock 679c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID || 680c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 681c717a561Smaybee return; 682ea8dc4b6Seschrock 683c717a561Smaybee /* free this block */ 684c717a561Smaybee if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) { 685c717a561Smaybee /* XXX can get silent EIO here */ 686088f3894Sahrens (void) dsl_free(NULL, 687088f3894Sahrens spa_get_dsl(db->db_dnode->dn_objset->os_spa), 688c717a561Smaybee txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT); 689fa9e4066Sahrens } 690c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 691c717a561Smaybee /* 692c717a561Smaybee * Release the already-written buffer, so we leave it in 693c717a561Smaybee * a consistent dirty state. Note that all callers are 694c717a561Smaybee * modifying the buffer, so they will immediately do 695c717a561Smaybee * another (redundant) arc_release(). Therefore, leave 696c717a561Smaybee * the buf thawed to save the effort of freezing & 697c717a561Smaybee * immediately re-thawing it. 698c717a561Smaybee */ 699c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 700fa9e4066Sahrens } 701fa9e4066Sahrens 702cdb0ab79Smaybee /* 703cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 704cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 705cdb0ab79Smaybee * empty blocks. Also, if we happen accross any level-1 dbufs in the 706cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 707cdb0ab79Smaybee * they stay in memory. 708cdb0ab79Smaybee */ 709fa9e4066Sahrens void 710cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 711fa9e4066Sahrens { 712fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 713fa9e4066Sahrens uint64_t txg = tx->tx_txg; 714cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 715cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 716cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 717fa9e4066Sahrens 718cdb0ab79Smaybee if (end > dn->dn_maxblkid) { 719cdb0ab79Smaybee end = dn->dn_maxblkid; 720cdb0ab79Smaybee last_l1 = end >> epbs; 721cdb0ab79Smaybee } 722cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 723fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 724fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 725fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 726ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 727cdb0ab79Smaybee 728cdb0ab79Smaybee if (db->db_level == 1 && 729cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 730cdb0ab79Smaybee mutex_enter(&db->db_mtx); 731cdb0ab79Smaybee if (db->db_last_dirty && 732cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 733cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 734cdb0ab79Smaybee mutex_exit(&db->db_mtx); 735cdb0ab79Smaybee dbuf_will_dirty(db, tx); 736cdb0ab79Smaybee dbuf_rele(db, FTAG); 737cdb0ab79Smaybee } else { 738cdb0ab79Smaybee mutex_exit(&db->db_mtx); 739cdb0ab79Smaybee } 740cdb0ab79Smaybee } 741cdb0ab79Smaybee 742ea8dc4b6Seschrock if (db->db_level != 0) 743fa9e4066Sahrens continue; 744fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 745cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 746fa9e4066Sahrens continue; 747fa9e4066Sahrens 748fa9e4066Sahrens /* found a level 0 buffer in the range */ 749fa9e4066Sahrens if (dbuf_undirty(db, tx)) 750fa9e4066Sahrens continue; 751fa9e4066Sahrens 752fa9e4066Sahrens mutex_enter(&db->db_mtx); 753ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 754ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 755fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 756fa9e4066Sahrens mutex_exit(&db->db_mtx); 757fa9e4066Sahrens continue; 758fa9e4066Sahrens } 759c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 760c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 761c717a561Smaybee db->db_freed_in_flight = TRUE; 762fa9e4066Sahrens mutex_exit(&db->db_mtx); 763fa9e4066Sahrens continue; 764fa9e4066Sahrens } 765ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 766ea8dc4b6Seschrock ASSERT(db->db_buf); 767ea8dc4b6Seschrock dbuf_clear(db); 768ea8dc4b6Seschrock continue; 769ea8dc4b6Seschrock } 770c717a561Smaybee /* The dbuf is referenced */ 771fa9e4066Sahrens 772c717a561Smaybee if (db->db_last_dirty != NULL) { 773c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 774c717a561Smaybee 775c717a561Smaybee if (dr->dr_txg == txg) { 77644eda4d7Smaybee /* 777c717a561Smaybee * This buffer is "in-use", re-adjust the file 778c717a561Smaybee * size to reflect that this buffer may 779c717a561Smaybee * contain new data when we sync. 78044eda4d7Smaybee */ 781c717a561Smaybee if (db->db_blkid > dn->dn_maxblkid) 782c717a561Smaybee dn->dn_maxblkid = db->db_blkid; 783c717a561Smaybee dbuf_unoverride(dr); 784c717a561Smaybee } else { 785c717a561Smaybee /* 786c717a561Smaybee * This dbuf is not dirty in the open context. 787c717a561Smaybee * Either uncache it (if its not referenced in 788c717a561Smaybee * the open context) or reset its contents to 789c717a561Smaybee * empty. 790c717a561Smaybee */ 791c717a561Smaybee dbuf_fix_old_data(db, txg); 79244eda4d7Smaybee } 793ea8dc4b6Seschrock } 794c717a561Smaybee /* clear the contents if its cached */ 795ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 796ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 797fa9e4066Sahrens arc_release(db->db_buf, db); 798fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 7996b4acc8bSahrens arc_buf_freeze(db->db_buf); 800fa9e4066Sahrens } 801ea8dc4b6Seschrock 802fa9e4066Sahrens mutex_exit(&db->db_mtx); 803fa9e4066Sahrens } 804fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 805fa9e4066Sahrens } 806fa9e4066Sahrens 807fa9e4066Sahrens static int 8081934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 809fa9e4066Sahrens { 810fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 811fa9e4066Sahrens uint64_t birth_txg = 0; 812fa9e4066Sahrens 813fa9e4066Sahrens /* 814fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 815c717a561Smaybee * If it's syncing, then db_last_dirty will be set 816c717a561Smaybee * so we'll ignore db_blkptr. 817fa9e4066Sahrens */ 818c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 819c717a561Smaybee if (db->db_last_dirty) 820c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 821fa9e4066Sahrens else if (db->db_blkptr) 822fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 823fa9e4066Sahrens 8241934e92fSmaybee /* If we don't exist or are in a snapshot, we can't be freed */ 825fa9e4066Sahrens if (birth_txg) 8261934e92fSmaybee return (ds == NULL || 8271934e92fSmaybee dsl_dataset_block_freeable(ds, birth_txg)); 828fa9e4066Sahrens else 8291934e92fSmaybee return (FALSE); 830fa9e4066Sahrens } 831fa9e4066Sahrens 832fa9e4066Sahrens void 833fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 834fa9e4066Sahrens { 835fa9e4066Sahrens arc_buf_t *buf, *obuf; 836fa9e4066Sahrens int osize = db->db.db_size; 837ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 838fa9e4066Sahrens 839ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 840ea8dc4b6Seschrock 841fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 842fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 843fa9e4066Sahrens 844fa9e4066Sahrens /* 845fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 846fa9e4066Sahrens * is OK, because there can be no other references to the db 847fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 848fa9e4066Sahrens * be happening. 849fa9e4066Sahrens */ 850ea8dc4b6Seschrock /* 851ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 852ea8dc4b6Seschrock * value and returning that up to our callers 853ea8dc4b6Seschrock */ 854fa9e4066Sahrens dbuf_will_dirty(db, tx); 855fa9e4066Sahrens 856fa9e4066Sahrens /* create the data buffer for the new block */ 857ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 858fa9e4066Sahrens 859fa9e4066Sahrens /* copy old block data to the new block */ 860fa9e4066Sahrens obuf = db->db_buf; 861f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 862fa9e4066Sahrens /* zero the remainder */ 863f65e61c0Sahrens if (size > osize) 864f65e61c0Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 865fa9e4066Sahrens 866fa9e4066Sahrens mutex_enter(&db->db_mtx); 867fa9e4066Sahrens dbuf_set_data(db, buf); 868ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 869fa9e4066Sahrens db->db.db_size = size; 870fa9e4066Sahrens 871c717a561Smaybee if (db->db_level == 0) { 872c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 873c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 874c717a561Smaybee } 875fa9e4066Sahrens mutex_exit(&db->db_mtx); 876fa9e4066Sahrens 877fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 878fa9e4066Sahrens } 879fa9e4066Sahrens 880c717a561Smaybee dbuf_dirty_record_t * 881fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 882fa9e4066Sahrens { 883fa9e4066Sahrens dnode_t *dn = db->db_dnode; 884fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 885c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 886fa9e4066Sahrens int drop_struct_lock = FALSE; 887fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 888fa9e4066Sahrens 889fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 890fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 8919c9dc39aSek DMU_TX_DIRTY_BUF(tx, db); 892fa9e4066Sahrens 893fa9e4066Sahrens /* 894fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 895fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 896fa9e4066Sahrens * were already pre-dirtied in open context. 897fa9e4066Sahrens * XXX We may want to prohibit dirtying in syncing context even 898fa9e4066Sahrens * if they did pre-dirty. 899fa9e4066Sahrens */ 900c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 901c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 902c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT || 903c717a561Smaybee dn->dn_objset->os_dsl_dataset == NULL || 904c717a561Smaybee dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir)); 905fa9e4066Sahrens 906fa9e4066Sahrens /* 907fa9e4066Sahrens * We make this assert for private objects as well, but after we 908fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 909fa9e4066Sahrens * in syncing context. 910fa9e4066Sahrens */ 911ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 912c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 913fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 914fa9e4066Sahrens 915fa9e4066Sahrens mutex_enter(&db->db_mtx); 916fa9e4066Sahrens /* 917c717a561Smaybee * XXX make this true for indirects too? The problem is that 918c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 919c717a561Smaybee * syncing context don't bother holding ahead. 920fa9e4066Sahrens */ 921c717a561Smaybee ASSERT(db->db_level != 0 || 922c717a561Smaybee db->db_state == DB_CACHED || db->db_state == DB_FILL); 923fa9e4066Sahrens 924fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 925fa9e4066Sahrens /* 926fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 927fa9e4066Sahrens * initialize the objset. 928fa9e4066Sahrens */ 929fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 930c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 931fa9e4066Sahrens dn->dn_dirtyctx = 932fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 933fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 934fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 935fa9e4066Sahrens } 936fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 937fa9e4066Sahrens 938fa9e4066Sahrens /* 939fa9e4066Sahrens * If this buffer is already dirty, we're done. 940fa9e4066Sahrens */ 941c717a561Smaybee drp = &db->db_last_dirty; 942c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 943c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 9447e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 9457e2186e3Sbonwick drp = &dr->dr_next; 9467e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 947c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 948c717a561Smaybee /* 949c717a561Smaybee * If this buffer has already been written out, 950c717a561Smaybee * we now need to reset its state. 951c717a561Smaybee */ 9527e2186e3Sbonwick dbuf_unoverride(dr); 953c717a561Smaybee if (db->db.db_object != DMU_META_DNODE_OBJECT) 954c717a561Smaybee arc_buf_thaw(db->db_buf); 955c717a561Smaybee } 956fa9e4066Sahrens mutex_exit(&db->db_mtx); 9577e2186e3Sbonwick return (dr); 958fa9e4066Sahrens } 959fa9e4066Sahrens 960fa9e4066Sahrens /* 961fa9e4066Sahrens * Only valid if not already dirty. 962fa9e4066Sahrens */ 963fa9e4066Sahrens ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 964fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 965fa9e4066Sahrens 966fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 967fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 968fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 969fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 970fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 971fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 972fa9e4066Sahrens 973fa9e4066Sahrens /* 974fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 975fa9e4066Sahrens * mos, a spa os, or we're initializing the os. However, we are 976fa9e4066Sahrens * allowed to dirty in syncing context provided we already 977fa9e4066Sahrens * dirtied it in open context. Hence we must make this 978fa9e4066Sahrens * assertion only if we're not already dirty. 979fa9e4066Sahrens */ 980fa9e4066Sahrens ASSERT(!dmu_tx_is_syncing(tx) || 981fa9e4066Sahrens os->os_dsl_dataset == NULL || 982fa9e4066Sahrens !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 983c717a561Smaybee !BP_IS_HOLE(os->os_rootbp)); 984fa9e4066Sahrens ASSERT(db->db.db_size != 0); 985fa9e4066Sahrens 986fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 987fa9e4066Sahrens 9881934e92fSmaybee if (db->db_blkid != DB_BONUS_BLKID) { 9891934e92fSmaybee /* 9901934e92fSmaybee * Update the accounting. 9911934e92fSmaybee */ 9921934e92fSmaybee if (dbuf_block_freeable(db)) { 9931934e92fSmaybee blkptr_t *bp = db->db_blkptr; 9941934e92fSmaybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 9951934e92fSmaybee bp_get_dasize(os->os_spa, bp) : db->db.db_size; 9961934e92fSmaybee /* 9971934e92fSmaybee * This is only a guess -- if the dbuf is dirty 9981934e92fSmaybee * in a previous txg, we don't know how much 9991934e92fSmaybee * space it will use on disk yet. We should 10001934e92fSmaybee * really have the struct_rwlock to access 10011934e92fSmaybee * db_blkptr, but since this is just a guess, 10021934e92fSmaybee * it's OK if we get an odd answer. 10031934e92fSmaybee */ 10041934e92fSmaybee dnode_willuse_space(dn, -willfree, tx); 10051934e92fSmaybee } 10061934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 10071934e92fSmaybee } 10081934e92fSmaybee 1009ea8dc4b6Seschrock /* 1010ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1011ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1012ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1013ea8dc4b6Seschrock */ 1014c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1015c717a561Smaybee if (db->db_level == 0) { 1016c717a561Smaybee void *data_old = db->db_buf; 1017c717a561Smaybee 1018c717a561Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 1019c717a561Smaybee dbuf_fix_old_data(db, tx->tx_txg); 1020c717a561Smaybee data_old = db->db.db_data; 1021c717a561Smaybee } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1022c717a561Smaybee /* 1023c717a561Smaybee * Release the data buffer from the cache so that we 1024c717a561Smaybee * can modify it without impacting possible other users 1025c717a561Smaybee * of this cached data block. Note that indirect 1026c717a561Smaybee * blocks and private objects are not released until the 1027c717a561Smaybee * syncing state (since they are only modified then). 1028c717a561Smaybee */ 1029fa9e4066Sahrens arc_release(db->db_buf, db); 1030fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1031c717a561Smaybee data_old = db->db_buf; 1032fa9e4066Sahrens } 1033c717a561Smaybee ASSERT(data_old != NULL); 1034c717a561Smaybee dr->dt.dl.dr_data = data_old; 1035c717a561Smaybee } else { 1036c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1037c717a561Smaybee list_create(&dr->dt.di.dr_children, 1038c717a561Smaybee sizeof (dbuf_dirty_record_t), 1039c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1040fa9e4066Sahrens } 1041c717a561Smaybee dr->dr_dbuf = db; 1042c717a561Smaybee dr->dr_txg = tx->tx_txg; 1043c717a561Smaybee dr->dr_next = *drp; 1044c717a561Smaybee *drp = dr; 1045fa9e4066Sahrens 1046fa9e4066Sahrens /* 1047fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1048fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1049fa9e4066Sahrens * happened after the free. 1050fa9e4066Sahrens */ 1051fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1052c717a561Smaybee mutex_enter(&dn->dn_mtx); 1053fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1054c717a561Smaybee mutex_exit(&dn->dn_mtx); 1055c717a561Smaybee db->db_freed_in_flight = FALSE; 1056fa9e4066Sahrens } 1057fa9e4066Sahrens 1058fa9e4066Sahrens /* 1059fa9e4066Sahrens * This buffer is now part of this txg 1060fa9e4066Sahrens */ 1061fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1062fa9e4066Sahrens db->db_dirtycnt += 1; 1063fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1064fa9e4066Sahrens 1065fa9e4066Sahrens mutex_exit(&db->db_mtx); 1066fa9e4066Sahrens 1067fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1068c717a561Smaybee mutex_enter(&dn->dn_mtx); 1069c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1070c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1071c717a561Smaybee mutex_exit(&dn->dn_mtx); 1072fa9e4066Sahrens dnode_setdirty(dn, tx); 1073c717a561Smaybee return (dr); 1074fa9e4066Sahrens } 1075fa9e4066Sahrens 1076fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1077fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1078fa9e4066Sahrens drop_struct_lock = TRUE; 1079fa9e4066Sahrens } 1080fa9e4066Sahrens 1081*8346f03fSJonathan W Adams if (db->db_level == 0) { 1082*8346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 1083*8346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 1084*8346f03fSJonathan W Adams } 1085*8346f03fSJonathan W Adams 108644eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1087c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1088c717a561Smaybee dbuf_dirty_record_t *di; 1089c717a561Smaybee int parent_held = FALSE; 1090c717a561Smaybee 1091c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1092c717a561Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1093c717a561Smaybee 1094c717a561Smaybee parent = dbuf_hold_level(dn, db->db_level+1, 1095c717a561Smaybee db->db_blkid >> epbs, FTAG); 1096c717a561Smaybee parent_held = TRUE; 1097c717a561Smaybee } 1098fa9e4066Sahrens if (drop_struct_lock) 1099fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1100c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1101c717a561Smaybee di = dbuf_dirty(parent, tx); 1102c717a561Smaybee if (parent_held) 1103c717a561Smaybee dbuf_rele(parent, FTAG); 1104c717a561Smaybee 1105c717a561Smaybee mutex_enter(&db->db_mtx); 1106c717a561Smaybee /* possible race with dbuf_undirty() */ 1107c717a561Smaybee if (db->db_last_dirty == dr || 1108c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1109c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1110c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1111c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1112c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1113c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1114c717a561Smaybee dr->dr_parent = di; 1115c717a561Smaybee } 1116c717a561Smaybee mutex_exit(&db->db_mtx); 1117fa9e4066Sahrens } else { 1118c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1119c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1120c717a561Smaybee ASSERT(db->db_parent == NULL || 1121c717a561Smaybee db->db_parent == db->db_dnode->dn_dbuf); 1122c717a561Smaybee mutex_enter(&dn->dn_mtx); 1123c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1124c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1125c717a561Smaybee mutex_exit(&dn->dn_mtx); 1126fa9e4066Sahrens if (drop_struct_lock) 1127fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1128fa9e4066Sahrens } 1129fa9e4066Sahrens 1130fa9e4066Sahrens dnode_setdirty(dn, tx); 1131c717a561Smaybee return (dr); 1132fa9e4066Sahrens } 1133fa9e4066Sahrens 1134fa9e4066Sahrens static int 1135fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1136fa9e4066Sahrens { 1137fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1138c717a561Smaybee uint64_t txg = tx->tx_txg; 113917f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1140fa9e4066Sahrens 1141c717a561Smaybee ASSERT(txg != 0); 1142ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1143fa9e4066Sahrens 1144fa9e4066Sahrens mutex_enter(&db->db_mtx); 1145fa9e4066Sahrens 1146fa9e4066Sahrens /* 1147fa9e4066Sahrens * If this buffer is not dirty, we're done. 1148fa9e4066Sahrens */ 114917f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1150c717a561Smaybee if (dr->dr_txg <= txg) 1151c717a561Smaybee break; 1152c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1153fa9e4066Sahrens mutex_exit(&db->db_mtx); 1154fa9e4066Sahrens return (0); 1155fa9e4066Sahrens } 1156c717a561Smaybee ASSERT(dr->dr_txg == txg); 1157fa9e4066Sahrens 1158fa9e4066Sahrens /* 1159fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1160fa9e4066Sahrens * it, since one of the current holders may be in the 1161fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1162fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1163fa9e4066Sahrens */ 1164fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1165fa9e4066Sahrens mutex_exit(&db->db_mtx); 116644eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1167fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1168fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1169fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1170fa9e4066Sahrens return (0); 1171fa9e4066Sahrens } 1172fa9e4066Sahrens 1173fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1174fa9e4066Sahrens 1175fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1176fa9e4066Sahrens 1177fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1178fa9e4066Sahrens 117917f17c2dSbonwick *drp = dr->dr_next; 1180c717a561Smaybee 1181c717a561Smaybee if (dr->dr_parent) { 1182c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1183c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1184c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1185c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1186cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1187c717a561Smaybee mutex_enter(&dn->dn_mtx); 1188c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1189c717a561Smaybee mutex_exit(&dn->dn_mtx); 1190c717a561Smaybee } 1191c717a561Smaybee 1192c717a561Smaybee if (db->db_level == 0) { 1193c717a561Smaybee dbuf_unoverride(dr); 1194c717a561Smaybee 1195c717a561Smaybee ASSERT(db->db_buf != NULL); 1196c717a561Smaybee ASSERT(dr->dt.dl.dr_data != NULL); 1197c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 1198c717a561Smaybee VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 1199c717a561Smaybee } else { 1200c717a561Smaybee ASSERT(db->db_buf != NULL); 1201c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1202c25056deSgw mutex_destroy(&dr->dt.di.dr_mtx); 1203c25056deSgw list_destroy(&dr->dt.di.dr_children); 1204c717a561Smaybee } 1205c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1206fa9e4066Sahrens 1207fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1208fa9e4066Sahrens db->db_dirtycnt -= 1; 1209fa9e4066Sahrens 1210c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1211ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1212fa9e4066Sahrens 1213ea8dc4b6Seschrock ASSERT(arc_released(buf)); 1214ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1215ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1216fa9e4066Sahrens dbuf_evict(db); 1217fa9e4066Sahrens return (1); 1218fa9e4066Sahrens } 1219fa9e4066Sahrens 1220fa9e4066Sahrens mutex_exit(&db->db_mtx); 1221fa9e4066Sahrens return (0); 1222fa9e4066Sahrens } 1223fa9e4066Sahrens 1224fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1225fa9e4066Sahrens void 1226fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1227fa9e4066Sahrens { 12281ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1229fa9e4066Sahrens 1230fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1231fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1232fa9e4066Sahrens 1233fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1234fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1235ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1236c717a561Smaybee (void) dbuf_dirty(db, tx); 1237fa9e4066Sahrens } 1238fa9e4066Sahrens 1239fa9e4066Sahrens void 1240ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1241fa9e4066Sahrens { 1242ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1243ea8dc4b6Seschrock 1244ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1245fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1246fa9e4066Sahrens ASSERT(db->db_level == 0); 1247fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1248fa9e4066Sahrens 1249ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1250fa9e4066Sahrens dmu_tx_private_ok(tx)); 1251fa9e4066Sahrens 1252fa9e4066Sahrens dbuf_noread(db); 1253c717a561Smaybee (void) dbuf_dirty(db, tx); 1254fa9e4066Sahrens } 1255fa9e4066Sahrens 1256fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1257fa9e4066Sahrens /* ARGSUSED */ 1258fa9e4066Sahrens void 1259fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1260fa9e4066Sahrens { 1261fa9e4066Sahrens mutex_enter(&db->db_mtx); 12629c9dc39aSek DBUF_VERIFY(db); 1263fa9e4066Sahrens 1264fa9e4066Sahrens if (db->db_state == DB_FILL) { 1265c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 1266ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1267fa9e4066Sahrens /* we were freed while filling */ 1268fa9e4066Sahrens /* XXX dbuf_undirty? */ 1269fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1270c717a561Smaybee db->db_freed_in_flight = FALSE; 1271fa9e4066Sahrens } 1272fa9e4066Sahrens db->db_state = DB_CACHED; 1273fa9e4066Sahrens cv_broadcast(&db->db_changed); 1274fa9e4066Sahrens } 1275fa9e4066Sahrens mutex_exit(&db->db_mtx); 1276fa9e4066Sahrens } 1277fa9e4066Sahrens 1278ea8dc4b6Seschrock /* 1279ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1280ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1281ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1282ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1283ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1284ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1285ea8dc4b6Seschrock * For the arc callback, we will usually see: 1286ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1287ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1288ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1289ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1290ea8dc4b6Seschrock */ 1291ea8dc4b6Seschrock void 1292fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1293fa9e4066Sahrens { 1294fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1295ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1296c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1297ea8dc4b6Seschrock int dbuf_gone = FALSE; 1298fa9e4066Sahrens 1299fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1300fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1301fa9e4066Sahrens 1302ea8dc4b6Seschrock dbuf_evict_user(db); 1303ea8dc4b6Seschrock 1304fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1305ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 13060e8c6158Smaybee if (db->db_blkid == DB_BONUS_BLKID) { 1307ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 13080e8c6158Smaybee arc_space_return(DN_MAX_BONUSLEN); 13090e8c6158Smaybee } 1310fa9e4066Sahrens db->db.db_data = NULL; 1311fa9e4066Sahrens db->db_state = DB_UNCACHED; 1312fa9e4066Sahrens } 1313fa9e4066Sahrens 1314fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_UNCACHED); 1315fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1316fa9e4066Sahrens 1317ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1318ea8dc4b6Seschrock db->db_blkptr = NULL; 1319ea8dc4b6Seschrock 1320ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1321ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1322ea8dc4b6Seschrock dnode_rele(dn, db); 13231934e92fSmaybee db->db_dnode = NULL; 1324ea8dc4b6Seschrock } 1325ea8dc4b6Seschrock 1326ea8dc4b6Seschrock if (db->db_buf) 1327ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1328ea8dc4b6Seschrock 1329ea8dc4b6Seschrock if (!dbuf_gone) 1330ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1331fa9e4066Sahrens 1332fa9e4066Sahrens /* 1333fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1334fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1335fa9e4066Sahrens */ 1336c543ec06Sahrens if (parent && parent != dndb) 1337ea8dc4b6Seschrock dbuf_rele(parent, db); 1338fa9e4066Sahrens } 1339fa9e4066Sahrens 1340fa9e4066Sahrens static int 1341fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1342fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1343fa9e4066Sahrens { 1344fa9e4066Sahrens int nlevels, epbs; 1345fa9e4066Sahrens 13460b69c2f0Sahrens *parentp = NULL; 13470b69c2f0Sahrens *bpp = NULL; 13480b69c2f0Sahrens 1349ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1350ea8dc4b6Seschrock 1351fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1352fa9e4066Sahrens nlevels = 1; 1353fa9e4066Sahrens else 1354fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1355fa9e4066Sahrens 1356fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1357fa9e4066Sahrens 1358fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1359fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1360ea8dc4b6Seschrock if (level >= nlevels || 1361fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1362fa9e4066Sahrens /* the buffer has no parent yet */ 1363fa9e4066Sahrens return (ENOENT); 1364fa9e4066Sahrens } else if (level < nlevels-1) { 1365fa9e4066Sahrens /* this block is referenced from an indirect block */ 1366fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1367fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1368fa9e4066Sahrens if (err) 1369fa9e4066Sahrens return (err); 1370ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1371ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1372c543ec06Sahrens if (err) { 1373c543ec06Sahrens dbuf_rele(*parentp, NULL); 1374c543ec06Sahrens *parentp = NULL; 1375c543ec06Sahrens return (err); 1376ea8dc4b6Seschrock } 1377c543ec06Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1378c543ec06Sahrens (blkid & ((1ULL << epbs) - 1)); 1379c543ec06Sahrens return (0); 1380fa9e4066Sahrens } else { 1381fa9e4066Sahrens /* the block is referenced from the dnode */ 1382fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1383fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1384fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1385c543ec06Sahrens if (dn->dn_dbuf) { 1386c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1387c543ec06Sahrens *parentp = dn->dn_dbuf; 1388c543ec06Sahrens } 1389fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1390fa9e4066Sahrens return (0); 1391fa9e4066Sahrens } 1392fa9e4066Sahrens } 1393fa9e4066Sahrens 1394fa9e4066Sahrens static dmu_buf_impl_t * 1395fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1396fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1397fa9e4066Sahrens { 1398fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1399fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1400fa9e4066Sahrens 1401fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1402fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1403fa9e4066Sahrens 1404fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1405fa9e4066Sahrens 1406fa9e4066Sahrens db->db_objset = os; 1407fa9e4066Sahrens db->db.db_object = dn->dn_object; 1408fa9e4066Sahrens db->db_level = level; 1409fa9e4066Sahrens db->db_blkid = blkid; 1410c717a561Smaybee db->db_last_dirty = NULL; 1411ea8dc4b6Seschrock db->db_dirtycnt = 0; 1412ea8dc4b6Seschrock db->db_dnode = dn; 1413ea8dc4b6Seschrock db->db_parent = parent; 1414ea8dc4b6Seschrock db->db_blkptr = blkptr; 1415fa9e4066Sahrens 1416c717a561Smaybee db->db_user_ptr = NULL; 1417c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1418c717a561Smaybee db->db_evict_func = NULL; 1419c717a561Smaybee db->db_immediate_evict = 0; 1420c717a561Smaybee db->db_freed_in_flight = 0; 1421ea8dc4b6Seschrock 1422ea8dc4b6Seschrock if (blkid == DB_BONUS_BLKID) { 1423ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 14241934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 14251934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 14261934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1427fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1428ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1429ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 14300e8c6158Smaybee arc_space_consume(sizeof (dmu_buf_impl_t)); 1431ea8dc4b6Seschrock return (db); 1432fa9e4066Sahrens } else { 1433fa9e4066Sahrens int blocksize = 1434fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1435fa9e4066Sahrens db->db.db_size = blocksize; 1436fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1437fa9e4066Sahrens } 1438fa9e4066Sahrens 1439fa9e4066Sahrens /* 1440fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1441fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1442fa9e4066Sahrens * This prevents a possible deadlock with someone 1443fa9e4066Sahrens * trying to look up this dbuf before its added to the 1444fa9e4066Sahrens * dn_dbufs list. 1445fa9e4066Sahrens */ 1446fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1447ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1448fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1449fa9e4066Sahrens /* someone else inserted it first */ 1450fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1451fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1452fa9e4066Sahrens return (odb); 1453fa9e4066Sahrens } 1454fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1455ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1456fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 14570e8c6158Smaybee arc_space_consume(sizeof (dmu_buf_impl_t)); 1458fa9e4066Sahrens 1459fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1460fa9e4066Sahrens dbuf_add_ref(parent, db); 1461fa9e4066Sahrens 1462ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1463ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1464fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1465fa9e4066Sahrens 1466fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1467fa9e4066Sahrens 1468fa9e4066Sahrens return (db); 1469fa9e4066Sahrens } 1470fa9e4066Sahrens 1471fa9e4066Sahrens static int 1472ea8dc4b6Seschrock dbuf_do_evict(void *private) 1473fa9e4066Sahrens { 1474ea8dc4b6Seschrock arc_buf_t *buf = private; 1475ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1476fa9e4066Sahrens 1477ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1478ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1479fa9e4066Sahrens 1480ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1481fa9e4066Sahrens 1482ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1483ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 1484ea8dc4b6Seschrock DBUF_VERIFY(db); 1485ea8dc4b6Seschrock db->db_buf = NULL; 1486ea8dc4b6Seschrock dbuf_evict(db); 1487ea8dc4b6Seschrock } else { 1488ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1489ea8dc4b6Seschrock dbuf_destroy(db); 1490fa9e4066Sahrens } 1491ea8dc4b6Seschrock return (0); 1492fa9e4066Sahrens } 1493fa9e4066Sahrens 1494fa9e4066Sahrens static void 1495fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1496fa9e4066Sahrens { 1497fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1498fa9e4066Sahrens 1499ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID) { 1500ea8dc4b6Seschrock /* 1501ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1502ea8dc4b6Seschrock * remove it from that list. 1503ea8dc4b6Seschrock */ 15041934e92fSmaybee if (db->db_dnode) { 15051934e92fSmaybee dnode_t *dn = db->db_dnode; 15061934e92fSmaybee 15071934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1508ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1509c543ec06Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1510ea8dc4b6Seschrock 1511ea8dc4b6Seschrock dnode_rele(dn, db); 15121934e92fSmaybee db->db_dnode = NULL; 1513ea8dc4b6Seschrock } 1514ea8dc4b6Seschrock dbuf_hash_remove(db); 1515ea8dc4b6Seschrock } 1516ea8dc4b6Seschrock db->db_parent = NULL; 1517ea8dc4b6Seschrock db->db_buf = NULL; 1518ea8dc4b6Seschrock 1519f82bfe17Sgw ASSERT(!list_link_active(&db->db_link)); 1520fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1521fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1522fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1523fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1524fa9e4066Sahrens 1525fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 15260e8c6158Smaybee arc_space_return(sizeof (dmu_buf_impl_t)); 1527fa9e4066Sahrens } 1528fa9e4066Sahrens 1529fa9e4066Sahrens void 1530fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1531fa9e4066Sahrens { 153213506d1eSmaybee dmu_buf_impl_t *db = NULL; 1533fa9e4066Sahrens blkptr_t *bp = NULL; 1534fa9e4066Sahrens 1535fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1536fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1537fa9e4066Sahrens 1538fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1539fa9e4066Sahrens return; 1540fa9e4066Sahrens 1541fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1542fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 154313506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 154413506d1eSmaybee /* 154513506d1eSmaybee * This dbuf is active. We assume that it is 154613506d1eSmaybee * already CACHED, or else about to be either 154713506d1eSmaybee * read or filled. 154813506d1eSmaybee */ 154913506d1eSmaybee mutex_exit(&db->db_mtx); 155013506d1eSmaybee return; 155113506d1eSmaybee } 1552fa9e4066Sahrens mutex_exit(&db->db_mtx); 15530b69c2f0Sahrens db = NULL; 1554fa9e4066Sahrens } 1555fa9e4066Sahrens 155613506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1557fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1558088f3894Sahrens arc_buf_t *pbuf; 155913506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1560ea8dc4b6Seschrock zbookmark_t zb; 1561ea8dc4b6Seschrock zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1562ea8dc4b6Seschrock dn->dn_objset->os_dsl_dataset->ds_object : 0; 1563ea8dc4b6Seschrock zb.zb_object = dn->dn_object; 1564ea8dc4b6Seschrock zb.zb_level = 0; 1565ea8dc4b6Seschrock zb.zb_blkid = blkid; 1566ea8dc4b6Seschrock 1567088f3894Sahrens if (db) 1568088f3894Sahrens pbuf = db->db_buf; 1569088f3894Sahrens else 1570088f3894Sahrens pbuf = dn->dn_objset->os_phys_buf; 1571088f3894Sahrens 1572088f3894Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, 1573088f3894Sahrens bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1574fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 157513506d1eSmaybee &aflags, &zb); 1576fa9e4066Sahrens } 157713506d1eSmaybee if (db) 157813506d1eSmaybee dbuf_rele(db, NULL); 1579fa9e4066Sahrens } 1580fa9e4066Sahrens } 1581fa9e4066Sahrens 1582fa9e4066Sahrens /* 1583fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1584fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1585fa9e4066Sahrens */ 1586fa9e4066Sahrens int 1587fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1588fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1589fa9e4066Sahrens { 1590fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1591fa9e4066Sahrens 1592ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1593fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1594fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1595fa9e4066Sahrens 1596fa9e4066Sahrens *dbp = NULL; 1597ea8dc4b6Seschrock top: 1598fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1599fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1600fa9e4066Sahrens 1601fa9e4066Sahrens if (db == NULL) { 1602fa9e4066Sahrens blkptr_t *bp = NULL; 1603fa9e4066Sahrens int err; 1604fa9e4066Sahrens 1605c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1606fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1607fa9e4066Sahrens if (fail_sparse) { 1608fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1609fa9e4066Sahrens err = ENOENT; 1610fa9e4066Sahrens if (err) { 1611c543ec06Sahrens if (parent) 1612ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1613fa9e4066Sahrens return (err); 1614fa9e4066Sahrens } 1615fa9e4066Sahrens } 1616ea8dc4b6Seschrock if (err && err != ENOENT) 1617ea8dc4b6Seschrock return (err); 1618fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1619fa9e4066Sahrens } 1620fa9e4066Sahrens 1621ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1622ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1623ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1624ea8dc4b6Seschrock dbuf_clear(db); 1625c543ec06Sahrens if (parent) { 1626c543ec06Sahrens dbuf_rele(parent, NULL); 1627c543ec06Sahrens parent = NULL; 1628c543ec06Sahrens } 1629ea8dc4b6Seschrock goto top; 1630ea8dc4b6Seschrock } 1631ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1632ea8dc4b6Seschrock } 1633ea8dc4b6Seschrock 1634ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1635ea8dc4b6Seschrock 1636fa9e4066Sahrens /* 1637c717a561Smaybee * If this buffer is currently syncing out, and we are are 1638c717a561Smaybee * still referencing it from db_data, we need to make a copy 1639c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1640fa9e4066Sahrens */ 1641c717a561Smaybee if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 1642ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1643c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1644c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1645fa9e4066Sahrens 1646c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1647c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1648c717a561Smaybee 1649c717a561Smaybee dbuf_set_data(db, 1650c717a561Smaybee arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1651c717a561Smaybee db->db.db_size, db, type)); 1652c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1653c717a561Smaybee db->db.db_size); 1654c717a561Smaybee } 1655fa9e4066Sahrens } 1656fa9e4066Sahrens 1657ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1658fa9e4066Sahrens dbuf_update_data(db); 16599c9dc39aSek DBUF_VERIFY(db); 1660fa9e4066Sahrens mutex_exit(&db->db_mtx); 1661fa9e4066Sahrens 1662fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1663c543ec06Sahrens if (parent) 1664ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1665fa9e4066Sahrens 1666fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1667fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1668fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1669fa9e4066Sahrens *dbp = db; 1670fa9e4066Sahrens 1671fa9e4066Sahrens return (0); 1672fa9e4066Sahrens } 1673fa9e4066Sahrens 1674fa9e4066Sahrens dmu_buf_impl_t * 1675ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1676fa9e4066Sahrens { 1677fa9e4066Sahrens dmu_buf_impl_t *db; 1678ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1679ea8dc4b6Seschrock return (err ? NULL : db); 1680fa9e4066Sahrens } 1681fa9e4066Sahrens 1682fa9e4066Sahrens dmu_buf_impl_t * 1683fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1684fa9e4066Sahrens { 1685fa9e4066Sahrens dmu_buf_impl_t *db; 1686ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1687ea8dc4b6Seschrock return (err ? NULL : db); 1688fa9e4066Sahrens } 1689fa9e4066Sahrens 16901934e92fSmaybee void 1691ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1692fa9e4066Sahrens { 1693ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1694ea8dc4b6Seschrock 1695ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 16961934e92fSmaybee dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1697fa9e4066Sahrens } 1698fa9e4066Sahrens 1699ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1700fa9e4066Sahrens void 1701fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1702fa9e4066Sahrens { 1703ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1704ea8dc4b6Seschrock ASSERT(holds > 1); 1705fa9e4066Sahrens } 1706fa9e4066Sahrens 1707ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1708fa9e4066Sahrens void 1709ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1710fa9e4066Sahrens { 1711fa9e4066Sahrens int64_t holds; 1712fa9e4066Sahrens 1713fa9e4066Sahrens mutex_enter(&db->db_mtx); 17149c9dc39aSek DBUF_VERIFY(db); 1715fa9e4066Sahrens 1716fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1717ea8dc4b6Seschrock ASSERT(holds >= 0); 1718ea8dc4b6Seschrock 1719c717a561Smaybee /* 1720c717a561Smaybee * We can't freeze indirects if there is a possibility that they 1721c717a561Smaybee * may be modified in the current syncing context. 1722c717a561Smaybee */ 1723c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 17246b4acc8bSahrens arc_buf_freeze(db->db_buf); 17256b4acc8bSahrens 1726ea8dc4b6Seschrock if (holds == db->db_dirtycnt && 1727c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 1728ea8dc4b6Seschrock dbuf_evict_user(db); 1729fa9e4066Sahrens 1730fa9e4066Sahrens if (holds == 0) { 1731ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1732ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1733ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1734ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1735ea8dc4b6Seschrock /* 1736ea8dc4b6Seschrock * This is a special case: we never associated this 1737ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1738ea8dc4b6Seschrock */ 1739ea8dc4b6Seschrock ASSERT3U(db->db_state, ==, DB_UNCACHED); 1740ea8dc4b6Seschrock dbuf_evict(db); 17416b4acc8bSahrens } else if (arc_released(db->db_buf)) { 1742ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1743ea8dc4b6Seschrock /* 1744ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1745ea8dc4b6Seschrock */ 1746ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1747ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1748ea8dc4b6Seschrock dbuf_evict(db); 1749ea8dc4b6Seschrock } else { 1750ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 17513baa08fcSek if (!DBUF_IS_CACHEABLE(db)) 17523baa08fcSek dbuf_clear(db); 17533baa08fcSek else 17543baa08fcSek mutex_exit(&db->db_mtx); 1755ea8dc4b6Seschrock } 1756fa9e4066Sahrens } else { 1757fa9e4066Sahrens mutex_exit(&db->db_mtx); 1758fa9e4066Sahrens } 1759fa9e4066Sahrens } 1760fa9e4066Sahrens 1761fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1762fa9e4066Sahrens uint64_t 1763fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1764fa9e4066Sahrens { 1765fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1766fa9e4066Sahrens } 1767fa9e4066Sahrens 1768fa9e4066Sahrens void * 1769fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1770fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1771fa9e4066Sahrens { 1772fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1773fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1774fa9e4066Sahrens } 1775fa9e4066Sahrens 1776fa9e4066Sahrens void * 1777fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1778fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1779fa9e4066Sahrens { 1780fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1781fa9e4066Sahrens 1782c717a561Smaybee db->db_immediate_evict = TRUE; 1783fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1784fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1785fa9e4066Sahrens } 1786fa9e4066Sahrens 1787fa9e4066Sahrens void * 1788fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1789fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1790fa9e4066Sahrens { 1791fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1792fa9e4066Sahrens ASSERT(db->db_level == 0); 1793fa9e4066Sahrens 1794fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1795fa9e4066Sahrens 1796fa9e4066Sahrens mutex_enter(&db->db_mtx); 1797fa9e4066Sahrens 1798c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 1799c717a561Smaybee db->db_user_ptr = user_ptr; 1800c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 1801c717a561Smaybee db->db_evict_func = evict_func; 1802fa9e4066Sahrens 1803fa9e4066Sahrens dbuf_update_data(db); 1804fa9e4066Sahrens } else { 1805c717a561Smaybee old_user_ptr = db->db_user_ptr; 1806fa9e4066Sahrens } 1807fa9e4066Sahrens 1808fa9e4066Sahrens mutex_exit(&db->db_mtx); 1809fa9e4066Sahrens return (old_user_ptr); 1810fa9e4066Sahrens } 1811fa9e4066Sahrens 1812fa9e4066Sahrens void * 1813fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1814fa9e4066Sahrens { 1815fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1816fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1817fa9e4066Sahrens 1818c717a561Smaybee return (db->db_user_ptr); 1819fa9e4066Sahrens } 1820fa9e4066Sahrens 1821c717a561Smaybee static void 1822c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 1823fa9e4066Sahrens { 1824c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 1825c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 1826c717a561Smaybee 1827c717a561Smaybee if (db->db_blkptr != NULL) 1828c717a561Smaybee return; 1829c717a561Smaybee 1830c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 1831c717a561Smaybee /* 1832c717a561Smaybee * This buffer was allocated at a time when there was 1833c717a561Smaybee * no available blkptrs from the dnode, or it was 1834c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 1835c717a561Smaybee */ 1836c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 1837c717a561Smaybee ASSERT(db->db_parent == NULL); 1838c717a561Smaybee db->db_parent = dn->dn_dbuf; 1839c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 1840c717a561Smaybee DBUF_VERIFY(db); 1841c717a561Smaybee } else { 1842c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1843c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1844c717a561Smaybee 1845c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 1846c717a561Smaybee if (parent == NULL) { 1847c717a561Smaybee mutex_exit(&db->db_mtx); 1848c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 1849c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 1850c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 1851c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 1852c717a561Smaybee mutex_enter(&db->db_mtx); 1853c717a561Smaybee db->db_parent = parent; 1854c717a561Smaybee } 1855c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 1856c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 1857c717a561Smaybee DBUF_VERIFY(db); 1858c717a561Smaybee } 1859c717a561Smaybee } 1860c717a561Smaybee 1861c717a561Smaybee static void 1862c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1863c717a561Smaybee { 1864c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1865c717a561Smaybee dnode_t *dn = db->db_dnode; 1866c717a561Smaybee zio_t *zio; 1867c717a561Smaybee 1868c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 1869c717a561Smaybee 1870c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1871c717a561Smaybee 1872c717a561Smaybee mutex_enter(&db->db_mtx); 1873c717a561Smaybee 1874c717a561Smaybee ASSERT(db->db_level > 0); 1875c717a561Smaybee DBUF_VERIFY(db); 1876c717a561Smaybee 1877c717a561Smaybee if (db->db_buf == NULL) { 1878c717a561Smaybee mutex_exit(&db->db_mtx); 1879c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 1880c717a561Smaybee mutex_enter(&db->db_mtx); 1881c717a561Smaybee } 1882c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 1883c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 1884c717a561Smaybee ASSERT(db->db_buf != NULL); 1885c717a561Smaybee 1886c717a561Smaybee dbuf_check_blkptr(dn, db); 1887c717a561Smaybee 1888c717a561Smaybee db->db_data_pending = dr; 1889c717a561Smaybee 1890af2c4821Smaybee mutex_exit(&db->db_mtx); 1891088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 1892c717a561Smaybee 1893c717a561Smaybee zio = dr->dr_zio; 1894c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 1895c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 1896c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1897c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 1898c717a561Smaybee zio_nowait(zio); 1899c717a561Smaybee } 1900c717a561Smaybee 1901c717a561Smaybee static void 1902c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 1903c717a561Smaybee { 1904c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 1905c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1906fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1907fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1908c717a561Smaybee uint64_t txg = tx->tx_txg; 1909fa9e4066Sahrens int blksz; 1910fa9e4066Sahrens 1911fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 1912fa9e4066Sahrens 1913fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1914fa9e4066Sahrens 1915fa9e4066Sahrens mutex_enter(&db->db_mtx); 1916fa9e4066Sahrens /* 1917fa9e4066Sahrens * To be synced, we must be dirtied. But we 1918fa9e4066Sahrens * might have been freed after the dirty. 1919fa9e4066Sahrens */ 1920fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 1921fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 1922fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1923fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 1924fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 1925c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 1926fa9e4066Sahrens } else { 1927fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 1928fa9e4066Sahrens } 19299c9dc39aSek DBUF_VERIFY(db); 1930fa9e4066Sahrens 1931fa9e4066Sahrens /* 1932c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 1933c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 1934c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 1935c717a561Smaybee * be called). 1936fa9e4066Sahrens */ 1937ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1938c717a561Smaybee dbuf_dirty_record_t **drp; 19391934e92fSmaybee 1940ea8dc4b6Seschrock ASSERT(*datap != NULL); 1941ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 1942ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1943ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 19440e8c6158Smaybee if (*datap != db->db.db_data) { 1945ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 19460e8c6158Smaybee arc_space_return(DN_MAX_BONUSLEN); 19470e8c6158Smaybee } 1948ea8dc4b6Seschrock db->db_data_pending = NULL; 1949c717a561Smaybee drp = &db->db_last_dirty; 1950c717a561Smaybee while (*drp != dr) 1951c717a561Smaybee drp = &(*drp)->dr_next; 195217f17c2dSbonwick ASSERT(dr->dr_next == NULL); 195317f17c2dSbonwick *drp = dr->dr_next; 1954c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1955ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 1956ea8dc4b6Seschrock db->db_dirtycnt -= 1; 1957ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1958ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 1959ea8dc4b6Seschrock return; 1960ea8dc4b6Seschrock } 1961ea8dc4b6Seschrock 1962f82bfe17Sgw /* 1963f82bfe17Sgw * This function may have dropped the db_mtx lock allowing a dmu_sync 1964f82bfe17Sgw * operation to sneak in. As a result, we need to ensure that we 1965f82bfe17Sgw * don't check the dr_override_state until we have returned from 1966f82bfe17Sgw * dbuf_check_blkptr. 1967f82bfe17Sgw */ 1968f82bfe17Sgw dbuf_check_blkptr(dn, db); 1969f82bfe17Sgw 1970c717a561Smaybee /* 1971c717a561Smaybee * If this buffer is in the middle of an immdiate write, 1972c717a561Smaybee * wait for the synchronous IO to complete. 1973c717a561Smaybee */ 1974c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 1975c717a561Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1976c717a561Smaybee cv_wait(&db->db_changed, &db->db_mtx); 1977c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 1978c717a561Smaybee } 1979c5c6ffa0Smaybee 1980c717a561Smaybee /* 1981c717a561Smaybee * If this dbuf has already been written out via an immediate write, 1982c717a561Smaybee * just complete the write by copying over the new block pointer and 1983c717a561Smaybee * updating the accounting via the write-completion functions. 1984c717a561Smaybee */ 1985c717a561Smaybee if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1986c717a561Smaybee zio_t zio_fake; 1987c717a561Smaybee 1988c717a561Smaybee zio_fake.io_private = &db; 1989c717a561Smaybee zio_fake.io_error = 0; 1990c717a561Smaybee zio_fake.io_bp = db->db_blkptr; 1991c717a561Smaybee zio_fake.io_bp_orig = *db->db_blkptr; 1992c717a561Smaybee zio_fake.io_txg = txg; 1993c717a561Smaybee 1994c717a561Smaybee *db->db_blkptr = dr->dt.dl.dr_overridden_by; 1995c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1996c717a561Smaybee db->db_data_pending = dr; 1997c717a561Smaybee dr->dr_zio = &zio_fake; 1998c717a561Smaybee mutex_exit(&db->db_mtx); 1999c717a561Smaybee 2000c717a561Smaybee if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg)) 2001cdb0ab79Smaybee (void) dsl_dataset_block_kill(os->os_dsl_dataset, 2002c717a561Smaybee &zio_fake.io_bp_orig, dn->dn_zio, tx); 2003c717a561Smaybee 2004c717a561Smaybee dbuf_write_ready(&zio_fake, db->db_buf, db); 2005c717a561Smaybee dbuf_write_done(&zio_fake, db->db_buf, db); 2006c717a561Smaybee 2007c717a561Smaybee return; 2008c717a561Smaybee } 2009c717a561Smaybee 2010c717a561Smaybee blksz = arc_buf_size(*datap); 2011c717a561Smaybee 2012c717a561Smaybee if (dn->dn_object != DMU_META_DNODE_OBJECT) { 2013fa9e4066Sahrens /* 2014fa9e4066Sahrens * If this buffer is currently "in use" (i.e., there are 2015fa9e4066Sahrens * active holds and db_data still references it), then make 2016fa9e4066Sahrens * a copy before we start the write so that any modifications 2017fa9e4066Sahrens * from the open txg will not leak into this write. 2018fa9e4066Sahrens * 2019fa9e4066Sahrens * NOTE: this copy does not need to be made for objects only 2020c717a561Smaybee * modified in the syncing context (e.g. DNONE_DNODE blocks). 2021fa9e4066Sahrens */ 2022c717a561Smaybee if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) { 2023c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2024c717a561Smaybee *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2025c717a561Smaybee bcopy(db->db.db_data, (*datap)->b_data, blksz); 2026fa9e4066Sahrens } 2027fa9e4066Sahrens } 2028fa9e4066Sahrens 2029c717a561Smaybee ASSERT(*datap != NULL); 2030c717a561Smaybee db->db_data_pending = dr; 2031fa9e4066Sahrens 2032c717a561Smaybee mutex_exit(&db->db_mtx); 2033fa9e4066Sahrens 2034088f3894Sahrens dbuf_write(dr, *datap, tx); 2035fa9e4066Sahrens 2036c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2037c717a561Smaybee if (dn->dn_object == DMU_META_DNODE_OBJECT) 2038c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2039c717a561Smaybee else 2040c717a561Smaybee zio_nowait(dr->dr_zio); 2041c717a561Smaybee } 204223b11526Smaybee 2043c717a561Smaybee void 2044c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2045c717a561Smaybee { 2046c717a561Smaybee dbuf_dirty_record_t *dr; 2047c717a561Smaybee 2048c717a561Smaybee while (dr = list_head(list)) { 2049c717a561Smaybee if (dr->dr_zio != NULL) { 2050c717a561Smaybee /* 2051c717a561Smaybee * If we find an already initialized zio then we 2052c717a561Smaybee * are processing the meta-dnode, and we have finished. 2053c717a561Smaybee * The dbufs for all dnodes are put back on the list 2054c717a561Smaybee * during processing, so that we can zio_wait() 2055c717a561Smaybee * these IOs after initiating all child IOs. 2056c717a561Smaybee */ 2057c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2058c717a561Smaybee DMU_META_DNODE_OBJECT); 2059c717a561Smaybee break; 206023b11526Smaybee } 2061c717a561Smaybee list_remove(list, dr); 2062c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2063c717a561Smaybee dbuf_sync_indirect(dr, tx); 2064c717a561Smaybee else 2065c717a561Smaybee dbuf_sync_leaf(dr, tx); 206623b11526Smaybee } 2067c717a561Smaybee } 206823b11526Smaybee 2069c717a561Smaybee static void 2070088f3894Sahrens dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2071c717a561Smaybee { 2072c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2073c717a561Smaybee dnode_t *dn = db->db_dnode; 2074c717a561Smaybee objset_impl_t *os = dn->dn_objset; 2075c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2076c717a561Smaybee uint64_t txg = tx->tx_txg; 2077c717a561Smaybee zbookmark_t zb; 2078088f3894Sahrens writeprops_t wp = { 0 }; 2079c717a561Smaybee zio_t *zio; 20809bc11082Sek int zio_flags; 2081fa9e4066Sahrens 2082088f3894Sahrens if (!BP_IS_HOLE(db->db_blkptr) && 2083088f3894Sahrens (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) { 2084088f3894Sahrens /* 2085088f3894Sahrens * Private object buffers are released here rather 2086088f3894Sahrens * than in dbuf_dirty() since they are only modified 2087088f3894Sahrens * in the syncing context and we don't want the 2088088f3894Sahrens * overhead of making multiple copies of the data. 2089088f3894Sahrens */ 2090088f3894Sahrens arc_release(data, db); 2091088f3894Sahrens } else { 2092088f3894Sahrens ASSERT(arc_released(data)); 2093088f3894Sahrens /* XXX why do we need to thaw here? */ 2094088f3894Sahrens arc_buf_thaw(data); 2095088f3894Sahrens } 2096088f3894Sahrens 2097c717a561Smaybee if (parent != dn->dn_dbuf) { 2098c717a561Smaybee ASSERT(parent && parent->db_data_pending); 2099fa9e4066Sahrens ASSERT(db->db_level == parent->db_level-1); 2100c717a561Smaybee ASSERT(arc_released(parent->db_buf)); 2101c717a561Smaybee zio = parent->db_data_pending->dr_zio; 2102ea8dc4b6Seschrock } else { 2103fa9e4066Sahrens ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2104fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 2105fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 2106c717a561Smaybee zio = dn->dn_zio; 2107ea8dc4b6Seschrock } 2108fa9e4066Sahrens 2109c717a561Smaybee ASSERT(db->db_level == 0 || data == db->db_buf); 2110c717a561Smaybee ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2111c717a561Smaybee ASSERT(zio); 2112fa9e4066Sahrens 2113ea8dc4b6Seschrock zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2114ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 2115ea8dc4b6Seschrock zb.zb_level = db->db_level; 2116ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 211744cd46caSbillm 21189bc11082Sek zio_flags = ZIO_FLAG_MUSTSUCCEED; 21199bc11082Sek if (dmu_ot[dn->dn_type].ot_metadata || zb.zb_level != 0) 21209bc11082Sek zio_flags |= ZIO_FLAG_METADATA; 2121088f3894Sahrens wp.wp_type = dn->dn_type; 2122088f3894Sahrens wp.wp_level = db->db_level; 2123088f3894Sahrens wp.wp_copies = os->os_copies; 2124088f3894Sahrens wp.wp_dncompress = dn->dn_compress; 2125088f3894Sahrens wp.wp_oscompress = os->os_compress; 2126088f3894Sahrens wp.wp_dnchecksum = dn->dn_checksum; 2127088f3894Sahrens wp.wp_oschecksum = os->os_checksum; 2128088f3894Sahrens 2129c717a561Smaybee if (BP_IS_OLDER(db->db_blkptr, txg)) 2130cdb0ab79Smaybee (void) dsl_dataset_block_kill( 2131c717a561Smaybee os->os_dsl_dataset, db->db_blkptr, zio, tx); 2132fa9e4066Sahrens 2133088f3894Sahrens dr->dr_zio = arc_write(zio, os->os_spa, &wp, 21343baa08fcSek DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr, 21353baa08fcSek data, dbuf_write_ready, dbuf_write_done, db, 21369bc11082Sek ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb); 2137fa9e4066Sahrens } 2138fa9e4066Sahrens 2139fa9e4066Sahrens /* ARGSUSED */ 2140fa9e4066Sahrens static void 2141c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2142fa9e4066Sahrens { 2143fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2144fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2145fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 2146c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2147fa9e4066Sahrens uint64_t fill = 0; 2148c717a561Smaybee int old_size, new_size, i; 2149fa9e4066Sahrens 2150c717a561Smaybee dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", ""); 2151fa9e4066Sahrens 2152c717a561Smaybee old_size = bp_get_dasize(os->os_spa, bp_orig); 215399653d4eSeschrock new_size = bp_get_dasize(os->os_spa, zio->io_bp); 2154fa9e4066Sahrens 2155fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 2156fa9e4066Sahrens 2157c717a561Smaybee if (BP_IS_HOLE(zio->io_bp)) { 2158c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2159c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2160fa9e4066Sahrens 2161c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2162cdb0ab79Smaybee (void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2163c717a561Smaybee ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2164c717a561Smaybee return; 2165c717a561Smaybee } 2166c5c6ffa0Smaybee 2167c717a561Smaybee mutex_enter(&db->db_mtx); 2168fa9e4066Sahrens 2169fa9e4066Sahrens if (db->db_level == 0) { 2170fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2171c717a561Smaybee if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2172fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2173fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2174fa9e4066Sahrens 2175fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2176fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2177fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2178fa9e4066Sahrens i--, dnp++) { 2179fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2180fa9e4066Sahrens fill++; 2181fa9e4066Sahrens } 2182fa9e4066Sahrens } else { 2183c717a561Smaybee fill = 1; 2184fa9e4066Sahrens } 2185fa9e4066Sahrens } else { 2186fa9e4066Sahrens blkptr_t *bp = db->db.db_data; 2187fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2188fa9e4066Sahrens for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2189fa9e4066Sahrens if (BP_IS_HOLE(bp)) 2190fa9e4066Sahrens continue; 2191fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(bp), ==, 2192fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2193fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2194fa9e4066Sahrens fill += bp->blk_fill; 2195fa9e4066Sahrens } 2196fa9e4066Sahrens } 2197fa9e4066Sahrens 2198c717a561Smaybee db->db_blkptr->blk_fill = fill; 2199c717a561Smaybee BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2200c717a561Smaybee BP_SET_LEVEL(db->db_blkptr, db->db_level); 2201c717a561Smaybee 2202c717a561Smaybee mutex_exit(&db->db_mtx); 2203c717a561Smaybee 2204c717a561Smaybee /* We must do this after we've set the bp's type and level */ 2205c717a561Smaybee if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), BP_IDENTITY(bp_orig))) { 2206c717a561Smaybee dsl_dataset_t *ds = os->os_dsl_dataset; 2207c717a561Smaybee dmu_tx_t *tx = os->os_synctx; 2208c717a561Smaybee 2209c717a561Smaybee if (bp_orig->blk_birth == tx->tx_txg) 2210cdb0ab79Smaybee (void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx); 2211c717a561Smaybee dsl_dataset_block_born(ds, zio->io_bp, tx); 2212fa9e4066Sahrens } 2213c717a561Smaybee } 2214fa9e4066Sahrens 2215c717a561Smaybee /* ARGSUSED */ 2216c717a561Smaybee static void 2217c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2218c717a561Smaybee { 2219c717a561Smaybee dmu_buf_impl_t *db = vdb; 2220c717a561Smaybee uint64_t txg = zio->io_txg; 2221c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2222c717a561Smaybee 2223c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2224c717a561Smaybee 2225c717a561Smaybee mutex_enter(&db->db_mtx); 2226c717a561Smaybee 2227c717a561Smaybee drp = &db->db_last_dirty; 222817f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 222917f17c2dSbonwick drp = &dr->dr_next; 223017f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 223117f17c2dSbonwick ASSERT(dr->dr_txg == txg); 223217f17c2dSbonwick ASSERT(dr->dr_next == NULL); 223317f17c2dSbonwick *drp = dr->dr_next; 2234c717a561Smaybee 2235c717a561Smaybee if (db->db_level == 0) { 2236c717a561Smaybee ASSERT(db->db_blkid != DB_BONUS_BLKID); 2237c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2238c717a561Smaybee 2239c717a561Smaybee if (dr->dt.dl.dr_data != db->db_buf) 2240c717a561Smaybee VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1); 2241c717a561Smaybee else if (!BP_IS_HOLE(db->db_blkptr)) 2242c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2243c717a561Smaybee else 2244c717a561Smaybee ASSERT(arc_released(db->db_buf)); 2245c717a561Smaybee } else { 2246c717a561Smaybee dnode_t *dn = db->db_dnode; 2247c717a561Smaybee 2248c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2249c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2250c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2251c717a561Smaybee int epbs = 2252c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2253c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2254c717a561Smaybee db->db.db_size); 2255c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2256c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2257c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2258c717a561Smaybee } 2259c25056deSgw mutex_destroy(&dr->dt.di.dr_mtx); 2260c25056deSgw list_destroy(&dr->dt.di.dr_children); 2261c717a561Smaybee } 2262c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2263fa9e4066Sahrens 2264fa9e4066Sahrens cv_broadcast(&db->db_changed); 2265fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2266fa9e4066Sahrens db->db_dirtycnt -= 1; 2267c717a561Smaybee db->db_data_pending = NULL; 2268fa9e4066Sahrens mutex_exit(&db->db_mtx); 2269fa9e4066Sahrens 2270c717a561Smaybee dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", ""); 2271fa9e4066Sahrens 2272ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2273fa9e4066Sahrens } 2274