1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2234f18512Seschrock * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29fa9e4066Sahrens #include <sys/dmu.h> 30fa9e4066Sahrens #include <sys/dmu_impl.h> 31fa9e4066Sahrens #include <sys/dbuf.h> 32fa9e4066Sahrens #include <sys/dmu_objset.h> 33fa9e4066Sahrens #include <sys/dsl_dataset.h> 34fa9e4066Sahrens #include <sys/dsl_dir.h> 35fa9e4066Sahrens #include <sys/dmu_tx.h> 36fa9e4066Sahrens #include <sys/spa.h> 37fa9e4066Sahrens #include <sys/zio.h> 38fa9e4066Sahrens #include <sys/dmu_zfetch.h> 39fa9e4066Sahrens 40fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 41fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42fa9e4066Sahrens static arc_done_func_t dbuf_write_done; 43fa9e4066Sahrens 44416e0cd8Sek int zfs_mdcomp_disable = 0; 45416e0cd8Sek 46fa9e4066Sahrens /* 47fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 48fa9e4066Sahrens */ 49fa9e4066Sahrens taskq_t *dbuf_tq; 50fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 51fa9e4066Sahrens 52fa9e4066Sahrens /* ARGSUSED */ 53fa9e4066Sahrens static int 54fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 55fa9e4066Sahrens { 56fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 57fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 58fa9e4066Sahrens 59fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 60fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 61fa9e4066Sahrens refcount_create(&db->db_holds); 62fa9e4066Sahrens return (0); 63fa9e4066Sahrens } 64fa9e4066Sahrens 65fa9e4066Sahrens /* ARGSUSED */ 66fa9e4066Sahrens static void 67fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 68fa9e4066Sahrens { 69fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 70fa9e4066Sahrens mutex_destroy(&db->db_mtx); 71fa9e4066Sahrens cv_destroy(&db->db_changed); 72fa9e4066Sahrens refcount_destroy(&db->db_holds); 73fa9e4066Sahrens } 74fa9e4066Sahrens 75fa9e4066Sahrens /* 76fa9e4066Sahrens * dbuf hash table routines 77fa9e4066Sahrens */ 78fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 79fa9e4066Sahrens 80fa9e4066Sahrens static uint64_t dbuf_hash_count; 81fa9e4066Sahrens 82fa9e4066Sahrens static uint64_t 83fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 84fa9e4066Sahrens { 85fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 86fa9e4066Sahrens uint64_t crc = -1ULL; 87fa9e4066Sahrens 88fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 91fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 92fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 93fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 94fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 95fa9e4066Sahrens 96fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 97fa9e4066Sahrens 98fa9e4066Sahrens return (crc); 99fa9e4066Sahrens } 100fa9e4066Sahrens 101fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 102fa9e4066Sahrens 103fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 104fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 105fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 106fa9e4066Sahrens (dbuf)->db_level == (level) && \ 107fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 108fa9e4066Sahrens 109fa9e4066Sahrens dmu_buf_impl_t * 110fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 111fa9e4066Sahrens { 112fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 113fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 114fa9e4066Sahrens uint64_t obj = dn->dn_object; 115fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 116fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 117fa9e4066Sahrens dmu_buf_impl_t *db; 118fa9e4066Sahrens 119fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 120fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 121fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 122fa9e4066Sahrens mutex_enter(&db->db_mtx); 123ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 124fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 125fa9e4066Sahrens return (db); 126fa9e4066Sahrens } 127fa9e4066Sahrens mutex_exit(&db->db_mtx); 128fa9e4066Sahrens } 129fa9e4066Sahrens } 130fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 131fa9e4066Sahrens return (NULL); 132fa9e4066Sahrens } 133fa9e4066Sahrens 134fa9e4066Sahrens /* 135fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 136fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 137fa9e4066Sahrens * will be returned and the new element will not be inserted. 138fa9e4066Sahrens * Otherwise returns NULL. 139fa9e4066Sahrens */ 140fa9e4066Sahrens static dmu_buf_impl_t * 141fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 142fa9e4066Sahrens { 143fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 144fa9e4066Sahrens objset_impl_t *os = db->db_objset; 145fa9e4066Sahrens uint64_t obj = db->db.db_object; 146fa9e4066Sahrens int level = db->db_level; 147fa9e4066Sahrens uint64_t blkid = db->db_blkid; 148fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 149fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 150fa9e4066Sahrens dmu_buf_impl_t *dbf; 151fa9e4066Sahrens 152fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 153fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 154fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 155fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 156ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 157fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 158fa9e4066Sahrens return (dbf); 159fa9e4066Sahrens } 160fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 161fa9e4066Sahrens } 162fa9e4066Sahrens } 163fa9e4066Sahrens 164fa9e4066Sahrens mutex_enter(&db->db_mtx); 165fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 166fa9e4066Sahrens h->hash_table[idx] = db; 167fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 168fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 169fa9e4066Sahrens 170fa9e4066Sahrens return (NULL); 171fa9e4066Sahrens } 172fa9e4066Sahrens 173fa9e4066Sahrens /* 174fa9e4066Sahrens * Remove an entry from the hash table. This operation will 175fa9e4066Sahrens * fail if there are any existing holds on the db. 176fa9e4066Sahrens */ 177fa9e4066Sahrens static void 178fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 179fa9e4066Sahrens { 180fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 181fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 182fa9e4066Sahrens db->db_level, db->db_blkid); 183fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 184fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 185fa9e4066Sahrens 186fa9e4066Sahrens /* 187fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 188fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 189fa9e4066Sahrens */ 190fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 191ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 192fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 193fa9e4066Sahrens 194fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 195fa9e4066Sahrens dbp = &h->hash_table[idx]; 196fa9e4066Sahrens while ((dbf = *dbp) != db) { 197fa9e4066Sahrens dbp = &dbf->db_hash_next; 198fa9e4066Sahrens ASSERT(dbf != NULL); 199fa9e4066Sahrens } 200fa9e4066Sahrens *dbp = db->db_hash_next; 201fa9e4066Sahrens db->db_hash_next = NULL; 202fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 203fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 204fa9e4066Sahrens } 205fa9e4066Sahrens 206ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 207fa9e4066Sahrens 208fa9e4066Sahrens static void 209fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 210fa9e4066Sahrens { 211fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 212fa9e4066Sahrens 213fa9e4066Sahrens if (db->db_level != 0 || db->db_d.db_evict_func == NULL) 214fa9e4066Sahrens return; 215fa9e4066Sahrens 216fa9e4066Sahrens if (db->db_d.db_user_data_ptr_ptr) 217fa9e4066Sahrens *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 218fa9e4066Sahrens db->db_d.db_evict_func(&db->db, db->db_d.db_user_ptr); 219fa9e4066Sahrens db->db_d.db_user_ptr = NULL; 220fa9e4066Sahrens db->db_d.db_user_data_ptr_ptr = NULL; 221fa9e4066Sahrens db->db_d.db_evict_func = NULL; 222fa9e4066Sahrens } 223fa9e4066Sahrens 224ea8dc4b6Seschrock void 225ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 226ea8dc4b6Seschrock { 227ea8dc4b6Seschrock int i; 228ea8dc4b6Seschrock 229ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 230ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 231ea8dc4b6Seschrock 232ea8dc4b6Seschrock #ifdef ZFS_DEBUG 233ea8dc4b6Seschrock for (i = 0; i < TXG_SIZE; i++) { 234ea8dc4b6Seschrock ASSERT(!list_link_active(&db->db_dirty_node[i])); 235ea8dc4b6Seschrock ASSERT(db->db_level != 0 || db->db_d.db_data_old[i] == NULL); 236ea8dc4b6Seschrock } 237ea8dc4b6Seschrock #endif 238ea8dc4b6Seschrock dbuf_clear(db); 239ea8dc4b6Seschrock dbuf_destroy(db); 240ea8dc4b6Seschrock } 241ea8dc4b6Seschrock 242fa9e4066Sahrens void 243fa9e4066Sahrens dbuf_init(void) 244fa9e4066Sahrens { 245ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 246fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 247fa9e4066Sahrens int i; 248fa9e4066Sahrens 249fa9e4066Sahrens /* 250fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 251ea8dc4b6Seschrock * with an average 4K block size. The table will take up 252ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 253fa9e4066Sahrens */ 254ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 255fa9e4066Sahrens hsize <<= 1; 256fa9e4066Sahrens 257ea8dc4b6Seschrock retry: 258fa9e4066Sahrens h->hash_table_mask = hsize - 1; 259ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 260ea8dc4b6Seschrock if (h->hash_table == NULL) { 261ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 262ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 263ea8dc4b6Seschrock hsize >>= 1; 264ea8dc4b6Seschrock goto retry; 265ea8dc4b6Seschrock } 266fa9e4066Sahrens 267fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 268fa9e4066Sahrens sizeof (dmu_buf_impl_t), 269fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 270fa9e4066Sahrens dbuf_tq = taskq_create("dbuf_tq", 8, maxclsyspri, 50, INT_MAX, 271fa9e4066Sahrens TASKQ_PREPOPULATE); 272fa9e4066Sahrens 273fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 274fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 275fa9e4066Sahrens } 276fa9e4066Sahrens 277fa9e4066Sahrens void 278fa9e4066Sahrens dbuf_fini(void) 279fa9e4066Sahrens { 280fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 281fa9e4066Sahrens int i; 282fa9e4066Sahrens 283fa9e4066Sahrens taskq_destroy(dbuf_tq); 284fa9e4066Sahrens dbuf_tq = NULL; 285fa9e4066Sahrens 286fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 287fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 288fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 289fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 290fa9e4066Sahrens } 291fa9e4066Sahrens 292fa9e4066Sahrens /* 293fa9e4066Sahrens * Other stuff. 294fa9e4066Sahrens */ 295fa9e4066Sahrens 2969c9dc39aSek #ifdef ZFS_DEBUG 297fa9e4066Sahrens static void 298fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 299fa9e4066Sahrens { 300fa9e4066Sahrens int i; 301fa9e4066Sahrens dnode_t *dn = db->db_dnode; 302fa9e4066Sahrens 303fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 304fa9e4066Sahrens 305fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 306fa9e4066Sahrens return; 307fa9e4066Sahrens 308fa9e4066Sahrens ASSERT(db->db_objset != NULL); 309fa9e4066Sahrens if (dn == NULL) { 310fa9e4066Sahrens ASSERT(db->db_parent == NULL); 311fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 312fa9e4066Sahrens } else { 313fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 314fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 315fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 316ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID || 317ea8dc4b6Seschrock list_head(&dn->dn_dbufs)); 318fa9e4066Sahrens } 319fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 320fa9e4066Sahrens ASSERT(dn != NULL); 321fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen); 322fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 323fa9e4066Sahrens } else { 324fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 325fa9e4066Sahrens } 326fa9e4066Sahrens 327fa9e4066Sahrens if (db->db_level == 0) { 328fa9e4066Sahrens /* we can be momentarily larger in dnode_set_blksz() */ 329fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID && dn) { 330fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 331fa9e4066Sahrens } 332ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) { 333fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 334fa9e4066Sahrens /* 335fa9e4066Sahrens * it should only be modified in syncing 336fa9e4066Sahrens * context, so make sure we only have 337fa9e4066Sahrens * one copy of the data. 338fa9e4066Sahrens */ 339fa9e4066Sahrens ASSERT(db->db_d.db_data_old[i] == NULL || 340fa9e4066Sahrens db->db_d.db_data_old[i] == db->db_buf); 341fa9e4066Sahrens } 342fa9e4066Sahrens } 343fa9e4066Sahrens } 344fa9e4066Sahrens 345fa9e4066Sahrens /* verify db->db_blkptr */ 346fa9e4066Sahrens if (db->db_blkptr) { 347fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 348fa9e4066Sahrens /* db is pointed to by the dnode */ 349fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 350ea8dc4b6Seschrock if (db->db.db_object == DMU_META_DNODE_OBJECT) 351fa9e4066Sahrens ASSERT(db->db_parent == NULL); 352fa9e4066Sahrens else 353fa9e4066Sahrens ASSERT(db->db_parent != NULL); 354fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 355fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 356fa9e4066Sahrens } else { 357fa9e4066Sahrens /* db is pointed to by an indirect block */ 358fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 359fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 360fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 361fa9e4066Sahrens db->db.db_object); 362fa9e4066Sahrens /* 363fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 364fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 365fa9e4066Sahrens * grows. safe to do this now? 366fa9e4066Sahrens */ 367fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 368fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 369fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 370fa9e4066Sahrens db->db_blkid % epb)); 371fa9e4066Sahrens } 372fa9e4066Sahrens } 373fa9e4066Sahrens } 374fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 375fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 376fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 377fa9e4066Sahrens /* 378fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 379fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 380fa9e4066Sahrens * data when we evict this buffer. 381fa9e4066Sahrens */ 382fa9e4066Sahrens if (db->db_dirtycnt == 0) { 383fa9e4066Sahrens uint64_t *buf = db->db.db_data; 384fa9e4066Sahrens int i; 385fa9e4066Sahrens 386fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 387fa9e4066Sahrens ASSERT(buf[i] == 0); 388fa9e4066Sahrens } 389fa9e4066Sahrens } 390fa9e4066Sahrens } 391fa9e4066Sahrens } 3929c9dc39aSek #endif 393fa9e4066Sahrens 394fa9e4066Sahrens static void 395fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 396fa9e4066Sahrens { 397fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 398fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_user_data_ptr_ptr) { 399fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 400fa9e4066Sahrens *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 401fa9e4066Sahrens } 402fa9e4066Sahrens } 403fa9e4066Sahrens 404fa9e4066Sahrens static void 405fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 406fa9e4066Sahrens { 407fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 408ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 409fa9e4066Sahrens db->db_buf = buf; 410ea8dc4b6Seschrock if (buf != NULL) { 411ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 412ea8dc4b6Seschrock db->db.db_data = buf->b_data; 413ea8dc4b6Seschrock if (!arc_released(buf)) 414ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 415ea8dc4b6Seschrock dbuf_update_data(db); 416ea8dc4b6Seschrock } else { 417ea8dc4b6Seschrock dbuf_evict_user(db); 418ea8dc4b6Seschrock db->db.db_data = NULL; 419ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 420ea8dc4b6Seschrock } 421fa9e4066Sahrens } 422fa9e4066Sahrens 423fa9e4066Sahrens uint64_t 424fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 425fa9e4066Sahrens { 426fa9e4066Sahrens if (dn->dn_datablkshift) { 427fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 428fa9e4066Sahrens } else { 429fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 430fa9e4066Sahrens return (0); 431fa9e4066Sahrens } 432fa9e4066Sahrens } 433fa9e4066Sahrens 434fa9e4066Sahrens static void 435fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 436fa9e4066Sahrens { 437fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 438fa9e4066Sahrens 439fa9e4066Sahrens mutex_enter(&db->db_mtx); 440fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 441fa9e4066Sahrens /* 442fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 443fa9e4066Sahrens */ 444fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 445ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 446fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 447fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 448fa9e4066Sahrens /* we were freed in flight; disregard any error */ 449fa9e4066Sahrens arc_release(buf, db); 450fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4516b4acc8bSahrens arc_buf_freeze(buf); 452fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 453fa9e4066Sahrens dbuf_set_data(db, buf); 454fa9e4066Sahrens db->db_state = DB_CACHED; 455fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 456fa9e4066Sahrens dbuf_set_data(db, buf); 457fa9e4066Sahrens db->db_state = DB_CACHED; 458fa9e4066Sahrens } else { 459fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 460fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 461ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 462ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 463fa9e4066Sahrens } 464fa9e4066Sahrens cv_broadcast(&db->db_changed); 465fa9e4066Sahrens mutex_exit(&db->db_mtx); 466ea8dc4b6Seschrock dbuf_rele(db, NULL); 467fa9e4066Sahrens } 468fa9e4066Sahrens 469ea8dc4b6Seschrock static void 47013506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 471fa9e4066Sahrens { 472fa9e4066Sahrens blkptr_t *bp; 473ea8dc4b6Seschrock zbookmark_t zb; 47413506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 475fa9e4066Sahrens 476fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 477fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 478fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 479ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 480ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 481ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 482fa9e4066Sahrens 483fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 484fa9e4066Sahrens ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size); 485ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 486fa9e4066Sahrens if (db->db.db_size < DN_MAX_BONUSLEN) 487ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 488ea8dc4b6Seschrock bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data, 489fa9e4066Sahrens db->db.db_size); 490ea8dc4b6Seschrock dbuf_update_data(db); 491fa9e4066Sahrens db->db_state = DB_CACHED; 492fa9e4066Sahrens mutex_exit(&db->db_mtx); 493fa9e4066Sahrens return; 494fa9e4066Sahrens } 495fa9e4066Sahrens 496fa9e4066Sahrens if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid)) 497fa9e4066Sahrens bp = NULL; 498fa9e4066Sahrens else 499fa9e4066Sahrens bp = db->db_blkptr; 500fa9e4066Sahrens 501fa9e4066Sahrens if (bp == NULL) 502fa9e4066Sahrens dprintf_dbuf(db, "blkptr: %s\n", "NULL"); 503fa9e4066Sahrens else 504fa9e4066Sahrens dprintf_dbuf_bp(db, bp, "%s", "blkptr:"); 505fa9e4066Sahrens 506fa9e4066Sahrens if (bp == NULL || BP_IS_HOLE(bp)) { 507*ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 508*ad23a2dbSjohansen 509fa9e4066Sahrens ASSERT(bp == NULL || BP_IS_HOLE(bp)); 510fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 511*ad23a2dbSjohansen db->db.db_size, db, type)); 512fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 513fa9e4066Sahrens db->db_state = DB_CACHED; 51413506d1eSmaybee *flags |= DB_RF_CACHED; 515fa9e4066Sahrens mutex_exit(&db->db_mtx); 516fa9e4066Sahrens return; 517fa9e4066Sahrens } 518fa9e4066Sahrens 519fa9e4066Sahrens db->db_state = DB_READ; 520fa9e4066Sahrens mutex_exit(&db->db_mtx); 521fa9e4066Sahrens 522ea8dc4b6Seschrock zb.zb_objset = db->db_objset->os_dsl_dataset ? 523ea8dc4b6Seschrock db->db_objset->os_dsl_dataset->ds_object : 0; 524ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 525ea8dc4b6Seschrock zb.zb_level = db->db_level; 526ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 527ea8dc4b6Seschrock 528ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 529fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 530fa9e4066Sahrens (void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp, 531fa9e4066Sahrens db->db_level > 0 ? byteswap_uint64_array : 532fa9e4066Sahrens dmu_ot[db->db_dnode->dn_type].ot_byteswap, 533fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 53413506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 53513506d1eSmaybee &aflags, &zb); 53613506d1eSmaybee if (aflags & ARC_CACHED) 53713506d1eSmaybee *flags |= DB_RF_CACHED; 538fa9e4066Sahrens } 539fa9e4066Sahrens 540ea8dc4b6Seschrock int 541ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 542fa9e4066Sahrens { 543ea8dc4b6Seschrock int err = 0; 544ea8dc4b6Seschrock int havepzio = (zio != NULL); 54513506d1eSmaybee int prefetch; 546fa9e4066Sahrens 547fa9e4066Sahrens /* 548fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 549fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 550fa9e4066Sahrens */ 551fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 552fa9e4066Sahrens 553ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 554ea8dc4b6Seschrock rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 555ea8dc4b6Seschrock 55613506d1eSmaybee prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 55713506d1eSmaybee (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL; 55813506d1eSmaybee 559ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 560ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 561ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 56213506d1eSmaybee if (prefetch) 56313506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 56413506d1eSmaybee db->db.db_size, TRUE); 565fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 566ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 567ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 568ea8dc4b6Seschrock if (zio == NULL) { 569ea8dc4b6Seschrock zio = zio_root(db->db_dnode->dn_objset->os_spa, 570ea8dc4b6Seschrock NULL, NULL, ZIO_FLAG_CANFAIL); 571ea8dc4b6Seschrock } 57213506d1eSmaybee dbuf_read_impl(db, zio, &flags); 57313506d1eSmaybee 574ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 575ea8dc4b6Seschrock 57613506d1eSmaybee if (prefetch) 577ea8dc4b6Seschrock dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 57813506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 579ea8dc4b6Seschrock 580fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 581fa9e4066Sahrens rw_exit(&db->db_dnode->dn_struct_rwlock); 582fa9e4066Sahrens 583ea8dc4b6Seschrock if (!havepzio) 584ea8dc4b6Seschrock err = zio_wait(zio); 585ea8dc4b6Seschrock } else { 58613506d1eSmaybee mutex_exit(&db->db_mtx); 58713506d1eSmaybee if (prefetch) 58813506d1eSmaybee dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 58913506d1eSmaybee db->db.db_size, TRUE); 590ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 591ea8dc4b6Seschrock rw_exit(&db->db_dnode->dn_struct_rwlock); 59213506d1eSmaybee 59313506d1eSmaybee mutex_enter(&db->db_mtx); 594ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 595ea8dc4b6Seschrock while (db->db_state == DB_READ || 596ea8dc4b6Seschrock db->db_state == DB_FILL) { 597ea8dc4b6Seschrock ASSERT(db->db_state == DB_READ || 598ea8dc4b6Seschrock (flags & DB_RF_HAVESTRUCT) == 0); 599ea8dc4b6Seschrock cv_wait(&db->db_changed, &db->db_mtx); 600ea8dc4b6Seschrock } 601ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 602ea8dc4b6Seschrock err = EIO; 603ea8dc4b6Seschrock } 604ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 605fa9e4066Sahrens } 606fa9e4066Sahrens 607ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 608ea8dc4b6Seschrock return (err); 609fa9e4066Sahrens } 610fa9e4066Sahrens 611fa9e4066Sahrens static void 612fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 613fa9e4066Sahrens { 614fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 615ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 616fa9e4066Sahrens mutex_enter(&db->db_mtx); 617fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 618fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 619fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 620*ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 621*ad23a2dbSjohansen 622ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 623fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 624fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 625*ad23a2dbSjohansen db->db.db_size, db, type)); 626fa9e4066Sahrens db->db_state = DB_FILL; 627fa9e4066Sahrens } else { 628fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 629fa9e4066Sahrens } 630fa9e4066Sahrens mutex_exit(&db->db_mtx); 631fa9e4066Sahrens } 632fa9e4066Sahrens 633fa9e4066Sahrens /* 634fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 635fa9e4066Sahrens * buffers, that have been modified in a previous transaction 636fa9e4066Sahrens * group, before we modify them in the current active group. 637fa9e4066Sahrens * 638fa9e4066Sahrens * This function is used in two places: when we are dirtying a 639fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 640fa9e4066Sahrens * a range in a dnode that includes this buffer. 641fa9e4066Sahrens * 642fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 643fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 644fa9e4066Sahrens * dbuf list for the dnode. 645fa9e4066Sahrens */ 646fa9e4066Sahrens static void 647fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 648fa9e4066Sahrens { 649fa9e4066Sahrens arc_buf_t **quiescing, **syncing; 650*ad23a2dbSjohansen arc_buf_contents_t type; 651fa9e4066Sahrens 652fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 653fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 654ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 655fa9e4066Sahrens 6566b4acc8bSahrens quiescing = &db->db_d.db_data_old[(txg-1)&TXG_MASK]; 6576b4acc8bSahrens syncing = &db->db_d.db_data_old[(txg-2)&TXG_MASK]; 658fa9e4066Sahrens 659fa9e4066Sahrens /* 660fa9e4066Sahrens * If this buffer is referenced from the current quiescing 661fa9e4066Sahrens * transaction group: either make a copy and reset the reference 662fa9e4066Sahrens * to point to the copy, or (if there a no active holders) just 663fa9e4066Sahrens * null out the current db_data pointer. 664fa9e4066Sahrens */ 665fa9e4066Sahrens if (*quiescing == db->db_buf) { 666fa9e4066Sahrens /* 667fa9e4066Sahrens * If the quiescing txg is "dirty", then we better not 668fa9e4066Sahrens * be referencing the same buffer from the syncing txg. 669fa9e4066Sahrens */ 670fa9e4066Sahrens ASSERT(*syncing != db->db_buf); 671fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 672ea8dc4b6Seschrock int size = db->db.db_size; 673*ad23a2dbSjohansen type = DBUF_GET_BUFC_TYPE(db); 674fa9e4066Sahrens *quiescing = arc_buf_alloc( 675*ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 676fa9e4066Sahrens bcopy(db->db.db_data, (*quiescing)->b_data, size); 677fa9e4066Sahrens } else { 678ea8dc4b6Seschrock dbuf_set_data(db, NULL); 679fa9e4066Sahrens } 680fa9e4066Sahrens return; 681fa9e4066Sahrens } 682fa9e4066Sahrens 683fa9e4066Sahrens /* 684fa9e4066Sahrens * If this buffer is referenced from the current syncing 685fa9e4066Sahrens * transaction group: either 686fa9e4066Sahrens * 1 - make a copy and reset the reference, or 687fa9e4066Sahrens * 2 - if there are no holders, just null the current db_data. 688fa9e4066Sahrens */ 689fa9e4066Sahrens if (*syncing == db->db_buf) { 690fa9e4066Sahrens ASSERT3P(*quiescing, ==, NULL); 691fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, ==, 1); 692fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 693ea8dc4b6Seschrock int size = db->db.db_size; 694*ad23a2dbSjohansen type = DBUF_GET_BUFC_TYPE(db); 695fa9e4066Sahrens /* we can't copy if we have already started a write */ 696fa9e4066Sahrens ASSERT(*syncing != db->db_data_pending); 697fa9e4066Sahrens *syncing = arc_buf_alloc( 698*ad23a2dbSjohansen db->db_dnode->dn_objset->os_spa, size, db, type); 699fa9e4066Sahrens bcopy(db->db.db_data, (*syncing)->b_data, size); 700fa9e4066Sahrens } else { 701ea8dc4b6Seschrock dbuf_set_data(db, NULL); 702fa9e4066Sahrens } 703fa9e4066Sahrens } 704fa9e4066Sahrens } 705fa9e4066Sahrens 706ea8dc4b6Seschrock /* 707ea8dc4b6Seschrock * This is the "bonus buffer" version of the above routine 708ea8dc4b6Seschrock */ 709ea8dc4b6Seschrock static void 710ea8dc4b6Seschrock dbuf_fix_old_bonus_data(dmu_buf_impl_t *db, uint64_t txg) 711ea8dc4b6Seschrock { 7126b4acc8bSahrens arc_buf_t **quiescing, **syncing; 713ea8dc4b6Seschrock 714ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 715ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 716ea8dc4b6Seschrock ASSERT(db->db_blkid == DB_BONUS_BLKID); 717ea8dc4b6Seschrock 718ea8dc4b6Seschrock quiescing = &db->db_d.db_data_old[(txg-1)&TXG_MASK]; 719ea8dc4b6Seschrock syncing = &db->db_d.db_data_old[(txg-2)&TXG_MASK]; 720ea8dc4b6Seschrock 721ea8dc4b6Seschrock if (*quiescing == db->db.db_data) { 722ea8dc4b6Seschrock ASSERT(*syncing != db->db.db_data); 723ea8dc4b6Seschrock *quiescing = zio_buf_alloc(DN_MAX_BONUSLEN); 724ea8dc4b6Seschrock bcopy(db->db.db_data, *quiescing, DN_MAX_BONUSLEN); 725ea8dc4b6Seschrock } else if (*syncing == db->db.db_data) { 726ea8dc4b6Seschrock ASSERT3P(*quiescing, ==, NULL); 727ea8dc4b6Seschrock ASSERT3U(db->db_dirtycnt, ==, 1); 728ea8dc4b6Seschrock *syncing = zio_buf_alloc(DN_MAX_BONUSLEN); 729ea8dc4b6Seschrock bcopy(db->db.db_data, *syncing, DN_MAX_BONUSLEN); 730ea8dc4b6Seschrock } 731ea8dc4b6Seschrock } 732ea8dc4b6Seschrock 733fa9e4066Sahrens void 734fa9e4066Sahrens dbuf_unoverride(dmu_buf_impl_t *db, uint64_t txg) 735fa9e4066Sahrens { 736ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 737fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 738c5c6ffa0Smaybee ASSERT(db->db_d.db_overridden_by[txg&TXG_MASK] != IN_DMU_SYNC); 739c5c6ffa0Smaybee 740c5c6ffa0Smaybee if (db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 741fa9e4066Sahrens /* free this block */ 742fa9e4066Sahrens ASSERT(list_link_active(&db->db_dirty_node[txg&TXG_MASK]) || 743fa9e4066Sahrens db->db_dnode->dn_free_txg == txg); 744fa9e4066Sahrens if (!BP_IS_HOLE(db->db_d.db_overridden_by[txg&TXG_MASK])) { 745fa9e4066Sahrens /* XXX can get silent EIO here */ 746fa9e4066Sahrens (void) arc_free(NULL, db->db_dnode->dn_objset->os_spa, 747fa9e4066Sahrens txg, db->db_d.db_overridden_by[txg&TXG_MASK], 748fa9e4066Sahrens NULL, NULL, ARC_WAIT); 749fa9e4066Sahrens } 750fa9e4066Sahrens kmem_free(db->db_d.db_overridden_by[txg&TXG_MASK], 751fa9e4066Sahrens sizeof (blkptr_t)); 752fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] = NULL; 7536b4acc8bSahrens /* 7546b4acc8bSahrens * Release the already-written buffer, so we leave it in 7556b4acc8bSahrens * a consistent dirty state. Note that all callers are 7566b4acc8bSahrens * modifying the buffer, so they will immediately do 7576b4acc8bSahrens * another (redundant) arc_release(). Therefore, leave 7586b4acc8bSahrens * the buf thawed to save the effort of freezing & 7596b4acc8bSahrens * immediately re-thawing it. 7606b4acc8bSahrens */ 761fa9e4066Sahrens arc_release(db->db_d.db_data_old[txg&TXG_MASK], db); 762fa9e4066Sahrens } 763fa9e4066Sahrens } 764fa9e4066Sahrens 765fa9e4066Sahrens void 766fa9e4066Sahrens dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) 767fa9e4066Sahrens { 768fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 769fa9e4066Sahrens uint64_t txg = tx->tx_txg; 770fa9e4066Sahrens 771fa9e4066Sahrens dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks); 772fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 773fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 774fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 775ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 776ea8dc4b6Seschrock if (db->db_level != 0) 777fa9e4066Sahrens continue; 778fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 779fa9e4066Sahrens if (db->db_blkid < blkid || 780fa9e4066Sahrens db->db_blkid >= blkid+nblks) 781fa9e4066Sahrens continue; 782fa9e4066Sahrens 783fa9e4066Sahrens /* found a level 0 buffer in the range */ 784fa9e4066Sahrens if (dbuf_undirty(db, tx)) 785fa9e4066Sahrens continue; 786fa9e4066Sahrens 787fa9e4066Sahrens mutex_enter(&db->db_mtx); 788ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 789ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 790fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 791fa9e4066Sahrens mutex_exit(&db->db_mtx); 792fa9e4066Sahrens continue; 793fa9e4066Sahrens } 794c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 795c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 796fa9e4066Sahrens db->db_d.db_freed_in_flight = TRUE; 797fa9e4066Sahrens mutex_exit(&db->db_mtx); 798fa9e4066Sahrens continue; 799fa9e4066Sahrens } 800ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 801ea8dc4b6Seschrock ASSERT(db->db_buf); 802ea8dc4b6Seschrock dbuf_clear(db); 803ea8dc4b6Seschrock continue; 804ea8dc4b6Seschrock } 805ea8dc4b6Seschrock /* The dbuf is CACHED and referenced */ 806fa9e4066Sahrens 807ea8dc4b6Seschrock if (!list_link_active(&db->db_dirty_node[txg & TXG_MASK])) { 808ea8dc4b6Seschrock /* 80944eda4d7Smaybee * This dbuf is not currently dirty. Either 810ea8dc4b6Seschrock * uncache it (if its not referenced in the open 811ea8dc4b6Seschrock * context) or reset its contents to empty. 812ea8dc4b6Seschrock */ 813ea8dc4b6Seschrock dbuf_fix_old_data(db, txg); 81444eda4d7Smaybee } else { 81544eda4d7Smaybee if (db->db_d.db_overridden_by[txg & TXG_MASK] != NULL) { 81644eda4d7Smaybee /* 81744eda4d7Smaybee * This dbuf is overridden. Clear that state. 81844eda4d7Smaybee */ 81944eda4d7Smaybee dbuf_unoverride(db, txg); 82044eda4d7Smaybee } 82144eda4d7Smaybee if (db->db_blkid > dn->dn_maxblkid) 82244eda4d7Smaybee dn->dn_maxblkid = db->db_blkid; 823ea8dc4b6Seschrock } 824ea8dc4b6Seschrock /* fill in with appropriate data */ 825ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 826ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 827fa9e4066Sahrens arc_release(db->db_buf, db); 828fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8296b4acc8bSahrens arc_buf_freeze(db->db_buf); 830fa9e4066Sahrens } 831ea8dc4b6Seschrock 832fa9e4066Sahrens mutex_exit(&db->db_mtx); 833fa9e4066Sahrens } 834fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 835fa9e4066Sahrens } 836fa9e4066Sahrens 837fa9e4066Sahrens static int 838ea8dc4b6Seschrock dbuf_new_block(dmu_buf_impl_t *db) 839fa9e4066Sahrens { 840fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 841fa9e4066Sahrens uint64_t birth_txg = 0; 842fa9e4066Sahrens 843fa9e4066Sahrens /* Don't count meta-objects */ 844fa9e4066Sahrens if (ds == NULL) 845fa9e4066Sahrens return (FALSE); 846fa9e4066Sahrens 847fa9e4066Sahrens /* 848fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 849fa9e4066Sahrens * If it's syncing, then db_dirtied will be set so we'll 850fa9e4066Sahrens * ignore db_blkptr. 851fa9e4066Sahrens */ 852fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); /* XXX strictly necessary? */ 853fa9e4066Sahrens /* If we have been dirtied since the last snapshot, its not new */ 854fa9e4066Sahrens if (db->db_dirtied) 855fa9e4066Sahrens birth_txg = db->db_dirtied; 856fa9e4066Sahrens else if (db->db_blkptr) 857fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 858fa9e4066Sahrens 859fa9e4066Sahrens if (birth_txg) 860ea8dc4b6Seschrock return (!dsl_dataset_block_freeable(ds, birth_txg)); 861fa9e4066Sahrens else 862fa9e4066Sahrens return (TRUE); 863fa9e4066Sahrens } 864fa9e4066Sahrens 865fa9e4066Sahrens void 866fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 867fa9e4066Sahrens { 868fa9e4066Sahrens arc_buf_t *buf, *obuf; 869fa9e4066Sahrens int osize = db->db.db_size; 870*ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 871fa9e4066Sahrens 872ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 873ea8dc4b6Seschrock 874fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 875fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 876fa9e4066Sahrens 877fa9e4066Sahrens /* 878fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 879fa9e4066Sahrens * is OK, because there can be no other references to the db 880fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 881fa9e4066Sahrens * be happening. 882fa9e4066Sahrens */ 883ea8dc4b6Seschrock /* 884ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 885ea8dc4b6Seschrock * value and returning that up to our callers 886ea8dc4b6Seschrock */ 887fa9e4066Sahrens dbuf_will_dirty(db, tx); 888fa9e4066Sahrens 889fa9e4066Sahrens /* create the data buffer for the new block */ 890*ad23a2dbSjohansen buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type); 891fa9e4066Sahrens 892fa9e4066Sahrens /* copy old block data to the new block */ 893fa9e4066Sahrens obuf = db->db_buf; 894f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 895fa9e4066Sahrens /* zero the remainder */ 896f65e61c0Sahrens if (size > osize) 897f65e61c0Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 898fa9e4066Sahrens 899fa9e4066Sahrens mutex_enter(&db->db_mtx); 900fa9e4066Sahrens dbuf_set_data(db, buf); 901ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 902fa9e4066Sahrens db->db.db_size = size; 903fa9e4066Sahrens 904fa9e4066Sahrens if (db->db_level == 0) 905fa9e4066Sahrens db->db_d.db_data_old[tx->tx_txg&TXG_MASK] = buf; 906fa9e4066Sahrens mutex_exit(&db->db_mtx); 907fa9e4066Sahrens 908fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 909fa9e4066Sahrens } 910fa9e4066Sahrens 911fa9e4066Sahrens void 912fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 913fa9e4066Sahrens { 914fa9e4066Sahrens dnode_t *dn = db->db_dnode; 915fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 916fa9e4066Sahrens int drop_struct_lock = FALSE; 917fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 918fa9e4066Sahrens 919fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 920fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 9219c9dc39aSek DMU_TX_DIRTY_BUF(tx, db); 922fa9e4066Sahrens 923fa9e4066Sahrens /* 924fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 925fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 926fa9e4066Sahrens * were already pre-dirtied in open context. 927fa9e4066Sahrens * XXX We may want to prohibit dirtying in syncing context even 928fa9e4066Sahrens * if they did pre-dirty. 929fa9e4066Sahrens */ 930fa9e4066Sahrens ASSERT(!(dmu_tx_is_syncing(tx) && 931fa9e4066Sahrens !BP_IS_HOLE(&dn->dn_objset->os_rootbp) && 932ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 933fa9e4066Sahrens dn->dn_objset->os_dsl_dataset != NULL && 934fa9e4066Sahrens !dsl_dir_is_private( 935fa9e4066Sahrens dn->dn_objset->os_dsl_dataset->ds_dir))); 936fa9e4066Sahrens 937fa9e4066Sahrens /* 938fa9e4066Sahrens * We make this assert for private objects as well, but after we 939fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 940fa9e4066Sahrens * in syncing context. 941fa9e4066Sahrens */ 942ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 943fa9e4066Sahrens dn->dn_dirtyctx == DN_UNDIRTIED || 944fa9e4066Sahrens dn->dn_dirtyctx == 945fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 946fa9e4066Sahrens 947fa9e4066Sahrens mutex_enter(&db->db_mtx); 948fa9e4066Sahrens /* XXX make this true for indirects too? */ 949fa9e4066Sahrens ASSERT(db->db_level != 0 || db->db_state == DB_CACHED || 950fa9e4066Sahrens db->db_state == DB_FILL); 951fa9e4066Sahrens 952fa9e4066Sahrens /* 953fa9e4066Sahrens * If this buffer is currently part of an "overridden" region, 954fa9e4066Sahrens * we now need to remove it from that region. 955fa9e4066Sahrens */ 956fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 957fa9e4066Sahrens db->db_d.db_overridden_by[txgoff] != NULL) { 958fa9e4066Sahrens dbuf_unoverride(db, tx->tx_txg); 959fa9e4066Sahrens } 960fa9e4066Sahrens 961fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 962fa9e4066Sahrens /* 963fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 964fa9e4066Sahrens * initialize the objset. 965fa9e4066Sahrens */ 966fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 967fa9e4066Sahrens !BP_IS_HOLE(&dn->dn_objset->os_rootbp)) { 968fa9e4066Sahrens dn->dn_dirtyctx = 969fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 970fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 971fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 972fa9e4066Sahrens } 973fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 974fa9e4066Sahrens 975fa9e4066Sahrens /* 976fa9e4066Sahrens * If this buffer is already dirty, we're done. 977fa9e4066Sahrens */ 978fa9e4066Sahrens if (list_link_active(&db->db_dirty_node[txgoff])) { 9796b4acc8bSahrens if (db->db_blkid != DB_BONUS_BLKID && db->db_level == 0 && 9806b4acc8bSahrens db->db.db_object != DMU_META_DNODE_OBJECT) 9816b4acc8bSahrens arc_buf_thaw(db->db_buf); 9826b4acc8bSahrens 983fa9e4066Sahrens mutex_exit(&db->db_mtx); 984fa9e4066Sahrens return; 985fa9e4066Sahrens } 986fa9e4066Sahrens 987fa9e4066Sahrens /* 988fa9e4066Sahrens * Only valid if not already dirty. 989fa9e4066Sahrens */ 990fa9e4066Sahrens ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 991fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 992fa9e4066Sahrens 993fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 994fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 995fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 996fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 997fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 998fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 999fa9e4066Sahrens 1000fa9e4066Sahrens /* 1001fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 1002fa9e4066Sahrens * mos, a spa os, or we're initializing the os. However, we are 1003fa9e4066Sahrens * allowed to dirty in syncing context provided we already 1004fa9e4066Sahrens * dirtied it in open context. Hence we must make this 1005fa9e4066Sahrens * assertion only if we're not already dirty. 1006fa9e4066Sahrens */ 1007fa9e4066Sahrens ASSERT(!dmu_tx_is_syncing(tx) || 1008fa9e4066Sahrens os->os_dsl_dataset == NULL || 1009fa9e4066Sahrens !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 1010fa9e4066Sahrens !BP_IS_HOLE(&os->os_rootbp)); 1011fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1012fa9e4066Sahrens 1013fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1014fa9e4066Sahrens 1015ea8dc4b6Seschrock /* 1016ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1017ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1018ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1019ea8dc4b6Seschrock */ 1020ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1021ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 1022ea8dc4b6Seschrock ASSERT(db->db_d.db_data_old[txgoff] == NULL); 1023ea8dc4b6Seschrock dbuf_fix_old_bonus_data(db, tx->tx_txg); 1024ea8dc4b6Seschrock db->db_d.db_data_old[txgoff] = db->db.db_data; 1025ea8dc4b6Seschrock } else if (db->db_level == 0) { 1026fa9e4066Sahrens /* 1027fa9e4066Sahrens * Release the data buffer from the cache so that we 1028fa9e4066Sahrens * can modify it without impacting possible other users 1029fa9e4066Sahrens * of this cached data block. Note that indirect blocks 1030fa9e4066Sahrens * and private objects are not released until the syncing 1031fa9e4066Sahrens * state (since they are only modified then). 1032fa9e4066Sahrens */ 1033fa9e4066Sahrens ASSERT(db->db_buf != NULL); 1034fa9e4066Sahrens ASSERT(db->db_d.db_data_old[txgoff] == NULL); 1035ea8dc4b6Seschrock if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1036fa9e4066Sahrens arc_release(db->db_buf, db); 1037fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 1038fa9e4066Sahrens ASSERT(db->db_buf != NULL); 1039fa9e4066Sahrens } 1040fa9e4066Sahrens db->db_d.db_data_old[txgoff] = db->db_buf; 1041fa9e4066Sahrens } 1042fa9e4066Sahrens 1043fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1044fa9e4066Sahrens /* 1045fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1046fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1047fa9e4066Sahrens * happened after the free. 1048fa9e4066Sahrens */ 1049fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 1050fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1051fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 1052fa9e4066Sahrens } 1053fa9e4066Sahrens 1054fa9e4066Sahrens db->db_dirtied = tx->tx_txg; 1055fa9e4066Sahrens list_insert_tail(&dn->dn_dirty_dbufs[txgoff], db); 1056fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1057fa9e4066Sahrens 1058fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID) { 1059ea8dc4b6Seschrock /* 1060ea8dc4b6Seschrock * Update the accounting. 1061ea8dc4b6Seschrock */ 1062ea8dc4b6Seschrock if (!dbuf_new_block(db) && db->db_blkptr) { 1063fa9e4066Sahrens /* 1064fa9e4066Sahrens * This is only a guess -- if the dbuf is dirty 1065fa9e4066Sahrens * in a previous txg, we don't know how much 1066fa9e4066Sahrens * space it will use on disk yet. We should 1067fa9e4066Sahrens * really have the struct_rwlock to access 1068fa9e4066Sahrens * db_blkptr, but since this is just a guess, 1069fa9e4066Sahrens * it's OK if we get an odd answer. 1070fa9e4066Sahrens */ 1071fa9e4066Sahrens dnode_willuse_space(dn, 107299653d4eSeschrock -bp_get_dasize(os->os_spa, db->db_blkptr), tx); 1073fa9e4066Sahrens } 1074fa9e4066Sahrens dnode_willuse_space(dn, db->db.db_size, tx); 1075fa9e4066Sahrens } 1076fa9e4066Sahrens 1077fa9e4066Sahrens /* 1078fa9e4066Sahrens * This buffer is now part of this txg 1079fa9e4066Sahrens */ 1080fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1081fa9e4066Sahrens db->db_dirtycnt += 1; 1082fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1083fa9e4066Sahrens 1084fa9e4066Sahrens mutex_exit(&db->db_mtx); 1085fa9e4066Sahrens 1086fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1087fa9e4066Sahrens dnode_setdirty(dn, tx); 1088fa9e4066Sahrens return; 1089fa9e4066Sahrens } 1090fa9e4066Sahrens 109144eda4d7Smaybee if (db->db_level == 0) { 1092fa9e4066Sahrens dnode_new_blkid(dn, db->db_blkid, tx); 109344eda4d7Smaybee ASSERT(dn->dn_maxblkid >= db->db_blkid); 109444eda4d7Smaybee } 1095fa9e4066Sahrens 1096fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1097fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1098fa9e4066Sahrens drop_struct_lock = TRUE; 1099fa9e4066Sahrens } 1100fa9e4066Sahrens 110144eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1102fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1103fa9e4066Sahrens dmu_buf_impl_t *parent; 1104fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1105fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1106fa9e4066Sahrens if (drop_struct_lock) 1107fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1108fa9e4066Sahrens dbuf_dirty(parent, tx); 1109ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1110fa9e4066Sahrens } else { 1111fa9e4066Sahrens if (drop_struct_lock) 1112fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1113fa9e4066Sahrens } 1114fa9e4066Sahrens 1115fa9e4066Sahrens dnode_setdirty(dn, tx); 1116fa9e4066Sahrens } 1117fa9e4066Sahrens 1118fa9e4066Sahrens static int 1119fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1120fa9e4066Sahrens { 1121fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1122fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1123ea8dc4b6Seschrock int64_t holds; 1124fa9e4066Sahrens 1125fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1126ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1127fa9e4066Sahrens 1128fa9e4066Sahrens mutex_enter(&db->db_mtx); 1129fa9e4066Sahrens 1130fa9e4066Sahrens /* 1131fa9e4066Sahrens * If this buffer is not dirty, we're done. 1132fa9e4066Sahrens */ 1133fa9e4066Sahrens if (!list_link_active(&db->db_dirty_node[txgoff])) { 1134fa9e4066Sahrens mutex_exit(&db->db_mtx); 1135fa9e4066Sahrens return (0); 1136fa9e4066Sahrens } 1137fa9e4066Sahrens 1138fa9e4066Sahrens /* 1139fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1140fa9e4066Sahrens * it, since one of the current holders may be in the 1141fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1142fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1143fa9e4066Sahrens */ 1144fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1145fa9e4066Sahrens mutex_exit(&db->db_mtx); 114644eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1147fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1148fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1149fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1150fa9e4066Sahrens return (0); 1151fa9e4066Sahrens } 1152fa9e4066Sahrens 1153fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1154fa9e4066Sahrens 1155fa9e4066Sahrens dbuf_unoverride(db, tx->tx_txg); 1156fa9e4066Sahrens 1157fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1158fa9e4066Sahrens if (db->db_level == 0) { 1159fa9e4066Sahrens ASSERT(db->db_buf != NULL); 1160fa9e4066Sahrens ASSERT(db->db_d.db_data_old[txgoff] != NULL); 1161fa9e4066Sahrens if (db->db_d.db_data_old[txgoff] != db->db_buf) 1162ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref( 1163ea8dc4b6Seschrock db->db_d.db_data_old[txgoff], db) == 1); 1164fa9e4066Sahrens db->db_d.db_data_old[txgoff] = NULL; 1165fa9e4066Sahrens } 1166fa9e4066Sahrens 1167fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1168fa9e4066Sahrens /* XXX undo db_dirtied? but how? */ 1169fa9e4066Sahrens /* db->db_dirtied = tx->tx_txg; */ 1170fa9e4066Sahrens 1171fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1172fa9e4066Sahrens list_remove(&dn->dn_dirty_dbufs[txgoff], db); 1173fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1174fa9e4066Sahrens 1175fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1176fa9e4066Sahrens db->db_dirtycnt -= 1; 1177fa9e4066Sahrens 1178ea8dc4b6Seschrock if ((holds = refcount_remove(&db->db_holds, 1179ea8dc4b6Seschrock (void *)(uintptr_t)tx->tx_txg)) == 0) { 1180ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1181fa9e4066Sahrens 1182ea8dc4b6Seschrock ASSERT(arc_released(buf)); 1183ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1184ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1185fa9e4066Sahrens dbuf_evict(db); 1186fa9e4066Sahrens return (1); 1187fa9e4066Sahrens } 1188ea8dc4b6Seschrock ASSERT(holds > 0); 1189fa9e4066Sahrens 1190fa9e4066Sahrens mutex_exit(&db->db_mtx); 1191fa9e4066Sahrens return (0); 1192fa9e4066Sahrens } 1193fa9e4066Sahrens 1194fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1195fa9e4066Sahrens void 1196fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1197fa9e4066Sahrens { 1198fa9e4066Sahrens int rf = DB_RF_MUST_SUCCEED; 1199fa9e4066Sahrens 1200fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1201fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1202fa9e4066Sahrens 1203fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1204fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1205ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1206fa9e4066Sahrens dbuf_dirty(db, tx); 1207fa9e4066Sahrens } 1208fa9e4066Sahrens 1209fa9e4066Sahrens void 1210ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1211fa9e4066Sahrens { 1212ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1213ea8dc4b6Seschrock 1214ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1215fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1216fa9e4066Sahrens ASSERT(db->db_level == 0); 1217fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1218fa9e4066Sahrens 1219ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1220fa9e4066Sahrens dmu_tx_private_ok(tx)); 1221fa9e4066Sahrens 1222fa9e4066Sahrens dbuf_noread(db); 1223fa9e4066Sahrens dbuf_dirty(db, tx); 1224fa9e4066Sahrens } 1225fa9e4066Sahrens 1226fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1227fa9e4066Sahrens /* ARGSUSED */ 1228fa9e4066Sahrens void 1229fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1230fa9e4066Sahrens { 1231fa9e4066Sahrens mutex_enter(&db->db_mtx); 12329c9dc39aSek DBUF_VERIFY(db); 1233fa9e4066Sahrens 1234fa9e4066Sahrens if (db->db_state == DB_FILL) { 1235fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 1236ea8dc4b6Seschrock ASSERT(db->db_blkid != DB_BONUS_BLKID); 1237fa9e4066Sahrens /* we were freed while filling */ 1238fa9e4066Sahrens /* XXX dbuf_undirty? */ 1239fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1240fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 1241fa9e4066Sahrens } 1242fa9e4066Sahrens db->db_state = DB_CACHED; 1243fa9e4066Sahrens cv_broadcast(&db->db_changed); 1244fa9e4066Sahrens } 1245fa9e4066Sahrens mutex_exit(&db->db_mtx); 1246fa9e4066Sahrens } 1247fa9e4066Sahrens 1248ea8dc4b6Seschrock /* 1249ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1250ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1251ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1252ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1253ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1254ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1255ea8dc4b6Seschrock * For the arc callback, we will usually see: 1256ea8dc4b6Seschrock * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1257ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1258ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1259ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1260ea8dc4b6Seschrock */ 1261ea8dc4b6Seschrock void 1262fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1263fa9e4066Sahrens { 1264fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1265ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1266c543ec06Sahrens dmu_buf_impl_t *dndb = dn->dn_dbuf; 1267ea8dc4b6Seschrock int dbuf_gone = FALSE; 1268fa9e4066Sahrens 1269fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1270fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1271fa9e4066Sahrens 1272ea8dc4b6Seschrock dbuf_evict_user(db); 1273ea8dc4b6Seschrock 1274fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1275ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 1276ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) 1277ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1278fa9e4066Sahrens db->db.db_data = NULL; 1279fa9e4066Sahrens db->db_state = DB_UNCACHED; 1280fa9e4066Sahrens } 1281fa9e4066Sahrens 1282fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_UNCACHED); 1283fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1284fa9e4066Sahrens 1285ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1286ea8dc4b6Seschrock db->db_blkptr = NULL; 1287ea8dc4b6Seschrock 1288ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1289ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1290ea8dc4b6Seschrock dnode_rele(dn, db); 1291ea8dc4b6Seschrock } 1292ea8dc4b6Seschrock 1293ea8dc4b6Seschrock if (db->db_buf) 1294ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1295ea8dc4b6Seschrock 1296ea8dc4b6Seschrock if (!dbuf_gone) 1297ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1298fa9e4066Sahrens 1299fa9e4066Sahrens /* 1300fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1301fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1302fa9e4066Sahrens */ 1303c543ec06Sahrens if (parent && parent != dndb) 1304ea8dc4b6Seschrock dbuf_rele(parent, db); 1305fa9e4066Sahrens } 1306fa9e4066Sahrens 1307fa9e4066Sahrens static int 1308fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1309fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1310fa9e4066Sahrens { 1311fa9e4066Sahrens int nlevels, epbs; 1312fa9e4066Sahrens 13130b69c2f0Sahrens *parentp = NULL; 13140b69c2f0Sahrens *bpp = NULL; 13150b69c2f0Sahrens 1316ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1317ea8dc4b6Seschrock 1318fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1319fa9e4066Sahrens nlevels = 1; 1320fa9e4066Sahrens else 1321fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1322fa9e4066Sahrens 1323fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1324fa9e4066Sahrens 1325fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1326fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1327ea8dc4b6Seschrock if (level >= nlevels || 1328fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1329fa9e4066Sahrens /* the buffer has no parent yet */ 1330fa9e4066Sahrens return (ENOENT); 1331fa9e4066Sahrens } else if (level < nlevels-1) { 1332fa9e4066Sahrens /* this block is referenced from an indirect block */ 1333fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1334fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1335fa9e4066Sahrens if (err) 1336fa9e4066Sahrens return (err); 1337ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1338ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1339c543ec06Sahrens if (err) { 1340c543ec06Sahrens dbuf_rele(*parentp, NULL); 1341c543ec06Sahrens *parentp = NULL; 1342c543ec06Sahrens return (err); 1343ea8dc4b6Seschrock } 1344c543ec06Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1345c543ec06Sahrens (blkid & ((1ULL << epbs) - 1)); 1346c543ec06Sahrens return (0); 1347fa9e4066Sahrens } else { 1348fa9e4066Sahrens /* the block is referenced from the dnode */ 1349fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1350fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1351fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1352c543ec06Sahrens if (dn->dn_dbuf) { 1353c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1354c543ec06Sahrens *parentp = dn->dn_dbuf; 1355c543ec06Sahrens } 1356fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1357fa9e4066Sahrens return (0); 1358fa9e4066Sahrens } 1359fa9e4066Sahrens } 1360fa9e4066Sahrens 1361fa9e4066Sahrens static dmu_buf_impl_t * 1362fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1363fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1364fa9e4066Sahrens { 1365fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1366fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1367fa9e4066Sahrens 1368fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1369fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1370fa9e4066Sahrens 1371fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1372fa9e4066Sahrens 1373fa9e4066Sahrens db->db_objset = os; 1374fa9e4066Sahrens db->db.db_object = dn->dn_object; 1375fa9e4066Sahrens db->db_level = level; 1376fa9e4066Sahrens db->db_blkid = blkid; 1377ea8dc4b6Seschrock db->db_dirtied = 0; 1378ea8dc4b6Seschrock db->db_dirtycnt = 0; 1379ea8dc4b6Seschrock db->db_dnode = dn; 1380ea8dc4b6Seschrock db->db_parent = parent; 1381ea8dc4b6Seschrock db->db_blkptr = blkptr; 1382fa9e4066Sahrens 1383ea8dc4b6Seschrock bzero(&db->db_d, sizeof (db->db_d)); 1384ea8dc4b6Seschrock 1385ea8dc4b6Seschrock if (blkid == DB_BONUS_BLKID) { 1386ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 1387fa9e4066Sahrens db->db.db_size = dn->dn_bonuslen; 1388fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1389ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1390ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 1391ea8dc4b6Seschrock return (db); 1392fa9e4066Sahrens } else { 1393fa9e4066Sahrens int blocksize = 1394fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1395fa9e4066Sahrens db->db.db_size = blocksize; 1396fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1397fa9e4066Sahrens } 1398fa9e4066Sahrens 1399fa9e4066Sahrens /* 1400fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1401fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1402fa9e4066Sahrens * This prevents a possible deadlock with someone 1403fa9e4066Sahrens * trying to look up this dbuf before its added to the 1404fa9e4066Sahrens * dn_dbufs list. 1405fa9e4066Sahrens */ 1406fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1407ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1408fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1409fa9e4066Sahrens /* someone else inserted it first */ 1410fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1411fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1412fa9e4066Sahrens return (odb); 1413fa9e4066Sahrens } 1414fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1415ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1416fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1417fa9e4066Sahrens 1418fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1419fa9e4066Sahrens dbuf_add_ref(parent, db); 1420fa9e4066Sahrens 1421ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1422ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1423fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1424fa9e4066Sahrens 1425fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1426fa9e4066Sahrens 1427fa9e4066Sahrens return (db); 1428fa9e4066Sahrens } 1429fa9e4066Sahrens 1430fa9e4066Sahrens static int 1431ea8dc4b6Seschrock dbuf_do_evict(void *private) 1432fa9e4066Sahrens { 1433ea8dc4b6Seschrock arc_buf_t *buf = private; 1434ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1435fa9e4066Sahrens 1436ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1437ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1438fa9e4066Sahrens 1439ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1440fa9e4066Sahrens 1441ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1442ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 1443ea8dc4b6Seschrock DBUF_VERIFY(db); 1444ea8dc4b6Seschrock db->db_buf = NULL; 1445ea8dc4b6Seschrock dbuf_evict(db); 1446ea8dc4b6Seschrock } else { 1447ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1448ea8dc4b6Seschrock dbuf_destroy(db); 1449fa9e4066Sahrens } 1450ea8dc4b6Seschrock return (0); 1451fa9e4066Sahrens } 1452fa9e4066Sahrens 1453fa9e4066Sahrens static void 1454fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1455fa9e4066Sahrens { 1456fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1457fa9e4066Sahrens 1458ea8dc4b6Seschrock if (db->db_blkid != DB_BONUS_BLKID) { 1459ea8dc4b6Seschrock dnode_t *dn = db->db_dnode; 1460ea8dc4b6Seschrock 1461ea8dc4b6Seschrock /* 1462ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1463ea8dc4b6Seschrock * remove it from that list. 1464ea8dc4b6Seschrock */ 1465ea8dc4b6Seschrock if (list_link_active(&db->db_link)) { 1466c543ec06Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1467ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1468c543ec06Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1469ea8dc4b6Seschrock 1470ea8dc4b6Seschrock dnode_rele(dn, db); 1471ea8dc4b6Seschrock } 1472ea8dc4b6Seschrock dbuf_hash_remove(db); 1473ea8dc4b6Seschrock } 1474ea8dc4b6Seschrock db->db_parent = NULL; 1475ea8dc4b6Seschrock db->db_dnode = NULL; 1476ea8dc4b6Seschrock db->db_buf = NULL; 1477ea8dc4b6Seschrock 1478fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1479fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1480fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1481fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1482fa9e4066Sahrens 1483fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1484fa9e4066Sahrens } 1485fa9e4066Sahrens 1486fa9e4066Sahrens void 1487fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1488fa9e4066Sahrens { 148913506d1eSmaybee dmu_buf_impl_t *db = NULL; 1490fa9e4066Sahrens blkptr_t *bp = NULL; 1491fa9e4066Sahrens 1492fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1493fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1494fa9e4066Sahrens 1495fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1496fa9e4066Sahrens return; 1497fa9e4066Sahrens 1498fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1499fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 150013506d1eSmaybee if (refcount_count(&db->db_holds) > 0) { 150113506d1eSmaybee /* 150213506d1eSmaybee * This dbuf is active. We assume that it is 150313506d1eSmaybee * already CACHED, or else about to be either 150413506d1eSmaybee * read or filled. 150513506d1eSmaybee */ 150613506d1eSmaybee mutex_exit(&db->db_mtx); 150713506d1eSmaybee return; 150813506d1eSmaybee } 1509fa9e4066Sahrens mutex_exit(&db->db_mtx); 15100b69c2f0Sahrens db = NULL; 1511fa9e4066Sahrens } 1512fa9e4066Sahrens 151313506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1514fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 151513506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1516ea8dc4b6Seschrock zbookmark_t zb; 1517ea8dc4b6Seschrock zb.zb_objset = dn->dn_objset->os_dsl_dataset ? 1518ea8dc4b6Seschrock dn->dn_objset->os_dsl_dataset->ds_object : 0; 1519ea8dc4b6Seschrock zb.zb_object = dn->dn_object; 1520ea8dc4b6Seschrock zb.zb_level = 0; 1521ea8dc4b6Seschrock zb.zb_blkid = blkid; 1522ea8dc4b6Seschrock 1523fa9e4066Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, bp, 1524fa9e4066Sahrens dmu_ot[dn->dn_type].ot_byteswap, 1525fa9e4066Sahrens NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1526fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 152713506d1eSmaybee &aflags, &zb); 1528fa9e4066Sahrens } 152913506d1eSmaybee if (db) 153013506d1eSmaybee dbuf_rele(db, NULL); 1531fa9e4066Sahrens } 1532fa9e4066Sahrens } 1533fa9e4066Sahrens 1534fa9e4066Sahrens /* 1535fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1536fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1537fa9e4066Sahrens */ 1538fa9e4066Sahrens int 1539fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1540fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1541fa9e4066Sahrens { 1542fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1543fa9e4066Sahrens 1544ea8dc4b6Seschrock ASSERT(blkid != DB_BONUS_BLKID); 1545fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1546fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1547fa9e4066Sahrens 1548fa9e4066Sahrens *dbp = NULL; 1549ea8dc4b6Seschrock top: 1550fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1551fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1552fa9e4066Sahrens 1553fa9e4066Sahrens if (db == NULL) { 1554fa9e4066Sahrens blkptr_t *bp = NULL; 1555fa9e4066Sahrens int err; 1556fa9e4066Sahrens 1557c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1558fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1559fa9e4066Sahrens if (fail_sparse) { 1560fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1561fa9e4066Sahrens err = ENOENT; 1562fa9e4066Sahrens if (err) { 1563c543ec06Sahrens if (parent) 1564ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1565fa9e4066Sahrens return (err); 1566fa9e4066Sahrens } 1567fa9e4066Sahrens } 1568ea8dc4b6Seschrock if (err && err != ENOENT) 1569ea8dc4b6Seschrock return (err); 1570fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1571fa9e4066Sahrens } 1572fa9e4066Sahrens 1573ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1574ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1575ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1576ea8dc4b6Seschrock dbuf_clear(db); 1577c543ec06Sahrens if (parent) { 1578c543ec06Sahrens dbuf_rele(parent, NULL); 1579c543ec06Sahrens parent = NULL; 1580c543ec06Sahrens } 1581ea8dc4b6Seschrock goto top; 1582ea8dc4b6Seschrock } 1583ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1584ea8dc4b6Seschrock } 1585ea8dc4b6Seschrock 1586ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1587ea8dc4b6Seschrock 1588fa9e4066Sahrens /* 1589fa9e4066Sahrens * If this buffer is currently syncing out, and we are 1590fa9e4066Sahrens * are still referencing it from db_data, we need to make 1591fa9e4066Sahrens * a copy of it in case we decide we want to dirty it 1592fa9e4066Sahrens * again in this txg. 1593fa9e4066Sahrens */ 1594fa9e4066Sahrens if (db->db_level == 0 && db->db_state == DB_CACHED && 1595ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1596fa9e4066Sahrens db->db_data_pending == db->db_buf) { 1597fa9e4066Sahrens int size = (db->db_blkid == DB_BONUS_BLKID) ? 1598fa9e4066Sahrens DN_MAX_BONUSLEN : db->db.db_size; 1599*ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1600fa9e4066Sahrens 1601fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1602*ad23a2dbSjohansen size, db, type)); 1603fa9e4066Sahrens bcopy(db->db_data_pending->b_data, db->db.db_data, 1604fa9e4066Sahrens db->db.db_size); 1605fa9e4066Sahrens } 1606fa9e4066Sahrens 1607ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1608fa9e4066Sahrens dbuf_update_data(db); 16099c9dc39aSek DBUF_VERIFY(db); 1610fa9e4066Sahrens mutex_exit(&db->db_mtx); 1611fa9e4066Sahrens 1612fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1613c543ec06Sahrens if (parent) 1614ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1615fa9e4066Sahrens 1616fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1617fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1618fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1619fa9e4066Sahrens *dbp = db; 1620fa9e4066Sahrens 1621fa9e4066Sahrens return (0); 1622fa9e4066Sahrens } 1623fa9e4066Sahrens 1624fa9e4066Sahrens dmu_buf_impl_t * 1625ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1626fa9e4066Sahrens { 1627fa9e4066Sahrens dmu_buf_impl_t *db; 1628ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1629ea8dc4b6Seschrock return (err ? NULL : db); 1630fa9e4066Sahrens } 1631fa9e4066Sahrens 1632fa9e4066Sahrens dmu_buf_impl_t * 1633fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1634fa9e4066Sahrens { 1635fa9e4066Sahrens dmu_buf_impl_t *db; 1636ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1637ea8dc4b6Seschrock return (err ? NULL : db); 1638fa9e4066Sahrens } 1639fa9e4066Sahrens 1640fa9e4066Sahrens dmu_buf_impl_t * 1641ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1642fa9e4066Sahrens { 1643ea8dc4b6Seschrock dmu_buf_impl_t *db = dn->dn_bonus; 1644ea8dc4b6Seschrock 1645ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1646ea8dc4b6Seschrock 1647ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 1648ea8dc4b6Seschrock db = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL); 1649fa9e4066Sahrens return (db); 1650fa9e4066Sahrens } 1651fa9e4066Sahrens 1652ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1653fa9e4066Sahrens void 1654fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1655fa9e4066Sahrens { 1656ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1657ea8dc4b6Seschrock ASSERT(holds > 1); 1658fa9e4066Sahrens } 1659fa9e4066Sahrens 1660ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 1661fa9e4066Sahrens void 1662ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 1663fa9e4066Sahrens { 1664fa9e4066Sahrens int64_t holds; 1665fa9e4066Sahrens 1666fa9e4066Sahrens mutex_enter(&db->db_mtx); 16679c9dc39aSek DBUF_VERIFY(db); 1668fa9e4066Sahrens 1669fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1670ea8dc4b6Seschrock ASSERT(holds >= 0); 1671ea8dc4b6Seschrock 16726b4acc8bSahrens if (db->db_buf && holds == db->db_dirtycnt) 16736b4acc8bSahrens arc_buf_freeze(db->db_buf); 16746b4acc8bSahrens 1675ea8dc4b6Seschrock if (holds == db->db_dirtycnt && 1676ea8dc4b6Seschrock db->db_level == 0 && db->db_d.db_immediate_evict) 1677ea8dc4b6Seschrock dbuf_evict_user(db); 1678fa9e4066Sahrens 1679fa9e4066Sahrens if (holds == 0) { 1680ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 1681ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1682ea8dc4b6Seschrock dnode_rele(db->db_dnode, db); 1683ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 1684ea8dc4b6Seschrock /* 1685ea8dc4b6Seschrock * This is a special case: we never associated this 1686ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 1687ea8dc4b6Seschrock */ 1688ea8dc4b6Seschrock ASSERT3U(db->db_state, ==, DB_UNCACHED); 1689ea8dc4b6Seschrock dbuf_evict(db); 16906b4acc8bSahrens } else if (arc_released(db->db_buf)) { 1691ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1692ea8dc4b6Seschrock /* 1693ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 1694ea8dc4b6Seschrock */ 1695ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1696ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1697ea8dc4b6Seschrock dbuf_evict(db); 1698ea8dc4b6Seschrock } else { 1699ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 1700ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1701ea8dc4b6Seschrock } 1702fa9e4066Sahrens } else { 1703fa9e4066Sahrens mutex_exit(&db->db_mtx); 1704fa9e4066Sahrens } 1705fa9e4066Sahrens } 1706fa9e4066Sahrens 1707fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1708fa9e4066Sahrens uint64_t 1709fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1710fa9e4066Sahrens { 1711fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1712fa9e4066Sahrens } 1713fa9e4066Sahrens 1714fa9e4066Sahrens void * 1715fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1716fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1717fa9e4066Sahrens { 1718fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1719fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1720fa9e4066Sahrens } 1721fa9e4066Sahrens 1722fa9e4066Sahrens void * 1723fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1724fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1725fa9e4066Sahrens { 1726fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1727fa9e4066Sahrens 1728fa9e4066Sahrens db->db_d.db_immediate_evict = TRUE; 1729fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1730fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1731fa9e4066Sahrens } 1732fa9e4066Sahrens 1733fa9e4066Sahrens void * 1734fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1735fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1736fa9e4066Sahrens { 1737fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1738fa9e4066Sahrens ASSERT(db->db_level == 0); 1739fa9e4066Sahrens 1740fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1741fa9e4066Sahrens 1742fa9e4066Sahrens mutex_enter(&db->db_mtx); 1743fa9e4066Sahrens 1744fa9e4066Sahrens if (db->db_d.db_user_ptr == old_user_ptr) { 1745fa9e4066Sahrens db->db_d.db_user_ptr = user_ptr; 1746fa9e4066Sahrens db->db_d.db_user_data_ptr_ptr = user_data_ptr_ptr; 1747fa9e4066Sahrens db->db_d.db_evict_func = evict_func; 1748fa9e4066Sahrens 1749fa9e4066Sahrens dbuf_update_data(db); 1750fa9e4066Sahrens } else { 1751fa9e4066Sahrens old_user_ptr = db->db_d.db_user_ptr; 1752fa9e4066Sahrens } 1753fa9e4066Sahrens 1754fa9e4066Sahrens mutex_exit(&db->db_mtx); 1755fa9e4066Sahrens return (old_user_ptr); 1756fa9e4066Sahrens } 1757fa9e4066Sahrens 1758fa9e4066Sahrens void * 1759fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1760fa9e4066Sahrens { 1761fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1762fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1763fa9e4066Sahrens 1764fa9e4066Sahrens return (db->db_d.db_user_ptr); 1765fa9e4066Sahrens } 1766fa9e4066Sahrens 1767fa9e4066Sahrens void 1768fa9e4066Sahrens dbuf_sync(dmu_buf_impl_t *db, zio_t *zio, dmu_tx_t *tx) 1769fa9e4066Sahrens { 1770fa9e4066Sahrens arc_buf_t **data; 1771fa9e4066Sahrens uint64_t txg = tx->tx_txg; 1772fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1773fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 177423b11526Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1775ea8dc4b6Seschrock int checksum, compress; 1776ea8dc4b6Seschrock zbookmark_t zb; 1777fa9e4066Sahrens int blksz; 1778*ad23a2dbSjohansen arc_buf_contents_t type; 1779fa9e4066Sahrens 1780fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 1781fa9e4066Sahrens 1782fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1783fa9e4066Sahrens 1784fa9e4066Sahrens mutex_enter(&db->db_mtx); 1785fa9e4066Sahrens /* 1786fa9e4066Sahrens * To be synced, we must be dirtied. But we 1787fa9e4066Sahrens * might have been freed after the dirty. 1788fa9e4066Sahrens */ 1789fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 1790fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 1791fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1792fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 1793fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 1794fa9e4066Sahrens ASSERT(db->db.db_data != db->db_d.db_data_old[txg&TXG_MASK]); 1795fa9e4066Sahrens } else { 1796fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 1797fa9e4066Sahrens } 17989c9dc39aSek DBUF_VERIFY(db); 1799fa9e4066Sahrens 1800fa9e4066Sahrens /* 1801fa9e4066Sahrens * Don't need a lock on db_dirty (dn_mtx), because it can't 1802fa9e4066Sahrens * be modified yet. 1803fa9e4066Sahrens */ 1804fa9e4066Sahrens 1805ea8dc4b6Seschrock if (db->db_blkid == DB_BONUS_BLKID) { 18066b4acc8bSahrens arc_buf_t **datap = &db->db_d.db_data_old[txg&TXG_MASK]; 1807ea8dc4b6Seschrock /* 1808ea8dc4b6Seschrock * Simply copy the bonus data into the dnode. It will 1809ea8dc4b6Seschrock * be written out when the dnode is synced (and it will 1810ea8dc4b6Seschrock * be synced, since it must have been dirty for dbuf_sync 1811ea8dc4b6Seschrock * to be called). 1812ea8dc4b6Seschrock */ 1813ea8dc4b6Seschrock /* 1814ea8dc4b6Seschrock * Use dn_phys->dn_bonuslen since db.db_size is the length 1815ea8dc4b6Seschrock * of the bonus buffer in the open transaction rather than 1816ea8dc4b6Seschrock * the syncing transaction. 1817ea8dc4b6Seschrock */ 1818ea8dc4b6Seschrock ASSERT(*datap != NULL); 1819ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 1820ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 1821ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 1822ea8dc4b6Seschrock if (*datap != db->db.db_data) 1823ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 1824ea8dc4b6Seschrock db->db_d.db_data_old[txg&TXG_MASK] = NULL; 1825ea8dc4b6Seschrock db->db_data_pending = NULL; 1826ea8dc4b6Seschrock if (db->db_dirtied == txg) 1827ea8dc4b6Seschrock db->db_dirtied = 0; 1828ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 1829ea8dc4b6Seschrock db->db_dirtycnt -= 1; 1830ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1831ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 1832ea8dc4b6Seschrock return; 1833ea8dc4b6Seschrock } 1834ea8dc4b6Seschrock 1835fa9e4066Sahrens if (db->db_level == 0) { 1836*ad23a2dbSjohansen type = DBUF_GET_BUFC_TYPE(db); 18376b4acc8bSahrens data = &db->db_d.db_data_old[txg&TXG_MASK]; 1838fa9e4066Sahrens blksz = arc_buf_size(*data); 1839c5c6ffa0Smaybee 1840c5c6ffa0Smaybee /* 1841c5c6ffa0Smaybee * This buffer is in the middle of an immdiate write. 1842c5c6ffa0Smaybee * Wait for the synchronous IO to complete. 1843c5c6ffa0Smaybee */ 1844c5c6ffa0Smaybee while (db->db_d.db_overridden_by[txg&TXG_MASK] == IN_DMU_SYNC) { 1845c5c6ffa0Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 1846c5c6ffa0Smaybee cv_wait(&db->db_changed, &db->db_mtx); 1847c5c6ffa0Smaybee ASSERT(db->db_d.db_overridden_by[txg&TXG_MASK]); 1848c5c6ffa0Smaybee } 1849fa9e4066Sahrens /* 1850fa9e4066Sahrens * If this buffer is currently "in use" (i.e., there are 1851fa9e4066Sahrens * active holds and db_data still references it), then make 1852fa9e4066Sahrens * a copy before we start the write so that any modifications 1853fa9e4066Sahrens * from the open txg will not leak into this write. 1854fa9e4066Sahrens * 1855fa9e4066Sahrens * NOTE: this copy does not need to be made for objects only 1856fa9e4066Sahrens * modified in the syncing context (e.g. DNONE_DNODE blocks) 1857fa9e4066Sahrens * or if there is no actual write involved (bonus blocks). 1858fa9e4066Sahrens */ 1859ea8dc4b6Seschrock if (dn->dn_object != DMU_META_DNODE_OBJECT && 1860ea8dc4b6Seschrock db->db_d.db_overridden_by[txg&TXG_MASK] == NULL) { 1861fa9e4066Sahrens if (refcount_count(&db->db_holds) > 1 && 1862fa9e4066Sahrens *data == db->db_buf) { 1863*ad23a2dbSjohansen *data = arc_buf_alloc(os->os_spa, blksz, db, 1864*ad23a2dbSjohansen type); 1865fa9e4066Sahrens bcopy(db->db.db_data, (*data)->b_data, blksz); 1866fa9e4066Sahrens } 1867fa9e4066Sahrens db->db_data_pending = *data; 1868ea8dc4b6Seschrock } else if (dn->dn_object == DMU_META_DNODE_OBJECT) { 1869fa9e4066Sahrens /* 1870fa9e4066Sahrens * Private object buffers are released here rather 1871fa9e4066Sahrens * than in dbuf_dirty() since they are only modified 1872fa9e4066Sahrens * in the syncing context and we don't want the 1873fa9e4066Sahrens * overhead of making multiple copies of the data. 1874fa9e4066Sahrens */ 1875fa9e4066Sahrens arc_release(db->db_buf, db); 1876fa9e4066Sahrens } 1877fa9e4066Sahrens } else { 1878fa9e4066Sahrens data = &db->db_buf; 1879fa9e4066Sahrens if (*data == NULL) { 1880fa9e4066Sahrens /* 1881fa9e4066Sahrens * This can happen if we dirty and then free 1882fa9e4066Sahrens * the level-0 data blocks in the same txg. So 1883fa9e4066Sahrens * this indirect remains unchanged. 1884fa9e4066Sahrens */ 1885fa9e4066Sahrens if (db->db_dirtied == txg) 1886fa9e4066Sahrens db->db_dirtied = 0; 1887fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1888fa9e4066Sahrens db->db_dirtycnt -= 1; 1889fa9e4066Sahrens mutex_exit(&db->db_mtx); 1890ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 1891fa9e4066Sahrens return; 1892fa9e4066Sahrens } 1893fa9e4066Sahrens blksz = db->db.db_size; 1894fa9e4066Sahrens ASSERT3U(blksz, ==, 1<<dn->dn_phys->dn_indblkshift); 1895fa9e4066Sahrens } 1896fa9e4066Sahrens 1897fa9e4066Sahrens ASSERT(*data != NULL); 1898fa9e4066Sahrens 1899ea8dc4b6Seschrock if (db->db_level > 0 && !arc_released(db->db_buf)) { 1900fa9e4066Sahrens /* 1901fa9e4066Sahrens * This indirect buffer was marked dirty, but 1902fa9e4066Sahrens * never modified (if it had been modified, then 1903fa9e4066Sahrens * we would have released the buffer). There is 1904fa9e4066Sahrens * no reason to write anything. 1905fa9e4066Sahrens */ 1906fa9e4066Sahrens db->db_data_pending = NULL; 1907fa9e4066Sahrens if (db->db_dirtied == txg) 1908fa9e4066Sahrens db->db_dirtied = 0; 1909fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1910fa9e4066Sahrens db->db_dirtycnt -= 1; 1911fa9e4066Sahrens mutex_exit(&db->db_mtx); 1912ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 1913fa9e4066Sahrens return; 1914fa9e4066Sahrens } else if (db->db_blkptr == NULL && 1915fa9e4066Sahrens db->db_level == dn->dn_phys->dn_nlevels-1 && 1916fa9e4066Sahrens db->db_blkid < dn->dn_phys->dn_nblkptr) { 1917fa9e4066Sahrens /* 1918fa9e4066Sahrens * This buffer was allocated at a time when there was 1919fa9e4066Sahrens * no available blkptrs from the dnode, or it was 1920fa9e4066Sahrens * inappropriate to hook it in (i.e., nlevels mis-match). 1921fa9e4066Sahrens */ 1922fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1923fa9e4066Sahrens ASSERT(db->db_parent == NULL); 1924fa9e4066Sahrens db->db_parent = dn->dn_dbuf; 1925fa9e4066Sahrens db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 19269c9dc39aSek DBUF_VERIFY(db); 1927fa9e4066Sahrens mutex_exit(&db->db_mtx); 1928fa9e4066Sahrens } else if (db->db_blkptr == NULL) { 1929fa9e4066Sahrens dmu_buf_impl_t *parent = db->db_parent; 1930fa9e4066Sahrens 1931fa9e4066Sahrens mutex_exit(&db->db_mtx); 1932fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nlevels > 1); 1933fa9e4066Sahrens if (parent == NULL) { 1934fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1935fa9e4066Sahrens (void) dbuf_hold_impl(dn, db->db_level+1, 1936ea8dc4b6Seschrock db->db_blkid >> epbs, FALSE, FTAG, &parent); 1937fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1938fa9e4066Sahrens dbuf_add_ref(parent, db); 1939fa9e4066Sahrens db->db_parent = parent; 1940ea8dc4b6Seschrock dbuf_rele(parent, FTAG); 1941fa9e4066Sahrens } 1942ea8dc4b6Seschrock (void) dbuf_read(parent, NULL, DB_RF_MUST_SUCCEED); 1943fa9e4066Sahrens } else { 1944fa9e4066Sahrens mutex_exit(&db->db_mtx); 1945fa9e4066Sahrens } 1946fa9e4066Sahrens 1947ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || db->db_parent != NULL); 1948fa9e4066Sahrens 194923b11526Smaybee if (db->db_level > 0 && 195023b11526Smaybee db->db_blkid > dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)) { 195123b11526Smaybee /* 195223b11526Smaybee * Don't write indirect blocks past EOF. 195323b11526Smaybee * We get these when we truncate a file *after* dirtying 195423b11526Smaybee * blocks in the truncate range (we undirty the level 0 195523b11526Smaybee * blocks in dbuf_free_range(), but not the indirects). 195623b11526Smaybee */ 195723b11526Smaybee #ifdef ZFS_DEBUG 195823b11526Smaybee /* 195923b11526Smaybee * Verify that this indirect block is empty. 196023b11526Smaybee */ 196123b11526Smaybee blkptr_t *bplist; 196223b11526Smaybee int i; 196323b11526Smaybee 196423b11526Smaybee mutex_enter(&db->db_mtx); 196523b11526Smaybee bplist = db->db.db_data; 196623b11526Smaybee for (i = 0; i < (1 << epbs); i++) { 196723b11526Smaybee if (!BP_IS_HOLE(&bplist[i])) { 196823b11526Smaybee panic("data past EOF: " 196934f18512Seschrock "db=%p level=%d id=%llu i=%d\n", 197034f18512Seschrock db, db->db_level, 197134f18512Seschrock (u_longlong_t)db->db_blkid, i); 197223b11526Smaybee } 197323b11526Smaybee } 197423b11526Smaybee mutex_exit(&db->db_mtx); 197523b11526Smaybee #endif 197623b11526Smaybee ASSERT(db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)); 197723b11526Smaybee mutex_enter(&db->db_mtx); 197823b11526Smaybee db->db_dirtycnt -= 1; 197923b11526Smaybee mutex_exit(&db->db_mtx); 1980ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 198123b11526Smaybee return; 198223b11526Smaybee } 198323b11526Smaybee 1984fa9e4066Sahrens if (db->db_parent != dn->dn_dbuf) { 1985fa9e4066Sahrens dmu_buf_impl_t *parent = db->db_parent; 1986fa9e4066Sahrens 1987fa9e4066Sahrens mutex_enter(&db->db_mtx); 1988fa9e4066Sahrens ASSERT(db->db_level == parent->db_level-1); 1989fa9e4066Sahrens ASSERT(list_link_active(&parent->db_dirty_node[txg&TXG_MASK])); 1990fa9e4066Sahrens /* 1991ea8dc4b6Seschrock * We may have read this indirect block after we dirtied it, 1992fa9e4066Sahrens * so never released it from the cache. 1993fa9e4066Sahrens */ 19946b4acc8bSahrens arc_release(parent->db_buf, parent); 1995fa9e4066Sahrens 1996fa9e4066Sahrens db->db_blkptr = (blkptr_t *)parent->db.db_data + 1997fa9e4066Sahrens (db->db_blkid & ((1ULL << epbs) - 1)); 19989c9dc39aSek DBUF_VERIFY(db); 1999fa9e4066Sahrens mutex_exit(&db->db_mtx); 2000fa9e4066Sahrens #ifdef ZFS_DEBUG 2001ea8dc4b6Seschrock } else { 2002fa9e4066Sahrens /* 2003fa9e4066Sahrens * We don't need to dnode_setdirty(dn) because if we got 2004fa9e4066Sahrens * here then the parent is already dirty. 2005fa9e4066Sahrens */ 2006fa9e4066Sahrens ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 2007fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 2008fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 2009fa9e4066Sahrens #endif 2010ea8dc4b6Seschrock } 2011ea8dc4b6Seschrock ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 2012ea8dc4b6Seschrock 2013fa9e4066Sahrens if (db->db_level == 0 && 2014fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 20156b4acc8bSahrens arc_buf_t **old = &db->db_d.db_data_old[txg&TXG_MASK]; 2016fa9e4066Sahrens blkptr_t **bpp = &db->db_d.db_overridden_by[txg&TXG_MASK]; 201799653d4eSeschrock int old_size = bp_get_dasize(os->os_spa, db->db_blkptr); 201899653d4eSeschrock int new_size = bp_get_dasize(os->os_spa, *bpp); 2019fa9e4066Sahrens 2020fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 2021fa9e4066Sahrens 2022fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 2023fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2024fa9e4066Sahrens if (db->db_blkid > dn->dn_phys->dn_maxblkid) 2025fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2026fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2027fa9e4066Sahrens 2028fa9e4066Sahrens dsl_dataset_block_born(os->os_dsl_dataset, *bpp, tx); 2029fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) 2030fa9e4066Sahrens dsl_dataset_block_kill(os->os_dsl_dataset, 2031fa9e4066Sahrens db->db_blkptr, os->os_synctx); 2032fa9e4066Sahrens 2033fa9e4066Sahrens mutex_enter(&db->db_mtx); 2034fa9e4066Sahrens *db->db_blkptr = **bpp; 2035fa9e4066Sahrens kmem_free(*bpp, sizeof (blkptr_t)); 2036fa9e4066Sahrens *bpp = NULL; 2037fa9e4066Sahrens 2038fa9e4066Sahrens if (*old != db->db_buf) 2039ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(*old, db) == 1); 2040ea8dc4b6Seschrock else if (!BP_IS_HOLE(db->db_blkptr)) 2041ea8dc4b6Seschrock arc_set_callback(db->db_buf, dbuf_do_evict, db); 2042ea8dc4b6Seschrock else 2043ea8dc4b6Seschrock ASSERT(arc_released(db->db_buf)); 2044fa9e4066Sahrens *old = NULL; 2045fa9e4066Sahrens db->db_data_pending = NULL; 2046fa9e4066Sahrens 2047fa9e4066Sahrens cv_broadcast(&db->db_changed); 2048fa9e4066Sahrens 2049fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2050fa9e4066Sahrens db->db_dirtycnt -= 1; 2051fa9e4066Sahrens mutex_exit(&db->db_mtx); 2052ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2053ea8dc4b6Seschrock return; 2054ea8dc4b6Seschrock } 2055fa9e4066Sahrens 2056ea8dc4b6Seschrock if (db->db_level > 0) { 2057ea8dc4b6Seschrock /* 2058ea8dc4b6Seschrock * XXX -- we should design a compression algorithm 2059ea8dc4b6Seschrock * that specializes in arrays of bps. 2060ea8dc4b6Seschrock */ 2061ea8dc4b6Seschrock checksum = ZIO_CHECKSUM_FLETCHER_4; 2062416e0cd8Sek if (zfs_mdcomp_disable) 2063416e0cd8Sek compress = ZIO_COMPRESS_EMPTY; 2064416e0cd8Sek else 2065416e0cd8Sek compress = ZIO_COMPRESS_LZJB; 2066ea8dc4b6Seschrock } else { 2067ea8dc4b6Seschrock /* 2068ea8dc4b6Seschrock * Allow dnode settings to override objset settings, 2069ea8dc4b6Seschrock * except for metadata checksums. 2070ea8dc4b6Seschrock */ 2071ea8dc4b6Seschrock if (dmu_ot[dn->dn_type].ot_metadata) { 2072ea8dc4b6Seschrock checksum = os->os_md_checksum; 2073ea8dc4b6Seschrock compress = zio_compress_select(dn->dn_compress, 2074ea8dc4b6Seschrock os->os_md_compress); 2075fa9e4066Sahrens } else { 2076ea8dc4b6Seschrock checksum = zio_checksum_select(dn->dn_checksum, 2077ea8dc4b6Seschrock os->os_checksum); 2078ea8dc4b6Seschrock compress = zio_compress_select(dn->dn_compress, 2079ea8dc4b6Seschrock os->os_compress); 2080fa9e4066Sahrens } 2081ea8dc4b6Seschrock } 2082fa9e4066Sahrens #ifdef ZFS_DEBUG 2083ea8dc4b6Seschrock if (db->db_parent) { 2084ea8dc4b6Seschrock ASSERT(list_link_active( 2085ea8dc4b6Seschrock &db->db_parent->db_dirty_node[txg&TXG_MASK])); 2086ea8dc4b6Seschrock ASSERT(db->db_parent == dn->dn_dbuf || 2087ea8dc4b6Seschrock db->db_parent->db_level > 0); 2088ea8dc4b6Seschrock if (dn->dn_object == DMU_META_DNODE_OBJECT || db->db_level > 0) 2089ea8dc4b6Seschrock ASSERT(*data == db->db_buf); 2090fa9e4066Sahrens } 2091ea8dc4b6Seschrock #endif 2092ea8dc4b6Seschrock ASSERT3U(db->db_blkptr->blk_birth, <=, tx->tx_txg); 2093ea8dc4b6Seschrock zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0; 2094ea8dc4b6Seschrock zb.zb_object = db->db.db_object; 2095ea8dc4b6Seschrock zb.zb_level = db->db_level; 2096ea8dc4b6Seschrock zb.zb_blkid = db->db_blkid; 209744cd46caSbillm 209844cd46caSbillm (void) arc_write(zio, os->os_spa, checksum, compress, 209944cd46caSbillm dmu_get_replication_level(os->os_spa, &zb, dn->dn_type), txg, 2100ea8dc4b6Seschrock db->db_blkptr, *data, dbuf_write_done, db, 2101ea8dc4b6Seschrock ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, ARC_NOWAIT, &zb); 2102ea8dc4b6Seschrock /* 2103ea8dc4b6Seschrock * We can't access db after arc_write, since it could finish 2104ea8dc4b6Seschrock * and be freed, and we have no locks on it. 2105ea8dc4b6Seschrock */ 2106fa9e4066Sahrens } 2107fa9e4066Sahrens 2108fa9e4066Sahrens struct dbuf_arg { 2109fa9e4066Sahrens objset_impl_t *os; 2110fa9e4066Sahrens blkptr_t bp; 2111fa9e4066Sahrens }; 2112fa9e4066Sahrens 2113fa9e4066Sahrens static void 2114fa9e4066Sahrens dbuf_do_born(void *arg) 2115fa9e4066Sahrens { 2116fa9e4066Sahrens struct dbuf_arg *da = arg; 2117fa9e4066Sahrens dsl_dataset_block_born(da->os->os_dsl_dataset, 2118fa9e4066Sahrens &da->bp, da->os->os_synctx); 2119fa9e4066Sahrens kmem_free(da, sizeof (struct dbuf_arg)); 2120fa9e4066Sahrens } 2121fa9e4066Sahrens 2122fa9e4066Sahrens static void 2123fa9e4066Sahrens dbuf_do_kill(void *arg) 2124fa9e4066Sahrens { 2125fa9e4066Sahrens struct dbuf_arg *da = arg; 2126fa9e4066Sahrens dsl_dataset_block_kill(da->os->os_dsl_dataset, 2127fa9e4066Sahrens &da->bp, da->os->os_synctx); 2128fa9e4066Sahrens kmem_free(da, sizeof (struct dbuf_arg)); 2129fa9e4066Sahrens } 2130fa9e4066Sahrens 2131fa9e4066Sahrens /* ARGSUSED */ 2132fa9e4066Sahrens static void 2133fa9e4066Sahrens dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2134fa9e4066Sahrens { 2135fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2136fa9e4066Sahrens dnode_t *dn = db->db_dnode; 2137fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 2138fa9e4066Sahrens uint64_t txg = zio->io_txg; 2139fa9e4066Sahrens uint64_t fill = 0; 2140fa9e4066Sahrens int i; 2141fa9e4066Sahrens int old_size, new_size; 2142fa9e4066Sahrens 2143fa9e4066Sahrens ASSERT3U(zio->io_error, ==, 0); 2144fa9e4066Sahrens 2145fa9e4066Sahrens dprintf_dbuf_bp(db, &zio->io_bp_orig, "bp_orig: %s", ""); 2146fa9e4066Sahrens 214799653d4eSeschrock old_size = bp_get_dasize(os->os_spa, &zio->io_bp_orig); 214899653d4eSeschrock new_size = bp_get_dasize(os->os_spa, zio->io_bp); 2149fa9e4066Sahrens 2150fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 2151fa9e4066Sahrens 2152fa9e4066Sahrens mutex_enter(&db->db_mtx); 2153fa9e4066Sahrens 2154c5c6ffa0Smaybee ASSERT(db->db_d.db_overridden_by[txg&TXG_MASK] == NULL); 2155c5c6ffa0Smaybee 2156fa9e4066Sahrens if (db->db_dirtied == txg) 2157fa9e4066Sahrens db->db_dirtied = 0; 2158fa9e4066Sahrens 2159fa9e4066Sahrens if (db->db_level == 0) { 21606b4acc8bSahrens arc_buf_t **old = &db->db_d.db_data_old[txg&TXG_MASK]; 2161fa9e4066Sahrens 2162fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 2163fa9e4066Sahrens 2164fa9e4066Sahrens if (*old != db->db_buf) 2165ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(*old, db) == 1); 2166ea8dc4b6Seschrock else if (!BP_IS_HOLE(db->db_blkptr)) 2167ea8dc4b6Seschrock arc_set_callback(db->db_buf, dbuf_do_evict, db); 2168ea8dc4b6Seschrock else 2169ea8dc4b6Seschrock ASSERT(arc_released(db->db_buf)); 2170fa9e4066Sahrens *old = NULL; 2171fa9e4066Sahrens db->db_data_pending = NULL; 2172fa9e4066Sahrens 2173fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 2174fa9e4066Sahrens if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2175fa9e4066Sahrens !BP_IS_HOLE(db->db_blkptr)) 2176fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2177fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2178fa9e4066Sahrens 2179fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2180fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2181fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2182fa9e4066Sahrens i--, dnp++) { 2183fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2184fa9e4066Sahrens fill++; 2185fa9e4066Sahrens } 2186fa9e4066Sahrens } else { 2187fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) 2188fa9e4066Sahrens fill = 1; 2189fa9e4066Sahrens } 2190fa9e4066Sahrens } else { 2191fa9e4066Sahrens blkptr_t *bp = db->db.db_data; 2192fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2193fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) { 219423b11526Smaybee int epbs = 219523b11526Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2196fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, db->db.db_size); 2197fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2198fa9e4066Sahrens db->db.db_size); 219923b11526Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 220023b11526Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2201ea8dc4b6Seschrock arc_set_callback(db->db_buf, dbuf_do_evict, db); 2202fa9e4066Sahrens } 2203fa9e4066Sahrens for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2204fa9e4066Sahrens if (BP_IS_HOLE(bp)) 2205fa9e4066Sahrens continue; 2206fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(bp), ==, 2207fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2208fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2209fa9e4066Sahrens fill += bp->blk_fill; 2210fa9e4066Sahrens } 2211fa9e4066Sahrens } 2212fa9e4066Sahrens 2213fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) { 2214fa9e4066Sahrens db->db_blkptr->blk_fill = fill; 2215fa9e4066Sahrens BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2216fa9e4066Sahrens BP_SET_LEVEL(db->db_blkptr, db->db_level); 2217fa9e4066Sahrens } else { 2218fa9e4066Sahrens ASSERT3U(fill, ==, 0); 2219fa9e4066Sahrens ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2220fa9e4066Sahrens } 2221fa9e4066Sahrens 2222fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, 2223fa9e4066Sahrens "wrote %llu bytes to blkptr:", zio->io_size); 2224fa9e4066Sahrens 2225fa9e4066Sahrens ASSERT(db->db_parent == NULL || 2226fa9e4066Sahrens list_link_active(&db->db_parent->db_dirty_node[txg&TXG_MASK])); 2227fa9e4066Sahrens cv_broadcast(&db->db_changed); 2228fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2229fa9e4066Sahrens db->db_dirtycnt -= 1; 2230fa9e4066Sahrens mutex_exit(&db->db_mtx); 2231fa9e4066Sahrens 2232fa9e4066Sahrens /* We must do this after we've set the bp's type and level */ 2233fa9e4066Sahrens if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), 2234fa9e4066Sahrens BP_IDENTITY(&zio->io_bp_orig))) { 2235fa9e4066Sahrens struct dbuf_arg *da; 2236fa9e4066Sahrens da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2237fa9e4066Sahrens da->os = os; 2238fa9e4066Sahrens da->bp = *zio->io_bp; 2239fa9e4066Sahrens (void) taskq_dispatch(dbuf_tq, dbuf_do_born, da, 0); 2240fa9e4066Sahrens if (!BP_IS_HOLE(&zio->io_bp_orig)) { 2241fa9e4066Sahrens da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2242fa9e4066Sahrens da->os = os; 2243fa9e4066Sahrens da->bp = zio->io_bp_orig; 2244fa9e4066Sahrens (void) taskq_dispatch(dbuf_tq, dbuf_do_kill, da, 0); 2245fa9e4066Sahrens } 2246fa9e4066Sahrens } 2247fa9e4066Sahrens 2248ea8dc4b6Seschrock dbuf_rele(db, (void *)(uintptr_t)txg); 2249fa9e4066Sahrens } 2250