1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5*f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6*f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2234f18512Seschrock * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23fa9e4066Sahrens * Use is subject to license terms. 24fa9e4066Sahrens */ 25fa9e4066Sahrens 26fa9e4066Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/zfs_context.h> 29fa9e4066Sahrens #include <sys/dmu.h> 30fa9e4066Sahrens #include <sys/dmu_impl.h> 31fa9e4066Sahrens #include <sys/dbuf.h> 32fa9e4066Sahrens #include <sys/dmu_objset.h> 33fa9e4066Sahrens #include <sys/dsl_dataset.h> 34fa9e4066Sahrens #include <sys/dsl_dir.h> 35fa9e4066Sahrens #include <sys/dmu_tx.h> 36fa9e4066Sahrens #include <sys/spa.h> 37fa9e4066Sahrens #include <sys/zio.h> 38fa9e4066Sahrens #include <sys/dmu_zfetch.h> 39fa9e4066Sahrens 40fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 41fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 42fa9e4066Sahrens static arc_done_func_t dbuf_write_done; 43fa9e4066Sahrens 44fa9e4066Sahrens /* 45fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 46fa9e4066Sahrens */ 47fa9e4066Sahrens taskq_t *dbuf_tq; 48fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 49fa9e4066Sahrens 50fa9e4066Sahrens /* ARGSUSED */ 51fa9e4066Sahrens static int 52fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 53fa9e4066Sahrens { 54fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 55fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 56fa9e4066Sahrens 57fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 58fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 59fa9e4066Sahrens refcount_create(&db->db_holds); 60fa9e4066Sahrens return (0); 61fa9e4066Sahrens } 62fa9e4066Sahrens 63fa9e4066Sahrens /* ARGSUSED */ 64fa9e4066Sahrens static void 65fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 66fa9e4066Sahrens { 67fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 68fa9e4066Sahrens mutex_destroy(&db->db_mtx); 69fa9e4066Sahrens cv_destroy(&db->db_changed); 70fa9e4066Sahrens refcount_destroy(&db->db_holds); 71fa9e4066Sahrens } 72fa9e4066Sahrens 73fa9e4066Sahrens /* 74fa9e4066Sahrens * dbuf hash table routines 75fa9e4066Sahrens */ 76fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 77fa9e4066Sahrens 78fa9e4066Sahrens static uint64_t dbuf_hash_count; 79fa9e4066Sahrens 80fa9e4066Sahrens static uint64_t 81fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 82fa9e4066Sahrens { 83fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 84fa9e4066Sahrens uint64_t crc = -1ULL; 85fa9e4066Sahrens 86fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 87fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 91fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 92fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 93fa9e4066Sahrens 94fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 95fa9e4066Sahrens 96fa9e4066Sahrens return (crc); 97fa9e4066Sahrens } 98fa9e4066Sahrens 99fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 100fa9e4066Sahrens 101fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 102fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 103fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 104fa9e4066Sahrens (dbuf)->db_level == (level) && \ 105fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 106fa9e4066Sahrens 107fa9e4066Sahrens dmu_buf_impl_t * 108fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 109fa9e4066Sahrens { 110fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 111fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 112fa9e4066Sahrens uint64_t obj = dn->dn_object; 113fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 114fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 115fa9e4066Sahrens dmu_buf_impl_t *db; 116fa9e4066Sahrens 117fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 118fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 119fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 120fa9e4066Sahrens mutex_enter(&db->db_mtx); 121fa9e4066Sahrens if (!refcount_is_zero(&db->db_holds)) { 122fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 123fa9e4066Sahrens return (db); 124fa9e4066Sahrens } 125fa9e4066Sahrens mutex_exit(&db->db_mtx); 126fa9e4066Sahrens } 127fa9e4066Sahrens } 128fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 129fa9e4066Sahrens return (NULL); 130fa9e4066Sahrens } 131fa9e4066Sahrens 132fa9e4066Sahrens /* 133fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 134fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 135fa9e4066Sahrens * will be returned and the new element will not be inserted. 136fa9e4066Sahrens * Otherwise returns NULL. 137fa9e4066Sahrens */ 138fa9e4066Sahrens static dmu_buf_impl_t * 139fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 140fa9e4066Sahrens { 141fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 142fa9e4066Sahrens objset_impl_t *os = db->db_objset; 143fa9e4066Sahrens uint64_t obj = db->db.db_object; 144fa9e4066Sahrens int level = db->db_level; 145fa9e4066Sahrens uint64_t blkid = db->db_blkid; 146fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 147fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 148fa9e4066Sahrens dmu_buf_impl_t *dbf; 149fa9e4066Sahrens 150fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 151fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 152fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 153fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 154fa9e4066Sahrens if (!refcount_is_zero(&dbf->db_holds)) { 155fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 156fa9e4066Sahrens return (dbf); 157fa9e4066Sahrens } 158fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 159fa9e4066Sahrens } 160fa9e4066Sahrens } 161fa9e4066Sahrens 162fa9e4066Sahrens mutex_enter(&db->db_mtx); 163fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 164fa9e4066Sahrens h->hash_table[idx] = db; 165fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 166fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 167fa9e4066Sahrens 168fa9e4066Sahrens return (NULL); 169fa9e4066Sahrens } 170fa9e4066Sahrens 171fa9e4066Sahrens /* 172fa9e4066Sahrens * Remove an entry from the hash table. This operation will 173fa9e4066Sahrens * fail if there are any existing holds on the db. 174fa9e4066Sahrens */ 175fa9e4066Sahrens static void 176fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 177fa9e4066Sahrens { 178fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 179fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 180fa9e4066Sahrens db->db_level, db->db_blkid); 181fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 182fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 183fa9e4066Sahrens 184fa9e4066Sahrens /* 185fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 186fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 187fa9e4066Sahrens */ 188fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 189fa9e4066Sahrens ASSERT(db->db_dnode != NULL); 190fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 191fa9e4066Sahrens 192fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 193fa9e4066Sahrens dbp = &h->hash_table[idx]; 194fa9e4066Sahrens while ((dbf = *dbp) != db) { 195fa9e4066Sahrens dbp = &dbf->db_hash_next; 196fa9e4066Sahrens ASSERT(dbf != NULL); 197fa9e4066Sahrens } 198fa9e4066Sahrens *dbp = db->db_hash_next; 199fa9e4066Sahrens db->db_hash_next = NULL; 200fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 201fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 202fa9e4066Sahrens } 203fa9e4066Sahrens 204fa9e4066Sahrens static int dbuf_evictable(dmu_buf_impl_t *db); 205fa9e4066Sahrens static void dbuf_clear(dmu_buf_impl_t *db); 206fa9e4066Sahrens 207fa9e4066Sahrens void 208fa9e4066Sahrens dbuf_evict(dmu_buf_impl_t *db) 209fa9e4066Sahrens { 210fa9e4066Sahrens int err; 211fa9e4066Sahrens 212fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 213fa9e4066Sahrens err = dbuf_evictable(db); 214fa9e4066Sahrens ASSERT(err == TRUE); 215fa9e4066Sahrens dbuf_clear(db); 216fa9e4066Sahrens dbuf_destroy(db); 217fa9e4066Sahrens } 218fa9e4066Sahrens 219fa9e4066Sahrens static void 220fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 221fa9e4066Sahrens { 222fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 223fa9e4066Sahrens 224fa9e4066Sahrens if (db->db_level != 0 || db->db_d.db_evict_func == NULL) 225fa9e4066Sahrens return; 226fa9e4066Sahrens 227fa9e4066Sahrens if (db->db_d.db_user_data_ptr_ptr) 228fa9e4066Sahrens *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 229fa9e4066Sahrens db->db_d.db_evict_func(&db->db, db->db_d.db_user_ptr); 230fa9e4066Sahrens db->db_d.db_user_ptr = NULL; 231fa9e4066Sahrens db->db_d.db_user_data_ptr_ptr = NULL; 232fa9e4066Sahrens db->db_d.db_evict_func = NULL; 233fa9e4066Sahrens } 234fa9e4066Sahrens 235fa9e4066Sahrens void 236fa9e4066Sahrens dbuf_init(void) 237fa9e4066Sahrens { 238fa9e4066Sahrens uint64_t hsize = 1; 239fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 240fa9e4066Sahrens int i; 241fa9e4066Sahrens 242fa9e4066Sahrens /* 243fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 244fa9e4066Sahrens * with an average 64k block size. The table will take up 245fa9e4066Sahrens * totalmem*sizeof(void*)/64k bytes (i.e. 128KB/GB with 8-byte 246fa9e4066Sahrens * pointers). 247fa9e4066Sahrens */ 248fa9e4066Sahrens while (hsize * 65536 < physmem * PAGESIZE) 249fa9e4066Sahrens hsize <<= 1; 250fa9e4066Sahrens 251fa9e4066Sahrens h->hash_table_mask = hsize - 1; 252fa9e4066Sahrens h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_SLEEP); 253fa9e4066Sahrens 254fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 255fa9e4066Sahrens sizeof (dmu_buf_impl_t), 256fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 257fa9e4066Sahrens dbuf_tq = taskq_create("dbuf_tq", 8, maxclsyspri, 50, INT_MAX, 258fa9e4066Sahrens TASKQ_PREPOPULATE); 259fa9e4066Sahrens 260fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 261fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 262fa9e4066Sahrens } 263fa9e4066Sahrens 264fa9e4066Sahrens void 265fa9e4066Sahrens dbuf_fini(void) 266fa9e4066Sahrens { 267fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 268fa9e4066Sahrens int i; 269fa9e4066Sahrens 270fa9e4066Sahrens taskq_destroy(dbuf_tq); 271fa9e4066Sahrens dbuf_tq = NULL; 272fa9e4066Sahrens 273fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 274fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 275fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 276fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 277fa9e4066Sahrens } 278fa9e4066Sahrens 279fa9e4066Sahrens /* 280fa9e4066Sahrens * Other stuff. 281fa9e4066Sahrens */ 282fa9e4066Sahrens 2839c9dc39aSek #ifdef ZFS_DEBUG 284fa9e4066Sahrens static void 285fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 286fa9e4066Sahrens { 287fa9e4066Sahrens int i; 288fa9e4066Sahrens dnode_t *dn = db->db_dnode; 289fa9e4066Sahrens 290fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 291fa9e4066Sahrens 292fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 293fa9e4066Sahrens return; 294fa9e4066Sahrens 295fa9e4066Sahrens ASSERT(db->db_objset != NULL); 296fa9e4066Sahrens if (dn == NULL) { 297fa9e4066Sahrens ASSERT(db->db_parent == NULL); 298fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 299fa9e4066Sahrens } else { 300fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 301fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 302fa9e4066Sahrens ASSERT(list_head(&dn->dn_dbufs)); 303fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 304fa9e4066Sahrens } 305fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 306fa9e4066Sahrens ASSERT(dn != NULL); 307fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen); 308fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID); 309fa9e4066Sahrens } else { 310fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 311fa9e4066Sahrens } 312fa9e4066Sahrens 313fa9e4066Sahrens if (db->db_level == 0) { 314fa9e4066Sahrens void **udpp = db->db_d.db_user_data_ptr_ptr; 315fa9e4066Sahrens /* we can be momentarily larger in dnode_set_blksz() */ 316fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID && dn) { 317fa9e4066Sahrens ASSERT3U(db->db.db_size, >=, dn->dn_datablksz); 318fa9e4066Sahrens } 319fa9e4066Sahrens if (udpp) { 320fa9e4066Sahrens ASSERT((refcount_is_zero(&db->db_holds) && 321fa9e4066Sahrens *udpp == NULL) || 322fa9e4066Sahrens (!refcount_is_zero(&db->db_holds) && 323fa9e4066Sahrens *udpp == db->db.db_data)); 324fa9e4066Sahrens } 325fa9e4066Sahrens 326fa9e4066Sahrens if (IS_DNODE_DNODE(db->db.db_object)) { 327fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 328fa9e4066Sahrens /* 329fa9e4066Sahrens * it should only be modified in syncing 330fa9e4066Sahrens * context, so make sure we only have 331fa9e4066Sahrens * one copy of the data. 332fa9e4066Sahrens */ 333fa9e4066Sahrens ASSERT(db->db_d.db_data_old[i] == NULL || 334fa9e4066Sahrens db->db_d.db_data_old[i] == db->db_buf); 335fa9e4066Sahrens } 336fa9e4066Sahrens } 337fa9e4066Sahrens } 338fa9e4066Sahrens 339fa9e4066Sahrens /* verify db->db_blkptr */ 340fa9e4066Sahrens if (db->db_blkptr) { 341fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 342fa9e4066Sahrens /* db is pointed to by the dnode */ 343fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 344fa9e4066Sahrens if (IS_DNODE_DNODE(db->db.db_object)) 345fa9e4066Sahrens ASSERT(db->db_parent == NULL); 346fa9e4066Sahrens else 347fa9e4066Sahrens ASSERT(db->db_parent != NULL); 348fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 349fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 350fa9e4066Sahrens } else { 351fa9e4066Sahrens /* db is pointed to by an indirect block */ 352fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 353fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 354fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 355fa9e4066Sahrens db->db.db_object); 356fa9e4066Sahrens /* 357fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 358fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 359fa9e4066Sahrens * grows. safe to do this now? 360fa9e4066Sahrens */ 361fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) { 362fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 363fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 364fa9e4066Sahrens db->db_blkid % epb)); 365fa9e4066Sahrens } 366fa9e4066Sahrens } 367fa9e4066Sahrens } 368fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 369fa9e4066Sahrens db->db.db_data && db->db_blkid != DB_BONUS_BLKID && 370fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 371fa9e4066Sahrens /* 372fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 373fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 374fa9e4066Sahrens * data when we evict this buffer. 375fa9e4066Sahrens */ 376fa9e4066Sahrens if (db->db_dirtycnt == 0) { 377fa9e4066Sahrens uint64_t *buf = db->db.db_data; 378fa9e4066Sahrens int i; 379fa9e4066Sahrens 380fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 381fa9e4066Sahrens ASSERT(buf[i] == 0); 382fa9e4066Sahrens } 383fa9e4066Sahrens } 384fa9e4066Sahrens } 385fa9e4066Sahrens } 3869c9dc39aSek #endif 387fa9e4066Sahrens 388fa9e4066Sahrens static void 389fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 390fa9e4066Sahrens { 391fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 392fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_user_data_ptr_ptr) { 393fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 394fa9e4066Sahrens *db->db_d.db_user_data_ptr_ptr = db->db.db_data; 395fa9e4066Sahrens } 396fa9e4066Sahrens } 397fa9e4066Sahrens 398fa9e4066Sahrens static void 399fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 400fa9e4066Sahrens { 401fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 402fa9e4066Sahrens ASSERT(buf->b_data != NULL); 403fa9e4066Sahrens db->db_buf = buf; 404fa9e4066Sahrens db->db.db_data = buf->b_data; 405fa9e4066Sahrens dbuf_update_data(db); 406fa9e4066Sahrens } 407fa9e4066Sahrens 408fa9e4066Sahrens uint64_t 409fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 410fa9e4066Sahrens { 411fa9e4066Sahrens if (dn->dn_datablkshift) { 412fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 413fa9e4066Sahrens } else { 414fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 415fa9e4066Sahrens return (0); 416fa9e4066Sahrens } 417fa9e4066Sahrens } 418fa9e4066Sahrens 419fa9e4066Sahrens static void 420fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 421fa9e4066Sahrens { 422fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 423fa9e4066Sahrens 424fa9e4066Sahrens mutex_enter(&db->db_mtx); 425fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 426fa9e4066Sahrens /* 427fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 428fa9e4066Sahrens */ 429fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 430fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 431fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 432fa9e4066Sahrens /* we were freed in flight; disregard any error */ 433fa9e4066Sahrens arc_release(buf, db); 434fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 435fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 436fa9e4066Sahrens dbuf_set_data(db, buf); 437fa9e4066Sahrens db->db_state = DB_CACHED; 438fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 439fa9e4066Sahrens dbuf_set_data(db, buf); 440fa9e4066Sahrens db->db_state = DB_CACHED; 441fa9e4066Sahrens } else { 442fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 443fa9e4066Sahrens arc_buf_free(buf, db); 444fa9e4066Sahrens db->db_state = DB_UNCACHED; 445fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 446fa9e4066Sahrens } 447fa9e4066Sahrens cv_broadcast(&db->db_changed); 448fa9e4066Sahrens mutex_exit(&db->db_mtx); 449fa9e4066Sahrens } 450fa9e4066Sahrens 451fa9e4066Sahrens void 452fa9e4066Sahrens dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 453fa9e4066Sahrens { 454fa9e4066Sahrens arc_buf_t *buf; 455fa9e4066Sahrens blkptr_t *bp; 456fa9e4066Sahrens 457fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 458fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 459fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 460fa9e4066Sahrens 461fa9e4066Sahrens /* 462fa9e4066Sahrens * prefetch only data blocks (level 0) -- don't prefetch indirect 463fa9e4066Sahrens * blocks 464fa9e4066Sahrens */ 465fa9e4066Sahrens if ((db->db_level > 0) || (db->db_blkid == DB_BONUS_BLKID)) { 466fa9e4066Sahrens flags |= DB_RF_NOPREFETCH; 467fa9e4066Sahrens } 468fa9e4066Sahrens 469fa9e4066Sahrens if (((flags & DB_RF_NOPREFETCH) == 0) && (db->db_dnode != NULL)) { 470fa9e4066Sahrens dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset, 471fa9e4066Sahrens db->db.db_size); 472fa9e4066Sahrens } 473fa9e4066Sahrens 474fa9e4066Sahrens if (db->db_state == DB_CACHED) { 475fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 476fa9e4066Sahrens return; 477fa9e4066Sahrens } 478fa9e4066Sahrens 479fa9e4066Sahrens mutex_enter(&db->db_mtx); 480fa9e4066Sahrens 481fa9e4066Sahrens if (db->db_state != DB_UNCACHED) { 482fa9e4066Sahrens mutex_exit(&db->db_mtx); 483fa9e4066Sahrens return; 484fa9e4066Sahrens } 485fa9e4066Sahrens 486fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_UNCACHED); 487fa9e4066Sahrens 488fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 489fa9e4066Sahrens ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size); 490fa9e4066Sahrens buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 491fa9e4066Sahrens DN_MAX_BONUSLEN, db); 492fa9e4066Sahrens if (db->db.db_size < DN_MAX_BONUSLEN) 493fa9e4066Sahrens bzero(buf->b_data, DN_MAX_BONUSLEN); 494fa9e4066Sahrens bcopy(DN_BONUS(db->db_dnode->dn_phys), buf->b_data, 495fa9e4066Sahrens db->db.db_size); 496fa9e4066Sahrens dbuf_set_data(db, buf); 497fa9e4066Sahrens db->db_state = DB_CACHED; 498fa9e4066Sahrens mutex_exit(&db->db_mtx); 499fa9e4066Sahrens return; 500fa9e4066Sahrens } 501fa9e4066Sahrens 502fa9e4066Sahrens if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid)) 503fa9e4066Sahrens bp = NULL; 504fa9e4066Sahrens else 505fa9e4066Sahrens bp = db->db_blkptr; 506fa9e4066Sahrens 507fa9e4066Sahrens if (bp == NULL) 508fa9e4066Sahrens dprintf_dbuf(db, "blkptr: %s\n", "NULL"); 509fa9e4066Sahrens else 510fa9e4066Sahrens dprintf_dbuf_bp(db, bp, "%s", "blkptr:"); 511fa9e4066Sahrens 512fa9e4066Sahrens if (bp == NULL || BP_IS_HOLE(bp)) { 513fa9e4066Sahrens ASSERT(bp == NULL || BP_IS_HOLE(bp)); 514fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 515fa9e4066Sahrens db->db.db_size, db)); 516fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 517fa9e4066Sahrens db->db_state = DB_CACHED; 518fa9e4066Sahrens mutex_exit(&db->db_mtx); 519fa9e4066Sahrens return; 520fa9e4066Sahrens } 521fa9e4066Sahrens 522fa9e4066Sahrens db->db_state = DB_READ; 523fa9e4066Sahrens mutex_exit(&db->db_mtx); 524fa9e4066Sahrens 525fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 526fa9e4066Sahrens (void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp, 527fa9e4066Sahrens db->db_level > 0 ? byteswap_uint64_array : 528fa9e4066Sahrens dmu_ot[db->db_dnode->dn_type].ot_byteswap, 529fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 530fa9e4066Sahrens (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 531fa9e4066Sahrens ARC_NOWAIT); 532fa9e4066Sahrens } 533fa9e4066Sahrens 534fa9e4066Sahrens static int 535fa9e4066Sahrens dbuf_read_generic(dmu_buf_impl_t *db, uint32_t flags) 536fa9e4066Sahrens { 537fa9e4066Sahrens zio_t *zio; 538fa9e4066Sahrens int err; 539fa9e4066Sahrens 540fa9e4066Sahrens /* 541fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 542fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 543fa9e4066Sahrens */ 544fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 545fa9e4066Sahrens if (db->db_state == DB_CACHED) 546fa9e4066Sahrens return (0); 547fa9e4066Sahrens 548fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 549fa9e4066Sahrens zio = zio_root(db->db_dnode->dn_objset->os_spa, NULL, NULL, 550fa9e4066Sahrens ZIO_FLAG_CANFAIL); 551fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 552fa9e4066Sahrens rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER); 553fa9e4066Sahrens dbuf_read_impl(db, zio, flags); 554fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 555fa9e4066Sahrens rw_exit(&db->db_dnode->dn_struct_rwlock); 556fa9e4066Sahrens err = zio_wait(zio); 557fa9e4066Sahrens if (err) 558fa9e4066Sahrens return (err); 559fa9e4066Sahrens } 560fa9e4066Sahrens 561fa9e4066Sahrens mutex_enter(&db->db_mtx); 562fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) { 563fa9e4066Sahrens ASSERT(db->db_state == DB_READ || 564fa9e4066Sahrens (flags & DB_RF_HAVESTRUCT) == 0); 565fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 566fa9e4066Sahrens } 567fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 568fa9e4066Sahrens mutex_exit(&db->db_mtx); 569fa9e4066Sahrens 570fa9e4066Sahrens return (0); 571fa9e4066Sahrens } 572fa9e4066Sahrens 573fa9e4066Sahrens #pragma weak dmu_buf_read = dbuf_read 574fa9e4066Sahrens void 575fa9e4066Sahrens dbuf_read(dmu_buf_impl_t *db) 576fa9e4066Sahrens { 577fa9e4066Sahrens int err; 578fa9e4066Sahrens 579fa9e4066Sahrens err = dbuf_read_generic(db, DB_RF_MUST_SUCCEED); 580fa9e4066Sahrens ASSERT(err == 0); 581fa9e4066Sahrens } 582fa9e4066Sahrens 583fa9e4066Sahrens #pragma weak dmu_buf_read_canfail = dbuf_read_canfail 584fa9e4066Sahrens int 585fa9e4066Sahrens dbuf_read_canfail(dmu_buf_impl_t *db) 586fa9e4066Sahrens { 587fa9e4066Sahrens return (dbuf_read_generic(db, DB_RF_CANFAIL)); 588fa9e4066Sahrens } 589fa9e4066Sahrens 590fa9e4066Sahrens void 591fa9e4066Sahrens dbuf_read_havestruct(dmu_buf_impl_t *db) 592fa9e4066Sahrens { 593fa9e4066Sahrens int err; 594fa9e4066Sahrens 595fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock)); 596fa9e4066Sahrens err = dbuf_read_generic(db, (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH)); 597fa9e4066Sahrens ASSERT(err == 0); 598fa9e4066Sahrens } 599fa9e4066Sahrens 600fa9e4066Sahrens static void 601fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 602fa9e4066Sahrens { 603fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 604fa9e4066Sahrens mutex_enter(&db->db_mtx); 605fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 606fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 607fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 608fa9e4066Sahrens int blksz = (db->db_blkid == DB_BONUS_BLKID) ? 609fa9e4066Sahrens DN_MAX_BONUSLEN : db->db.db_size; 610fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 611fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 612fa9e4066Sahrens blksz, db)); 613fa9e4066Sahrens db->db_state = DB_FILL; 614fa9e4066Sahrens } else { 615fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 616fa9e4066Sahrens } 617fa9e4066Sahrens mutex_exit(&db->db_mtx); 618fa9e4066Sahrens } 619fa9e4066Sahrens 620fa9e4066Sahrens /* 621fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 622fa9e4066Sahrens * buffers, that have been modified in a previous transaction 623fa9e4066Sahrens * group, before we modify them in the current active group. 624fa9e4066Sahrens * 625fa9e4066Sahrens * This function is used in two places: when we are dirtying a 626fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 627fa9e4066Sahrens * a range in a dnode that includes this buffer. 628fa9e4066Sahrens * 629fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 630fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 631fa9e4066Sahrens * dbuf list for the dnode. 632fa9e4066Sahrens */ 633fa9e4066Sahrens static void 634fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 635fa9e4066Sahrens { 636fa9e4066Sahrens arc_buf_t **quiescing, **syncing; 637fa9e4066Sahrens int size = (db->db_blkid == DB_BONUS_BLKID) ? 638fa9e4066Sahrens DN_MAX_BONUSLEN : db->db.db_size; 639fa9e4066Sahrens 640fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 641fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 642fa9e4066Sahrens 643fa9e4066Sahrens quiescing = &db->db_d.db_data_old[(txg-1)&TXG_MASK]; 644fa9e4066Sahrens syncing = &db->db_d.db_data_old[(txg-2)&TXG_MASK]; 645fa9e4066Sahrens 646fa9e4066Sahrens /* 647fa9e4066Sahrens * If this buffer is referenced from the current quiescing 648fa9e4066Sahrens * transaction group: either make a copy and reset the reference 649fa9e4066Sahrens * to point to the copy, or (if there a no active holders) just 650fa9e4066Sahrens * null out the current db_data pointer. 651fa9e4066Sahrens */ 652fa9e4066Sahrens if (*quiescing == db->db_buf) { 653fa9e4066Sahrens /* 654fa9e4066Sahrens * If the quiescing txg is "dirty", then we better not 655fa9e4066Sahrens * be referencing the same buffer from the syncing txg. 656fa9e4066Sahrens */ 657fa9e4066Sahrens ASSERT(*syncing != db->db_buf); 658fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 659fa9e4066Sahrens *quiescing = arc_buf_alloc( 660fa9e4066Sahrens db->db_dnode->dn_objset->os_spa, size, db); 661fa9e4066Sahrens bcopy(db->db.db_data, (*quiescing)->b_data, size); 662fa9e4066Sahrens } else { 663fa9e4066Sahrens db->db.db_data = NULL; 664fa9e4066Sahrens db->db_buf = NULL; 665fa9e4066Sahrens db->db_state = DB_UNCACHED; 666fa9e4066Sahrens } 667fa9e4066Sahrens return; 668fa9e4066Sahrens } 669fa9e4066Sahrens 670fa9e4066Sahrens /* 671fa9e4066Sahrens * If this buffer is referenced from the current syncing 672fa9e4066Sahrens * transaction group: either 673fa9e4066Sahrens * 1 - make a copy and reset the reference, or 674fa9e4066Sahrens * 2 - if there are no holders, just null the current db_data. 675fa9e4066Sahrens */ 676fa9e4066Sahrens if (*syncing == db->db_buf) { 677fa9e4066Sahrens ASSERT3P(*quiescing, ==, NULL); 678fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, ==, 1); 679fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 680fa9e4066Sahrens /* we can't copy if we have already started a write */ 681fa9e4066Sahrens ASSERT(*syncing != db->db_data_pending); 682fa9e4066Sahrens *syncing = arc_buf_alloc( 683fa9e4066Sahrens db->db_dnode->dn_objset->os_spa, size, db); 684fa9e4066Sahrens bcopy(db->db.db_data, (*syncing)->b_data, size); 685fa9e4066Sahrens } else { 686fa9e4066Sahrens db->db.db_data = NULL; 687fa9e4066Sahrens db->db_buf = NULL; 688fa9e4066Sahrens db->db_state = DB_UNCACHED; 689fa9e4066Sahrens } 690fa9e4066Sahrens } 691fa9e4066Sahrens } 692fa9e4066Sahrens 693fa9e4066Sahrens void 694fa9e4066Sahrens dbuf_unoverride(dmu_buf_impl_t *db, uint64_t txg) 695fa9e4066Sahrens { 696fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 697fa9e4066Sahrens if (db->db_d.db_overridden_by[txg&TXG_MASK] == IN_DMU_SYNC) { 698fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] = NULL; 699fa9e4066Sahrens } else if (db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 700fa9e4066Sahrens /* free this block */ 701fa9e4066Sahrens ASSERT(list_link_active(&db->db_dirty_node[txg&TXG_MASK]) || 702fa9e4066Sahrens db->db_dnode->dn_free_txg == txg); 703fa9e4066Sahrens if (!BP_IS_HOLE(db->db_d.db_overridden_by[txg&TXG_MASK])) { 704fa9e4066Sahrens /* XXX can get silent EIO here */ 705fa9e4066Sahrens (void) arc_free(NULL, db->db_dnode->dn_objset->os_spa, 706fa9e4066Sahrens txg, db->db_d.db_overridden_by[txg&TXG_MASK], 707fa9e4066Sahrens NULL, NULL, ARC_WAIT); 708fa9e4066Sahrens } 709fa9e4066Sahrens kmem_free(db->db_d.db_overridden_by[txg&TXG_MASK], 710fa9e4066Sahrens sizeof (blkptr_t)); 711fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] = NULL; 712fa9e4066Sahrens /* release the already-written buffer */ 713fa9e4066Sahrens arc_release(db->db_d.db_data_old[txg&TXG_MASK], db); 714fa9e4066Sahrens } 715fa9e4066Sahrens } 716fa9e4066Sahrens 717fa9e4066Sahrens void 718fa9e4066Sahrens dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) 719fa9e4066Sahrens { 720fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 721fa9e4066Sahrens uint64_t txg = tx->tx_txg; 722fa9e4066Sahrens 723fa9e4066Sahrens dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks); 724fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 725fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 726fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 727fa9e4066Sahrens if ((db->db_level != 0) || (db->db_blkid == DB_BONUS_BLKID)) 728fa9e4066Sahrens continue; 729fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 730fa9e4066Sahrens if (db->db_blkid < blkid || 731fa9e4066Sahrens db->db_blkid >= blkid+nblks) 732fa9e4066Sahrens continue; 733fa9e4066Sahrens 734fa9e4066Sahrens /* found a level 0 buffer in the range */ 735fa9e4066Sahrens if (dbuf_undirty(db, tx)) 736fa9e4066Sahrens continue; 737fa9e4066Sahrens 738fa9e4066Sahrens mutex_enter(&db->db_mtx); 739fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 740fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 741fa9e4066Sahrens mutex_exit(&db->db_mtx); 742fa9e4066Sahrens continue; 743fa9e4066Sahrens } 744fa9e4066Sahrens if (db->db_state == DB_READ) { 745fa9e4066Sahrens /* this will be handled in dbuf_read_done() */ 746fa9e4066Sahrens db->db_d.db_freed_in_flight = TRUE; 747fa9e4066Sahrens mutex_exit(&db->db_mtx); 748fa9e4066Sahrens continue; 749fa9e4066Sahrens } 750fa9e4066Sahrens if (db->db_state == DB_FILL) { 751fa9e4066Sahrens /* this will be handled in dbuf_rele() */ 752fa9e4066Sahrens db->db_d.db_freed_in_flight = TRUE; 753fa9e4066Sahrens mutex_exit(&db->db_mtx); 754fa9e4066Sahrens continue; 755fa9e4066Sahrens } 756fa9e4066Sahrens 757fa9e4066Sahrens /* make a copy of the data if necessary */ 758fa9e4066Sahrens dbuf_fix_old_data(db, txg); 759fa9e4066Sahrens 760fa9e4066Sahrens if (db->db.db_data) { 761fa9e4066Sahrens /* fill in with appropriate data */ 762fa9e4066Sahrens arc_release(db->db_buf, db); 763fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 764fa9e4066Sahrens } 765fa9e4066Sahrens mutex_exit(&db->db_mtx); 766fa9e4066Sahrens } 767fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 768fa9e4066Sahrens } 769fa9e4066Sahrens 770fa9e4066Sahrens static int 771fa9e4066Sahrens dbuf_new_block(dmu_buf_impl_t *db, dmu_tx_t *tx) 772fa9e4066Sahrens { 773fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 774fa9e4066Sahrens uint64_t birth_txg = 0; 775fa9e4066Sahrens 776fa9e4066Sahrens /* Don't count meta-objects */ 777fa9e4066Sahrens if (ds == NULL) 778fa9e4066Sahrens return (FALSE); 779fa9e4066Sahrens 780fa9e4066Sahrens /* 781fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 782fa9e4066Sahrens * If it's syncing, then db_dirtied will be set so we'll 783fa9e4066Sahrens * ignore db_blkptr. 784fa9e4066Sahrens */ 785fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); /* XXX strictly necessary? */ 786fa9e4066Sahrens /* If we have been dirtied since the last snapshot, its not new */ 787fa9e4066Sahrens if (db->db_dirtied) 788fa9e4066Sahrens birth_txg = db->db_dirtied; 789fa9e4066Sahrens else if (db->db_blkptr) 790fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 791fa9e4066Sahrens 792fa9e4066Sahrens if (birth_txg) 793fa9e4066Sahrens return (!dsl_dataset_block_freeable(ds, birth_txg, tx)); 794fa9e4066Sahrens else 795fa9e4066Sahrens return (TRUE); 796fa9e4066Sahrens } 797fa9e4066Sahrens 798fa9e4066Sahrens void 799fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 800fa9e4066Sahrens { 801fa9e4066Sahrens arc_buf_t *buf, *obuf; 802fa9e4066Sahrens int osize = db->db.db_size; 803fa9e4066Sahrens 804fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 805fa9e4066Sahrens ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)); 806fa9e4066Sahrens 807fa9e4066Sahrens if (osize == size) 808fa9e4066Sahrens return; 809fa9e4066Sahrens 810fa9e4066Sahrens /* 811fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 812fa9e4066Sahrens * is OK, because there can be no other references to the db 813fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 814fa9e4066Sahrens * be happening. 815fa9e4066Sahrens */ 816fa9e4066Sahrens /* Make a copy of the data if necessary */ 817fa9e4066Sahrens dbuf_will_dirty(db, tx); 818fa9e4066Sahrens 819fa9e4066Sahrens /* create the data buffer for the new block */ 820fa9e4066Sahrens buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db); 821fa9e4066Sahrens 822fa9e4066Sahrens /* copy old block data to the new block */ 823fa9e4066Sahrens obuf = db->db_buf; 824*f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 825fa9e4066Sahrens /* zero the remainder */ 826*f65e61c0Sahrens if (size > osize) 827*f65e61c0Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 828fa9e4066Sahrens 829fa9e4066Sahrens mutex_enter(&db->db_mtx); 830fa9e4066Sahrens /* ASSERT3U(refcount_count(&db->db_holds), ==, 1); */ 831fa9e4066Sahrens dbuf_set_data(db, buf); 832fa9e4066Sahrens arc_buf_free(obuf, db); 833fa9e4066Sahrens db->db.db_size = size; 834fa9e4066Sahrens 835fa9e4066Sahrens /* fix up the dirty info */ 836fa9e4066Sahrens if (db->db_level == 0) 837fa9e4066Sahrens db->db_d.db_data_old[tx->tx_txg&TXG_MASK] = buf; 838fa9e4066Sahrens mutex_exit(&db->db_mtx); 839fa9e4066Sahrens 840fa9e4066Sahrens dnode_willuse_space(db->db_dnode, size-osize, tx); 841fa9e4066Sahrens } 842fa9e4066Sahrens 843fa9e4066Sahrens void 844fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 845fa9e4066Sahrens { 846fa9e4066Sahrens dnode_t *dn = db->db_dnode; 847fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 848fa9e4066Sahrens int drop_struct_lock = FALSE; 849fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 850fa9e4066Sahrens 851fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 852fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 8539c9dc39aSek DMU_TX_DIRTY_BUF(tx, db); 854fa9e4066Sahrens 855fa9e4066Sahrens /* 856fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 857fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 858fa9e4066Sahrens * were already pre-dirtied in open context. 859fa9e4066Sahrens * XXX We may want to prohibit dirtying in syncing context even 860fa9e4066Sahrens * if they did pre-dirty. 861fa9e4066Sahrens */ 862fa9e4066Sahrens ASSERT(!(dmu_tx_is_syncing(tx) && 863fa9e4066Sahrens !BP_IS_HOLE(&dn->dn_objset->os_rootbp) && 864fa9e4066Sahrens !(dn->dn_object & DMU_PRIVATE_OBJECT) && 865fa9e4066Sahrens dn->dn_objset->os_dsl_dataset != NULL && 866fa9e4066Sahrens !dsl_dir_is_private( 867fa9e4066Sahrens dn->dn_objset->os_dsl_dataset->ds_dir))); 868fa9e4066Sahrens 869fa9e4066Sahrens /* 870fa9e4066Sahrens * We make this assert for private objects as well, but after we 871fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 872fa9e4066Sahrens * in syncing context. 873fa9e4066Sahrens */ 874fa9e4066Sahrens ASSERT(dn->dn_object & DMU_PRIVATE_OBJECT || 875fa9e4066Sahrens dn->dn_dirtyctx == DN_UNDIRTIED || 876fa9e4066Sahrens dn->dn_dirtyctx == 877fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 878fa9e4066Sahrens 879fa9e4066Sahrens mutex_enter(&db->db_mtx); 880fa9e4066Sahrens /* XXX make this true for indirects too? */ 881fa9e4066Sahrens ASSERT(db->db_level != 0 || db->db_state == DB_CACHED || 882fa9e4066Sahrens db->db_state == DB_FILL); 883fa9e4066Sahrens 884fa9e4066Sahrens /* 885fa9e4066Sahrens * If this buffer is currently part of an "overridden" region, 886fa9e4066Sahrens * we now need to remove it from that region. 887fa9e4066Sahrens */ 888fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID && 889fa9e4066Sahrens db->db_d.db_overridden_by[txgoff] != NULL) { 890fa9e4066Sahrens dbuf_unoverride(db, tx->tx_txg); 891fa9e4066Sahrens } 892fa9e4066Sahrens 893fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 894fa9e4066Sahrens /* 895fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 896fa9e4066Sahrens * initialize the objset. 897fa9e4066Sahrens */ 898fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 899fa9e4066Sahrens !BP_IS_HOLE(&dn->dn_objset->os_rootbp)) { 900fa9e4066Sahrens dn->dn_dirtyctx = 901fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 902fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 903fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 904fa9e4066Sahrens } 905fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 906fa9e4066Sahrens 907fa9e4066Sahrens /* 908fa9e4066Sahrens * If this buffer is already dirty, we're done. 909fa9e4066Sahrens */ 910fa9e4066Sahrens if (list_link_active(&db->db_dirty_node[txgoff])) { 911fa9e4066Sahrens mutex_exit(&db->db_mtx); 912fa9e4066Sahrens return; 913fa9e4066Sahrens } 914fa9e4066Sahrens 915fa9e4066Sahrens /* 916fa9e4066Sahrens * Only valid if not already dirty. 917fa9e4066Sahrens */ 918fa9e4066Sahrens ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 919fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 920fa9e4066Sahrens 921fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 922fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 923fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 924fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 925fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 926fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 927fa9e4066Sahrens 928fa9e4066Sahrens /* 929fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 930fa9e4066Sahrens * mos, a spa os, or we're initializing the os. However, we are 931fa9e4066Sahrens * allowed to dirty in syncing context provided we already 932fa9e4066Sahrens * dirtied it in open context. Hence we must make this 933fa9e4066Sahrens * assertion only if we're not already dirty. 934fa9e4066Sahrens */ 935fa9e4066Sahrens ASSERT(!dmu_tx_is_syncing(tx) || 936fa9e4066Sahrens os->os_dsl_dataset == NULL || 937fa9e4066Sahrens !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) || 938fa9e4066Sahrens !BP_IS_HOLE(&os->os_rootbp)); 939fa9e4066Sahrens ASSERT(db->db.db_size != 0); 940fa9e4066Sahrens 941fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 942fa9e4066Sahrens 943fa9e4066Sahrens if (db->db_level == 0) { 944fa9e4066Sahrens /* 945fa9e4066Sahrens * Release the data buffer from the cache so that we 946fa9e4066Sahrens * can modify it without impacting possible other users 947fa9e4066Sahrens * of this cached data block. Note that indirect blocks 948fa9e4066Sahrens * and private objects are not released until the syncing 949fa9e4066Sahrens * state (since they are only modified then). 950fa9e4066Sahrens * 951fa9e4066Sahrens * If this buffer is dirty in an old transaction group we need 952fa9e4066Sahrens * to make a copy of it so that the changes we make in this 953fa9e4066Sahrens * transaction group won't leak out when we sync the older txg. 954fa9e4066Sahrens */ 955fa9e4066Sahrens ASSERT(db->db_buf != NULL); 956fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 957fa9e4066Sahrens ASSERT(db->db_d.db_data_old[txgoff] == NULL); 958fa9e4066Sahrens if (!(db->db.db_object & DMU_PRIVATE_OBJECT)) { 959fa9e4066Sahrens arc_release(db->db_buf, db); 960fa9e4066Sahrens dbuf_fix_old_data(db, tx->tx_txg); 961fa9e4066Sahrens ASSERT(db->db_buf != NULL); 962fa9e4066Sahrens } 963fa9e4066Sahrens db->db_d.db_data_old[txgoff] = db->db_buf; 964fa9e4066Sahrens } 965fa9e4066Sahrens 966fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 967fa9e4066Sahrens /* 968fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 969fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 970fa9e4066Sahrens * happened after the free. 971fa9e4066Sahrens */ 972fa9e4066Sahrens if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) { 973fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 974fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 975fa9e4066Sahrens } 976fa9e4066Sahrens 977fa9e4066Sahrens db->db_dirtied = tx->tx_txg; 978fa9e4066Sahrens list_insert_tail(&dn->dn_dirty_dbufs[txgoff], db); 979fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 980fa9e4066Sahrens 981fa9e4066Sahrens /* 982fa9e4066Sahrens * If writting this buffer will consume a new block on disk, 983fa9e4066Sahrens * then update the accounting. 984fa9e4066Sahrens */ 985fa9e4066Sahrens if (db->db_blkid != DB_BONUS_BLKID) { 986fa9e4066Sahrens if (!dbuf_new_block(db, tx) && db->db_blkptr) { 987fa9e4066Sahrens /* 988fa9e4066Sahrens * This is only a guess -- if the dbuf is dirty 989fa9e4066Sahrens * in a previous txg, we don't know how much 990fa9e4066Sahrens * space it will use on disk yet. We should 991fa9e4066Sahrens * really have the struct_rwlock to access 992fa9e4066Sahrens * db_blkptr, but since this is just a guess, 993fa9e4066Sahrens * it's OK if we get an odd answer. 994fa9e4066Sahrens */ 995fa9e4066Sahrens dnode_willuse_space(dn, 996fa9e4066Sahrens -BP_GET_ASIZE(db->db_blkptr), tx); 997fa9e4066Sahrens } 998fa9e4066Sahrens dnode_willuse_space(dn, db->db.db_size, tx); 999fa9e4066Sahrens } 1000fa9e4066Sahrens 1001fa9e4066Sahrens /* 1002fa9e4066Sahrens * This buffer is now part of this txg 1003fa9e4066Sahrens */ 1004fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1005fa9e4066Sahrens db->db_dirtycnt += 1; 1006fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1007fa9e4066Sahrens 1008fa9e4066Sahrens mutex_exit(&db->db_mtx); 1009fa9e4066Sahrens 1010fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1011fa9e4066Sahrens dnode_setdirty(dn, tx); 1012fa9e4066Sahrens return; 1013fa9e4066Sahrens } 1014fa9e4066Sahrens 1015fa9e4066Sahrens if (db->db_level == 0) 1016fa9e4066Sahrens dnode_new_blkid(dn, db->db_blkid, tx); 1017fa9e4066Sahrens 1018fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1019fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1020fa9e4066Sahrens drop_struct_lock = TRUE; 1021fa9e4066Sahrens } 1022fa9e4066Sahrens 1023fa9e4066Sahrens if (db->db_level < dn->dn_nlevels-1) { 1024fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1025fa9e4066Sahrens dmu_buf_impl_t *parent; 1026fa9e4066Sahrens parent = dbuf_hold_level(dn, db->db_level+1, 1027fa9e4066Sahrens db->db_blkid >> epbs, FTAG); 1028fa9e4066Sahrens if (drop_struct_lock) 1029fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1030fa9e4066Sahrens dbuf_dirty(parent, tx); 1031fa9e4066Sahrens dbuf_remove_ref(parent, FTAG); 1032fa9e4066Sahrens } else { 1033fa9e4066Sahrens if (drop_struct_lock) 1034fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1035fa9e4066Sahrens } 1036fa9e4066Sahrens 1037fa9e4066Sahrens dnode_setdirty(dn, tx); 1038fa9e4066Sahrens } 1039fa9e4066Sahrens 1040fa9e4066Sahrens static int 1041fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1042fa9e4066Sahrens { 1043fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1044fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1045fa9e4066Sahrens 1046fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1047fa9e4066Sahrens 1048fa9e4066Sahrens mutex_enter(&db->db_mtx); 1049fa9e4066Sahrens 1050fa9e4066Sahrens /* 1051fa9e4066Sahrens * If this buffer is not dirty, we're done. 1052fa9e4066Sahrens */ 1053fa9e4066Sahrens if (!list_link_active(&db->db_dirty_node[txgoff])) { 1054fa9e4066Sahrens mutex_exit(&db->db_mtx); 1055fa9e4066Sahrens return (0); 1056fa9e4066Sahrens } 1057fa9e4066Sahrens 1058fa9e4066Sahrens /* 1059fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1060fa9e4066Sahrens * it, since one of the current holders may be in the 1061fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1062fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1063fa9e4066Sahrens * XXX - this check assumes we are being called from 1064fa9e4066Sahrens * dbuf_free_range(), perhaps we should move it there? 1065fa9e4066Sahrens */ 1066fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1067fa9e4066Sahrens mutex_exit(&db->db_mtx); 1068fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1069fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1070fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1071fa9e4066Sahrens return (0); 1072fa9e4066Sahrens } 1073fa9e4066Sahrens 1074fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1075fa9e4066Sahrens 1076fa9e4066Sahrens dbuf_unoverride(db, tx->tx_txg); 1077fa9e4066Sahrens 1078fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1079fa9e4066Sahrens if (db->db_level == 0) { 1080fa9e4066Sahrens ASSERT(db->db_buf != NULL); 1081fa9e4066Sahrens ASSERT(db->db_d.db_data_old[txgoff] != NULL); 1082fa9e4066Sahrens if (db->db_d.db_data_old[txgoff] != db->db_buf) 1083fa9e4066Sahrens arc_buf_free(db->db_d.db_data_old[txgoff], db); 1084fa9e4066Sahrens db->db_d.db_data_old[txgoff] = NULL; 1085fa9e4066Sahrens } 1086fa9e4066Sahrens 1087fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1088fa9e4066Sahrens /* XXX undo db_dirtied? but how? */ 1089fa9e4066Sahrens /* db->db_dirtied = tx->tx_txg; */ 1090fa9e4066Sahrens 1091fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1092fa9e4066Sahrens list_remove(&dn->dn_dirty_dbufs[txgoff], db); 1093fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1094fa9e4066Sahrens 1095fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1096fa9e4066Sahrens db->db_dirtycnt -= 1; 1097fa9e4066Sahrens 1098fa9e4066Sahrens if (refcount_remove(&db->db_holds, 1099fa9e4066Sahrens (void *)(uintptr_t)tx->tx_txg) == 0) { 1100fa9e4066Sahrens /* make duf_verify() happy */ 1101fa9e4066Sahrens if (db->db.db_data) 1102fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1103fa9e4066Sahrens 1104fa9e4066Sahrens dbuf_evict(db); 1105fa9e4066Sahrens return (1); 1106fa9e4066Sahrens } 1107fa9e4066Sahrens 1108fa9e4066Sahrens mutex_exit(&db->db_mtx); 1109fa9e4066Sahrens return (0); 1110fa9e4066Sahrens } 1111fa9e4066Sahrens 1112fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1113fa9e4066Sahrens void 1114fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1115fa9e4066Sahrens { 1116fa9e4066Sahrens int rf = DB_RF_MUST_SUCCEED; 1117fa9e4066Sahrens 1118fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1119fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1120fa9e4066Sahrens 1121fa9e4066Sahrens if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) 1122fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1123fa9e4066Sahrens (void) dbuf_read_generic(db, rf); 1124fa9e4066Sahrens dbuf_dirty(db, tx); 1125fa9e4066Sahrens } 1126fa9e4066Sahrens 1127fa9e4066Sahrens #pragma weak dmu_buf_will_fill = dbuf_will_fill 1128fa9e4066Sahrens void 1129fa9e4066Sahrens dbuf_will_fill(dmu_buf_impl_t *db, dmu_tx_t *tx) 1130fa9e4066Sahrens { 1131fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1132fa9e4066Sahrens ASSERT(db->db_level == 0); 1133fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1134fa9e4066Sahrens 1135fa9e4066Sahrens ASSERT(!(db->db.db_object & DMU_PRIVATE_OBJECT) || 1136fa9e4066Sahrens dmu_tx_private_ok(tx)); 1137fa9e4066Sahrens 1138fa9e4066Sahrens dbuf_noread(db); 1139fa9e4066Sahrens dbuf_dirty(db, tx); 1140fa9e4066Sahrens } 1141fa9e4066Sahrens 1142fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1143fa9e4066Sahrens /* ARGSUSED */ 1144fa9e4066Sahrens void 1145fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1146fa9e4066Sahrens { 1147fa9e4066Sahrens mutex_enter(&db->db_mtx); 11489c9dc39aSek DBUF_VERIFY(db); 1149fa9e4066Sahrens 1150fa9e4066Sahrens if (db->db_state == DB_FILL) { 1151fa9e4066Sahrens if (db->db_level == 0 && db->db_d.db_freed_in_flight) { 1152fa9e4066Sahrens /* we were freed while filling */ 1153fa9e4066Sahrens /* XXX dbuf_undirty? */ 1154fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1155fa9e4066Sahrens db->db_d.db_freed_in_flight = FALSE; 1156fa9e4066Sahrens } 1157fa9e4066Sahrens db->db_state = DB_CACHED; 1158fa9e4066Sahrens cv_broadcast(&db->db_changed); 1159fa9e4066Sahrens } 1160fa9e4066Sahrens mutex_exit(&db->db_mtx); 1161fa9e4066Sahrens } 1162fa9e4066Sahrens 1163fa9e4066Sahrens 1164fa9e4066Sahrens static void 1165fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1166fa9e4066Sahrens { 1167fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1168fa9e4066Sahrens 1169fa9e4066Sahrens ASSERT(MUTEX_HELD(&dn->dn_dbufs_mtx)); 1170fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1171fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1172fa9e4066Sahrens 1173fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1174fa9e4066Sahrens ASSERT(db->db_buf != NULL); 1175fa9e4066Sahrens arc_buf_free(db->db_buf, db); 1176fa9e4066Sahrens db->db.db_data = NULL; 1177fa9e4066Sahrens db->db_buf = NULL; 1178fa9e4066Sahrens db->db_state = DB_UNCACHED; 1179fa9e4066Sahrens } 1180fa9e4066Sahrens 1181fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_UNCACHED); 1182fa9e4066Sahrens ASSERT(db->db_buf == NULL); 1183fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1184fa9e4066Sahrens 1185fa9e4066Sahrens mutex_exit(&db->db_mtx); 1186fa9e4066Sahrens 1187fa9e4066Sahrens /* 1188fa9e4066Sahrens * If this dbuf is referened from an indirect dbuf, 1189fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1190fa9e4066Sahrens */ 1191fa9e4066Sahrens if (db->db_parent && db->db_parent != dn->dn_dbuf) 1192fa9e4066Sahrens dbuf_remove_ref(db->db_parent, db); 1193fa9e4066Sahrens 1194fa9e4066Sahrens /* remove from dn_dbufs */ 1195fa9e4066Sahrens list_remove(&dn->dn_dbufs, db); 1196fa9e4066Sahrens 1197fa9e4066Sahrens dnode_rele(dn, db); 1198fa9e4066Sahrens 1199fa9e4066Sahrens dbuf_hash_remove(db); 1200fa9e4066Sahrens 1201fa9e4066Sahrens db->db_dnode = NULL; 1202fa9e4066Sahrens db->db_parent = NULL; 1203fa9e4066Sahrens db->db_blkptr = NULL; 1204fa9e4066Sahrens } 1205fa9e4066Sahrens 1206fa9e4066Sahrens static int 1207fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1208fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1209fa9e4066Sahrens { 1210fa9e4066Sahrens int nlevels, epbs; 1211fa9e4066Sahrens 1212fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1213fa9e4066Sahrens nlevels = 1; 1214fa9e4066Sahrens else 1215fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1216fa9e4066Sahrens 1217fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1218fa9e4066Sahrens 1219fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1220fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1221fa9e4066Sahrens if (blkid == DB_BONUS_BLKID) { 1222fa9e4066Sahrens /* this is the bonus buffer */ 1223fa9e4066Sahrens *parentp = NULL; 1224fa9e4066Sahrens *bpp = NULL; 1225fa9e4066Sahrens return (0); 1226fa9e4066Sahrens } else if (level >= nlevels || 1227fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1228fa9e4066Sahrens /* the buffer has no parent yet */ 1229fa9e4066Sahrens *parentp = NULL; 1230fa9e4066Sahrens *bpp = NULL; 1231fa9e4066Sahrens return (ENOENT); 1232fa9e4066Sahrens } else if (level < nlevels-1) { 1233fa9e4066Sahrens /* this block is referenced from an indirect block */ 1234fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1235fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1236fa9e4066Sahrens if (err) 1237fa9e4066Sahrens return (err); 1238fa9e4066Sahrens dbuf_read_havestruct(*parentp); 1239fa9e4066Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1240fa9e4066Sahrens (blkid & ((1ULL << epbs) - 1)); 1241fa9e4066Sahrens return (0); 1242fa9e4066Sahrens } else { 1243fa9e4066Sahrens /* the block is referenced from the dnode */ 1244fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1245fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1246fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1247fa9e4066Sahrens *parentp = dn->dn_dbuf; 1248fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1249fa9e4066Sahrens return (0); 1250fa9e4066Sahrens } 1251fa9e4066Sahrens } 1252fa9e4066Sahrens 1253fa9e4066Sahrens static dmu_buf_impl_t * 1254fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1255fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1256fa9e4066Sahrens { 1257fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1258fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1259fa9e4066Sahrens 1260fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1261fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1262fa9e4066Sahrens 1263fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1264fa9e4066Sahrens 1265fa9e4066Sahrens db->db_objset = os; 1266fa9e4066Sahrens db->db.db_object = dn->dn_object; 1267fa9e4066Sahrens db->db_level = level; 1268fa9e4066Sahrens db->db_blkid = blkid; 1269fa9e4066Sahrens db->db_state = DB_UNCACHED; 1270fa9e4066Sahrens 1271fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1272fa9e4066Sahrens db->db.db_size = dn->dn_bonuslen; 1273fa9e4066Sahrens db->db.db_offset = DB_BONUS_BLKID; 1274fa9e4066Sahrens } else { 1275fa9e4066Sahrens int blocksize = 1276fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1277fa9e4066Sahrens db->db.db_size = blocksize; 1278fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1279fa9e4066Sahrens } 1280fa9e4066Sahrens 1281fa9e4066Sahrens db->db_dirtied = 0; 1282fa9e4066Sahrens db->db_dirtycnt = 0; 1283fa9e4066Sahrens 1284fa9e4066Sahrens bzero(&db->db_d, sizeof (db->db_d)); 1285fa9e4066Sahrens 1286fa9e4066Sahrens /* 1287fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1288fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1289fa9e4066Sahrens * This prevents a possible deadlock with someone 1290fa9e4066Sahrens * trying to look up this dbuf before its added to the 1291fa9e4066Sahrens * dn_dbufs list. 1292fa9e4066Sahrens */ 1293fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1294fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1295fa9e4066Sahrens /* someone else inserted it first */ 1296fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1297fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1298fa9e4066Sahrens return (odb); 1299fa9e4066Sahrens } 1300fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1301fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1302fa9e4066Sahrens 1303fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1304fa9e4066Sahrens dbuf_add_ref(parent, db); 1305fa9e4066Sahrens 1306fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1307fa9e4066Sahrens 1308fa9e4066Sahrens db->db_dnode = dn; 1309fa9e4066Sahrens db->db_parent = parent; 1310fa9e4066Sahrens db->db_blkptr = blkptr; 1311fa9e4066Sahrens 1312fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1313fa9e4066Sahrens 1314fa9e4066Sahrens return (db); 1315fa9e4066Sahrens } 1316fa9e4066Sahrens 1317fa9e4066Sahrens static int 1318fa9e4066Sahrens dbuf_evictable(dmu_buf_impl_t *db) 1319fa9e4066Sahrens { 1320fa9e4066Sahrens int i; 1321fa9e4066Sahrens 1322fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 13239c9dc39aSek DBUF_VERIFY(db); 1324fa9e4066Sahrens 1325fa9e4066Sahrens if (db->db_state != DB_UNCACHED && db->db_state != DB_CACHED) 1326fa9e4066Sahrens return (FALSE); 1327fa9e4066Sahrens 1328fa9e4066Sahrens if (!refcount_is_zero(&db->db_holds)) 1329fa9e4066Sahrens return (FALSE); 1330fa9e4066Sahrens 1331fa9e4066Sahrens #ifdef ZFS_DEBUG 1332fa9e4066Sahrens for (i = 0; i < TXG_SIZE; i++) { 1333fa9e4066Sahrens ASSERT(!list_link_active(&db->db_dirty_node[i])); 1334fa9e4066Sahrens ASSERT(db->db_level != 0 || db->db_d.db_data_old[i] == NULL); 1335fa9e4066Sahrens } 1336fa9e4066Sahrens #endif 1337fa9e4066Sahrens 1338fa9e4066Sahrens /* 1339fa9e4066Sahrens * Now we know we want to free it. 1340fa9e4066Sahrens * This call must be done last, since it has side effects - 1341fa9e4066Sahrens * calling the db_evict_func(). 1342fa9e4066Sahrens */ 1343fa9e4066Sahrens dbuf_evict_user(db); 1344fa9e4066Sahrens return (TRUE); 1345fa9e4066Sahrens } 1346fa9e4066Sahrens 1347fa9e4066Sahrens static void 1348fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1349fa9e4066Sahrens { 1350fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1351fa9e4066Sahrens 1352fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1353fa9e4066Sahrens ASSERT(db->db_dnode == NULL); 1354fa9e4066Sahrens ASSERT(db->db_parent == NULL); 1355fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1356fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1357fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1358fa9e4066Sahrens 1359fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1360fa9e4066Sahrens } 1361fa9e4066Sahrens 1362fa9e4066Sahrens void 1363fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1364fa9e4066Sahrens { 1365fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1366fa9e4066Sahrens blkptr_t *bp = NULL; 1367fa9e4066Sahrens 1368fa9e4066Sahrens ASSERT(blkid != DB_BONUS_BLKID); 1369fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1370fa9e4066Sahrens 1371fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1372fa9e4066Sahrens return; 1373fa9e4066Sahrens 1374fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1375fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 1376fa9e4066Sahrens /* 1377fa9e4066Sahrens * This dbuf is already in the cache. We assume that 1378fa9e4066Sahrens * it is already CACHED, or else about to be either 1379fa9e4066Sahrens * read or filled. 1380fa9e4066Sahrens */ 1381fa9e4066Sahrens mutex_exit(&db->db_mtx); 1382fa9e4066Sahrens return; 1383fa9e4066Sahrens } 1384fa9e4066Sahrens 1385fa9e4066Sahrens if (dbuf_findbp(dn, 0, blkid, TRUE, &parent, &bp) == 0) { 1386fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1387fa9e4066Sahrens (void) arc_read(NULL, dn->dn_objset->os_spa, bp, 1388fa9e4066Sahrens dmu_ot[dn->dn_type].ot_byteswap, 1389fa9e4066Sahrens NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 1390fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1391fa9e4066Sahrens (ARC_NOWAIT | ARC_PREFETCH)); 1392fa9e4066Sahrens } 1393fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1394fa9e4066Sahrens dbuf_rele(parent); 1395fa9e4066Sahrens } 1396fa9e4066Sahrens } 1397fa9e4066Sahrens 1398fa9e4066Sahrens /* 1399fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1400fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1401fa9e4066Sahrens */ 1402fa9e4066Sahrens int 1403fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1404fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1405fa9e4066Sahrens { 1406fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1407fa9e4066Sahrens 1408fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1409fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1410fa9e4066Sahrens 1411fa9e4066Sahrens *dbp = NULL; 1412fa9e4066Sahrens 1413fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1414fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1415fa9e4066Sahrens 1416fa9e4066Sahrens if (db == NULL) { 1417fa9e4066Sahrens blkptr_t *bp = NULL; 1418fa9e4066Sahrens int err; 1419fa9e4066Sahrens 1420fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1421fa9e4066Sahrens if (fail_sparse) { 1422fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1423fa9e4066Sahrens err = ENOENT; 1424fa9e4066Sahrens if (err) { 1425fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1426fa9e4066Sahrens dbuf_rele(parent); 1427fa9e4066Sahrens return (err); 1428fa9e4066Sahrens } 1429fa9e4066Sahrens } 1430fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1431fa9e4066Sahrens } 1432fa9e4066Sahrens 1433fa9e4066Sahrens /* 1434fa9e4066Sahrens * If this buffer is currently syncing out, and we are 1435fa9e4066Sahrens * are still referencing it from db_data, we need to make 1436fa9e4066Sahrens * a copy of it in case we decide we want to dirty it 1437fa9e4066Sahrens * again in this txg. 1438fa9e4066Sahrens */ 1439fa9e4066Sahrens if (db->db_level == 0 && db->db_state == DB_CACHED && 1440fa9e4066Sahrens !(dn->dn_object & DMU_PRIVATE_OBJECT) && 1441fa9e4066Sahrens db->db_data_pending == db->db_buf) { 1442fa9e4066Sahrens int size = (db->db_blkid == DB_BONUS_BLKID) ? 1443fa9e4066Sahrens DN_MAX_BONUSLEN : db->db.db_size; 1444fa9e4066Sahrens 1445fa9e4066Sahrens dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa, 1446fa9e4066Sahrens size, db)); 1447fa9e4066Sahrens bcopy(db->db_data_pending->b_data, db->db.db_data, 1448fa9e4066Sahrens db->db.db_size); 1449fa9e4066Sahrens } 1450fa9e4066Sahrens 1451fa9e4066Sahrens dbuf_add_ref(db, tag); 1452fa9e4066Sahrens dbuf_update_data(db); 14539c9dc39aSek DBUF_VERIFY(db); 1454fa9e4066Sahrens mutex_exit(&db->db_mtx); 1455fa9e4066Sahrens 1456fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1457fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1458fa9e4066Sahrens dbuf_rele(parent); 1459fa9e4066Sahrens 1460fa9e4066Sahrens ASSERT3P(db->db_dnode, ==, dn); 1461fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1462fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1463fa9e4066Sahrens *dbp = db; 1464fa9e4066Sahrens 1465fa9e4066Sahrens return (0); 1466fa9e4066Sahrens } 1467fa9e4066Sahrens 1468fa9e4066Sahrens dmu_buf_impl_t * 1469fa9e4066Sahrens dbuf_hold(dnode_t *dn, uint64_t blkid) 1470fa9e4066Sahrens { 1471fa9e4066Sahrens dmu_buf_impl_t *db; 1472fa9e4066Sahrens (void) dbuf_hold_impl(dn, 0, blkid, FALSE, NULL, &db); 1473fa9e4066Sahrens return (db); 1474fa9e4066Sahrens } 1475fa9e4066Sahrens 1476fa9e4066Sahrens dmu_buf_impl_t * 1477fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1478fa9e4066Sahrens { 1479fa9e4066Sahrens dmu_buf_impl_t *db; 1480fa9e4066Sahrens (void) dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1481fa9e4066Sahrens return (db); 1482fa9e4066Sahrens } 1483fa9e4066Sahrens 1484fa9e4066Sahrens dmu_buf_impl_t * 1485fa9e4066Sahrens dbuf_hold_bonus(dnode_t *dn, void *tag) 1486fa9e4066Sahrens { 1487fa9e4066Sahrens dmu_buf_impl_t *db; 1488fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1489fa9e4066Sahrens (void) dbuf_hold_impl(dn, 0, DB_BONUS_BLKID, FALSE, tag, &db); 1490fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1491fa9e4066Sahrens return (db); 1492fa9e4066Sahrens } 1493fa9e4066Sahrens 1494fa9e4066Sahrens void 1495fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1496fa9e4066Sahrens { 1497fa9e4066Sahrens (void) refcount_add(&db->db_holds, tag); 1498fa9e4066Sahrens /* dprintf_dbuf(db, "adding ref %p; holds up to %lld\n", tag, holds); */ 1499fa9e4066Sahrens } 1500fa9e4066Sahrens 1501fa9e4066Sahrens void 1502fa9e4066Sahrens dbuf_remove_ref(dmu_buf_impl_t *db, void *tag) 1503fa9e4066Sahrens { 1504fa9e4066Sahrens int64_t holds; 1505fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1506fa9e4066Sahrens int need_mutex; 1507fa9e4066Sahrens 1508fa9e4066Sahrens ASSERT(dn != NULL); 1509fa9e4066Sahrens need_mutex = !MUTEX_HELD(&dn->dn_dbufs_mtx); 1510fa9e4066Sahrens 1511fa9e4066Sahrens if (need_mutex) { 1512fa9e4066Sahrens dnode_add_ref(dn, FTAG); 1513fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1514fa9e4066Sahrens } 1515fa9e4066Sahrens 1516fa9e4066Sahrens mutex_enter(&db->db_mtx); 15179c9dc39aSek DBUF_VERIFY(db); 1518fa9e4066Sahrens 1519fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 1520fa9e4066Sahrens 1521fa9e4066Sahrens if (holds == 0) { 1522fa9e4066Sahrens ASSERT3U(db->db_state, !=, DB_FILL); 1523fa9e4066Sahrens if (db->db_level == 0 && 1524fa9e4066Sahrens db->db_d.db_user_data_ptr_ptr != NULL) 1525fa9e4066Sahrens *db->db_d.db_user_data_ptr_ptr = NULL; 1526fa9e4066Sahrens dbuf_evict(db); 1527fa9e4066Sahrens } else { 1528fa9e4066Sahrens if (holds == db->db_dirtycnt && 1529fa9e4066Sahrens db->db_level == 0 && db->db_d.db_immediate_evict) 1530fa9e4066Sahrens dbuf_evict_user(db); 1531fa9e4066Sahrens mutex_exit(&db->db_mtx); 1532fa9e4066Sahrens } 1533fa9e4066Sahrens 1534fa9e4066Sahrens if (need_mutex) { 1535fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1536fa9e4066Sahrens dnode_rele(dn, FTAG); 1537fa9e4066Sahrens } 1538fa9e4066Sahrens } 1539fa9e4066Sahrens 1540fa9e4066Sahrens void 1541fa9e4066Sahrens dbuf_rele(dmu_buf_impl_t *db) 1542fa9e4066Sahrens { 1543fa9e4066Sahrens dbuf_remove_ref(db, NULL); 1544fa9e4066Sahrens } 1545fa9e4066Sahrens 1546fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 1547fa9e4066Sahrens uint64_t 1548fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 1549fa9e4066Sahrens { 1550fa9e4066Sahrens return (refcount_count(&db->db_holds)); 1551fa9e4066Sahrens } 1552fa9e4066Sahrens 1553fa9e4066Sahrens void * 1554fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1555fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1556fa9e4066Sahrens { 1557fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1558fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1559fa9e4066Sahrens } 1560fa9e4066Sahrens 1561fa9e4066Sahrens void * 1562fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 1563fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 1564fa9e4066Sahrens { 1565fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1566fa9e4066Sahrens 1567fa9e4066Sahrens db->db_d.db_immediate_evict = TRUE; 1568fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 1569fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 1570fa9e4066Sahrens } 1571fa9e4066Sahrens 1572fa9e4066Sahrens void * 1573fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 1574fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 1575fa9e4066Sahrens { 1576fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1577fa9e4066Sahrens ASSERT(db->db_level == 0); 1578fa9e4066Sahrens 1579fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 1580fa9e4066Sahrens 1581fa9e4066Sahrens mutex_enter(&db->db_mtx); 1582fa9e4066Sahrens 1583fa9e4066Sahrens if (db->db_d.db_user_ptr == old_user_ptr) { 1584fa9e4066Sahrens db->db_d.db_user_ptr = user_ptr; 1585fa9e4066Sahrens db->db_d.db_user_data_ptr_ptr = user_data_ptr_ptr; 1586fa9e4066Sahrens db->db_d.db_evict_func = evict_func; 1587fa9e4066Sahrens 1588fa9e4066Sahrens dbuf_update_data(db); 1589fa9e4066Sahrens } else { 1590fa9e4066Sahrens old_user_ptr = db->db_d.db_user_ptr; 1591fa9e4066Sahrens } 1592fa9e4066Sahrens 1593fa9e4066Sahrens mutex_exit(&db->db_mtx); 1594fa9e4066Sahrens return (old_user_ptr); 1595fa9e4066Sahrens } 1596fa9e4066Sahrens 1597fa9e4066Sahrens void * 1598fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 1599fa9e4066Sahrens { 1600fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1601fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1602fa9e4066Sahrens 1603fa9e4066Sahrens return (db->db_d.db_user_ptr); 1604fa9e4066Sahrens } 1605fa9e4066Sahrens 1606fa9e4066Sahrens void 1607fa9e4066Sahrens dbuf_sync(dmu_buf_impl_t *db, zio_t *zio, dmu_tx_t *tx) 1608fa9e4066Sahrens { 1609fa9e4066Sahrens arc_buf_t **data; 1610fa9e4066Sahrens uint64_t txg = tx->tx_txg; 1611fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1612fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 161323b11526Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 1614fa9e4066Sahrens int blksz; 1615fa9e4066Sahrens 1616fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 1617fa9e4066Sahrens 1618fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 1619fa9e4066Sahrens 1620fa9e4066Sahrens mutex_enter(&db->db_mtx); 1621fa9e4066Sahrens /* 1622fa9e4066Sahrens * To be synced, we must be dirtied. But we 1623fa9e4066Sahrens * might have been freed after the dirty. 1624fa9e4066Sahrens */ 1625fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 1626fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 1627fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1628fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 1629fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 1630fa9e4066Sahrens ASSERT(db->db.db_data != db->db_d.db_data_old[txg&TXG_MASK]); 1631fa9e4066Sahrens } else { 1632fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 1633fa9e4066Sahrens } 16349c9dc39aSek DBUF_VERIFY(db); 1635fa9e4066Sahrens 1636fa9e4066Sahrens /* 1637fa9e4066Sahrens * Don't need a lock on db_dirty (dn_mtx), because it can't 1638fa9e4066Sahrens * be modified yet. 1639fa9e4066Sahrens */ 1640fa9e4066Sahrens 1641fa9e4066Sahrens if (db->db_level == 0) { 1642fa9e4066Sahrens data = &db->db_d.db_data_old[txg&TXG_MASK]; 1643fa9e4066Sahrens blksz = arc_buf_size(*data); 1644fa9e4066Sahrens /* 1645fa9e4066Sahrens * If this buffer is currently "in use" (i.e., there are 1646fa9e4066Sahrens * active holds and db_data still references it), then make 1647fa9e4066Sahrens * a copy before we start the write so that any modifications 1648fa9e4066Sahrens * from the open txg will not leak into this write. 1649fa9e4066Sahrens * 1650fa9e4066Sahrens * NOTE: this copy does not need to be made for objects only 1651fa9e4066Sahrens * modified in the syncing context (e.g. DNONE_DNODE blocks) 1652fa9e4066Sahrens * or if there is no actual write involved (bonus blocks). 1653fa9e4066Sahrens */ 1654fa9e4066Sahrens if (!(dn->dn_object & DMU_PRIVATE_OBJECT) && 1655fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] == NULL && 1656fa9e4066Sahrens db->db_blkid != DB_BONUS_BLKID) { 1657fa9e4066Sahrens if (refcount_count(&db->db_holds) > 1 && 1658fa9e4066Sahrens *data == db->db_buf) { 1659fa9e4066Sahrens *data = arc_buf_alloc( 1660fa9e4066Sahrens db->db_dnode->dn_objset->os_spa, blksz, db); 1661fa9e4066Sahrens bcopy(db->db.db_data, (*data)->b_data, blksz); 1662fa9e4066Sahrens } 1663fa9e4066Sahrens db->db_data_pending = *data; 1664fa9e4066Sahrens } else if (dn->dn_object & DMU_PRIVATE_OBJECT) { 1665fa9e4066Sahrens /* 1666fa9e4066Sahrens * Private object buffers are released here rather 1667fa9e4066Sahrens * than in dbuf_dirty() since they are only modified 1668fa9e4066Sahrens * in the syncing context and we don't want the 1669fa9e4066Sahrens * overhead of making multiple copies of the data. 1670fa9e4066Sahrens */ 1671fa9e4066Sahrens arc_release(db->db_buf, db); 1672fa9e4066Sahrens } 1673fa9e4066Sahrens } else { 1674fa9e4066Sahrens data = &db->db_buf; 1675fa9e4066Sahrens if (*data == NULL) { 1676fa9e4066Sahrens /* 1677fa9e4066Sahrens * This can happen if we dirty and then free 1678fa9e4066Sahrens * the level-0 data blocks in the same txg. So 1679fa9e4066Sahrens * this indirect remains unchanged. 1680fa9e4066Sahrens */ 1681fa9e4066Sahrens if (db->db_dirtied == txg) 1682fa9e4066Sahrens db->db_dirtied = 0; 1683fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1684fa9e4066Sahrens db->db_dirtycnt -= 1; 1685fa9e4066Sahrens mutex_exit(&db->db_mtx); 1686fa9e4066Sahrens dbuf_remove_ref(db, (void *)(uintptr_t)txg); 1687fa9e4066Sahrens return; 1688fa9e4066Sahrens } 1689fa9e4066Sahrens blksz = db->db.db_size; 1690fa9e4066Sahrens ASSERT3U(blksz, ==, 1<<dn->dn_phys->dn_indblkshift); 1691fa9e4066Sahrens } 1692fa9e4066Sahrens 1693fa9e4066Sahrens ASSERT(*data != NULL); 1694fa9e4066Sahrens 1695fa9e4066Sahrens if (db->db_blkid == DB_BONUS_BLKID) { 1696fa9e4066Sahrens /* 1697fa9e4066Sahrens * Simply copy the bonus data into the dnode. It will 1698fa9e4066Sahrens * be written out when the dnode is synced (and it will 1699fa9e4066Sahrens * be synced, since it must have been dirty for dbuf_sync 1700fa9e4066Sahrens * to be called). The bonus data will be byte swapped 1701fa9e4066Sahrens * in dnode_byteswap. 1702fa9e4066Sahrens */ 1703fa9e4066Sahrens /* 1704fa9e4066Sahrens * Use dn_phys->dn_bonuslen since db.db_size is the length 1705fa9e4066Sahrens * of the bonus buffer in the open transaction rather than 1706fa9e4066Sahrens * the syncing transaction. 1707fa9e4066Sahrens */ 1708fa9e4066Sahrens ASSERT3U(db->db_level, ==, 0); 1709fa9e4066Sahrens ASSERT3U(dn->dn_phys->dn_bonuslen, <=, blksz); 1710fa9e4066Sahrens bcopy((*data)->b_data, DN_BONUS(dn->dn_phys), 1711fa9e4066Sahrens dn->dn_phys->dn_bonuslen); 1712fa9e4066Sahrens if (*data != db->db_buf) 1713fa9e4066Sahrens arc_buf_free(*data, db); 1714fa9e4066Sahrens db->db_d.db_data_old[txg&TXG_MASK] = NULL; 1715fa9e4066Sahrens db->db_data_pending = NULL; 1716fa9e4066Sahrens if (db->db_dirtied == txg) 1717fa9e4066Sahrens db->db_dirtied = 0; 1718fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1719fa9e4066Sahrens db->db_dirtycnt -= 1; 1720fa9e4066Sahrens mutex_exit(&db->db_mtx); 1721fa9e4066Sahrens dbuf_remove_ref(db, (void *)(uintptr_t)txg); 1722fa9e4066Sahrens return; 1723fa9e4066Sahrens } else if (db->db_level > 0 && !arc_released(db->db_buf)) { 1724fa9e4066Sahrens /* 1725fa9e4066Sahrens * This indirect buffer was marked dirty, but 1726fa9e4066Sahrens * never modified (if it had been modified, then 1727fa9e4066Sahrens * we would have released the buffer). There is 1728fa9e4066Sahrens * no reason to write anything. 1729fa9e4066Sahrens */ 1730fa9e4066Sahrens db->db_data_pending = NULL; 1731fa9e4066Sahrens if (db->db_dirtied == txg) 1732fa9e4066Sahrens db->db_dirtied = 0; 1733fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1734fa9e4066Sahrens db->db_dirtycnt -= 1; 1735fa9e4066Sahrens mutex_exit(&db->db_mtx); 1736fa9e4066Sahrens dbuf_remove_ref(db, (void *)(uintptr_t)txg); 1737fa9e4066Sahrens return; 1738fa9e4066Sahrens } else if (db->db_blkptr == NULL && 1739fa9e4066Sahrens db->db_level == dn->dn_phys->dn_nlevels-1 && 1740fa9e4066Sahrens db->db_blkid < dn->dn_phys->dn_nblkptr) { 1741fa9e4066Sahrens /* 1742fa9e4066Sahrens * This buffer was allocated at a time when there was 1743fa9e4066Sahrens * no available blkptrs from the dnode, or it was 1744fa9e4066Sahrens * inappropriate to hook it in (i.e., nlevels mis-match). 1745fa9e4066Sahrens */ 1746fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1747fa9e4066Sahrens ASSERT(db->db_parent == NULL); 1748fa9e4066Sahrens db->db_parent = dn->dn_dbuf; 1749fa9e4066Sahrens db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 17509c9dc39aSek DBUF_VERIFY(db); 1751fa9e4066Sahrens mutex_exit(&db->db_mtx); 1752fa9e4066Sahrens } else if (db->db_blkptr == NULL) { 1753fa9e4066Sahrens dmu_buf_impl_t *parent = db->db_parent; 1754fa9e4066Sahrens 1755fa9e4066Sahrens mutex_exit(&db->db_mtx); 1756fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nlevels > 1); 1757fa9e4066Sahrens if (parent == NULL) { 1758fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1759fa9e4066Sahrens (void) dbuf_hold_impl(dn, db->db_level+1, 1760fa9e4066Sahrens db->db_blkid >> epbs, FALSE, NULL, &parent); 1761fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1762fa9e4066Sahrens dbuf_add_ref(parent, db); 1763fa9e4066Sahrens db->db_parent = parent; 1764fa9e4066Sahrens dbuf_rele(parent); 1765fa9e4066Sahrens } 1766fa9e4066Sahrens dbuf_read(parent); 1767fa9e4066Sahrens } else { 1768fa9e4066Sahrens mutex_exit(&db->db_mtx); 1769fa9e4066Sahrens } 1770fa9e4066Sahrens 1771fa9e4066Sahrens ASSERT(IS_DNODE_DNODE(dn->dn_object) || db->db_parent != NULL); 1772fa9e4066Sahrens 177323b11526Smaybee if (db->db_level > 0 && 177423b11526Smaybee db->db_blkid > dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)) { 177523b11526Smaybee /* 177623b11526Smaybee * Don't write indirect blocks past EOF. 177723b11526Smaybee * We get these when we truncate a file *after* dirtying 177823b11526Smaybee * blocks in the truncate range (we undirty the level 0 177923b11526Smaybee * blocks in dbuf_free_range(), but not the indirects). 178023b11526Smaybee */ 178123b11526Smaybee #ifdef ZFS_DEBUG 178223b11526Smaybee /* 178323b11526Smaybee * Verify that this indirect block is empty. 178423b11526Smaybee */ 178523b11526Smaybee blkptr_t *bplist; 178623b11526Smaybee int i; 178723b11526Smaybee 178823b11526Smaybee mutex_enter(&db->db_mtx); 178923b11526Smaybee bplist = db->db.db_data; 179023b11526Smaybee for (i = 0; i < (1 << epbs); i++) { 179123b11526Smaybee if (!BP_IS_HOLE(&bplist[i])) { 179223b11526Smaybee panic("data past EOF: " 179334f18512Seschrock "db=%p level=%d id=%llu i=%d\n", 179434f18512Seschrock db, db->db_level, 179534f18512Seschrock (u_longlong_t)db->db_blkid, i); 179623b11526Smaybee } 179723b11526Smaybee } 179823b11526Smaybee mutex_exit(&db->db_mtx); 179923b11526Smaybee #endif 180023b11526Smaybee ASSERT(db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)); 180123b11526Smaybee mutex_enter(&db->db_mtx); 180223b11526Smaybee db->db_dirtycnt -= 1; 180323b11526Smaybee mutex_exit(&db->db_mtx); 180423b11526Smaybee dbuf_remove_ref(db, (void *)(uintptr_t)txg); 180523b11526Smaybee return; 180623b11526Smaybee } 180723b11526Smaybee 1808fa9e4066Sahrens if (db->db_parent != dn->dn_dbuf) { 1809fa9e4066Sahrens dmu_buf_impl_t *parent = db->db_parent; 1810fa9e4066Sahrens 1811fa9e4066Sahrens mutex_enter(&db->db_mtx); 1812fa9e4066Sahrens ASSERT(db->db_level == parent->db_level-1); 1813fa9e4066Sahrens ASSERT(list_link_active(&parent->db_dirty_node[txg&TXG_MASK])); 1814fa9e4066Sahrens /* 1815fa9e4066Sahrens * We may have read this block after we dirtied it, 1816fa9e4066Sahrens * so never released it from the cache. 1817fa9e4066Sahrens */ 1818fa9e4066Sahrens arc_release(parent->db_buf, parent); 1819fa9e4066Sahrens 1820fa9e4066Sahrens db->db_blkptr = (blkptr_t *)parent->db.db_data + 1821fa9e4066Sahrens (db->db_blkid & ((1ULL << epbs) - 1)); 18229c9dc39aSek DBUF_VERIFY(db); 1823fa9e4066Sahrens mutex_exit(&db->db_mtx); 1824fa9e4066Sahrens } 1825fa9e4066Sahrens ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1826fa9e4066Sahrens 1827fa9e4066Sahrens #ifdef ZFS_DEBUG 1828fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 1829fa9e4066Sahrens /* 1830fa9e4066Sahrens * We don't need to dnode_setdirty(dn) because if we got 1831fa9e4066Sahrens * here then the parent is already dirty. 1832fa9e4066Sahrens */ 1833fa9e4066Sahrens ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1); 1834fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 1835fa9e4066Sahrens &dn->dn_phys->dn_blkptr[db->db_blkid]); 1836fa9e4066Sahrens } 1837fa9e4066Sahrens #endif 1838fa9e4066Sahrens if (db->db_level == 0 && 1839fa9e4066Sahrens db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) { 1840fa9e4066Sahrens arc_buf_t **old = &db->db_d.db_data_old[txg&TXG_MASK]; 1841fa9e4066Sahrens blkptr_t **bpp = &db->db_d.db_overridden_by[txg&TXG_MASK]; 1842fa9e4066Sahrens int old_size = BP_GET_ASIZE(db->db_blkptr); 1843fa9e4066Sahrens int new_size = BP_GET_ASIZE(*bpp); 1844fa9e4066Sahrens 1845fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 1846fa9e4066Sahrens 1847fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 1848fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1849fa9e4066Sahrens if (db->db_blkid > dn->dn_phys->dn_maxblkid) 1850fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 1851fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1852fa9e4066Sahrens 1853fa9e4066Sahrens dsl_dataset_block_born(os->os_dsl_dataset, *bpp, tx); 1854fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) 1855fa9e4066Sahrens dsl_dataset_block_kill(os->os_dsl_dataset, 1856fa9e4066Sahrens db->db_blkptr, os->os_synctx); 1857fa9e4066Sahrens 1858fa9e4066Sahrens mutex_enter(&db->db_mtx); 1859fa9e4066Sahrens *db->db_blkptr = **bpp; 1860fa9e4066Sahrens kmem_free(*bpp, sizeof (blkptr_t)); 1861fa9e4066Sahrens *bpp = NULL; 1862fa9e4066Sahrens 1863fa9e4066Sahrens if (*old != db->db_buf) 1864fa9e4066Sahrens arc_buf_free(*old, db); 1865fa9e4066Sahrens *old = NULL; 1866fa9e4066Sahrens db->db_data_pending = NULL; 1867fa9e4066Sahrens 1868fa9e4066Sahrens cv_broadcast(&db->db_changed); 1869fa9e4066Sahrens 1870fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1871fa9e4066Sahrens db->db_dirtycnt -= 1; 1872fa9e4066Sahrens mutex_exit(&db->db_mtx); 1873fa9e4066Sahrens dbuf_remove_ref(db, (void *)(uintptr_t)txg); 1874fa9e4066Sahrens } else { 1875fa9e4066Sahrens int checksum, compress; 1876fa9e4066Sahrens 1877fa9e4066Sahrens if (db->db_level > 0) { 1878fa9e4066Sahrens /* 1879fa9e4066Sahrens * XXX -- we should design a compression algorithm 1880fa9e4066Sahrens * that specializes in arrays of bps. 1881fa9e4066Sahrens */ 1882fa9e4066Sahrens checksum = ZIO_CHECKSUM_FLETCHER_4; 1883112fe045Smaybee /* XXX - disable compresssion for now */ 1884112fe045Smaybee compress = ZIO_COMPRESS_OFF; 1885fa9e4066Sahrens } else { 1886fa9e4066Sahrens /* 1887fa9e4066Sahrens * Allow dnode settings to override objset settings, 1888fa9e4066Sahrens * except for metadata checksums. 1889fa9e4066Sahrens */ 1890fa9e4066Sahrens if (dmu_ot[dn->dn_type].ot_metadata) { 1891fa9e4066Sahrens checksum = os->os_md_checksum; 1892fa9e4066Sahrens compress = zio_compress_select(dn->dn_compress, 1893fa9e4066Sahrens os->os_md_compress); 1894fa9e4066Sahrens } else { 1895fa9e4066Sahrens checksum = zio_checksum_select(dn->dn_checksum, 1896fa9e4066Sahrens os->os_checksum); 1897fa9e4066Sahrens compress = zio_compress_select(dn->dn_compress, 1898fa9e4066Sahrens os->os_compress); 1899fa9e4066Sahrens } 1900fa9e4066Sahrens } 1901fa9e4066Sahrens #ifdef ZFS_DEBUG 1902fa9e4066Sahrens if (db->db_parent) { 1903fa9e4066Sahrens ASSERT(list_link_active( 1904fa9e4066Sahrens &db->db_parent->db_dirty_node[txg&TXG_MASK])); 1905fa9e4066Sahrens ASSERT(db->db_parent == dn->dn_dbuf || 1906fa9e4066Sahrens db->db_parent->db_level > 0); 1907fa9e4066Sahrens if (dn->dn_object & DMU_PRIVATE_OBJECT || 1908fa9e4066Sahrens db->db_level > 0) 1909fa9e4066Sahrens ASSERT(*data == db->db_buf); 1910fa9e4066Sahrens } 1911fa9e4066Sahrens #endif 1912fa9e4066Sahrens ASSERT3U(db->db_blkptr->blk_birth, <=, tx->tx_txg); 1913fa9e4066Sahrens (void) arc_write(zio, os->os_spa, checksum, compress, txg, 1914fa9e4066Sahrens db->db_blkptr, *data, dbuf_write_done, db, 1915fa9e4066Sahrens ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, ARC_NOWAIT); 1916fa9e4066Sahrens /* 1917fa9e4066Sahrens * We can't access db after arc_write, since it could finish 1918fa9e4066Sahrens * and be freed, and we have no locks on it. 1919fa9e4066Sahrens */ 1920fa9e4066Sahrens } 1921fa9e4066Sahrens } 1922fa9e4066Sahrens 1923fa9e4066Sahrens struct dbuf_arg { 1924fa9e4066Sahrens objset_impl_t *os; 1925fa9e4066Sahrens blkptr_t bp; 1926fa9e4066Sahrens }; 1927fa9e4066Sahrens 1928fa9e4066Sahrens static void 1929fa9e4066Sahrens dbuf_do_born(void *arg) 1930fa9e4066Sahrens { 1931fa9e4066Sahrens struct dbuf_arg *da = arg; 1932fa9e4066Sahrens dsl_dataset_block_born(da->os->os_dsl_dataset, 1933fa9e4066Sahrens &da->bp, da->os->os_synctx); 1934fa9e4066Sahrens kmem_free(da, sizeof (struct dbuf_arg)); 1935fa9e4066Sahrens } 1936fa9e4066Sahrens 1937fa9e4066Sahrens static void 1938fa9e4066Sahrens dbuf_do_kill(void *arg) 1939fa9e4066Sahrens { 1940fa9e4066Sahrens struct dbuf_arg *da = arg; 1941fa9e4066Sahrens dsl_dataset_block_kill(da->os->os_dsl_dataset, 1942fa9e4066Sahrens &da->bp, da->os->os_synctx); 1943fa9e4066Sahrens kmem_free(da, sizeof (struct dbuf_arg)); 1944fa9e4066Sahrens } 1945fa9e4066Sahrens 1946fa9e4066Sahrens /* ARGSUSED */ 1947fa9e4066Sahrens static void 1948fa9e4066Sahrens dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 1949fa9e4066Sahrens { 1950fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 1951fa9e4066Sahrens dnode_t *dn = db->db_dnode; 1952fa9e4066Sahrens objset_impl_t *os = dn->dn_objset; 1953fa9e4066Sahrens uint64_t txg = zio->io_txg; 1954fa9e4066Sahrens uint64_t fill = 0; 1955fa9e4066Sahrens int i; 1956fa9e4066Sahrens int old_size, new_size; 1957fa9e4066Sahrens 1958fa9e4066Sahrens ASSERT3U(zio->io_error, ==, 0); 1959fa9e4066Sahrens 1960fa9e4066Sahrens dprintf_dbuf_bp(db, &zio->io_bp_orig, "bp_orig: %s", ""); 1961fa9e4066Sahrens 1962fa9e4066Sahrens old_size = BP_GET_ASIZE(&zio->io_bp_orig); 1963fa9e4066Sahrens new_size = BP_GET_ASIZE(zio->io_bp); 1964fa9e4066Sahrens 1965fa9e4066Sahrens dnode_diduse_space(dn, new_size-old_size); 1966fa9e4066Sahrens 1967fa9e4066Sahrens mutex_enter(&db->db_mtx); 1968fa9e4066Sahrens 1969fa9e4066Sahrens if (db->db_dirtied == txg) 1970fa9e4066Sahrens db->db_dirtied = 0; 1971fa9e4066Sahrens 1972fa9e4066Sahrens if (db->db_level == 0) { 1973fa9e4066Sahrens arc_buf_t **old = &db->db_d.db_data_old[txg&TXG_MASK]; 1974fa9e4066Sahrens 1975fa9e4066Sahrens ASSERT(db->db_blkid != DB_BONUS_BLKID); 1976fa9e4066Sahrens 1977fa9e4066Sahrens if (*old != db->db_buf) 1978fa9e4066Sahrens arc_buf_free(*old, db); 1979fa9e4066Sahrens *old = NULL; 1980fa9e4066Sahrens db->db_data_pending = NULL; 1981fa9e4066Sahrens 1982fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1983fa9e4066Sahrens if (db->db_blkid > dn->dn_phys->dn_maxblkid && 1984fa9e4066Sahrens !BP_IS_HOLE(db->db_blkptr)) 1985fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 1986fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1987fa9e4066Sahrens 1988fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 1989fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 1990fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 1991fa9e4066Sahrens i--, dnp++) { 1992fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 1993fa9e4066Sahrens fill++; 1994fa9e4066Sahrens } 1995fa9e4066Sahrens } else { 1996fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) 1997fa9e4066Sahrens fill = 1; 1998fa9e4066Sahrens } 1999fa9e4066Sahrens } else { 2000fa9e4066Sahrens blkptr_t *bp = db->db.db_data; 2001fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2002fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) { 200323b11526Smaybee int epbs = 200423b11526Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2005fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, db->db.db_size); 2006fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2007fa9e4066Sahrens db->db.db_size); 200823b11526Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 200923b11526Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2010fa9e4066Sahrens } 2011fa9e4066Sahrens for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) { 2012fa9e4066Sahrens if (BP_IS_HOLE(bp)) 2013fa9e4066Sahrens continue; 2014fa9e4066Sahrens ASSERT3U(BP_GET_LSIZE(bp), ==, 2015fa9e4066Sahrens db->db_level == 1 ? dn->dn_datablksz : 2016fa9e4066Sahrens (1<<dn->dn_phys->dn_indblkshift)); 2017fa9e4066Sahrens fill += bp->blk_fill; 2018fa9e4066Sahrens } 2019fa9e4066Sahrens } 2020fa9e4066Sahrens 2021fa9e4066Sahrens if (!BP_IS_HOLE(db->db_blkptr)) { 2022fa9e4066Sahrens db->db_blkptr->blk_fill = fill; 2023fa9e4066Sahrens BP_SET_TYPE(db->db_blkptr, dn->dn_type); 2024fa9e4066Sahrens BP_SET_LEVEL(db->db_blkptr, db->db_level); 2025fa9e4066Sahrens } else { 2026fa9e4066Sahrens ASSERT3U(fill, ==, 0); 2027fa9e4066Sahrens ASSERT3U(db->db_blkptr->blk_fill, ==, 0); 2028fa9e4066Sahrens } 2029fa9e4066Sahrens 2030fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, 2031fa9e4066Sahrens "wrote %llu bytes to blkptr:", zio->io_size); 2032fa9e4066Sahrens 2033fa9e4066Sahrens ASSERT(db->db_parent == NULL || 2034fa9e4066Sahrens list_link_active(&db->db_parent->db_dirty_node[txg&TXG_MASK])); 2035fa9e4066Sahrens cv_broadcast(&db->db_changed); 2036fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2037fa9e4066Sahrens db->db_dirtycnt -= 1; 2038fa9e4066Sahrens mutex_exit(&db->db_mtx); 2039fa9e4066Sahrens 2040fa9e4066Sahrens /* We must do this after we've set the bp's type and level */ 2041fa9e4066Sahrens if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), 2042fa9e4066Sahrens BP_IDENTITY(&zio->io_bp_orig))) { 2043fa9e4066Sahrens struct dbuf_arg *da; 2044fa9e4066Sahrens da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2045fa9e4066Sahrens da->os = os; 2046fa9e4066Sahrens da->bp = *zio->io_bp; 2047fa9e4066Sahrens (void) taskq_dispatch(dbuf_tq, dbuf_do_born, da, 0); 2048fa9e4066Sahrens if (!BP_IS_HOLE(&zio->io_bp_orig)) { 2049fa9e4066Sahrens da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP); 2050fa9e4066Sahrens da->os = os; 2051fa9e4066Sahrens da->bp = zio->io_bp_orig; 2052fa9e4066Sahrens (void) taskq_dispatch(dbuf_tq, dbuf_do_kill, da, 0); 2053fa9e4066Sahrens } 2054fa9e4066Sahrens } 2055fa9e4066Sahrens 2056fa9e4066Sahrens dbuf_remove_ref(db, (void *)(uintptr_t)txg); 2057fa9e4066Sahrens } 2058