1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23fa9e4066Sahrens */ 24fa9e4066Sahrens 25fa9e4066Sahrens #include <sys/zfs_context.h> 26fa9e4066Sahrens #include <sys/dmu.h> 27fa9e4066Sahrens #include <sys/dmu_impl.h> 28fa9e4066Sahrens #include <sys/dbuf.h> 29fa9e4066Sahrens #include <sys/dmu_objset.h> 30fa9e4066Sahrens #include <sys/dsl_dataset.h> 31fa9e4066Sahrens #include <sys/dsl_dir.h> 32fa9e4066Sahrens #include <sys/dmu_tx.h> 33fa9e4066Sahrens #include <sys/spa.h> 34fa9e4066Sahrens #include <sys/zio.h> 35fa9e4066Sahrens #include <sys/dmu_zfetch.h> 360a586ceaSMark Shellenbaum #include <sys/sa.h> 370a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 38fa9e4066Sahrens 39fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 40fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 41088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 42fa9e4066Sahrens 43fa9e4066Sahrens /* 44fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 45fa9e4066Sahrens */ 46fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 47fa9e4066Sahrens 48fa9e4066Sahrens /* ARGSUSED */ 49fa9e4066Sahrens static int 50fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 51fa9e4066Sahrens { 52fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 53fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 54fa9e4066Sahrens 55fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 56fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 57fa9e4066Sahrens refcount_create(&db->db_holds); 58fa9e4066Sahrens return (0); 59fa9e4066Sahrens } 60fa9e4066Sahrens 61fa9e4066Sahrens /* ARGSUSED */ 62fa9e4066Sahrens static void 63fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 64fa9e4066Sahrens { 65fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 66fa9e4066Sahrens mutex_destroy(&db->db_mtx); 67fa9e4066Sahrens cv_destroy(&db->db_changed); 68fa9e4066Sahrens refcount_destroy(&db->db_holds); 69fa9e4066Sahrens } 70fa9e4066Sahrens 71fa9e4066Sahrens /* 72fa9e4066Sahrens * dbuf hash table routines 73fa9e4066Sahrens */ 74fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 75fa9e4066Sahrens 76fa9e4066Sahrens static uint64_t dbuf_hash_count; 77fa9e4066Sahrens 78fa9e4066Sahrens static uint64_t 79fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 80fa9e4066Sahrens { 81fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 82fa9e4066Sahrens uint64_t crc = -1ULL; 83fa9e4066Sahrens 84fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 85fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 86fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 87fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 88fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 89fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 90fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 91fa9e4066Sahrens 92fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 93fa9e4066Sahrens 94fa9e4066Sahrens return (crc); 95fa9e4066Sahrens } 96fa9e4066Sahrens 97fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 98fa9e4066Sahrens 99fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 100fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 101fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 102fa9e4066Sahrens (dbuf)->db_level == (level) && \ 103fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 104fa9e4066Sahrens 105fa9e4066Sahrens dmu_buf_impl_t * 106fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 107fa9e4066Sahrens { 108fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 109503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 110fa9e4066Sahrens uint64_t obj = dn->dn_object; 111fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 112fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 113fa9e4066Sahrens dmu_buf_impl_t *db; 114fa9e4066Sahrens 115fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 116fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 117fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 118fa9e4066Sahrens mutex_enter(&db->db_mtx); 119ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 120fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 121fa9e4066Sahrens return (db); 122fa9e4066Sahrens } 123fa9e4066Sahrens mutex_exit(&db->db_mtx); 124fa9e4066Sahrens } 125fa9e4066Sahrens } 126fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 127fa9e4066Sahrens return (NULL); 128fa9e4066Sahrens } 129fa9e4066Sahrens 130fa9e4066Sahrens /* 131fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 132fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 133fa9e4066Sahrens * will be returned and the new element will not be inserted. 134fa9e4066Sahrens * Otherwise returns NULL. 135fa9e4066Sahrens */ 136fa9e4066Sahrens static dmu_buf_impl_t * 137fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 138fa9e4066Sahrens { 139fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 140503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 141fa9e4066Sahrens uint64_t obj = db->db.db_object; 142fa9e4066Sahrens int level = db->db_level; 143fa9e4066Sahrens uint64_t blkid = db->db_blkid; 144fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 145fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 146fa9e4066Sahrens dmu_buf_impl_t *dbf; 147fa9e4066Sahrens 148fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 149fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 150fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 151fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 152ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 153fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 154fa9e4066Sahrens return (dbf); 155fa9e4066Sahrens } 156fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 157fa9e4066Sahrens } 158fa9e4066Sahrens } 159fa9e4066Sahrens 160fa9e4066Sahrens mutex_enter(&db->db_mtx); 161fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 162fa9e4066Sahrens h->hash_table[idx] = db; 163fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 164fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 165fa9e4066Sahrens 166fa9e4066Sahrens return (NULL); 167fa9e4066Sahrens } 168fa9e4066Sahrens 169fa9e4066Sahrens /* 170fa9e4066Sahrens * Remove an entry from the hash table. This operation will 171fa9e4066Sahrens * fail if there are any existing holds on the db. 172fa9e4066Sahrens */ 173fa9e4066Sahrens static void 174fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 175fa9e4066Sahrens { 176fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 177fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 178fa9e4066Sahrens db->db_level, db->db_blkid); 179fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 180fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 181fa9e4066Sahrens 182fa9e4066Sahrens /* 183fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 184fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 185fa9e4066Sahrens */ 186fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 187ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 188fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 189fa9e4066Sahrens 190fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 191fa9e4066Sahrens dbp = &h->hash_table[idx]; 192fa9e4066Sahrens while ((dbf = *dbp) != db) { 193fa9e4066Sahrens dbp = &dbf->db_hash_next; 194fa9e4066Sahrens ASSERT(dbf != NULL); 195fa9e4066Sahrens } 196fa9e4066Sahrens *dbp = db->db_hash_next; 197fa9e4066Sahrens db->db_hash_next = NULL; 198fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 199fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 200fa9e4066Sahrens } 201fa9e4066Sahrens 202ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 203fa9e4066Sahrens 204fa9e4066Sahrens static void 205fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 206fa9e4066Sahrens { 207fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 208fa9e4066Sahrens 209c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 210fa9e4066Sahrens return; 211fa9e4066Sahrens 212c717a561Smaybee if (db->db_user_data_ptr_ptr) 213c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 214c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 215c717a561Smaybee db->db_user_ptr = NULL; 216c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 217c717a561Smaybee db->db_evict_func = NULL; 218fa9e4066Sahrens } 219fa9e4066Sahrens 220744947dcSTom Erickson boolean_t 221744947dcSTom Erickson dbuf_is_metadata(dmu_buf_impl_t *db) 222744947dcSTom Erickson { 223744947dcSTom Erickson if (db->db_level > 0) { 224744947dcSTom Erickson return (B_TRUE); 225744947dcSTom Erickson } else { 226744947dcSTom Erickson boolean_t is_metadata; 227744947dcSTom Erickson 228744947dcSTom Erickson DB_DNODE_ENTER(db); 229744947dcSTom Erickson is_metadata = dmu_ot[DB_DNODE(db)->dn_type].ot_metadata; 230744947dcSTom Erickson DB_DNODE_EXIT(db); 231744947dcSTom Erickson 232744947dcSTom Erickson return (is_metadata); 233744947dcSTom Erickson } 234744947dcSTom Erickson } 235744947dcSTom Erickson 236ea8dc4b6Seschrock void 237ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 238ea8dc4b6Seschrock { 239ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 240ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 241c717a561Smaybee ASSERT(db->db_data_pending == NULL); 242ea8dc4b6Seschrock 243ea8dc4b6Seschrock dbuf_clear(db); 244ea8dc4b6Seschrock dbuf_destroy(db); 245ea8dc4b6Seschrock } 246ea8dc4b6Seschrock 247fa9e4066Sahrens void 248fa9e4066Sahrens dbuf_init(void) 249fa9e4066Sahrens { 250ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 251fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 252fa9e4066Sahrens int i; 253fa9e4066Sahrens 254fa9e4066Sahrens /* 255fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 256ea8dc4b6Seschrock * with an average 4K block size. The table will take up 257ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 258fa9e4066Sahrens */ 259ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 260fa9e4066Sahrens hsize <<= 1; 261fa9e4066Sahrens 262ea8dc4b6Seschrock retry: 263fa9e4066Sahrens h->hash_table_mask = hsize - 1; 264ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 265ea8dc4b6Seschrock if (h->hash_table == NULL) { 266ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 267ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 268ea8dc4b6Seschrock hsize >>= 1; 269ea8dc4b6Seschrock goto retry; 270ea8dc4b6Seschrock } 271fa9e4066Sahrens 272fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 273fa9e4066Sahrens sizeof (dmu_buf_impl_t), 274fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 275fa9e4066Sahrens 276fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 277fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 278fa9e4066Sahrens } 279fa9e4066Sahrens 280fa9e4066Sahrens void 281fa9e4066Sahrens dbuf_fini(void) 282fa9e4066Sahrens { 283fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 284fa9e4066Sahrens int i; 285fa9e4066Sahrens 286fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 287fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 288fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 289fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 290fa9e4066Sahrens } 291fa9e4066Sahrens 292fa9e4066Sahrens /* 293fa9e4066Sahrens * Other stuff. 294fa9e4066Sahrens */ 295fa9e4066Sahrens 2969c9dc39aSek #ifdef ZFS_DEBUG 297fa9e4066Sahrens static void 298fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 299fa9e4066Sahrens { 300744947dcSTom Erickson dnode_t *dn; 301b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 302fa9e4066Sahrens 303fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 304fa9e4066Sahrens 305fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 306fa9e4066Sahrens return; 307fa9e4066Sahrens 308fa9e4066Sahrens ASSERT(db->db_objset != NULL); 309744947dcSTom Erickson DB_DNODE_ENTER(db); 310744947dcSTom Erickson dn = DB_DNODE(db); 311fa9e4066Sahrens if (dn == NULL) { 312fa9e4066Sahrens ASSERT(db->db_parent == NULL); 313fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 314fa9e4066Sahrens } else { 315fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 316fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 317fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 318744947dcSTom Erickson ASSERT(db->db_blkid == DMU_BONUS_BLKID || 319744947dcSTom Erickson db->db_blkid == DMU_SPILL_BLKID || 320744947dcSTom Erickson !list_is_empty(&dn->dn_dbufs)); 321fa9e4066Sahrens } 3220a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 323fa9e4066Sahrens ASSERT(dn != NULL); 3241934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3250a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 3260a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 3270a586ceaSMark Shellenbaum ASSERT(dn != NULL); 3280a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3290a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, 0); 330fa9e4066Sahrens } else { 331fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 332fa9e4066Sahrens } 333fa9e4066Sahrens 334b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 335b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 336b24ab676SJeff Bonwick 337b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 338b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 339b24ab676SJeff Bonwick 34088b7b0f2SMatthew Ahrens /* 34188b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 34288b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 34388b7b0f2SMatthew Ahrens * dnode_set_blksz(). 34488b7b0f2SMatthew Ahrens */ 34588b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 346b24ab676SJeff Bonwick dr = db->db_data_pending; 34788b7b0f2SMatthew Ahrens /* 34888b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 34988b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 35088b7b0f2SMatthew Ahrens */ 35188b7b0f2SMatthew Ahrens ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 352fa9e4066Sahrens } 353fa9e4066Sahrens 354fa9e4066Sahrens /* verify db->db_blkptr */ 355fa9e4066Sahrens if (db->db_blkptr) { 356fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 357fa9e4066Sahrens /* db is pointed to by the dnode */ 358fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 35914843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 360fa9e4066Sahrens ASSERT(db->db_parent == NULL); 361fa9e4066Sahrens else 362fa9e4066Sahrens ASSERT(db->db_parent != NULL); 3630a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 3640a586ceaSMark Shellenbaum ASSERT3P(db->db_blkptr, ==, 3650a586ceaSMark Shellenbaum &dn->dn_phys->dn_blkptr[db->db_blkid]); 366fa9e4066Sahrens } else { 367fa9e4066Sahrens /* db is pointed to by an indirect block */ 368fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 369fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 370fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 371fa9e4066Sahrens db->db.db_object); 372fa9e4066Sahrens /* 373fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 374fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 375fa9e4066Sahrens * grows. safe to do this now? 376fa9e4066Sahrens */ 377744947dcSTom Erickson if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 378fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 379fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 380fa9e4066Sahrens db->db_blkid % epb)); 381fa9e4066Sahrens } 382fa9e4066Sahrens } 383fa9e4066Sahrens } 384fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 3853f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 3860a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 387fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 388fa9e4066Sahrens /* 389fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 390fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 391fa9e4066Sahrens * data when we evict this buffer. 392fa9e4066Sahrens */ 393fa9e4066Sahrens if (db->db_dirtycnt == 0) { 394fa9e4066Sahrens uint64_t *buf = db->db.db_data; 395fa9e4066Sahrens int i; 396fa9e4066Sahrens 397fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 398fa9e4066Sahrens ASSERT(buf[i] == 0); 399fa9e4066Sahrens } 400fa9e4066Sahrens } 401fa9e4066Sahrens } 402744947dcSTom Erickson DB_DNODE_EXIT(db); 403fa9e4066Sahrens } 4049c9dc39aSek #endif 405fa9e4066Sahrens 406fa9e4066Sahrens static void 407fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 408fa9e4066Sahrens { 409fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 410c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 411fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 412c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 413fa9e4066Sahrens } 414fa9e4066Sahrens } 415fa9e4066Sahrens 416fa9e4066Sahrens static void 417fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 418fa9e4066Sahrens { 419fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 420ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 421fa9e4066Sahrens db->db_buf = buf; 422ea8dc4b6Seschrock if (buf != NULL) { 423ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 424ea8dc4b6Seschrock db->db.db_data = buf->b_data; 425ea8dc4b6Seschrock if (!arc_released(buf)) 426ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 427ea8dc4b6Seschrock dbuf_update_data(db); 428ea8dc4b6Seschrock } else { 429ea8dc4b6Seschrock dbuf_evict_user(db); 430ea8dc4b6Seschrock db->db.db_data = NULL; 43182c9918fSTim Haley if (db->db_state != DB_NOFILL) 43282c9918fSTim Haley db->db_state = DB_UNCACHED; 433ea8dc4b6Seschrock } 434fa9e4066Sahrens } 435fa9e4066Sahrens 436c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 437c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 438c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 439c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 440c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 441c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 442c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 443c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 444c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 445c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 446c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 447744947dcSTom Erickson spa_t *spa; 448744947dcSTom Erickson 449c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 450744947dcSTom Erickson DB_GET_SPA(&spa, db); 451744947dcSTom Erickson abuf = arc_loan_buf(spa, blksz); 452c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 453c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 454c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 455c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 456c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_set_data(db, NULL); 457c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 458c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 459c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 460c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 461c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 462fa9e4066Sahrens uint64_t 463fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 464fa9e4066Sahrens { 465fa9e4066Sahrens if (dn->dn_datablkshift) { 466fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 467fa9e4066Sahrens } else { 468fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 469fa9e4066Sahrens return (0); 470fa9e4066Sahrens } 471fa9e4066Sahrens } 472fa9e4066Sahrens 473fa9e4066Sahrens static void 474fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 475fa9e4066Sahrens { 476fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 477fa9e4066Sahrens 478fa9e4066Sahrens mutex_enter(&db->db_mtx); 479fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 480fa9e4066Sahrens /* 481fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 482fa9e4066Sahrens */ 483fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 484ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 485fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 486c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 487fa9e4066Sahrens /* we were freed in flight; disregard any error */ 488fa9e4066Sahrens arc_release(buf, db); 489fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 4906b4acc8bSahrens arc_buf_freeze(buf); 491c717a561Smaybee db->db_freed_in_flight = FALSE; 492fa9e4066Sahrens dbuf_set_data(db, buf); 493fa9e4066Sahrens db->db_state = DB_CACHED; 494fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 495fa9e4066Sahrens dbuf_set_data(db, buf); 496fa9e4066Sahrens db->db_state = DB_CACHED; 497fa9e4066Sahrens } else { 4980a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 499fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 500ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 501ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 502fa9e4066Sahrens } 503fa9e4066Sahrens cv_broadcast(&db->db_changed); 5043f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 505fa9e4066Sahrens } 506fa9e4066Sahrens 507ea8dc4b6Seschrock static void 50813506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 509fa9e4066Sahrens { 510744947dcSTom Erickson dnode_t *dn; 511744947dcSTom Erickson spa_t *spa; 512ea8dc4b6Seschrock zbookmark_t zb; 51313506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 514088f3894Sahrens arc_buf_t *pbuf; 515fa9e4066Sahrens 516744947dcSTom Erickson DB_DNODE_ENTER(db); 517744947dcSTom Erickson dn = DB_DNODE(db); 518fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 519fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 520088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 521ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 522ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 523ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 524fa9e4066Sahrens 5250a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 526cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 5271934e92fSmaybee 5281934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 529ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 5305a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 5311934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 532ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 533cf04dda1SMark Maybee if (bonuslen) 534cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 535744947dcSTom Erickson DB_DNODE_EXIT(db); 536ea8dc4b6Seschrock dbuf_update_data(db); 537fa9e4066Sahrens db->db_state = DB_CACHED; 538fa9e4066Sahrens mutex_exit(&db->db_mtx); 539fa9e4066Sahrens return; 540fa9e4066Sahrens } 541fa9e4066Sahrens 5421c8564a7SMark Maybee /* 5431c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 5441c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 5451c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 5461c8564a7SMark Maybee */ 547088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 5481c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 5491c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 550ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 551ad23a2dbSjohansen 552088f3894Sahrens dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa, 553ad23a2dbSjohansen db->db.db_size, db, type)); 554744947dcSTom Erickson DB_DNODE_EXIT(db); 555fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 556fa9e4066Sahrens db->db_state = DB_CACHED; 55713506d1eSmaybee *flags |= DB_RF_CACHED; 558fa9e4066Sahrens mutex_exit(&db->db_mtx); 559fa9e4066Sahrens return; 560fa9e4066Sahrens } 561fa9e4066Sahrens 562744947dcSTom Erickson spa = dn->dn_objset->os_spa; 563744947dcSTom Erickson DB_DNODE_EXIT(db); 564744947dcSTom Erickson 565fa9e4066Sahrens db->db_state = DB_READ; 566fa9e4066Sahrens mutex_exit(&db->db_mtx); 567fa9e4066Sahrens 5683baa08fcSek if (DBUF_IS_L2CACHEABLE(db)) 5693baa08fcSek aflags |= ARC_L2CACHE; 5703baa08fcSek 571b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 572b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 573b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 574ea8dc4b6Seschrock 575ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 576fa9e4066Sahrens /* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */ 577088f3894Sahrens 578088f3894Sahrens if (db->db_parent) 579088f3894Sahrens pbuf = db->db_parent->db_buf; 580088f3894Sahrens else 581088f3894Sahrens pbuf = db->db_objset->os_phys_buf; 582088f3894Sahrens 583744947dcSTom Erickson (void) dsl_read(zio, spa, db->db_blkptr, pbuf, 584fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 58513506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 58613506d1eSmaybee &aflags, &zb); 58713506d1eSmaybee if (aflags & ARC_CACHED) 58813506d1eSmaybee *flags |= DB_RF_CACHED; 589fa9e4066Sahrens } 590fa9e4066Sahrens 591ea8dc4b6Seschrock int 592ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 593fa9e4066Sahrens { 594ea8dc4b6Seschrock int err = 0; 595ea8dc4b6Seschrock int havepzio = (zio != NULL); 59613506d1eSmaybee int prefetch; 597744947dcSTom Erickson dnode_t *dn; 598fa9e4066Sahrens 599fa9e4066Sahrens /* 600fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 601fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 602fa9e4066Sahrens */ 603fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 604fa9e4066Sahrens 60582c9918fSTim Haley if (db->db_state == DB_NOFILL) 60682c9918fSTim Haley return (EIO); 60782c9918fSTim Haley 608744947dcSTom Erickson DB_DNODE_ENTER(db); 609744947dcSTom Erickson dn = DB_DNODE(db); 610ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 611744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 612ea8dc4b6Seschrock 6130a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 614744947dcSTom Erickson (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 6153baa08fcSek DBUF_IS_CACHEABLE(db); 61613506d1eSmaybee 617ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 618ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 619ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 62013506d1eSmaybee if (prefetch) 621744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 62213506d1eSmaybee db->db.db_size, TRUE); 623fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 624744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 625744947dcSTom Erickson DB_DNODE_EXIT(db); 626ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 627744947dcSTom Erickson spa_t *spa = dn->dn_objset->os_spa; 628744947dcSTom Erickson 629744947dcSTom Erickson if (zio == NULL) 630744947dcSTom Erickson zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 63113506d1eSmaybee dbuf_read_impl(db, zio, &flags); 63213506d1eSmaybee 633ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 634ea8dc4b6Seschrock 63513506d1eSmaybee if (prefetch) 636744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 63713506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 638ea8dc4b6Seschrock 639fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 640744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 641744947dcSTom Erickson DB_DNODE_EXIT(db); 642fa9e4066Sahrens 643ea8dc4b6Seschrock if (!havepzio) 644ea8dc4b6Seschrock err = zio_wait(zio); 645ea8dc4b6Seschrock } else { 64613506d1eSmaybee mutex_exit(&db->db_mtx); 64713506d1eSmaybee if (prefetch) 648744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 64913506d1eSmaybee db->db.db_size, TRUE); 650ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 651744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 652744947dcSTom Erickson DB_DNODE_EXIT(db); 65313506d1eSmaybee 65413506d1eSmaybee mutex_enter(&db->db_mtx); 655ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 656ea8dc4b6Seschrock while (db->db_state == DB_READ || 657ea8dc4b6Seschrock db->db_state == DB_FILL) { 658ea8dc4b6Seschrock ASSERT(db->db_state == DB_READ || 659ea8dc4b6Seschrock (flags & DB_RF_HAVESTRUCT) == 0); 660ea8dc4b6Seschrock cv_wait(&db->db_changed, &db->db_mtx); 661ea8dc4b6Seschrock } 662ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 663ea8dc4b6Seschrock err = EIO; 664ea8dc4b6Seschrock } 665ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 666fa9e4066Sahrens } 667fa9e4066Sahrens 668ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 669ea8dc4b6Seschrock return (err); 670fa9e4066Sahrens } 671fa9e4066Sahrens 672fa9e4066Sahrens static void 673fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 674fa9e4066Sahrens { 675fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 6760a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 677fa9e4066Sahrens mutex_enter(&db->db_mtx); 678fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 679fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 680fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 681ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 682744947dcSTom Erickson spa_t *spa; 683ad23a2dbSjohansen 684ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 685fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 686744947dcSTom Erickson DB_GET_SPA(&spa, db); 687744947dcSTom Erickson dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 688fa9e4066Sahrens db->db_state = DB_FILL; 68982c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 69082c9918fSTim Haley dbuf_set_data(db, NULL); 691fa9e4066Sahrens } else { 692fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 693fa9e4066Sahrens } 694fa9e4066Sahrens mutex_exit(&db->db_mtx); 695fa9e4066Sahrens } 696fa9e4066Sahrens 697fa9e4066Sahrens /* 698fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 699fa9e4066Sahrens * buffers, that have been modified in a previous transaction 700fa9e4066Sahrens * group, before we modify them in the current active group. 701fa9e4066Sahrens * 702fa9e4066Sahrens * This function is used in two places: when we are dirtying a 703fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 704fa9e4066Sahrens * a range in a dnode that includes this buffer. 705fa9e4066Sahrens * 706fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 707fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 708fa9e4066Sahrens * dbuf list for the dnode. 709fa9e4066Sahrens */ 710fa9e4066Sahrens static void 711fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 712fa9e4066Sahrens { 713c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 714fa9e4066Sahrens 715fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 716fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 717c717a561Smaybee ASSERT(db->db_level == 0); 718c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 719fa9e4066Sahrens 7204d31c452Smaybee if (dr == NULL || 7214d31c452Smaybee (dr->dt.dl.dr_data != 7220a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 723fa9e4066Sahrens return; 724fa9e4066Sahrens 725fa9e4066Sahrens /* 726c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 727c717a561Smaybee * and its referencing the dbuf data, either: 728744947dcSTom Erickson * reset the reference to point to a new copy, 729c717a561Smaybee * or (if there a no active holders) 730c717a561Smaybee * just null out the current db_data pointer. 731fa9e4066Sahrens */ 732c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 7330a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 734c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 735c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 7365a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 737c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 738c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 739c717a561Smaybee int size = db->db.db_size; 740c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 741744947dcSTom Erickson spa_t *spa; 742744947dcSTom Erickson 743744947dcSTom Erickson DB_GET_SPA(&spa, db); 744744947dcSTom Erickson dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 745c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 746c717a561Smaybee } else { 747c717a561Smaybee dbuf_set_data(db, NULL); 748fa9e4066Sahrens } 749fa9e4066Sahrens } 750fa9e4066Sahrens 751c717a561Smaybee void 752c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 753ea8dc4b6Seschrock { 754c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 755b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 756c717a561Smaybee uint64_t txg = dr->dr_txg; 757ea8dc4b6Seschrock 758ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 759c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 760c717a561Smaybee ASSERT(db->db_level == 0); 761ea8dc4b6Seschrock 7620a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 763c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 764c717a561Smaybee return; 765ea8dc4b6Seschrock 766b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 767b24ab676SJeff Bonwick 768c717a561Smaybee /* free this block */ 769744947dcSTom Erickson if (!BP_IS_HOLE(bp)) { 770744947dcSTom Erickson spa_t *spa; 771b24ab676SJeff Bonwick 772744947dcSTom Erickson DB_GET_SPA(&spa, db); 773744947dcSTom Erickson zio_free(spa, txg, bp); 774744947dcSTom Erickson } 775c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 776c717a561Smaybee /* 777c717a561Smaybee * Release the already-written buffer, so we leave it in 778c717a561Smaybee * a consistent dirty state. Note that all callers are 779c717a561Smaybee * modifying the buffer, so they will immediately do 780c717a561Smaybee * another (redundant) arc_release(). Therefore, leave 781c717a561Smaybee * the buf thawed to save the effort of freezing & 782c717a561Smaybee * immediately re-thawing it. 783c717a561Smaybee */ 784c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 785fa9e4066Sahrens } 786fa9e4066Sahrens 787cdb0ab79Smaybee /* 788cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 789cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 790cdb0ab79Smaybee * empty blocks. Also, if we happen accross any level-1 dbufs in the 791cdb0ab79Smaybee * range that have not already been marked dirty, mark them dirty so 792cdb0ab79Smaybee * they stay in memory. 793cdb0ab79Smaybee */ 794fa9e4066Sahrens void 795cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 796fa9e4066Sahrens { 797fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 798fa9e4066Sahrens uint64_t txg = tx->tx_txg; 799cdb0ab79Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 800cdb0ab79Smaybee uint64_t first_l1 = start >> epbs; 801cdb0ab79Smaybee uint64_t last_l1 = end >> epbs; 802fa9e4066Sahrens 8030a586ceaSMark Shellenbaum if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) { 804cdb0ab79Smaybee end = dn->dn_maxblkid; 805cdb0ab79Smaybee last_l1 = end >> epbs; 806cdb0ab79Smaybee } 807cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 808fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 809fa9e4066Sahrens for (db = list_head(&dn->dn_dbufs); db; db = db_next) { 810fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 8110a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 812cdb0ab79Smaybee 813cdb0ab79Smaybee if (db->db_level == 1 && 814cdb0ab79Smaybee db->db_blkid >= first_l1 && db->db_blkid <= last_l1) { 815cdb0ab79Smaybee mutex_enter(&db->db_mtx); 816cdb0ab79Smaybee if (db->db_last_dirty && 817cdb0ab79Smaybee db->db_last_dirty->dr_txg < txg) { 818cdb0ab79Smaybee dbuf_add_ref(db, FTAG); 819cdb0ab79Smaybee mutex_exit(&db->db_mtx); 820cdb0ab79Smaybee dbuf_will_dirty(db, tx); 821cdb0ab79Smaybee dbuf_rele(db, FTAG); 822cdb0ab79Smaybee } else { 823cdb0ab79Smaybee mutex_exit(&db->db_mtx); 824cdb0ab79Smaybee } 825cdb0ab79Smaybee } 826cdb0ab79Smaybee 827ea8dc4b6Seschrock if (db->db_level != 0) 828fa9e4066Sahrens continue; 829fa9e4066Sahrens dprintf_dbuf(db, "found buf %s\n", ""); 830cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 831fa9e4066Sahrens continue; 832fa9e4066Sahrens 833fa9e4066Sahrens /* found a level 0 buffer in the range */ 834fa9e4066Sahrens if (dbuf_undirty(db, tx)) 835fa9e4066Sahrens continue; 836fa9e4066Sahrens 837fa9e4066Sahrens mutex_enter(&db->db_mtx); 838ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 83982c9918fSTim Haley db->db_state == DB_NOFILL || 840ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 841fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 842fa9e4066Sahrens mutex_exit(&db->db_mtx); 843fa9e4066Sahrens continue; 844fa9e4066Sahrens } 845c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 846c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 847c717a561Smaybee db->db_freed_in_flight = TRUE; 848fa9e4066Sahrens mutex_exit(&db->db_mtx); 849fa9e4066Sahrens continue; 850fa9e4066Sahrens } 851ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 852ea8dc4b6Seschrock ASSERT(db->db_buf); 853ea8dc4b6Seschrock dbuf_clear(db); 854ea8dc4b6Seschrock continue; 855ea8dc4b6Seschrock } 856c717a561Smaybee /* The dbuf is referenced */ 857fa9e4066Sahrens 858c717a561Smaybee if (db->db_last_dirty != NULL) { 859c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 860c717a561Smaybee 861c717a561Smaybee if (dr->dr_txg == txg) { 86244eda4d7Smaybee /* 863c717a561Smaybee * This buffer is "in-use", re-adjust the file 864c717a561Smaybee * size to reflect that this buffer may 865c717a561Smaybee * contain new data when we sync. 86644eda4d7Smaybee */ 86706e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 86806e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 869c717a561Smaybee dn->dn_maxblkid = db->db_blkid; 870c717a561Smaybee dbuf_unoverride(dr); 871c717a561Smaybee } else { 872c717a561Smaybee /* 873c717a561Smaybee * This dbuf is not dirty in the open context. 874c717a561Smaybee * Either uncache it (if its not referenced in 875c717a561Smaybee * the open context) or reset its contents to 876c717a561Smaybee * empty. 877c717a561Smaybee */ 878c717a561Smaybee dbuf_fix_old_data(db, txg); 87944eda4d7Smaybee } 880ea8dc4b6Seschrock } 881c717a561Smaybee /* clear the contents if its cached */ 882ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 883ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 884fa9e4066Sahrens arc_release(db->db_buf, db); 885fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8866b4acc8bSahrens arc_buf_freeze(db->db_buf); 887fa9e4066Sahrens } 888ea8dc4b6Seschrock 889fa9e4066Sahrens mutex_exit(&db->db_mtx); 890fa9e4066Sahrens } 891fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 892fa9e4066Sahrens } 893fa9e4066Sahrens 894fa9e4066Sahrens static int 8951934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 896fa9e4066Sahrens { 897fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 898fa9e4066Sahrens uint64_t birth_txg = 0; 899fa9e4066Sahrens 900fa9e4066Sahrens /* 901fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 902c717a561Smaybee * If it's syncing, then db_last_dirty will be set 903c717a561Smaybee * so we'll ignore db_blkptr. 904fa9e4066Sahrens */ 905c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 906c717a561Smaybee if (db->db_last_dirty) 907c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 908fa9e4066Sahrens else if (db->db_blkptr) 909fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 910fa9e4066Sahrens 911837b568bSGeorge Wilson /* 912837b568bSGeorge Wilson * If we don't exist or are in a snapshot, we can't be freed. 913837b568bSGeorge Wilson * Don't pass the bp to dsl_dataset_block_freeable() since we 914837b568bSGeorge Wilson * are holding the db_mtx lock and might deadlock if we are 915837b568bSGeorge Wilson * prefetching a dedup-ed block. 916837b568bSGeorge Wilson */ 917fa9e4066Sahrens if (birth_txg) 9181934e92fSmaybee return (ds == NULL || 919837b568bSGeorge Wilson dsl_dataset_block_freeable(ds, NULL, birth_txg)); 920fa9e4066Sahrens else 9211934e92fSmaybee return (FALSE); 922fa9e4066Sahrens } 923fa9e4066Sahrens 924fa9e4066Sahrens void 925fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 926fa9e4066Sahrens { 927fa9e4066Sahrens arc_buf_t *buf, *obuf; 928fa9e4066Sahrens int osize = db->db.db_size; 929ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 930744947dcSTom Erickson dnode_t *dn; 931fa9e4066Sahrens 9320a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 933ea8dc4b6Seschrock 934744947dcSTom Erickson DB_DNODE_ENTER(db); 935744947dcSTom Erickson dn = DB_DNODE(db); 936744947dcSTom Erickson 937fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 938744947dcSTom Erickson ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 939fa9e4066Sahrens 940fa9e4066Sahrens /* 941fa9e4066Sahrens * This call to dbuf_will_dirty() with the dn_struct_rwlock held 942fa9e4066Sahrens * is OK, because there can be no other references to the db 943fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 944fa9e4066Sahrens * be happening. 945fa9e4066Sahrens */ 946ea8dc4b6Seschrock /* 947ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 948ea8dc4b6Seschrock * value and returning that up to our callers 949ea8dc4b6Seschrock */ 950fa9e4066Sahrens dbuf_will_dirty(db, tx); 951fa9e4066Sahrens 952fa9e4066Sahrens /* create the data buffer for the new block */ 953744947dcSTom Erickson buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 954fa9e4066Sahrens 955fa9e4066Sahrens /* copy old block data to the new block */ 956fa9e4066Sahrens obuf = db->db_buf; 957f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 958fa9e4066Sahrens /* zero the remainder */ 959f65e61c0Sahrens if (size > osize) 960f65e61c0Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 961fa9e4066Sahrens 962fa9e4066Sahrens mutex_enter(&db->db_mtx); 963fa9e4066Sahrens dbuf_set_data(db, buf); 964ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(obuf, db) == 1); 965fa9e4066Sahrens db->db.db_size = size; 966fa9e4066Sahrens 967c717a561Smaybee if (db->db_level == 0) { 968c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 969c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 970c717a561Smaybee } 971fa9e4066Sahrens mutex_exit(&db->db_mtx); 972fa9e4066Sahrens 973744947dcSTom Erickson dnode_willuse_space(dn, size-osize, tx); 974744947dcSTom Erickson DB_DNODE_EXIT(db); 975fa9e4066Sahrens } 976fa9e4066Sahrens 9773f9d6ad7SLin Ling void 9783f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 9793f9d6ad7SLin Ling { 980744947dcSTom Erickson objset_t *os; 9813f9d6ad7SLin Ling zbookmark_t zb; 9823f9d6ad7SLin Ling 983744947dcSTom Erickson DB_GET_OBJSET(&os, db); 9843f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 9853f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 9863f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 9873f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 9883f9d6ad7SLin Ling 9893f9d6ad7SLin Ling zb.zb_objset = os->os_dsl_dataset ? 9903f9d6ad7SLin Ling os->os_dsl_dataset->ds_object : 0; 9913f9d6ad7SLin Ling zb.zb_object = db->db.db_object; 9923f9d6ad7SLin Ling zb.zb_level = db->db_level; 9933f9d6ad7SLin Ling zb.zb_blkid = db->db_blkid; 9943f9d6ad7SLin Ling (void) arc_release_bp(db->db_buf, db, 9953f9d6ad7SLin Ling db->db_blkptr, os->os_spa, &zb); 9963f9d6ad7SLin Ling } 9973f9d6ad7SLin Ling 998c717a561Smaybee dbuf_dirty_record_t * 999fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1000fa9e4066Sahrens { 1001744947dcSTom Erickson dnode_t *dn; 1002744947dcSTom Erickson objset_t *os; 1003c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 1004fa9e4066Sahrens int drop_struct_lock = FALSE; 1005d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 1006fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1007fa9e4066Sahrens 1008fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1009fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 10109c9dc39aSek DMU_TX_DIRTY_BUF(tx, db); 1011fa9e4066Sahrens 1012744947dcSTom Erickson DB_DNODE_ENTER(db); 1013744947dcSTom Erickson dn = DB_DNODE(db); 1014fa9e4066Sahrens /* 1015fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 1016fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 1017fa9e4066Sahrens * were already pre-dirtied in open context. 1018fa9e4066Sahrens */ 1019c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 1020c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 102114843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 102214843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 1023fa9e4066Sahrens /* 1024fa9e4066Sahrens * We make this assert for private objects as well, but after we 1025fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 1026fa9e4066Sahrens * in syncing context. 1027fa9e4066Sahrens */ 1028ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1029c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1030fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1031fa9e4066Sahrens 1032fa9e4066Sahrens mutex_enter(&db->db_mtx); 1033fa9e4066Sahrens /* 1034c717a561Smaybee * XXX make this true for indirects too? The problem is that 1035c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 1036c717a561Smaybee * syncing context don't bother holding ahead. 1037fa9e4066Sahrens */ 1038c717a561Smaybee ASSERT(db->db_level != 0 || 103982c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 104082c9918fSTim Haley db->db_state == DB_NOFILL); 1041fa9e4066Sahrens 1042fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1043fa9e4066Sahrens /* 1044fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 1045fa9e4066Sahrens * initialize the objset. 1046fa9e4066Sahrens */ 1047fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 1048c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1049fa9e4066Sahrens dn->dn_dirtyctx = 1050fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1051fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 1052fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1053fa9e4066Sahrens } 1054fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1055fa9e4066Sahrens 10560a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 10570a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 10580a586ceaSMark Shellenbaum 1059fa9e4066Sahrens /* 1060fa9e4066Sahrens * If this buffer is already dirty, we're done. 1061fa9e4066Sahrens */ 1062c717a561Smaybee drp = &db->db_last_dirty; 1063c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1064c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 10657e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 10667e2186e3Sbonwick drp = &dr->dr_next; 10677e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 1068744947dcSTom Erickson DB_DNODE_EXIT(db); 1069744947dcSTom Erickson 10700a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1071c717a561Smaybee /* 1072c717a561Smaybee * If this buffer has already been written out, 1073c717a561Smaybee * we now need to reset its state. 1074c717a561Smaybee */ 10757e2186e3Sbonwick dbuf_unoverride(dr); 1076b24ab676SJeff Bonwick if (db->db.db_object != DMU_META_DNODE_OBJECT && 1077b24ab676SJeff Bonwick db->db_state != DB_NOFILL) 1078c717a561Smaybee arc_buf_thaw(db->db_buf); 1079c717a561Smaybee } 1080fa9e4066Sahrens mutex_exit(&db->db_mtx); 10817e2186e3Sbonwick return (dr); 1082fa9e4066Sahrens } 1083fa9e4066Sahrens 1084fa9e4066Sahrens /* 1085fa9e4066Sahrens * Only valid if not already dirty. 1086fa9e4066Sahrens */ 108714843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 108814843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1089fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1090fa9e4066Sahrens 1091fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1092fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1093fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1094fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1095fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1096fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1097fa9e4066Sahrens 1098fa9e4066Sahrens /* 1099fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 110014843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 110114843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 110214843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 110314843421SMatthew Ahrens * this assertion only if we're not already dirty. 1104fa9e4066Sahrens */ 1105744947dcSTom Erickson os = dn->dn_objset; 110614843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 110714843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1108fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1109fa9e4066Sahrens 1110fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1111fa9e4066Sahrens 11120a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 11131934e92fSmaybee /* 11141934e92fSmaybee * Update the accounting. 1115d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1116d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1117b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1118d3469faaSMark Maybee * also holding the db_mtx. 11191934e92fSmaybee */ 11201934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1121d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 11221934e92fSmaybee } 11231934e92fSmaybee 1124ea8dc4b6Seschrock /* 1125ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1126ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1127ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1128ea8dc4b6Seschrock */ 1129c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1130c717a561Smaybee if (db->db_level == 0) { 1131c717a561Smaybee void *data_old = db->db_buf; 1132c717a561Smaybee 113382c9918fSTim Haley if (db->db_state != DB_NOFILL) { 11340a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 113582c9918fSTim Haley dbuf_fix_old_data(db, tx->tx_txg); 113682c9918fSTim Haley data_old = db->db.db_data; 113782c9918fSTim Haley } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 113882c9918fSTim Haley /* 113982c9918fSTim Haley * Release the data buffer from the cache so 114082c9918fSTim Haley * that we can modify it without impacting 114182c9918fSTim Haley * possible other users of this cached data 114282c9918fSTim Haley * block. Note that indirect blocks and 114382c9918fSTim Haley * private objects are not released until the 114482c9918fSTim Haley * syncing state (since they are only modified 114582c9918fSTim Haley * then). 114682c9918fSTim Haley */ 114782c9918fSTim Haley arc_release(db->db_buf, db); 114882c9918fSTim Haley dbuf_fix_old_data(db, tx->tx_txg); 114982c9918fSTim Haley data_old = db->db_buf; 115082c9918fSTim Haley } 115182c9918fSTim Haley ASSERT(data_old != NULL); 1152fa9e4066Sahrens } 1153c717a561Smaybee dr->dt.dl.dr_data = data_old; 1154c717a561Smaybee } else { 1155c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1156c717a561Smaybee list_create(&dr->dt.di.dr_children, 1157c717a561Smaybee sizeof (dbuf_dirty_record_t), 1158c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1159fa9e4066Sahrens } 1160c717a561Smaybee dr->dr_dbuf = db; 1161c717a561Smaybee dr->dr_txg = tx->tx_txg; 1162c717a561Smaybee dr->dr_next = *drp; 1163c717a561Smaybee *drp = dr; 1164fa9e4066Sahrens 1165fa9e4066Sahrens /* 1166fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1167fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1168fa9e4066Sahrens * happened after the free. 1169fa9e4066Sahrens */ 11700a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 11710a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1172c717a561Smaybee mutex_enter(&dn->dn_mtx); 1173fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1174c717a561Smaybee mutex_exit(&dn->dn_mtx); 1175c717a561Smaybee db->db_freed_in_flight = FALSE; 1176fa9e4066Sahrens } 1177fa9e4066Sahrens 1178fa9e4066Sahrens /* 1179fa9e4066Sahrens * This buffer is now part of this txg 1180fa9e4066Sahrens */ 1181fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1182fa9e4066Sahrens db->db_dirtycnt += 1; 1183fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1184fa9e4066Sahrens 1185fa9e4066Sahrens mutex_exit(&db->db_mtx); 1186fa9e4066Sahrens 11870a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 11880a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1189c717a561Smaybee mutex_enter(&dn->dn_mtx); 1190c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1191c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1192c717a561Smaybee mutex_exit(&dn->dn_mtx); 1193fa9e4066Sahrens dnode_setdirty(dn, tx); 1194744947dcSTom Erickson DB_DNODE_EXIT(db); 1195c717a561Smaybee return (dr); 1196d3469faaSMark Maybee } else if (do_free_accounting) { 1197d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1198d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1199b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1200d3469faaSMark Maybee /* 1201d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1202d3469faaSMark Maybee * in a previous txg, we don't know how much 1203d3469faaSMark Maybee * space it will use on disk yet. We should 1204d3469faaSMark Maybee * really have the struct_rwlock to access 1205d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1206d3469faaSMark Maybee * it's OK if we get an odd answer. 1207d3469faaSMark Maybee */ 1208837b568bSGeorge Wilson ddt_prefetch(os->os_spa, bp); 1209d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1210fa9e4066Sahrens } 1211fa9e4066Sahrens 1212fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1213fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1214fa9e4066Sahrens drop_struct_lock = TRUE; 1215fa9e4066Sahrens } 1216fa9e4066Sahrens 12178346f03fSJonathan W Adams if (db->db_level == 0) { 12188346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 12198346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 12208346f03fSJonathan W Adams } 12218346f03fSJonathan W Adams 122244eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1223c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1224c717a561Smaybee dbuf_dirty_record_t *di; 1225c717a561Smaybee int parent_held = FALSE; 1226c717a561Smaybee 1227c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1228c717a561Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1229c717a561Smaybee 1230c717a561Smaybee parent = dbuf_hold_level(dn, db->db_level+1, 1231c717a561Smaybee db->db_blkid >> epbs, FTAG); 123201025c89SJohn Harres ASSERT(parent != NULL); 1233c717a561Smaybee parent_held = TRUE; 1234c717a561Smaybee } 1235fa9e4066Sahrens if (drop_struct_lock) 1236fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1237c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1238c717a561Smaybee di = dbuf_dirty(parent, tx); 1239c717a561Smaybee if (parent_held) 1240c717a561Smaybee dbuf_rele(parent, FTAG); 1241c717a561Smaybee 1242c717a561Smaybee mutex_enter(&db->db_mtx); 1243c717a561Smaybee /* possible race with dbuf_undirty() */ 1244c717a561Smaybee if (db->db_last_dirty == dr || 1245c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1246c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1247c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1248c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1249c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1250c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1251c717a561Smaybee dr->dr_parent = di; 1252c717a561Smaybee } 1253c717a561Smaybee mutex_exit(&db->db_mtx); 1254fa9e4066Sahrens } else { 1255c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1256c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1257744947dcSTom Erickson ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1258c717a561Smaybee mutex_enter(&dn->dn_mtx); 1259c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1260c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1261c717a561Smaybee mutex_exit(&dn->dn_mtx); 1262fa9e4066Sahrens if (drop_struct_lock) 1263fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1264fa9e4066Sahrens } 1265fa9e4066Sahrens 1266fa9e4066Sahrens dnode_setdirty(dn, tx); 1267744947dcSTom Erickson DB_DNODE_EXIT(db); 1268c717a561Smaybee return (dr); 1269fa9e4066Sahrens } 1270fa9e4066Sahrens 1271fa9e4066Sahrens static int 1272fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1273fa9e4066Sahrens { 1274744947dcSTom Erickson dnode_t *dn; 1275c717a561Smaybee uint64_t txg = tx->tx_txg; 127617f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1277fa9e4066Sahrens 1278c717a561Smaybee ASSERT(txg != 0); 12790a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1280fa9e4066Sahrens 1281fa9e4066Sahrens mutex_enter(&db->db_mtx); 1282fa9e4066Sahrens /* 1283fa9e4066Sahrens * If this buffer is not dirty, we're done. 1284fa9e4066Sahrens */ 128517f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1286c717a561Smaybee if (dr->dr_txg <= txg) 1287c717a561Smaybee break; 1288c717a561Smaybee if (dr == NULL || dr->dr_txg < txg) { 1289fa9e4066Sahrens mutex_exit(&db->db_mtx); 1290fa9e4066Sahrens return (0); 1291fa9e4066Sahrens } 1292c717a561Smaybee ASSERT(dr->dr_txg == txg); 1293b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1294fa9e4066Sahrens 1295744947dcSTom Erickson DB_DNODE_ENTER(db); 1296744947dcSTom Erickson dn = DB_DNODE(db); 1297744947dcSTom Erickson 1298fa9e4066Sahrens /* 1299fa9e4066Sahrens * If this buffer is currently held, we cannot undirty 1300fa9e4066Sahrens * it, since one of the current holders may be in the 1301fa9e4066Sahrens * middle of an update. Note that users of dbuf_undirty() 1302fa9e4066Sahrens * should not place a hold on the dbuf before the call. 1303fa9e4066Sahrens */ 1304fa9e4066Sahrens if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 1305fa9e4066Sahrens mutex_exit(&db->db_mtx); 130644eda4d7Smaybee /* Make sure we don't toss this buffer at sync phase */ 1307fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1308fa9e4066Sahrens dnode_clear_range(dn, db->db_blkid, 1, tx); 1309fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1310744947dcSTom Erickson DB_DNODE_EXIT(db); 1311fa9e4066Sahrens return (0); 1312fa9e4066Sahrens } 1313fa9e4066Sahrens 1314fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1315fa9e4066Sahrens 1316fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1317fa9e4066Sahrens 1318fa9e4066Sahrens /* XXX would be nice to fix up dn_towrite_space[] */ 1319fa9e4066Sahrens 132017f17c2dSbonwick *drp = dr->dr_next; 1321c717a561Smaybee 1322c717a561Smaybee if (dr->dr_parent) { 1323c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1324c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1325c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1326c717a561Smaybee } else if (db->db_level+1 == dn->dn_nlevels) { 1327cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1328c717a561Smaybee mutex_enter(&dn->dn_mtx); 1329c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1330c717a561Smaybee mutex_exit(&dn->dn_mtx); 1331c717a561Smaybee } 1332744947dcSTom Erickson DB_DNODE_EXIT(db); 1333c717a561Smaybee 1334c717a561Smaybee if (db->db_level == 0) { 133582c9918fSTim Haley if (db->db_state != DB_NOFILL) { 133682c9918fSTim Haley dbuf_unoverride(dr); 1337c717a561Smaybee 133882c9918fSTim Haley ASSERT(db->db_buf != NULL); 133982c9918fSTim Haley ASSERT(dr->dt.dl.dr_data != NULL); 134082c9918fSTim Haley if (dr->dt.dl.dr_data != db->db_buf) 134182c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 134282c9918fSTim Haley db) == 1); 134382c9918fSTim Haley } 1344c717a561Smaybee } else { 1345c717a561Smaybee ASSERT(db->db_buf != NULL); 1346c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 1347c25056deSgw mutex_destroy(&dr->dt.di.dr_mtx); 1348c25056deSgw list_destroy(&dr->dt.di.dr_children); 1349c717a561Smaybee } 1350c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1351fa9e4066Sahrens 1352fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1353fa9e4066Sahrens db->db_dirtycnt -= 1; 1354fa9e4066Sahrens 1355c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1356ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1357fa9e4066Sahrens 1358b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1359ea8dc4b6Seschrock dbuf_set_data(db, NULL); 1360ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 1361fa9e4066Sahrens dbuf_evict(db); 1362fa9e4066Sahrens return (1); 1363fa9e4066Sahrens } 1364fa9e4066Sahrens 1365fa9e4066Sahrens mutex_exit(&db->db_mtx); 1366fa9e4066Sahrens return (0); 1367fa9e4066Sahrens } 1368fa9e4066Sahrens 1369fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty 1370fa9e4066Sahrens void 1371fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1372fa9e4066Sahrens { 13731ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1374fa9e4066Sahrens 1375fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1376fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1377fa9e4066Sahrens 1378744947dcSTom Erickson DB_DNODE_ENTER(db); 1379744947dcSTom Erickson if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1380fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1381744947dcSTom Erickson DB_DNODE_EXIT(db); 1382ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1383c717a561Smaybee (void) dbuf_dirty(db, tx); 1384fa9e4066Sahrens } 1385fa9e4066Sahrens 138682c9918fSTim Haley void 138782c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 138882c9918fSTim Haley { 138982c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 139082c9918fSTim Haley 139182c9918fSTim Haley db->db_state = DB_NOFILL; 139282c9918fSTim Haley 139382c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 139482c9918fSTim Haley } 139582c9918fSTim Haley 1396fa9e4066Sahrens void 1397ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1398fa9e4066Sahrens { 1399ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1400ea8dc4b6Seschrock 14010a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1402fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1403fa9e4066Sahrens ASSERT(db->db_level == 0); 1404fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1405fa9e4066Sahrens 1406ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1407fa9e4066Sahrens dmu_tx_private_ok(tx)); 1408fa9e4066Sahrens 1409fa9e4066Sahrens dbuf_noread(db); 1410c717a561Smaybee (void) dbuf_dirty(db, tx); 1411fa9e4066Sahrens } 1412fa9e4066Sahrens 1413fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1414fa9e4066Sahrens /* ARGSUSED */ 1415fa9e4066Sahrens void 1416fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1417fa9e4066Sahrens { 1418fa9e4066Sahrens mutex_enter(&db->db_mtx); 14199c9dc39aSek DBUF_VERIFY(db); 1420fa9e4066Sahrens 1421fa9e4066Sahrens if (db->db_state == DB_FILL) { 1422c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 14230a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1424fa9e4066Sahrens /* we were freed while filling */ 1425fa9e4066Sahrens /* XXX dbuf_undirty? */ 1426fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1427c717a561Smaybee db->db_freed_in_flight = FALSE; 1428fa9e4066Sahrens } 1429fa9e4066Sahrens db->db_state = DB_CACHED; 1430fa9e4066Sahrens cv_broadcast(&db->db_changed); 1431fa9e4066Sahrens } 1432fa9e4066Sahrens mutex_exit(&db->db_mtx); 1433fa9e4066Sahrens } 1434fa9e4066Sahrens 14352fdbea25SAleksandr Guzovskiy /* 14362fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 14372fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 14382fdbea25SAleksandr Guzovskiy */ 14392fdbea25SAleksandr Guzovskiy void 14402fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 14412fdbea25SAleksandr Guzovskiy { 14422fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 14430a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 14442fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 14452fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 14462fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 14472fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 14482fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 14492fdbea25SAleksandr Guzovskiy 14502fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 14512fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 14522fdbea25SAleksandr Guzovskiy 14532fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 14542fdbea25SAleksandr Guzovskiy 14552fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 14562fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 14572fdbea25SAleksandr Guzovskiy 14582fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 14592fdbea25SAleksandr Guzovskiy 14602fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 14612fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 14622fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 14632fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 14642fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 14652fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(buf, db) == 1); 1466c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 14672fdbea25SAleksandr Guzovskiy return; 14682fdbea25SAleksandr Guzovskiy } 14692fdbea25SAleksandr Guzovskiy 1470c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 14712fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 14722fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 14732fdbea25SAleksandr Guzovskiy 14742fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 14752fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 14762fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 14772fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 14782fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 14792fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 14802fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 14812fdbea25SAleksandr Guzovskiy } 14822fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 14832fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 14842fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 14852fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 14862fdbea25SAleksandr Guzovskiy VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1); 14872fdbea25SAleksandr Guzovskiy } 14882fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 14892fdbea25SAleksandr Guzovskiy } 14902fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 14912fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 14922fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 14932fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 14942fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 14952fdbea25SAleksandr Guzovskiy dbuf_fill_done(db, tx); 14962fdbea25SAleksandr Guzovskiy } 14972fdbea25SAleksandr Guzovskiy 1498ea8dc4b6Seschrock /* 1499ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 1500ea8dc4b6Seschrock * EVICTING and clear *most* of its references. Unfortunetely, 1501ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1502ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1503ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1504ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1505ea8dc4b6Seschrock * For the arc callback, we will usually see: 1506744947dcSTom Erickson * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1507ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1508ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1509ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1510ea8dc4b6Seschrock */ 1511ea8dc4b6Seschrock void 1512fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1513fa9e4066Sahrens { 1514744947dcSTom Erickson dnode_t *dn; 1515ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1516744947dcSTom Erickson dmu_buf_impl_t *dndb; 1517ea8dc4b6Seschrock int dbuf_gone = FALSE; 1518fa9e4066Sahrens 1519fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1520fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1521fa9e4066Sahrens 1522ea8dc4b6Seschrock dbuf_evict_user(db); 1523ea8dc4b6Seschrock 1524fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1525ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 15260a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1527ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 15285a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 15290e8c6158Smaybee } 1530fa9e4066Sahrens db->db.db_data = NULL; 1531fa9e4066Sahrens db->db_state = DB_UNCACHED; 1532fa9e4066Sahrens } 1533fa9e4066Sahrens 153482c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1535fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1536fa9e4066Sahrens 1537ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1538ea8dc4b6Seschrock db->db_blkptr = NULL; 1539ea8dc4b6Seschrock 1540744947dcSTom Erickson DB_DNODE_ENTER(db); 1541744947dcSTom Erickson dn = DB_DNODE(db); 1542744947dcSTom Erickson dndb = dn->dn_dbuf; 15430a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1544ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1545744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1546744947dcSTom Erickson membar_producer(); 1547744947dcSTom Erickson DB_DNODE_EXIT(db); 1548744947dcSTom Erickson /* 1549744947dcSTom Erickson * Decrementing the dbuf count means that the hold corresponding 1550744947dcSTom Erickson * to the removed dbuf is no longer discounted in dnode_move(), 1551744947dcSTom Erickson * so the dnode cannot be moved until after we release the hold. 1552744947dcSTom Erickson * The membar_producer() ensures visibility of the decremented 1553744947dcSTom Erickson * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1554744947dcSTom Erickson * release any lock. 1555744947dcSTom Erickson */ 1556ea8dc4b6Seschrock dnode_rele(dn, db); 1557744947dcSTom Erickson db->db_dnode_handle = NULL; 1558744947dcSTom Erickson } else { 1559744947dcSTom Erickson DB_DNODE_EXIT(db); 1560ea8dc4b6Seschrock } 1561ea8dc4b6Seschrock 1562ea8dc4b6Seschrock if (db->db_buf) 1563ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1564ea8dc4b6Seschrock 1565ea8dc4b6Seschrock if (!dbuf_gone) 1566ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1567fa9e4066Sahrens 1568fa9e4066Sahrens /* 1569744947dcSTom Erickson * If this dbuf is referenced from an indirect dbuf, 1570fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1571fa9e4066Sahrens */ 1572c543ec06Sahrens if (parent && parent != dndb) 1573ea8dc4b6Seschrock dbuf_rele(parent, db); 1574fa9e4066Sahrens } 1575fa9e4066Sahrens 1576fa9e4066Sahrens static int 1577fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1578fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1579fa9e4066Sahrens { 1580fa9e4066Sahrens int nlevels, epbs; 1581fa9e4066Sahrens 15820b69c2f0Sahrens *parentp = NULL; 15830b69c2f0Sahrens *bpp = NULL; 15840b69c2f0Sahrens 15850a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 15860a586ceaSMark Shellenbaum 15870a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 15880a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 158906e0070dSMark Shellenbaum if (dn->dn_have_spill && 159006e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 15910a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 15920a586ceaSMark Shellenbaum else 15930a586ceaSMark Shellenbaum *bpp = NULL; 15940a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 15950a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 15960a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 15970a586ceaSMark Shellenbaum return (0); 15980a586ceaSMark Shellenbaum } 1599ea8dc4b6Seschrock 1600fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1601fa9e4066Sahrens nlevels = 1; 1602fa9e4066Sahrens else 1603fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1604fa9e4066Sahrens 1605fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1606fa9e4066Sahrens 1607fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1608fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1609ea8dc4b6Seschrock if (level >= nlevels || 1610fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1611fa9e4066Sahrens /* the buffer has no parent yet */ 1612fa9e4066Sahrens return (ENOENT); 1613fa9e4066Sahrens } else if (level < nlevels-1) { 1614fa9e4066Sahrens /* this block is referenced from an indirect block */ 1615fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1616fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1617fa9e4066Sahrens if (err) 1618fa9e4066Sahrens return (err); 1619ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1620ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1621c543ec06Sahrens if (err) { 1622c543ec06Sahrens dbuf_rele(*parentp, NULL); 1623c543ec06Sahrens *parentp = NULL; 1624c543ec06Sahrens return (err); 1625ea8dc4b6Seschrock } 1626c543ec06Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1627c543ec06Sahrens (blkid & ((1ULL << epbs) - 1)); 1628c543ec06Sahrens return (0); 1629fa9e4066Sahrens } else { 1630fa9e4066Sahrens /* the block is referenced from the dnode */ 1631fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1632fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1633fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1634c543ec06Sahrens if (dn->dn_dbuf) { 1635c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1636c543ec06Sahrens *parentp = dn->dn_dbuf; 1637c543ec06Sahrens } 1638fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1639fa9e4066Sahrens return (0); 1640fa9e4066Sahrens } 1641fa9e4066Sahrens } 1642fa9e4066Sahrens 1643fa9e4066Sahrens static dmu_buf_impl_t * 1644fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1645fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1646fa9e4066Sahrens { 1647503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1648fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1649fa9e4066Sahrens 1650fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1651fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1652fa9e4066Sahrens 1653fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1654fa9e4066Sahrens 1655fa9e4066Sahrens db->db_objset = os; 1656fa9e4066Sahrens db->db.db_object = dn->dn_object; 1657fa9e4066Sahrens db->db_level = level; 1658fa9e4066Sahrens db->db_blkid = blkid; 1659c717a561Smaybee db->db_last_dirty = NULL; 1660ea8dc4b6Seschrock db->db_dirtycnt = 0; 1661744947dcSTom Erickson db->db_dnode_handle = dn->dn_handle; 1662ea8dc4b6Seschrock db->db_parent = parent; 1663ea8dc4b6Seschrock db->db_blkptr = blkptr; 1664fa9e4066Sahrens 1665c717a561Smaybee db->db_user_ptr = NULL; 1666c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1667c717a561Smaybee db->db_evict_func = NULL; 1668c717a561Smaybee db->db_immediate_evict = 0; 1669c717a561Smaybee db->db_freed_in_flight = 0; 1670ea8dc4b6Seschrock 16710a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 1672ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 16731934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 16741934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 16751934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 16760a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 1677ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1678ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 16795a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1680ea8dc4b6Seschrock return (db); 16810a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 16820a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 16830a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 16840a586ceaSMark Shellenbaum db->db.db_offset = 0; 1685fa9e4066Sahrens } else { 1686fa9e4066Sahrens int blocksize = 1687fa9e4066Sahrens db->db_level ? 1<<dn->dn_indblkshift : dn->dn_datablksz; 1688fa9e4066Sahrens db->db.db_size = blocksize; 1689fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1690fa9e4066Sahrens } 1691fa9e4066Sahrens 1692fa9e4066Sahrens /* 1693fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1694fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1695fa9e4066Sahrens * This prevents a possible deadlock with someone 1696fa9e4066Sahrens * trying to look up this dbuf before its added to the 1697fa9e4066Sahrens * dn_dbufs list. 1698fa9e4066Sahrens */ 1699fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1700ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1701fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1702fa9e4066Sahrens /* someone else inserted it first */ 1703fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1704fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1705fa9e4066Sahrens return (odb); 1706fa9e4066Sahrens } 1707fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1708ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1709fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 17105a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1711fa9e4066Sahrens 1712fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1713fa9e4066Sahrens dbuf_add_ref(parent, db); 1714fa9e4066Sahrens 1715ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1716ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1717fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1718744947dcSTom Erickson (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1719fa9e4066Sahrens 1720fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1721fa9e4066Sahrens 1722fa9e4066Sahrens return (db); 1723fa9e4066Sahrens } 1724fa9e4066Sahrens 1725fa9e4066Sahrens static int 1726ea8dc4b6Seschrock dbuf_do_evict(void *private) 1727fa9e4066Sahrens { 1728ea8dc4b6Seschrock arc_buf_t *buf = private; 1729ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1730fa9e4066Sahrens 1731ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1732ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1733fa9e4066Sahrens 1734ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1735fa9e4066Sahrens 1736ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1737ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 1738ea8dc4b6Seschrock DBUF_VERIFY(db); 1739ea8dc4b6Seschrock db->db_buf = NULL; 1740ea8dc4b6Seschrock dbuf_evict(db); 1741ea8dc4b6Seschrock } else { 1742ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1743ea8dc4b6Seschrock dbuf_destroy(db); 1744fa9e4066Sahrens } 1745ea8dc4b6Seschrock return (0); 1746fa9e4066Sahrens } 1747fa9e4066Sahrens 1748fa9e4066Sahrens static void 1749fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1750fa9e4066Sahrens { 1751fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1752fa9e4066Sahrens 17530a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 1754ea8dc4b6Seschrock /* 1755ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1756ea8dc4b6Seschrock * remove it from that list. 1757ea8dc4b6Seschrock */ 1758744947dcSTom Erickson if (db->db_dnode_handle != NULL) { 1759744947dcSTom Erickson dnode_t *dn; 17601934e92fSmaybee 1761744947dcSTom Erickson DB_DNODE_ENTER(db); 1762744947dcSTom Erickson dn = DB_DNODE(db); 17631934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1764ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1765744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1766c543ec06Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1767744947dcSTom Erickson DB_DNODE_EXIT(db); 1768744947dcSTom Erickson /* 1769744947dcSTom Erickson * Decrementing the dbuf count means that the hold 1770744947dcSTom Erickson * corresponding to the removed dbuf is no longer 1771744947dcSTom Erickson * discounted in dnode_move(), so the dnode cannot be 1772744947dcSTom Erickson * moved until after we release the hold. 1773744947dcSTom Erickson */ 1774ea8dc4b6Seschrock dnode_rele(dn, db); 1775744947dcSTom Erickson db->db_dnode_handle = NULL; 1776ea8dc4b6Seschrock } 1777ea8dc4b6Seschrock dbuf_hash_remove(db); 1778ea8dc4b6Seschrock } 1779ea8dc4b6Seschrock db->db_parent = NULL; 1780ea8dc4b6Seschrock db->db_buf = NULL; 1781ea8dc4b6Seschrock 1782f82bfe17Sgw ASSERT(!list_link_active(&db->db_link)); 1783fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1784fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1785fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1786fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1787fa9e4066Sahrens 1788fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 17895a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1790fa9e4066Sahrens } 1791fa9e4066Sahrens 1792fa9e4066Sahrens void 1793fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid) 1794fa9e4066Sahrens { 179513506d1eSmaybee dmu_buf_impl_t *db = NULL; 1796fa9e4066Sahrens blkptr_t *bp = NULL; 1797fa9e4066Sahrens 17980a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1799fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1800fa9e4066Sahrens 1801fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1802fa9e4066Sahrens return; 1803fa9e4066Sahrens 1804fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1805fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 1806*9e9c486fSGeorge Wilson /* 1807*9e9c486fSGeorge Wilson * This dbuf is already in the cache. We assume that 1808*9e9c486fSGeorge Wilson * it is already CACHED, or else about to be either 1809*9e9c486fSGeorge Wilson * read or filled. 1810*9e9c486fSGeorge Wilson */ 1811fa9e4066Sahrens mutex_exit(&db->db_mtx); 1812*9e9c486fSGeorge Wilson return; 1813fa9e4066Sahrens } 1814fa9e4066Sahrens 181513506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1816fa9e4066Sahrens if (bp && !BP_IS_HOLE(bp)) { 1817c7cd2421SGeorge Wilson int priority = dn->dn_type == DMU_OT_DDT_ZAP ? 1818c7cd2421SGeorge Wilson ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ; 1819088f3894Sahrens arc_buf_t *pbuf; 1820b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 182113506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1822ea8dc4b6Seschrock zbookmark_t zb; 1823b24ab676SJeff Bonwick 1824b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1825b24ab676SJeff Bonwick dn->dn_object, 0, blkid); 1826ea8dc4b6Seschrock 1827088f3894Sahrens if (db) 1828088f3894Sahrens pbuf = db->db_buf; 1829088f3894Sahrens else 1830088f3894Sahrens pbuf = dn->dn_objset->os_phys_buf; 1831088f3894Sahrens 18323f9d6ad7SLin Ling (void) dsl_read(NULL, dn->dn_objset->os_spa, 1833c7cd2421SGeorge Wilson bp, pbuf, NULL, NULL, priority, 1834fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 183513506d1eSmaybee &aflags, &zb); 1836fa9e4066Sahrens } 183713506d1eSmaybee if (db) 183813506d1eSmaybee dbuf_rele(db, NULL); 1839fa9e4066Sahrens } 1840fa9e4066Sahrens } 1841fa9e4066Sahrens 1842fa9e4066Sahrens /* 1843fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1844fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1845fa9e4066Sahrens */ 1846fa9e4066Sahrens int 1847fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1848fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1849fa9e4066Sahrens { 1850fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1851fa9e4066Sahrens 18520a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1853fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1854fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1855fa9e4066Sahrens 1856fa9e4066Sahrens *dbp = NULL; 1857ea8dc4b6Seschrock top: 1858fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1859fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1860fa9e4066Sahrens 1861fa9e4066Sahrens if (db == NULL) { 1862fa9e4066Sahrens blkptr_t *bp = NULL; 1863fa9e4066Sahrens int err; 1864fa9e4066Sahrens 1865c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1866fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1867fa9e4066Sahrens if (fail_sparse) { 1868fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1869fa9e4066Sahrens err = ENOENT; 1870fa9e4066Sahrens if (err) { 1871c543ec06Sahrens if (parent) 1872ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1873fa9e4066Sahrens return (err); 1874fa9e4066Sahrens } 1875fa9e4066Sahrens } 1876ea8dc4b6Seschrock if (err && err != ENOENT) 1877ea8dc4b6Seschrock return (err); 1878fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1879fa9e4066Sahrens } 1880fa9e4066Sahrens 1881ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1882ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1883ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1884ea8dc4b6Seschrock dbuf_clear(db); 1885c543ec06Sahrens if (parent) { 1886c543ec06Sahrens dbuf_rele(parent, NULL); 1887c543ec06Sahrens parent = NULL; 1888c543ec06Sahrens } 1889ea8dc4b6Seschrock goto top; 1890ea8dc4b6Seschrock } 1891ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1892ea8dc4b6Seschrock } 1893ea8dc4b6Seschrock 1894ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1895ea8dc4b6Seschrock 1896fa9e4066Sahrens /* 1897c717a561Smaybee * If this buffer is currently syncing out, and we are are 1898c717a561Smaybee * still referencing it from db_data, we need to make a copy 1899c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1900fa9e4066Sahrens */ 19010a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1902ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1903c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1904c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1905fa9e4066Sahrens 1906c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1907c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1908c717a561Smaybee 1909c717a561Smaybee dbuf_set_data(db, 1910744947dcSTom Erickson arc_buf_alloc(dn->dn_objset->os_spa, 1911c717a561Smaybee db->db.db_size, db, type)); 1912c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1913c717a561Smaybee db->db.db_size); 1914c717a561Smaybee } 1915fa9e4066Sahrens } 1916fa9e4066Sahrens 1917ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1918fa9e4066Sahrens dbuf_update_data(db); 19199c9dc39aSek DBUF_VERIFY(db); 1920fa9e4066Sahrens mutex_exit(&db->db_mtx); 1921fa9e4066Sahrens 1922fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1923c543ec06Sahrens if (parent) 1924ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1925fa9e4066Sahrens 1926744947dcSTom Erickson ASSERT3P(DB_DNODE(db), ==, dn); 1927fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1928fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1929fa9e4066Sahrens *dbp = db; 1930fa9e4066Sahrens 1931fa9e4066Sahrens return (0); 1932fa9e4066Sahrens } 1933fa9e4066Sahrens 1934fa9e4066Sahrens dmu_buf_impl_t * 1935ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1936fa9e4066Sahrens { 1937fa9e4066Sahrens dmu_buf_impl_t *db; 1938ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1939ea8dc4b6Seschrock return (err ? NULL : db); 1940fa9e4066Sahrens } 1941fa9e4066Sahrens 1942fa9e4066Sahrens dmu_buf_impl_t * 1943fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1944fa9e4066Sahrens { 1945fa9e4066Sahrens dmu_buf_impl_t *db; 1946ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1947ea8dc4b6Seschrock return (err ? NULL : db); 1948fa9e4066Sahrens } 1949fa9e4066Sahrens 19501934e92fSmaybee void 1951ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1952fa9e4066Sahrens { 1953ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1954ea8dc4b6Seschrock 1955ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 19560a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 19570a586ceaSMark Shellenbaum } 19580a586ceaSMark Shellenbaum 19590a586ceaSMark Shellenbaum int 19600a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 19610a586ceaSMark Shellenbaum { 19620a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1963744947dcSTom Erickson dnode_t *dn; 1964744947dcSTom Erickson 19650a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 19660a586ceaSMark Shellenbaum return (ENOTSUP); 19670a586ceaSMark Shellenbaum if (blksz == 0) 19680a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 19690a586ceaSMark Shellenbaum if (blksz > SPA_MAXBLOCKSIZE) 19700a586ceaSMark Shellenbaum blksz = SPA_MAXBLOCKSIZE; 19710a586ceaSMark Shellenbaum else 19720a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 19730a586ceaSMark Shellenbaum 1974744947dcSTom Erickson DB_DNODE_ENTER(db); 1975744947dcSTom Erickson dn = DB_DNODE(db); 1976744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 19770a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 1978744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 1979744947dcSTom Erickson DB_DNODE_EXIT(db); 19800a586ceaSMark Shellenbaum 19810a586ceaSMark Shellenbaum return (0); 19820a586ceaSMark Shellenbaum } 19830a586ceaSMark Shellenbaum 19840a586ceaSMark Shellenbaum void 19850a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 19860a586ceaSMark Shellenbaum { 19870a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 1988fa9e4066Sahrens } 1989fa9e4066Sahrens 1990ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 1991fa9e4066Sahrens void 1992fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 1993fa9e4066Sahrens { 1994ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 1995ea8dc4b6Seschrock ASSERT(holds > 1); 1996fa9e4066Sahrens } 1997fa9e4066Sahrens 1998744947dcSTom Erickson /* 1999744947dcSTom Erickson * If you call dbuf_rele() you had better not be referencing the dnode handle 2000744947dcSTom Erickson * unless you have some other direct or indirect hold on the dnode. (An indirect 2001744947dcSTom Erickson * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2002744947dcSTom Erickson * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2003744947dcSTom Erickson * dnode's parent dbuf evicting its dnode handles. 2004744947dcSTom Erickson */ 2005ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele 2006fa9e4066Sahrens void 2007ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 2008b24ab676SJeff Bonwick { 2009b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2010b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 2011b24ab676SJeff Bonwick } 2012b24ab676SJeff Bonwick 2013b24ab676SJeff Bonwick /* 2014b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2015b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 2016b24ab676SJeff Bonwick */ 2017b24ab676SJeff Bonwick void 2018b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2019fa9e4066Sahrens { 2020fa9e4066Sahrens int64_t holds; 2021fa9e4066Sahrens 2022b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 20239c9dc39aSek DBUF_VERIFY(db); 2024fa9e4066Sahrens 2025744947dcSTom Erickson /* 2026744947dcSTom Erickson * Remove the reference to the dbuf before removing its hold on the 2027744947dcSTom Erickson * dnode so we can guarantee in dnode_move() that a referenced bonus 2028744947dcSTom Erickson * buffer has a corresponding dnode hold. 2029744947dcSTom Erickson */ 2030fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 2031ea8dc4b6Seschrock ASSERT(holds >= 0); 2032ea8dc4b6Seschrock 2033c717a561Smaybee /* 2034c717a561Smaybee * We can't freeze indirects if there is a possibility that they 2035c717a561Smaybee * may be modified in the current syncing context. 2036c717a561Smaybee */ 2037c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 20386b4acc8bSahrens arc_buf_freeze(db->db_buf); 20396b4acc8bSahrens 2040ea8dc4b6Seschrock if (holds == db->db_dirtycnt && 2041c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 2042ea8dc4b6Seschrock dbuf_evict_user(db); 2043fa9e4066Sahrens 2044fa9e4066Sahrens if (holds == 0) { 20450a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2046ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2047744947dcSTom Erickson 2048744947dcSTom Erickson /* 2049744947dcSTom Erickson * If the dnode moves here, we cannot cross this barrier 2050744947dcSTom Erickson * until the move completes. 2051744947dcSTom Erickson */ 2052744947dcSTom Erickson DB_DNODE_ENTER(db); 2053744947dcSTom Erickson (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2054744947dcSTom Erickson DB_DNODE_EXIT(db); 2055744947dcSTom Erickson /* 2056744947dcSTom Erickson * The bonus buffer's dnode hold is no longer discounted 2057744947dcSTom Erickson * in dnode_move(). The dnode cannot move until after 2058744947dcSTom Erickson * the dnode_rele(). 2059744947dcSTom Erickson */ 2060744947dcSTom Erickson dnode_rele(DB_DNODE(db), db); 2061ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 2062ea8dc4b6Seschrock /* 2063ea8dc4b6Seschrock * This is a special case: we never associated this 2064ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 2065ea8dc4b6Seschrock */ 206682c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 206782c9918fSTim Haley db->db_state == DB_NOFILL); 2068ea8dc4b6Seschrock dbuf_evict(db); 20696b4acc8bSahrens } else if (arc_released(db->db_buf)) { 2070ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 2071ea8dc4b6Seschrock /* 2072ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 2073ea8dc4b6Seschrock */ 2074ea8dc4b6Seschrock dbuf_set_data(db, NULL); 2075ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(buf, db) == 1); 2076ea8dc4b6Seschrock dbuf_evict(db); 2077ea8dc4b6Seschrock } else { 2078ea8dc4b6Seschrock VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0); 20793baa08fcSek if (!DBUF_IS_CACHEABLE(db)) 20803baa08fcSek dbuf_clear(db); 20813baa08fcSek else 20823baa08fcSek mutex_exit(&db->db_mtx); 2083ea8dc4b6Seschrock } 2084fa9e4066Sahrens } else { 2085fa9e4066Sahrens mutex_exit(&db->db_mtx); 2086fa9e4066Sahrens } 2087fa9e4066Sahrens } 2088fa9e4066Sahrens 2089fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 2090fa9e4066Sahrens uint64_t 2091fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 2092fa9e4066Sahrens { 2093fa9e4066Sahrens return (refcount_count(&db->db_holds)); 2094fa9e4066Sahrens } 2095fa9e4066Sahrens 2096fa9e4066Sahrens void * 2097fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2098fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2099fa9e4066Sahrens { 2100fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2101fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2102fa9e4066Sahrens } 2103fa9e4066Sahrens 2104fa9e4066Sahrens void * 2105fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2106fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2107fa9e4066Sahrens { 2108fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2109fa9e4066Sahrens 2110c717a561Smaybee db->db_immediate_evict = TRUE; 2111fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2112fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2113fa9e4066Sahrens } 2114fa9e4066Sahrens 2115fa9e4066Sahrens void * 2116fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2117fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2118fa9e4066Sahrens { 2119fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2120fa9e4066Sahrens ASSERT(db->db_level == 0); 2121fa9e4066Sahrens 2122fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2123fa9e4066Sahrens 2124fa9e4066Sahrens mutex_enter(&db->db_mtx); 2125fa9e4066Sahrens 2126c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 2127c717a561Smaybee db->db_user_ptr = user_ptr; 2128c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2129c717a561Smaybee db->db_evict_func = evict_func; 2130fa9e4066Sahrens 2131fa9e4066Sahrens dbuf_update_data(db); 2132fa9e4066Sahrens } else { 2133c717a561Smaybee old_user_ptr = db->db_user_ptr; 2134fa9e4066Sahrens } 2135fa9e4066Sahrens 2136fa9e4066Sahrens mutex_exit(&db->db_mtx); 2137fa9e4066Sahrens return (old_user_ptr); 2138fa9e4066Sahrens } 2139fa9e4066Sahrens 2140fa9e4066Sahrens void * 2141fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2142fa9e4066Sahrens { 2143fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2144fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 2145fa9e4066Sahrens 2146c717a561Smaybee return (db->db_user_ptr); 2147fa9e4066Sahrens } 2148fa9e4066Sahrens 21493d692628SSanjeev Bagewadi boolean_t 21503d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 21513d692628SSanjeev Bagewadi { 21523d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 21533d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 21543d692628SSanjeev Bagewadi 21553d692628SSanjeev Bagewadi if (db->db_blkptr) 21563d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2157c7cd2421SGeorge Wilson db->db_blkptr, db->db_blkptr->blk_birth); 21583d692628SSanjeev Bagewadi 21593d692628SSanjeev Bagewadi return (res); 21603d692628SSanjeev Bagewadi } 21613d692628SSanjeev Bagewadi 2162c717a561Smaybee static void 2163c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2164fa9e4066Sahrens { 2165c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2166c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2167c717a561Smaybee 2168c717a561Smaybee if (db->db_blkptr != NULL) 2169c717a561Smaybee return; 2170c717a561Smaybee 21710a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 21720a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 21730a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 21740a586ceaSMark Shellenbaum return; 21750a586ceaSMark Shellenbaum } 2176c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2177c717a561Smaybee /* 2178c717a561Smaybee * This buffer was allocated at a time when there was 2179c717a561Smaybee * no available blkptrs from the dnode, or it was 2180c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2181c717a561Smaybee */ 2182c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2183c717a561Smaybee ASSERT(db->db_parent == NULL); 2184c717a561Smaybee db->db_parent = dn->dn_dbuf; 2185c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2186c717a561Smaybee DBUF_VERIFY(db); 2187c717a561Smaybee } else { 2188c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2189c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2190c717a561Smaybee 2191c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2192c717a561Smaybee if (parent == NULL) { 2193c717a561Smaybee mutex_exit(&db->db_mtx); 2194c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2195c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 2196c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 2197c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2198c717a561Smaybee mutex_enter(&db->db_mtx); 2199c717a561Smaybee db->db_parent = parent; 2200c717a561Smaybee } 2201c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2202c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2203c717a561Smaybee DBUF_VERIFY(db); 2204c717a561Smaybee } 2205c717a561Smaybee } 2206c717a561Smaybee 2207c717a561Smaybee static void 2208c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2209c717a561Smaybee { 2210c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2211744947dcSTom Erickson dnode_t *dn; 2212c717a561Smaybee zio_t *zio; 2213c717a561Smaybee 2214c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2215c717a561Smaybee 2216c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2217c717a561Smaybee 2218c717a561Smaybee mutex_enter(&db->db_mtx); 2219c717a561Smaybee 2220c717a561Smaybee ASSERT(db->db_level > 0); 2221c717a561Smaybee DBUF_VERIFY(db); 2222c717a561Smaybee 2223c717a561Smaybee if (db->db_buf == NULL) { 2224c717a561Smaybee mutex_exit(&db->db_mtx); 2225c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2226c717a561Smaybee mutex_enter(&db->db_mtx); 2227c717a561Smaybee } 2228c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2229c717a561Smaybee ASSERT(db->db_buf != NULL); 2230c717a561Smaybee 2231744947dcSTom Erickson DB_DNODE_ENTER(db); 2232744947dcSTom Erickson dn = DB_DNODE(db); 2233744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2234c717a561Smaybee dbuf_check_blkptr(dn, db); 2235744947dcSTom Erickson DB_DNODE_EXIT(db); 2236c717a561Smaybee 2237c717a561Smaybee db->db_data_pending = dr; 2238c717a561Smaybee 2239af2c4821Smaybee mutex_exit(&db->db_mtx); 2240088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2241c717a561Smaybee 2242c717a561Smaybee zio = dr->dr_zio; 2243c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 2244c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 2245c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2246c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2247c717a561Smaybee zio_nowait(zio); 2248c717a561Smaybee } 2249c717a561Smaybee 2250c717a561Smaybee static void 2251c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2252c717a561Smaybee { 2253c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2254c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2255744947dcSTom Erickson dnode_t *dn; 2256744947dcSTom Erickson objset_t *os; 2257c717a561Smaybee uint64_t txg = tx->tx_txg; 2258fa9e4066Sahrens 2259fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2260fa9e4066Sahrens 2261fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2262fa9e4066Sahrens 2263fa9e4066Sahrens mutex_enter(&db->db_mtx); 2264fa9e4066Sahrens /* 2265fa9e4066Sahrens * To be synced, we must be dirtied. But we 2266fa9e4066Sahrens * might have been freed after the dirty. 2267fa9e4066Sahrens */ 2268fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2269fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2270fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2271fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2272fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2273c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2274fa9e4066Sahrens } else { 227582c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2276fa9e4066Sahrens } 22779c9dc39aSek DBUF_VERIFY(db); 2278fa9e4066Sahrens 2279744947dcSTom Erickson DB_DNODE_ENTER(db); 2280744947dcSTom Erickson dn = DB_DNODE(db); 2281744947dcSTom Erickson 22820a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 22830a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 22840a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 22850a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 22860a586ceaSMark Shellenbaum } 22870a586ceaSMark Shellenbaum 2288fa9e4066Sahrens /* 2289c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2290c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2291c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2292c717a561Smaybee * be called). 2293fa9e4066Sahrens */ 22940a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2295c717a561Smaybee dbuf_dirty_record_t **drp; 22961934e92fSmaybee 2297ea8dc4b6Seschrock ASSERT(*datap != NULL); 2298ea8dc4b6Seschrock ASSERT3U(db->db_level, ==, 0); 2299ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2300ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2301744947dcSTom Erickson DB_DNODE_EXIT(db); 2302744947dcSTom Erickson 23030e8c6158Smaybee if (*datap != db->db.db_data) { 2304ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 23055a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 23060e8c6158Smaybee } 2307ea8dc4b6Seschrock db->db_data_pending = NULL; 2308c717a561Smaybee drp = &db->db_last_dirty; 2309c717a561Smaybee while (*drp != dr) 2310c717a561Smaybee drp = &(*drp)->dr_next; 231117f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2312b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 231317f17c2dSbonwick *drp = dr->dr_next; 2314c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2315ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2316ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2317b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2318ea8dc4b6Seschrock return; 2319ea8dc4b6Seschrock } 2320ea8dc4b6Seschrock 2321744947dcSTom Erickson os = dn->dn_objset; 2322744947dcSTom Erickson 2323f82bfe17Sgw /* 2324f82bfe17Sgw * This function may have dropped the db_mtx lock allowing a dmu_sync 2325f82bfe17Sgw * operation to sneak in. As a result, we need to ensure that we 2326f82bfe17Sgw * don't check the dr_override_state until we have returned from 2327f82bfe17Sgw * dbuf_check_blkptr. 2328f82bfe17Sgw */ 2329f82bfe17Sgw dbuf_check_blkptr(dn, db); 2330f82bfe17Sgw 2331c717a561Smaybee /* 2332744947dcSTom Erickson * If this buffer is in the middle of an immediate write, 2333c717a561Smaybee * wait for the synchronous IO to complete. 2334c717a561Smaybee */ 2335c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2336c717a561Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2337c717a561Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2338c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2339c717a561Smaybee } 2340c5c6ffa0Smaybee 2341ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2342ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2343ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2344b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2345ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2346ab69d62fSMatthew Ahrens /* 2347ab69d62fSMatthew Ahrens * If this buffer is currently "in use" (i.e., there 2348ab69d62fSMatthew Ahrens * are active holds and db_data still references it), 2349ab69d62fSMatthew Ahrens * then make a copy before we start the write so that 2350ab69d62fSMatthew Ahrens * any modifications from the open txg will not leak 2351ab69d62fSMatthew Ahrens * into this write. 2352ab69d62fSMatthew Ahrens * 2353ab69d62fSMatthew Ahrens * NOTE: this copy does not need to be made for 2354ab69d62fSMatthew Ahrens * objects only modified in the syncing context (e.g. 2355ab69d62fSMatthew Ahrens * DNONE_DNODE blocks). 2356ab69d62fSMatthew Ahrens */ 2357ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2358ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2359ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2360ab69d62fSMatthew Ahrens bcopy(db->db.db_data, (*datap)->b_data, blksz); 236182c9918fSTim Haley } 2362c717a561Smaybee db->db_data_pending = dr; 2363fa9e4066Sahrens 2364c717a561Smaybee mutex_exit(&db->db_mtx); 2365fa9e4066Sahrens 2366088f3894Sahrens dbuf_write(dr, *datap, tx); 2367fa9e4066Sahrens 2368c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2369744947dcSTom Erickson if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2370c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2371744947dcSTom Erickson DB_DNODE_EXIT(db); 2372744947dcSTom Erickson } else { 2373744947dcSTom Erickson /* 2374744947dcSTom Erickson * Although zio_nowait() does not "wait for an IO", it does 2375744947dcSTom Erickson * initiate the IO. If this is an empty write it seems plausible 2376744947dcSTom Erickson * that the IO could actually be completed before the nowait 2377744947dcSTom Erickson * returns. We need to DB_DNODE_EXIT() first in case 2378744947dcSTom Erickson * zio_nowait() invalidates the dbuf. 2379744947dcSTom Erickson */ 2380744947dcSTom Erickson DB_DNODE_EXIT(db); 2381c717a561Smaybee zio_nowait(dr->dr_zio); 2382744947dcSTom Erickson } 2383c717a561Smaybee } 238423b11526Smaybee 2385c717a561Smaybee void 2386c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2387c717a561Smaybee { 2388c717a561Smaybee dbuf_dirty_record_t *dr; 2389c717a561Smaybee 2390c717a561Smaybee while (dr = list_head(list)) { 2391c717a561Smaybee if (dr->dr_zio != NULL) { 2392c717a561Smaybee /* 2393c717a561Smaybee * If we find an already initialized zio then we 2394c717a561Smaybee * are processing the meta-dnode, and we have finished. 2395c717a561Smaybee * The dbufs for all dnodes are put back on the list 2396c717a561Smaybee * during processing, so that we can zio_wait() 2397c717a561Smaybee * these IOs after initiating all child IOs. 2398c717a561Smaybee */ 2399c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2400c717a561Smaybee DMU_META_DNODE_OBJECT); 2401c717a561Smaybee break; 240223b11526Smaybee } 2403c717a561Smaybee list_remove(list, dr); 2404c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2405c717a561Smaybee dbuf_sync_indirect(dr, tx); 2406c717a561Smaybee else 2407c717a561Smaybee dbuf_sync_leaf(dr, tx); 240823b11526Smaybee } 2409c717a561Smaybee } 241023b11526Smaybee 2411fa9e4066Sahrens /* ARGSUSED */ 2412fa9e4066Sahrens static void 2413c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2414fa9e4066Sahrens { 2415fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2416744947dcSTom Erickson dnode_t *dn; 2417e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2418c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2419b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2420b24ab676SJeff Bonwick int64_t delta; 2421fa9e4066Sahrens uint64_t fill = 0; 2422b24ab676SJeff Bonwick int i; 2423fa9e4066Sahrens 2424e14bb325SJeff Bonwick ASSERT(db->db_blkptr == bp); 2425e14bb325SJeff Bonwick 2426744947dcSTom Erickson DB_DNODE_ENTER(db); 2427744947dcSTom Erickson dn = DB_DNODE(db); 2428b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2429b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2430b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2431fa9e4066Sahrens 2432e14bb325SJeff Bonwick if (BP_IS_HOLE(bp)) { 2433b24ab676SJeff Bonwick ASSERT(bp->blk_fill == 0); 2434744947dcSTom Erickson DB_DNODE_EXIT(db); 2435c717a561Smaybee return; 2436c717a561Smaybee } 2437c5c6ffa0Smaybee 24380a586ceaSMark Shellenbaum ASSERT((db->db_blkid != DMU_SPILL_BLKID && 24390a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_type) || 24400a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && 24410a586ceaSMark Shellenbaum BP_GET_TYPE(bp) == dn->dn_bonustype)); 2442e14bb325SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2443e14bb325SJeff Bonwick 2444c717a561Smaybee mutex_enter(&db->db_mtx); 2445fa9e4066Sahrens 24460a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 24470a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 24480a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 24490a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 24500a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 24510a586ceaSMark Shellenbaum } 24520a586ceaSMark Shellenbaum #endif 24530a586ceaSMark Shellenbaum 2454fa9e4066Sahrens if (db->db_level == 0) { 2455fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 24560a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 24570a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 2458fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2459fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2460fa9e4066Sahrens 2461fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2462fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2463fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2464fa9e4066Sahrens i--, dnp++) { 2465fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2466fa9e4066Sahrens fill++; 2467fa9e4066Sahrens } 2468fa9e4066Sahrens } else { 2469c717a561Smaybee fill = 1; 2470fa9e4066Sahrens } 2471fa9e4066Sahrens } else { 2472e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2473fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2474e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2475e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2476fa9e4066Sahrens continue; 2477e14bb325SJeff Bonwick fill += ibp->blk_fill; 2478fa9e4066Sahrens } 2479fa9e4066Sahrens } 2480744947dcSTom Erickson DB_DNODE_EXIT(db); 2481fa9e4066Sahrens 2482e14bb325SJeff Bonwick bp->blk_fill = fill; 2483c717a561Smaybee 2484c717a561Smaybee mutex_exit(&db->db_mtx); 2485c717a561Smaybee } 2486fa9e4066Sahrens 2487c717a561Smaybee /* ARGSUSED */ 2488c717a561Smaybee static void 2489c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2490c717a561Smaybee { 2491c717a561Smaybee dmu_buf_impl_t *db = vdb; 2492b24ab676SJeff Bonwick blkptr_t *bp = zio->io_bp; 2493b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 2494c717a561Smaybee uint64_t txg = zio->io_txg; 2495c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2496c717a561Smaybee 2497c717a561Smaybee ASSERT3U(zio->io_error, ==, 0); 2498b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 2499b24ab676SJeff Bonwick 2500b24ab676SJeff Bonwick if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 2501b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 2502b24ab676SJeff Bonwick } else { 2503744947dcSTom Erickson objset_t *os; 2504744947dcSTom Erickson dsl_dataset_t *ds; 2505744947dcSTom Erickson dmu_tx_t *tx; 2506744947dcSTom Erickson 2507744947dcSTom Erickson DB_GET_OBJSET(&os, db); 2508744947dcSTom Erickson ds = os->os_dsl_dataset; 2509744947dcSTom Erickson tx = os->os_synctx; 2510b24ab676SJeff Bonwick 2511b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2512b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2513b24ab676SJeff Bonwick } 2514c717a561Smaybee 2515c717a561Smaybee mutex_enter(&db->db_mtx); 2516c717a561Smaybee 2517b24ab676SJeff Bonwick DBUF_VERIFY(db); 2518b24ab676SJeff Bonwick 2519c717a561Smaybee drp = &db->db_last_dirty; 252017f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 252117f17c2dSbonwick drp = &dr->dr_next; 252217f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 252317f17c2dSbonwick ASSERT(dr->dr_txg == txg); 2524b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 252517f17c2dSbonwick ASSERT(dr->dr_next == NULL); 252617f17c2dSbonwick *drp = dr->dr_next; 2527c717a561Smaybee 25280a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 25290a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 2530744947dcSTom Erickson dnode_t *dn; 2531744947dcSTom Erickson 2532744947dcSTom Erickson DB_DNODE_ENTER(db); 2533744947dcSTom Erickson dn = DB_DNODE(db); 25340a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 25350a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 25360a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 2537744947dcSTom Erickson DB_DNODE_EXIT(db); 25380a586ceaSMark Shellenbaum } 25390a586ceaSMark Shellenbaum #endif 25400a586ceaSMark Shellenbaum 2541c717a561Smaybee if (db->db_level == 0) { 25420a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2543c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 254482c9918fSTim Haley if (db->db_state != DB_NOFILL) { 254582c9918fSTim Haley if (dr->dt.dl.dr_data != db->db_buf) 254682c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 254782c9918fSTim Haley db) == 1); 2548b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 254982c9918fSTim Haley arc_set_callback(db->db_buf, dbuf_do_evict, db); 255082c9918fSTim Haley } 2551c717a561Smaybee } else { 2552744947dcSTom Erickson dnode_t *dn; 2553744947dcSTom Erickson 2554744947dcSTom Erickson DB_DNODE_ENTER(db); 2555744947dcSTom Erickson dn = DB_DNODE(db); 2556c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2557c717a561Smaybee ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2558c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2559c717a561Smaybee int epbs = 2560c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2561c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2562c717a561Smaybee db->db.db_size); 2563c717a561Smaybee ASSERT3U(dn->dn_phys->dn_maxblkid 2564c717a561Smaybee >> (db->db_level * epbs), >=, db->db_blkid); 2565c717a561Smaybee arc_set_callback(db->db_buf, dbuf_do_evict, db); 2566c717a561Smaybee } 2567744947dcSTom Erickson DB_DNODE_EXIT(db); 2568c25056deSgw mutex_destroy(&dr->dt.di.dr_mtx); 2569c25056deSgw list_destroy(&dr->dt.di.dr_children); 2570c717a561Smaybee } 2571c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2572fa9e4066Sahrens 2573fa9e4066Sahrens cv_broadcast(&db->db_changed); 2574fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2575fa9e4066Sahrens db->db_dirtycnt -= 1; 2576c717a561Smaybee db->db_data_pending = NULL; 2577b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2578b24ab676SJeff Bonwick } 2579b24ab676SJeff Bonwick 2580b24ab676SJeff Bonwick static void 2581b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 2582b24ab676SJeff Bonwick { 2583b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 2584b24ab676SJeff Bonwick } 2585b24ab676SJeff Bonwick 2586b24ab676SJeff Bonwick static void 2587b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 2588b24ab676SJeff Bonwick { 2589b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 2590b24ab676SJeff Bonwick } 2591b24ab676SJeff Bonwick 2592b24ab676SJeff Bonwick static void 2593b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 2594b24ab676SJeff Bonwick { 2595b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2596b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2597b24ab676SJeff Bonwick 2598b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 2599b24ab676SJeff Bonwick } 2600b24ab676SJeff Bonwick 2601b24ab676SJeff Bonwick static void 2602b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 2603b24ab676SJeff Bonwick { 2604b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2605b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2606b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2607b24ab676SJeff Bonwick 2608b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2609b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 2610b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 2611b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2612b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 2613b24ab676SJeff Bonwick } 2614fa9e4066Sahrens mutex_exit(&db->db_mtx); 2615fa9e4066Sahrens 2616b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 2617b24ab676SJeff Bonwick } 2618b24ab676SJeff Bonwick 2619b24ab676SJeff Bonwick static void 2620b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2621b24ab676SJeff Bonwick { 2622b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2623744947dcSTom Erickson dnode_t *dn; 2624744947dcSTom Erickson objset_t *os; 2625b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 2626b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 2627b24ab676SJeff Bonwick zbookmark_t zb; 2628b24ab676SJeff Bonwick zio_prop_t zp; 2629b24ab676SJeff Bonwick zio_t *zio; 26300a586ceaSMark Shellenbaum int wp_flag = 0; 2631b24ab676SJeff Bonwick 2632744947dcSTom Erickson DB_DNODE_ENTER(db); 2633744947dcSTom Erickson dn = DB_DNODE(db); 2634744947dcSTom Erickson os = dn->dn_objset; 2635744947dcSTom Erickson 2636b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 2637b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2638b24ab676SJeff Bonwick /* 2639b24ab676SJeff Bonwick * Private object buffers are released here rather 2640b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 2641b24ab676SJeff Bonwick * in the syncing context and we don't want the 2642b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 2643b24ab676SJeff Bonwick */ 2644b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 2645b24ab676SJeff Bonwick arc_buf_thaw(data); 2646b24ab676SJeff Bonwick } else { 26473f9d6ad7SLin Ling dbuf_release_bp(db); 2648b24ab676SJeff Bonwick } 2649b24ab676SJeff Bonwick } 2650b24ab676SJeff Bonwick } 2651b24ab676SJeff Bonwick 2652b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 2653b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 2654b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 2655b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 2656b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 2657b24ab676SJeff Bonwick } else { 26580a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 26590a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 26600a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 26610a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 26620a586ceaSMark Shellenbaum ASSERT3P(db->db_blkptr, ==, 26630a586ceaSMark Shellenbaum &dn->dn_phys->dn_blkptr[db->db_blkid]); 2664b24ab676SJeff Bonwick zio = dn->dn_zio; 2665b24ab676SJeff Bonwick } 2666b24ab676SJeff Bonwick 2667b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 2668b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2669b24ab676SJeff Bonwick ASSERT(zio); 2670fa9e4066Sahrens 2671b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2672b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2673b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 2674b24ab676SJeff Bonwick 26750a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 26760a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 26770a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 26780a586ceaSMark Shellenbaum 26790a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2680744947dcSTom Erickson DB_DNODE_EXIT(db); 2681b24ab676SJeff Bonwick 2682b24ab676SJeff Bonwick if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2683b24ab676SJeff Bonwick ASSERT(db->db_state != DB_NOFILL); 2684b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2685b24ab676SJeff Bonwick db->db_blkptr, data->b_data, arc_buf_size(data), &zp, 2686b24ab676SJeff Bonwick dbuf_write_override_ready, dbuf_write_override_done, dr, 2687b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2688b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2689b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2690b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2691b24ab676SJeff Bonwick dr->dt.dl.dr_copies); 2692b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 2693b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 2694b24ab676SJeff Bonwick ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); 2695b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2696b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 2697b24ab676SJeff Bonwick dbuf_write_nofill_ready, dbuf_write_nofill_done, db, 2698b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 2699b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2700b24ab676SJeff Bonwick } else { 2701b24ab676SJeff Bonwick ASSERT(arc_released(data)); 2702b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 2703b24ab676SJeff Bonwick db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp, 2704b24ab676SJeff Bonwick dbuf_write_ready, dbuf_write_done, db, 2705b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2706b24ab676SJeff Bonwick } 2707fa9e4066Sahrens } 2708