1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2206e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 233f2366c2SGordon Ross * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24bf16b11eSMatthew Ahrens * Copyright (c) 2012, 2014 by Delphix. All rights reserved. 25aad02571SSaso Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26810e43b2SBill Pijewski * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27fa9e4066Sahrens */ 28fa9e4066Sahrens 29fa9e4066Sahrens #include <sys/zfs_context.h> 30fa9e4066Sahrens #include <sys/dmu.h> 312f3d8780SMatthew Ahrens #include <sys/dmu_send.h> 32fa9e4066Sahrens #include <sys/dmu_impl.h> 33fa9e4066Sahrens #include <sys/dbuf.h> 34fa9e4066Sahrens #include <sys/dmu_objset.h> 35fa9e4066Sahrens #include <sys/dsl_dataset.h> 36fa9e4066Sahrens #include <sys/dsl_dir.h> 37fa9e4066Sahrens #include <sys/dmu_tx.h> 38fa9e4066Sahrens #include <sys/spa.h> 39fa9e4066Sahrens #include <sys/zio.h> 40fa9e4066Sahrens #include <sys/dmu_zfetch.h> 410a586ceaSMark Shellenbaum #include <sys/sa.h> 420a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 435d7b4d43SMatthew Ahrens #include <sys/zfeature.h> 445d7b4d43SMatthew Ahrens #include <sys/blkptr.h> 45bf16b11eSMatthew Ahrens #include <sys/range_tree.h> 46fa9e4066Sahrens 47713d6c20SMatthew Ahrens /* 48713d6c20SMatthew Ahrens * Number of times that zfs_free_range() took the slow path while doing 49713d6c20SMatthew Ahrens * a zfs receive. A nonzero value indicates a potential performance problem. 50713d6c20SMatthew Ahrens */ 51713d6c20SMatthew Ahrens uint64_t zfs_free_range_recv_miss; 52713d6c20SMatthew Ahrens 53fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db); 543b2aab18SMatthew Ahrens static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 55088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 56fa9e4066Sahrens 57fa9e4066Sahrens /* 58fa9e4066Sahrens * Global data structures and functions for the dbuf cache. 59fa9e4066Sahrens */ 60fa9e4066Sahrens static kmem_cache_t *dbuf_cache; 61fa9e4066Sahrens 62fa9e4066Sahrens /* ARGSUSED */ 63fa9e4066Sahrens static int 64fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag) 65fa9e4066Sahrens { 66fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 67fa9e4066Sahrens bzero(db, sizeof (dmu_buf_impl_t)); 68fa9e4066Sahrens 69fa9e4066Sahrens mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 70fa9e4066Sahrens cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 71fa9e4066Sahrens refcount_create(&db->db_holds); 72fa9e4066Sahrens return (0); 73fa9e4066Sahrens } 74fa9e4066Sahrens 75fa9e4066Sahrens /* ARGSUSED */ 76fa9e4066Sahrens static void 77fa9e4066Sahrens dbuf_dest(void *vdb, void *unused) 78fa9e4066Sahrens { 79fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 80fa9e4066Sahrens mutex_destroy(&db->db_mtx); 81fa9e4066Sahrens cv_destroy(&db->db_changed); 82fa9e4066Sahrens refcount_destroy(&db->db_holds); 83fa9e4066Sahrens } 84fa9e4066Sahrens 85fa9e4066Sahrens /* 86fa9e4066Sahrens * dbuf hash table routines 87fa9e4066Sahrens */ 88fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table; 89fa9e4066Sahrens 90fa9e4066Sahrens static uint64_t dbuf_hash_count; 91fa9e4066Sahrens 92fa9e4066Sahrens static uint64_t 93fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 94fa9e4066Sahrens { 95fa9e4066Sahrens uintptr_t osv = (uintptr_t)os; 96fa9e4066Sahrens uint64_t crc = -1ULL; 97fa9e4066Sahrens 98fa9e4066Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 99fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 100fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 101fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 102fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 103fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 104fa9e4066Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 105fa9e4066Sahrens 106fa9e4066Sahrens crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 107fa9e4066Sahrens 108fa9e4066Sahrens return (crc); 109fa9e4066Sahrens } 110fa9e4066Sahrens 111fa9e4066Sahrens #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 112fa9e4066Sahrens 113fa9e4066Sahrens #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 114fa9e4066Sahrens ((dbuf)->db.db_object == (obj) && \ 115fa9e4066Sahrens (dbuf)->db_objset == (os) && \ 116fa9e4066Sahrens (dbuf)->db_level == (level) && \ 117fa9e4066Sahrens (dbuf)->db_blkid == (blkid)) 118fa9e4066Sahrens 119fa9e4066Sahrens dmu_buf_impl_t * 120fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 121fa9e4066Sahrens { 122fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 123503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 124fa9e4066Sahrens uint64_t obj = dn->dn_object; 125fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 126fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 127fa9e4066Sahrens dmu_buf_impl_t *db; 128fa9e4066Sahrens 129fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 130fa9e4066Sahrens for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 131fa9e4066Sahrens if (DBUF_EQUAL(db, os, obj, level, blkid)) { 132fa9e4066Sahrens mutex_enter(&db->db_mtx); 133ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 134fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 135fa9e4066Sahrens return (db); 136fa9e4066Sahrens } 137fa9e4066Sahrens mutex_exit(&db->db_mtx); 138fa9e4066Sahrens } 139fa9e4066Sahrens } 140fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 141fa9e4066Sahrens return (NULL); 142fa9e4066Sahrens } 143fa9e4066Sahrens 144fa9e4066Sahrens /* 145fa9e4066Sahrens * Insert an entry into the hash table. If there is already an element 146fa9e4066Sahrens * equal to elem in the hash table, then the already existing element 147fa9e4066Sahrens * will be returned and the new element will not be inserted. 148fa9e4066Sahrens * Otherwise returns NULL. 149fa9e4066Sahrens */ 150fa9e4066Sahrens static dmu_buf_impl_t * 151fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db) 152fa9e4066Sahrens { 153fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 154503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 155fa9e4066Sahrens uint64_t obj = db->db.db_object; 156fa9e4066Sahrens int level = db->db_level; 157fa9e4066Sahrens uint64_t blkid = db->db_blkid; 158fa9e4066Sahrens uint64_t hv = DBUF_HASH(os, obj, level, blkid); 159fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 160fa9e4066Sahrens dmu_buf_impl_t *dbf; 161fa9e4066Sahrens 162fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 163fa9e4066Sahrens for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 164fa9e4066Sahrens if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 165fa9e4066Sahrens mutex_enter(&dbf->db_mtx); 166ea8dc4b6Seschrock if (dbf->db_state != DB_EVICTING) { 167fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 168fa9e4066Sahrens return (dbf); 169fa9e4066Sahrens } 170fa9e4066Sahrens mutex_exit(&dbf->db_mtx); 171fa9e4066Sahrens } 172fa9e4066Sahrens } 173fa9e4066Sahrens 174fa9e4066Sahrens mutex_enter(&db->db_mtx); 175fa9e4066Sahrens db->db_hash_next = h->hash_table[idx]; 176fa9e4066Sahrens h->hash_table[idx] = db; 177fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 178fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, 1); 179fa9e4066Sahrens 180fa9e4066Sahrens return (NULL); 181fa9e4066Sahrens } 182fa9e4066Sahrens 183fa9e4066Sahrens /* 184fa9e4066Sahrens * Remove an entry from the hash table. This operation will 185fa9e4066Sahrens * fail if there are any existing holds on the db. 186fa9e4066Sahrens */ 187fa9e4066Sahrens static void 188fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db) 189fa9e4066Sahrens { 190fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 191fa9e4066Sahrens uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 192fa9e4066Sahrens db->db_level, db->db_blkid); 193fa9e4066Sahrens uint64_t idx = hv & h->hash_table_mask; 194fa9e4066Sahrens dmu_buf_impl_t *dbf, **dbp; 195fa9e4066Sahrens 196fa9e4066Sahrens /* 197fa9e4066Sahrens * We musn't hold db_mtx to maintin lock ordering: 198fa9e4066Sahrens * DBUF_HASH_MUTEX > db_mtx. 199fa9e4066Sahrens */ 200fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 201ea8dc4b6Seschrock ASSERT(db->db_state == DB_EVICTING); 202fa9e4066Sahrens ASSERT(!MUTEX_HELD(&db->db_mtx)); 203fa9e4066Sahrens 204fa9e4066Sahrens mutex_enter(DBUF_HASH_MUTEX(h, idx)); 205fa9e4066Sahrens dbp = &h->hash_table[idx]; 206fa9e4066Sahrens while ((dbf = *dbp) != db) { 207fa9e4066Sahrens dbp = &dbf->db_hash_next; 208fa9e4066Sahrens ASSERT(dbf != NULL); 209fa9e4066Sahrens } 210fa9e4066Sahrens *dbp = db->db_hash_next; 211fa9e4066Sahrens db->db_hash_next = NULL; 212fa9e4066Sahrens mutex_exit(DBUF_HASH_MUTEX(h, idx)); 213fa9e4066Sahrens atomic_add_64(&dbuf_hash_count, -1); 214fa9e4066Sahrens } 215fa9e4066Sahrens 216ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict; 217fa9e4066Sahrens 218fa9e4066Sahrens static void 219fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db) 220fa9e4066Sahrens { 221fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 222fa9e4066Sahrens 223c717a561Smaybee if (db->db_level != 0 || db->db_evict_func == NULL) 224fa9e4066Sahrens return; 225fa9e4066Sahrens 226c717a561Smaybee if (db->db_user_data_ptr_ptr) 227c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 228c717a561Smaybee db->db_evict_func(&db->db, db->db_user_ptr); 229c717a561Smaybee db->db_user_ptr = NULL; 230c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 231c717a561Smaybee db->db_evict_func = NULL; 232fa9e4066Sahrens } 233fa9e4066Sahrens 234744947dcSTom Erickson boolean_t 235744947dcSTom Erickson dbuf_is_metadata(dmu_buf_impl_t *db) 236744947dcSTom Erickson { 237744947dcSTom Erickson if (db->db_level > 0) { 238744947dcSTom Erickson return (B_TRUE); 239744947dcSTom Erickson } else { 240744947dcSTom Erickson boolean_t is_metadata; 241744947dcSTom Erickson 242744947dcSTom Erickson DB_DNODE_ENTER(db); 243ad135b5dSChristopher Siden is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 244744947dcSTom Erickson DB_DNODE_EXIT(db); 245744947dcSTom Erickson 246744947dcSTom Erickson return (is_metadata); 247744947dcSTom Erickson } 248744947dcSTom Erickson } 249744947dcSTom Erickson 250ea8dc4b6Seschrock void 251ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db) 252ea8dc4b6Seschrock { 253ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 254ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 255c717a561Smaybee ASSERT(db->db_data_pending == NULL); 256ea8dc4b6Seschrock 257ea8dc4b6Seschrock dbuf_clear(db); 258ea8dc4b6Seschrock dbuf_destroy(db); 259ea8dc4b6Seschrock } 260ea8dc4b6Seschrock 261fa9e4066Sahrens void 262fa9e4066Sahrens dbuf_init(void) 263fa9e4066Sahrens { 264ea8dc4b6Seschrock uint64_t hsize = 1ULL << 16; 265fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 266fa9e4066Sahrens int i; 267fa9e4066Sahrens 268fa9e4066Sahrens /* 269fa9e4066Sahrens * The hash table is big enough to fill all of physical memory 270ea8dc4b6Seschrock * with an average 4K block size. The table will take up 271ea8dc4b6Seschrock * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 272fa9e4066Sahrens */ 273ea8dc4b6Seschrock while (hsize * 4096 < physmem * PAGESIZE) 274fa9e4066Sahrens hsize <<= 1; 275fa9e4066Sahrens 276ea8dc4b6Seschrock retry: 277fa9e4066Sahrens h->hash_table_mask = hsize - 1; 278ea8dc4b6Seschrock h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 279ea8dc4b6Seschrock if (h->hash_table == NULL) { 280ea8dc4b6Seschrock /* XXX - we should really return an error instead of assert */ 281ea8dc4b6Seschrock ASSERT(hsize > (1ULL << 10)); 282ea8dc4b6Seschrock hsize >>= 1; 283ea8dc4b6Seschrock goto retry; 284ea8dc4b6Seschrock } 285fa9e4066Sahrens 286fa9e4066Sahrens dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 287fa9e4066Sahrens sizeof (dmu_buf_impl_t), 288fa9e4066Sahrens 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 289fa9e4066Sahrens 290fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 291fa9e4066Sahrens mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL); 292fa9e4066Sahrens } 293fa9e4066Sahrens 294fa9e4066Sahrens void 295fa9e4066Sahrens dbuf_fini(void) 296fa9e4066Sahrens { 297fa9e4066Sahrens dbuf_hash_table_t *h = &dbuf_hash_table; 298fa9e4066Sahrens int i; 299fa9e4066Sahrens 300fa9e4066Sahrens for (i = 0; i < DBUF_MUTEXES; i++) 301fa9e4066Sahrens mutex_destroy(&h->hash_mutexes[i]); 302fa9e4066Sahrens kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 303fa9e4066Sahrens kmem_cache_destroy(dbuf_cache); 304fa9e4066Sahrens } 305fa9e4066Sahrens 306fa9e4066Sahrens /* 307fa9e4066Sahrens * Other stuff. 308fa9e4066Sahrens */ 309fa9e4066Sahrens 3109c9dc39aSek #ifdef ZFS_DEBUG 311fa9e4066Sahrens static void 312fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db) 313fa9e4066Sahrens { 314744947dcSTom Erickson dnode_t *dn; 315b24ab676SJeff Bonwick dbuf_dirty_record_t *dr; 316fa9e4066Sahrens 317fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 318fa9e4066Sahrens 319fa9e4066Sahrens if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 320fa9e4066Sahrens return; 321fa9e4066Sahrens 322fa9e4066Sahrens ASSERT(db->db_objset != NULL); 323744947dcSTom Erickson DB_DNODE_ENTER(db); 324744947dcSTom Erickson dn = DB_DNODE(db); 325fa9e4066Sahrens if (dn == NULL) { 326fa9e4066Sahrens ASSERT(db->db_parent == NULL); 327fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 328fa9e4066Sahrens } else { 329fa9e4066Sahrens ASSERT3U(db->db.db_object, ==, dn->dn_object); 330fa9e4066Sahrens ASSERT3P(db->db_objset, ==, dn->dn_objset); 331fa9e4066Sahrens ASSERT3U(db->db_level, <, dn->dn_nlevels); 332744947dcSTom Erickson ASSERT(db->db_blkid == DMU_BONUS_BLKID || 333744947dcSTom Erickson db->db_blkid == DMU_SPILL_BLKID || 334744947dcSTom Erickson !list_is_empty(&dn->dn_dbufs)); 335fa9e4066Sahrens } 3360a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 337fa9e4066Sahrens ASSERT(dn != NULL); 3381934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 3390a586ceaSMark Shellenbaum ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 3400a586ceaSMark Shellenbaum } else if (db->db_blkid == DMU_SPILL_BLKID) { 3410a586ceaSMark Shellenbaum ASSERT(dn != NULL); 3420a586ceaSMark Shellenbaum ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 343fb09f5aaSMadhav Suresh ASSERT0(db->db.db_offset); 344fa9e4066Sahrens } else { 345fa9e4066Sahrens ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 346fa9e4066Sahrens } 347fa9e4066Sahrens 348b24ab676SJeff Bonwick for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 349b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 350b24ab676SJeff Bonwick 351b24ab676SJeff Bonwick for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 352b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 353b24ab676SJeff Bonwick 35488b7b0f2SMatthew Ahrens /* 35588b7b0f2SMatthew Ahrens * We can't assert that db_size matches dn_datablksz because it 35688b7b0f2SMatthew Ahrens * can be momentarily different when another thread is doing 35788b7b0f2SMatthew Ahrens * dnode_set_blksz(). 35888b7b0f2SMatthew Ahrens */ 35988b7b0f2SMatthew Ahrens if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 360b24ab676SJeff Bonwick dr = db->db_data_pending; 36188b7b0f2SMatthew Ahrens /* 36288b7b0f2SMatthew Ahrens * It should only be modified in syncing context, so 36388b7b0f2SMatthew Ahrens * make sure we only have one copy of the data. 36488b7b0f2SMatthew Ahrens */ 36588b7b0f2SMatthew Ahrens ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 366fa9e4066Sahrens } 367fa9e4066Sahrens 368fa9e4066Sahrens /* verify db->db_blkptr */ 369fa9e4066Sahrens if (db->db_blkptr) { 370fa9e4066Sahrens if (db->db_parent == dn->dn_dbuf) { 371fa9e4066Sahrens /* db is pointed to by the dnode */ 372fa9e4066Sahrens /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 37314843421SMatthew Ahrens if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 374fa9e4066Sahrens ASSERT(db->db_parent == NULL); 375fa9e4066Sahrens else 376fa9e4066Sahrens ASSERT(db->db_parent != NULL); 3770a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 3780a586ceaSMark Shellenbaum ASSERT3P(db->db_blkptr, ==, 3790a586ceaSMark Shellenbaum &dn->dn_phys->dn_blkptr[db->db_blkid]); 380fa9e4066Sahrens } else { 381fa9e4066Sahrens /* db is pointed to by an indirect block */ 382fa9e4066Sahrens int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 383fa9e4066Sahrens ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 384fa9e4066Sahrens ASSERT3U(db->db_parent->db.db_object, ==, 385fa9e4066Sahrens db->db.db_object); 386fa9e4066Sahrens /* 387fa9e4066Sahrens * dnode_grow_indblksz() can make this fail if we don't 388fa9e4066Sahrens * have the struct_rwlock. XXX indblksz no longer 389fa9e4066Sahrens * grows. safe to do this now? 390fa9e4066Sahrens */ 391744947dcSTom Erickson if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 392fa9e4066Sahrens ASSERT3P(db->db_blkptr, ==, 393fa9e4066Sahrens ((blkptr_t *)db->db_parent->db.db_data + 394fa9e4066Sahrens db->db_blkid % epb)); 395fa9e4066Sahrens } 396fa9e4066Sahrens } 397fa9e4066Sahrens } 398fa9e4066Sahrens if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 3993f9d6ad7SLin Ling (db->db_buf == NULL || db->db_buf->b_data) && 4000a586ceaSMark Shellenbaum db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 401fa9e4066Sahrens db->db_state != DB_FILL && !dn->dn_free_txg) { 402fa9e4066Sahrens /* 403fa9e4066Sahrens * If the blkptr isn't set but they have nonzero data, 404fa9e4066Sahrens * it had better be dirty, otherwise we'll lose that 405fa9e4066Sahrens * data when we evict this buffer. 406fa9e4066Sahrens */ 407fa9e4066Sahrens if (db->db_dirtycnt == 0) { 408fa9e4066Sahrens uint64_t *buf = db->db.db_data; 409fa9e4066Sahrens int i; 410fa9e4066Sahrens 411fa9e4066Sahrens for (i = 0; i < db->db.db_size >> 3; i++) { 412fa9e4066Sahrens ASSERT(buf[i] == 0); 413fa9e4066Sahrens } 414fa9e4066Sahrens } 415fa9e4066Sahrens } 416744947dcSTom Erickson DB_DNODE_EXIT(db); 417fa9e4066Sahrens } 4189c9dc39aSek #endif 419fa9e4066Sahrens 420fa9e4066Sahrens static void 421fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db) 422fa9e4066Sahrens { 423fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 424c717a561Smaybee if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 425fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 426c717a561Smaybee *db->db_user_data_ptr_ptr = db->db.db_data; 427fa9e4066Sahrens } 428fa9e4066Sahrens } 429fa9e4066Sahrens 430fa9e4066Sahrens static void 431fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 432fa9e4066Sahrens { 433fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 434ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf)); 435fa9e4066Sahrens db->db_buf = buf; 436ea8dc4b6Seschrock if (buf != NULL) { 437ea8dc4b6Seschrock ASSERT(buf->b_data != NULL); 438ea8dc4b6Seschrock db->db.db_data = buf->b_data; 439ea8dc4b6Seschrock if (!arc_released(buf)) 440ea8dc4b6Seschrock arc_set_callback(buf, dbuf_do_evict, db); 441ea8dc4b6Seschrock dbuf_update_data(db); 442ea8dc4b6Seschrock } else { 443ea8dc4b6Seschrock dbuf_evict_user(db); 444ea8dc4b6Seschrock db->db.db_data = NULL; 44582c9918fSTim Haley if (db->db_state != DB_NOFILL) 44682c9918fSTim Haley db->db_state = DB_UNCACHED; 447ea8dc4b6Seschrock } 448fa9e4066Sahrens } 449fa9e4066Sahrens 450c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 451c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Loan out an arc_buf for read. Return the loaned arc_buf. 452c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 453c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 454c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_loan_arcbuf(dmu_buf_impl_t *db) 455c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 456c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf; 457c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 458c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_enter(&db->db_mtx); 459c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 460c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int blksz = db->db.db_size; 46143466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 462744947dcSTom Erickson 463c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 464744947dcSTom Erickson abuf = arc_loan_buf(spa, blksz); 465c242f9a0Schunli zhang - Sun Microsystems - Irvine United States bcopy(db->db.db_data, abuf->b_data, blksz); 466c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 467c242f9a0Schunli zhang - Sun Microsystems - Irvine United States abuf = db->db_buf; 468c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_loan_inuse_buf(abuf, db); 469c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dbuf_set_data(db, NULL); 470c242f9a0Schunli zhang - Sun Microsystems - Irvine United States mutex_exit(&db->db_mtx); 471c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 472c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (abuf); 473c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 474c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 475fa9e4066Sahrens uint64_t 476fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset) 477fa9e4066Sahrens { 478fa9e4066Sahrens if (dn->dn_datablkshift) { 479fa9e4066Sahrens return (offset >> dn->dn_datablkshift); 480fa9e4066Sahrens } else { 481fa9e4066Sahrens ASSERT3U(offset, <, dn->dn_datablksz); 482fa9e4066Sahrens return (0); 483fa9e4066Sahrens } 484fa9e4066Sahrens } 485fa9e4066Sahrens 486fa9e4066Sahrens static void 487fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 488fa9e4066Sahrens { 489fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 490fa9e4066Sahrens 491fa9e4066Sahrens mutex_enter(&db->db_mtx); 492fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_READ); 493fa9e4066Sahrens /* 494fa9e4066Sahrens * All reads are synchronous, so we must have a hold on the dbuf 495fa9e4066Sahrens */ 496fa9e4066Sahrens ASSERT(refcount_count(&db->db_holds) > 0); 497ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 498fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 499c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 500fa9e4066Sahrens /* we were freed in flight; disregard any error */ 501fa9e4066Sahrens arc_release(buf, db); 502fa9e4066Sahrens bzero(buf->b_data, db->db.db_size); 5036b4acc8bSahrens arc_buf_freeze(buf); 504c717a561Smaybee db->db_freed_in_flight = FALSE; 505fa9e4066Sahrens dbuf_set_data(db, buf); 506fa9e4066Sahrens db->db_state = DB_CACHED; 507fa9e4066Sahrens } else if (zio == NULL || zio->io_error == 0) { 508fa9e4066Sahrens dbuf_set_data(db, buf); 509fa9e4066Sahrens db->db_state = DB_CACHED; 510fa9e4066Sahrens } else { 5110a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 512fa9e4066Sahrens ASSERT3P(db->db_buf, ==, NULL); 5133b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 514ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 515fa9e4066Sahrens } 516fa9e4066Sahrens cv_broadcast(&db->db_changed); 5173f9d6ad7SLin Ling dbuf_rele_and_unlock(db, NULL); 518fa9e4066Sahrens } 519fa9e4066Sahrens 520ea8dc4b6Seschrock static void 52113506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 522fa9e4066Sahrens { 523744947dcSTom Erickson dnode_t *dn; 524*7802d7bfSMatthew Ahrens zbookmark_phys_t zb; 52513506d1eSmaybee uint32_t aflags = ARC_NOWAIT; 526fa9e4066Sahrens 527744947dcSTom Erickson DB_DNODE_ENTER(db); 528744947dcSTom Erickson dn = DB_DNODE(db); 529fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 530fa9e4066Sahrens /* We need the struct_rwlock to prevent db_blkptr from changing. */ 531088f3894Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 532ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 533ea8dc4b6Seschrock ASSERT(db->db_state == DB_UNCACHED); 534ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 535fa9e4066Sahrens 5360a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 537cf04dda1SMark Maybee int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 5381934e92fSmaybee 5391934e92fSmaybee ASSERT3U(bonuslen, <=, db->db.db_size); 540ea8dc4b6Seschrock db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 5415a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 5421934e92fSmaybee if (bonuslen < DN_MAX_BONUSLEN) 543ea8dc4b6Seschrock bzero(db->db.db_data, DN_MAX_BONUSLEN); 544cf04dda1SMark Maybee if (bonuslen) 545cf04dda1SMark Maybee bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 546744947dcSTom Erickson DB_DNODE_EXIT(db); 547ea8dc4b6Seschrock dbuf_update_data(db); 548fa9e4066Sahrens db->db_state = DB_CACHED; 549fa9e4066Sahrens mutex_exit(&db->db_mtx); 550fa9e4066Sahrens return; 551fa9e4066Sahrens } 552fa9e4066Sahrens 5531c8564a7SMark Maybee /* 5541c8564a7SMark Maybee * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 5551c8564a7SMark Maybee * processes the delete record and clears the bp while we are waiting 5561c8564a7SMark Maybee * for the dn_mtx (resulting in a "no" from block_freed). 5571c8564a7SMark Maybee */ 558088f3894Sahrens if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 5591c8564a7SMark Maybee (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 5601c8564a7SMark Maybee BP_IS_HOLE(db->db_blkptr)))) { 561ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 562ad23a2dbSjohansen 563744947dcSTom Erickson DB_DNODE_EXIT(db); 56443466aaeSMax Grossman dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa, 56543466aaeSMax Grossman db->db.db_size, db, type)); 566fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 567fa9e4066Sahrens db->db_state = DB_CACHED; 56813506d1eSmaybee *flags |= DB_RF_CACHED; 569fa9e4066Sahrens mutex_exit(&db->db_mtx); 570fa9e4066Sahrens return; 571fa9e4066Sahrens } 572fa9e4066Sahrens 573744947dcSTom Erickson DB_DNODE_EXIT(db); 574744947dcSTom Erickson 575fa9e4066Sahrens db->db_state = DB_READ; 576fa9e4066Sahrens mutex_exit(&db->db_mtx); 577fa9e4066Sahrens 5783baa08fcSek if (DBUF_IS_L2CACHEABLE(db)) 5793baa08fcSek aflags |= ARC_L2CACHE; 580aad02571SSaso Kiselkov if (DBUF_IS_L2COMPRESSIBLE(db)) 581aad02571SSaso Kiselkov aflags |= ARC_L2COMPRESS; 5823baa08fcSek 583b24ab676SJeff Bonwick SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 584b24ab676SJeff Bonwick db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 585b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 586ea8dc4b6Seschrock 587ea8dc4b6Seschrock dbuf_add_ref(db, NULL); 588088f3894Sahrens 58943466aaeSMax Grossman (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 590fa9e4066Sahrens dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 59113506d1eSmaybee (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 59213506d1eSmaybee &aflags, &zb); 59313506d1eSmaybee if (aflags & ARC_CACHED) 59413506d1eSmaybee *flags |= DB_RF_CACHED; 595fa9e4066Sahrens } 596fa9e4066Sahrens 597ea8dc4b6Seschrock int 598ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 599fa9e4066Sahrens { 600ea8dc4b6Seschrock int err = 0; 60143466aaeSMax Grossman boolean_t havepzio = (zio != NULL); 60243466aaeSMax Grossman boolean_t prefetch; 603744947dcSTom Erickson dnode_t *dn; 604fa9e4066Sahrens 605fa9e4066Sahrens /* 606fa9e4066Sahrens * We don't have to hold the mutex to check db_state because it 607fa9e4066Sahrens * can't be freed while we have a hold on the buffer. 608fa9e4066Sahrens */ 609fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 610fa9e4066Sahrens 61182c9918fSTim Haley if (db->db_state == DB_NOFILL) 612be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 61382c9918fSTim Haley 614744947dcSTom Erickson DB_DNODE_ENTER(db); 615744947dcSTom Erickson dn = DB_DNODE(db); 616ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 617744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 618ea8dc4b6Seschrock 6190a586ceaSMark Shellenbaum prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 620744947dcSTom Erickson (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 6213baa08fcSek DBUF_IS_CACHEABLE(db); 62213506d1eSmaybee 623ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 624ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 625ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 62613506d1eSmaybee if (prefetch) 627744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 62813506d1eSmaybee db->db.db_size, TRUE); 629fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 630744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 631744947dcSTom Erickson DB_DNODE_EXIT(db); 632ea8dc4b6Seschrock } else if (db->db_state == DB_UNCACHED) { 633744947dcSTom Erickson spa_t *spa = dn->dn_objset->os_spa; 634744947dcSTom Erickson 635744947dcSTom Erickson if (zio == NULL) 636744947dcSTom Erickson zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 63713506d1eSmaybee dbuf_read_impl(db, zio, &flags); 63813506d1eSmaybee 639ea8dc4b6Seschrock /* dbuf_read_impl has dropped db_mtx for us */ 640ea8dc4b6Seschrock 64113506d1eSmaybee if (prefetch) 642744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 64313506d1eSmaybee db->db.db_size, flags & DB_RF_CACHED); 644ea8dc4b6Seschrock 645fa9e4066Sahrens if ((flags & DB_RF_HAVESTRUCT) == 0) 646744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 647744947dcSTom Erickson DB_DNODE_EXIT(db); 648fa9e4066Sahrens 649ea8dc4b6Seschrock if (!havepzio) 650ea8dc4b6Seschrock err = zio_wait(zio); 651ea8dc4b6Seschrock } else { 6523e30c24aSWill Andrews /* 6533e30c24aSWill Andrews * Another reader came in while the dbuf was in flight 6543e30c24aSWill Andrews * between UNCACHED and CACHED. Either a writer will finish 6553e30c24aSWill Andrews * writing the buffer (sending the dbuf to CACHED) or the 6563e30c24aSWill Andrews * first reader's request will reach the read_done callback 6573e30c24aSWill Andrews * and send the dbuf to CACHED. Otherwise, a failure 6583e30c24aSWill Andrews * occurred and the dbuf went to UNCACHED. 6593e30c24aSWill Andrews */ 66013506d1eSmaybee mutex_exit(&db->db_mtx); 66113506d1eSmaybee if (prefetch) 662744947dcSTom Erickson dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 66313506d1eSmaybee db->db.db_size, TRUE); 664ea8dc4b6Seschrock if ((flags & DB_RF_HAVESTRUCT) == 0) 665744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 666744947dcSTom Erickson DB_DNODE_EXIT(db); 66713506d1eSmaybee 6683e30c24aSWill Andrews /* Skip the wait per the caller's request. */ 66913506d1eSmaybee mutex_enter(&db->db_mtx); 670ea8dc4b6Seschrock if ((flags & DB_RF_NEVERWAIT) == 0) { 671ea8dc4b6Seschrock while (db->db_state == DB_READ || 672ea8dc4b6Seschrock db->db_state == DB_FILL) { 673ea8dc4b6Seschrock ASSERT(db->db_state == DB_READ || 674ea8dc4b6Seschrock (flags & DB_RF_HAVESTRUCT) == 0); 675ea8dc4b6Seschrock cv_wait(&db->db_changed, &db->db_mtx); 676ea8dc4b6Seschrock } 677ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 678be6fd75aSMatthew Ahrens err = SET_ERROR(EIO); 679ea8dc4b6Seschrock } 680ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 681fa9e4066Sahrens } 682fa9e4066Sahrens 683ea8dc4b6Seschrock ASSERT(err || havepzio || db->db_state == DB_CACHED); 684ea8dc4b6Seschrock return (err); 685fa9e4066Sahrens } 686fa9e4066Sahrens 687fa9e4066Sahrens static void 688fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db) 689fa9e4066Sahrens { 690fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 6910a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 692fa9e4066Sahrens mutex_enter(&db->db_mtx); 693fa9e4066Sahrens while (db->db_state == DB_READ || db->db_state == DB_FILL) 694fa9e4066Sahrens cv_wait(&db->db_changed, &db->db_mtx); 695fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 696ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 69743466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 698ad23a2dbSjohansen 699ea8dc4b6Seschrock ASSERT(db->db_buf == NULL); 700fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 701744947dcSTom Erickson dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 702fa9e4066Sahrens db->db_state = DB_FILL; 70382c9918fSTim Haley } else if (db->db_state == DB_NOFILL) { 70482c9918fSTim Haley dbuf_set_data(db, NULL); 705fa9e4066Sahrens } else { 706fa9e4066Sahrens ASSERT3U(db->db_state, ==, DB_CACHED); 707fa9e4066Sahrens } 708fa9e4066Sahrens mutex_exit(&db->db_mtx); 709fa9e4066Sahrens } 710fa9e4066Sahrens 711fa9e4066Sahrens /* 712fa9e4066Sahrens * This is our just-in-time copy function. It makes a copy of 713fa9e4066Sahrens * buffers, that have been modified in a previous transaction 714fa9e4066Sahrens * group, before we modify them in the current active group. 715fa9e4066Sahrens * 716fa9e4066Sahrens * This function is used in two places: when we are dirtying a 717fa9e4066Sahrens * buffer for the first time in a txg, and when we are freeing 718fa9e4066Sahrens * a range in a dnode that includes this buffer. 719fa9e4066Sahrens * 720fa9e4066Sahrens * Note that when we are called from dbuf_free_range() we do 721fa9e4066Sahrens * not put a hold on the buffer, we just traverse the active 722fa9e4066Sahrens * dbuf list for the dnode. 723fa9e4066Sahrens */ 724fa9e4066Sahrens static void 725fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 726fa9e4066Sahrens { 727c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 728fa9e4066Sahrens 729fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 730fa9e4066Sahrens ASSERT(db->db.db_data != NULL); 731c717a561Smaybee ASSERT(db->db_level == 0); 732c717a561Smaybee ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 733fa9e4066Sahrens 7344d31c452Smaybee if (dr == NULL || 7354d31c452Smaybee (dr->dt.dl.dr_data != 7360a586ceaSMark Shellenbaum ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 737fa9e4066Sahrens return; 738fa9e4066Sahrens 739fa9e4066Sahrens /* 740c717a561Smaybee * If the last dirty record for this dbuf has not yet synced 741c717a561Smaybee * and its referencing the dbuf data, either: 742744947dcSTom Erickson * reset the reference to point to a new copy, 743c717a561Smaybee * or (if there a no active holders) 744c717a561Smaybee * just null out the current db_data pointer. 745fa9e4066Sahrens */ 746c717a561Smaybee ASSERT(dr->dr_txg >= txg - 2); 7470a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 748c717a561Smaybee /* Note that the data bufs here are zio_bufs */ 749c717a561Smaybee dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 7505a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 751c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 752c717a561Smaybee } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 753c717a561Smaybee int size = db->db.db_size; 754c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 75543466aaeSMax Grossman spa_t *spa = db->db_objset->os_spa; 756744947dcSTom Erickson 757744947dcSTom Erickson dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 758c717a561Smaybee bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 759c717a561Smaybee } else { 760c717a561Smaybee dbuf_set_data(db, NULL); 761fa9e4066Sahrens } 762fa9e4066Sahrens } 763fa9e4066Sahrens 764c717a561Smaybee void 765c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr) 766ea8dc4b6Seschrock { 767c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 768b24ab676SJeff Bonwick blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 769c717a561Smaybee uint64_t txg = dr->dr_txg; 770ea8dc4b6Seschrock 771ea8dc4b6Seschrock ASSERT(MUTEX_HELD(&db->db_mtx)); 772c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 773c717a561Smaybee ASSERT(db->db_level == 0); 774ea8dc4b6Seschrock 7750a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 776c717a561Smaybee dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 777c717a561Smaybee return; 778ea8dc4b6Seschrock 779b24ab676SJeff Bonwick ASSERT(db->db_data_pending != dr); 780b24ab676SJeff Bonwick 781c717a561Smaybee /* free this block */ 78243466aaeSMax Grossman if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 78343466aaeSMax Grossman zio_free(db->db_objset->os_spa, txg, bp); 784b24ab676SJeff Bonwick 785c717a561Smaybee dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 78680901aeaSGeorge Wilson dr->dt.dl.dr_nopwrite = B_FALSE; 78780901aeaSGeorge Wilson 788c717a561Smaybee /* 789c717a561Smaybee * Release the already-written buffer, so we leave it in 790c717a561Smaybee * a consistent dirty state. Note that all callers are 791c717a561Smaybee * modifying the buffer, so they will immediately do 792c717a561Smaybee * another (redundant) arc_release(). Therefore, leave 793c717a561Smaybee * the buf thawed to save the effort of freezing & 794c717a561Smaybee * immediately re-thawing it. 795c717a561Smaybee */ 796c717a561Smaybee arc_release(dr->dt.dl.dr_data, db); 797fa9e4066Sahrens } 798fa9e4066Sahrens 799cdb0ab79Smaybee /* 800cdb0ab79Smaybee * Evict (if its unreferenced) or clear (if its referenced) any level-0 801cdb0ab79Smaybee * data blocks in the free range, so that any future readers will find 80243466aaeSMax Grossman * empty blocks. 8032f3d8780SMatthew Ahrens * 8042f3d8780SMatthew Ahrens * This is a no-op if the dataset is in the middle of an incremental 8052f3d8780SMatthew Ahrens * receive; see comment below for details. 806cdb0ab79Smaybee */ 807fa9e4066Sahrens void 808cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx) 809fa9e4066Sahrens { 810fa9e4066Sahrens dmu_buf_impl_t *db, *db_next; 811fa9e4066Sahrens uint64_t txg = tx->tx_txg; 812fa9e4066Sahrens 81343466aaeSMax Grossman if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) 814cdb0ab79Smaybee end = dn->dn_maxblkid; 815cdb0ab79Smaybee dprintf_dnode(dn, "start=%llu end=%llu\n", start, end); 8162f3d8780SMatthew Ahrens 817713d6c20SMatthew Ahrens mutex_enter(&dn->dn_dbufs_mtx); 818713d6c20SMatthew Ahrens if (start >= dn->dn_unlisted_l0_blkid * dn->dn_datablksz) { 819713d6c20SMatthew Ahrens /* There can't be any dbufs in this range; no need to search. */ 820713d6c20SMatthew Ahrens mutex_exit(&dn->dn_dbufs_mtx); 821713d6c20SMatthew Ahrens return; 822713d6c20SMatthew Ahrens } else if (dmu_objset_is_receiving(dn->dn_objset)) { 8232f3d8780SMatthew Ahrens /* 824713d6c20SMatthew Ahrens * If we are receiving, we expect there to be no dbufs in 825713d6c20SMatthew Ahrens * the range to be freed, because receive modifies each 826713d6c20SMatthew Ahrens * block at most once, and in offset order. If this is 827713d6c20SMatthew Ahrens * not the case, it can lead to performance problems, 828713d6c20SMatthew Ahrens * so note that we unexpectedly took the slow path. 8292f3d8780SMatthew Ahrens */ 830713d6c20SMatthew Ahrens atomic_inc_64(&zfs_free_range_recv_miss); 8312f3d8780SMatthew Ahrens } 8322f3d8780SMatthew Ahrens 83369962b56SMatthew Ahrens for (db = list_head(&dn->dn_dbufs); db != NULL; db = db_next) { 834fa9e4066Sahrens db_next = list_next(&dn->dn_dbufs, db); 8350a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 836cdb0ab79Smaybee 837ea8dc4b6Seschrock if (db->db_level != 0) 838fa9e4066Sahrens continue; 839cdb0ab79Smaybee if (db->db_blkid < start || db->db_blkid > end) 840fa9e4066Sahrens continue; 841fa9e4066Sahrens 842fa9e4066Sahrens /* found a level 0 buffer in the range */ 8433b2aab18SMatthew Ahrens mutex_enter(&db->db_mtx); 8443b2aab18SMatthew Ahrens if (dbuf_undirty(db, tx)) { 8453b2aab18SMatthew Ahrens /* mutex has been dropped and dbuf destroyed */ 846fa9e4066Sahrens continue; 8473b2aab18SMatthew Ahrens } 848fa9e4066Sahrens 849ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED || 85082c9918fSTim Haley db->db_state == DB_NOFILL || 851ea8dc4b6Seschrock db->db_state == DB_EVICTING) { 852fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 853fa9e4066Sahrens mutex_exit(&db->db_mtx); 854fa9e4066Sahrens continue; 855fa9e4066Sahrens } 856c543ec06Sahrens if (db->db_state == DB_READ || db->db_state == DB_FILL) { 857c543ec06Sahrens /* will be handled in dbuf_read_done or dbuf_rele */ 858c717a561Smaybee db->db_freed_in_flight = TRUE; 859fa9e4066Sahrens mutex_exit(&db->db_mtx); 860fa9e4066Sahrens continue; 861fa9e4066Sahrens } 862ea8dc4b6Seschrock if (refcount_count(&db->db_holds) == 0) { 863ea8dc4b6Seschrock ASSERT(db->db_buf); 864ea8dc4b6Seschrock dbuf_clear(db); 865ea8dc4b6Seschrock continue; 866ea8dc4b6Seschrock } 867c717a561Smaybee /* The dbuf is referenced */ 868fa9e4066Sahrens 869c717a561Smaybee if (db->db_last_dirty != NULL) { 870c717a561Smaybee dbuf_dirty_record_t *dr = db->db_last_dirty; 871c717a561Smaybee 872c717a561Smaybee if (dr->dr_txg == txg) { 87344eda4d7Smaybee /* 874c717a561Smaybee * This buffer is "in-use", re-adjust the file 875c717a561Smaybee * size to reflect that this buffer may 876c717a561Smaybee * contain new data when we sync. 87744eda4d7Smaybee */ 87806e0070dSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID && 87906e0070dSMark Shellenbaum db->db_blkid > dn->dn_maxblkid) 880c717a561Smaybee dn->dn_maxblkid = db->db_blkid; 881c717a561Smaybee dbuf_unoverride(dr); 882c717a561Smaybee } else { 883c717a561Smaybee /* 884c717a561Smaybee * This dbuf is not dirty in the open context. 885c717a561Smaybee * Either uncache it (if its not referenced in 886c717a561Smaybee * the open context) or reset its contents to 887c717a561Smaybee * empty. 888c717a561Smaybee */ 889c717a561Smaybee dbuf_fix_old_data(db, txg); 89044eda4d7Smaybee } 891ea8dc4b6Seschrock } 892c717a561Smaybee /* clear the contents if its cached */ 893ea8dc4b6Seschrock if (db->db_state == DB_CACHED) { 894ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 895fa9e4066Sahrens arc_release(db->db_buf, db); 896fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 8976b4acc8bSahrens arc_buf_freeze(db->db_buf); 898fa9e4066Sahrens } 899ea8dc4b6Seschrock 900fa9e4066Sahrens mutex_exit(&db->db_mtx); 901fa9e4066Sahrens } 902fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 903fa9e4066Sahrens } 904fa9e4066Sahrens 905fa9e4066Sahrens static int 9061934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db) 907fa9e4066Sahrens { 908fa9e4066Sahrens dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 909fa9e4066Sahrens uint64_t birth_txg = 0; 910fa9e4066Sahrens 911fa9e4066Sahrens /* 912fa9e4066Sahrens * We don't need any locking to protect db_blkptr: 913c717a561Smaybee * If it's syncing, then db_last_dirty will be set 914c717a561Smaybee * so we'll ignore db_blkptr. 91543466aaeSMax Grossman * 91643466aaeSMax Grossman * This logic ensures that only block births for 91743466aaeSMax Grossman * filled blocks are considered. 918fa9e4066Sahrens */ 919c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 92043466aaeSMax Grossman if (db->db_last_dirty && (db->db_blkptr == NULL || 92143466aaeSMax Grossman !BP_IS_HOLE(db->db_blkptr))) { 922c717a561Smaybee birth_txg = db->db_last_dirty->dr_txg; 92343466aaeSMax Grossman } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 924fa9e4066Sahrens birth_txg = db->db_blkptr->blk_birth; 92543466aaeSMax Grossman } 926fa9e4066Sahrens 927837b568bSGeorge Wilson /* 92843466aaeSMax Grossman * If this block don't exist or is in a snapshot, it can't be freed. 929837b568bSGeorge Wilson * Don't pass the bp to dsl_dataset_block_freeable() since we 930837b568bSGeorge Wilson * are holding the db_mtx lock and might deadlock if we are 931837b568bSGeorge Wilson * prefetching a dedup-ed block. 932837b568bSGeorge Wilson */ 93343466aaeSMax Grossman if (birth_txg != 0) 9341934e92fSmaybee return (ds == NULL || 935837b568bSGeorge Wilson dsl_dataset_block_freeable(ds, NULL, birth_txg)); 936fa9e4066Sahrens else 93743466aaeSMax Grossman return (B_FALSE); 938fa9e4066Sahrens } 939fa9e4066Sahrens 940fa9e4066Sahrens void 941fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 942fa9e4066Sahrens { 943fa9e4066Sahrens arc_buf_t *buf, *obuf; 944fa9e4066Sahrens int osize = db->db.db_size; 945ad23a2dbSjohansen arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 946744947dcSTom Erickson dnode_t *dn; 947fa9e4066Sahrens 9480a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 949ea8dc4b6Seschrock 950744947dcSTom Erickson DB_DNODE_ENTER(db); 951744947dcSTom Erickson dn = DB_DNODE(db); 952744947dcSTom Erickson 953fa9e4066Sahrens /* XXX does *this* func really need the lock? */ 954744947dcSTom Erickson ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 955fa9e4066Sahrens 956fa9e4066Sahrens /* 95743466aaeSMax Grossman * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 958fa9e4066Sahrens * is OK, because there can be no other references to the db 959fa9e4066Sahrens * when we are changing its size, so no concurrent DB_FILL can 960fa9e4066Sahrens * be happening. 961fa9e4066Sahrens */ 962ea8dc4b6Seschrock /* 963ea8dc4b6Seschrock * XXX we should be doing a dbuf_read, checking the return 964ea8dc4b6Seschrock * value and returning that up to our callers 965ea8dc4b6Seschrock */ 96643466aaeSMax Grossman dmu_buf_will_dirty(&db->db, tx); 967fa9e4066Sahrens 968fa9e4066Sahrens /* create the data buffer for the new block */ 969744947dcSTom Erickson buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 970fa9e4066Sahrens 971fa9e4066Sahrens /* copy old block data to the new block */ 972fa9e4066Sahrens obuf = db->db_buf; 973f65e61c0Sahrens bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 974fa9e4066Sahrens /* zero the remainder */ 975f65e61c0Sahrens if (size > osize) 976f65e61c0Sahrens bzero((uint8_t *)buf->b_data + osize, size - osize); 977fa9e4066Sahrens 978fa9e4066Sahrens mutex_enter(&db->db_mtx); 979fa9e4066Sahrens dbuf_set_data(db, buf); 9803b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(obuf, db)); 981fa9e4066Sahrens db->db.db_size = size; 982fa9e4066Sahrens 983c717a561Smaybee if (db->db_level == 0) { 984c717a561Smaybee ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 985c717a561Smaybee db->db_last_dirty->dt.dl.dr_data = buf; 986c717a561Smaybee } 987fa9e4066Sahrens mutex_exit(&db->db_mtx); 988fa9e4066Sahrens 989744947dcSTom Erickson dnode_willuse_space(dn, size-osize, tx); 990744947dcSTom Erickson DB_DNODE_EXIT(db); 991fa9e4066Sahrens } 992fa9e4066Sahrens 9933f9d6ad7SLin Ling void 9943f9d6ad7SLin Ling dbuf_release_bp(dmu_buf_impl_t *db) 9953f9d6ad7SLin Ling { 99643466aaeSMax Grossman objset_t *os = db->db_objset; 9973f9d6ad7SLin Ling 9983f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 9993f9d6ad7SLin Ling ASSERT(arc_released(os->os_phys_buf) || 10003f9d6ad7SLin Ling list_link_active(&os->os_dsl_dataset->ds_synced_link)); 10013f9d6ad7SLin Ling ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 10023f9d6ad7SLin Ling 10031b912ec7SGeorge Wilson (void) arc_release(db->db_buf, db); 10043f9d6ad7SLin Ling } 10053f9d6ad7SLin Ling 1006c717a561Smaybee dbuf_dirty_record_t * 1007fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1008fa9e4066Sahrens { 1009744947dcSTom Erickson dnode_t *dn; 1010744947dcSTom Erickson objset_t *os; 1011c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 1012fa9e4066Sahrens int drop_struct_lock = FALSE; 1013d3469faaSMark Maybee boolean_t do_free_accounting = B_FALSE; 1014fa9e4066Sahrens int txgoff = tx->tx_txg & TXG_MASK; 1015fa9e4066Sahrens 1016fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1017fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 10189c9dc39aSek DMU_TX_DIRTY_BUF(tx, db); 1019fa9e4066Sahrens 1020744947dcSTom Erickson DB_DNODE_ENTER(db); 1021744947dcSTom Erickson dn = DB_DNODE(db); 1022fa9e4066Sahrens /* 1023fa9e4066Sahrens * Shouldn't dirty a regular buffer in syncing context. Private 1024fa9e4066Sahrens * objects may be dirtied in syncing context, but only if they 1025fa9e4066Sahrens * were already pre-dirtied in open context. 1026fa9e4066Sahrens */ 1027c717a561Smaybee ASSERT(!dmu_tx_is_syncing(tx) || 1028c717a561Smaybee BP_IS_HOLE(dn->dn_objset->os_rootbp) || 102914843421SMatthew Ahrens DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 103014843421SMatthew Ahrens dn->dn_objset->os_dsl_dataset == NULL); 1031fa9e4066Sahrens /* 1032fa9e4066Sahrens * We make this assert for private objects as well, but after we 1033fa9e4066Sahrens * check if we're already dirty. They are allowed to re-dirty 1034fa9e4066Sahrens * in syncing context. 1035fa9e4066Sahrens */ 1036ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1037c717a561Smaybee dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1038fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1039fa9e4066Sahrens 1040fa9e4066Sahrens mutex_enter(&db->db_mtx); 1041fa9e4066Sahrens /* 1042c717a561Smaybee * XXX make this true for indirects too? The problem is that 1043c717a561Smaybee * transactions created with dmu_tx_create_assigned() from 1044c717a561Smaybee * syncing context don't bother holding ahead. 1045fa9e4066Sahrens */ 1046c717a561Smaybee ASSERT(db->db_level != 0 || 104782c9918fSTim Haley db->db_state == DB_CACHED || db->db_state == DB_FILL || 104882c9918fSTim Haley db->db_state == DB_NOFILL); 1049fa9e4066Sahrens 1050fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1051fa9e4066Sahrens /* 1052fa9e4066Sahrens * Don't set dirtyctx to SYNC if we're just modifying this as we 1053fa9e4066Sahrens * initialize the objset. 1054fa9e4066Sahrens */ 1055fa9e4066Sahrens if (dn->dn_dirtyctx == DN_UNDIRTIED && 1056c717a561Smaybee !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1057fa9e4066Sahrens dn->dn_dirtyctx = 1058fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1059fa9e4066Sahrens ASSERT(dn->dn_dirtyctx_firstset == NULL); 1060fa9e4066Sahrens dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1061fa9e4066Sahrens } 1062fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1063fa9e4066Sahrens 10640a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 10650a586ceaSMark Shellenbaum dn->dn_have_spill = B_TRUE; 10660a586ceaSMark Shellenbaum 1067fa9e4066Sahrens /* 1068fa9e4066Sahrens * If this buffer is already dirty, we're done. 1069fa9e4066Sahrens */ 1070c717a561Smaybee drp = &db->db_last_dirty; 1071c717a561Smaybee ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1072c717a561Smaybee db->db.db_object == DMU_META_DNODE_OBJECT); 10737e2186e3Sbonwick while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 10747e2186e3Sbonwick drp = &dr->dr_next; 10757e2186e3Sbonwick if (dr && dr->dr_txg == tx->tx_txg) { 1076744947dcSTom Erickson DB_DNODE_EXIT(db); 1077744947dcSTom Erickson 10780a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1079c717a561Smaybee /* 1080c717a561Smaybee * If this buffer has already been written out, 1081c717a561Smaybee * we now need to reset its state. 1082c717a561Smaybee */ 10837e2186e3Sbonwick dbuf_unoverride(dr); 1084b24ab676SJeff Bonwick if (db->db.db_object != DMU_META_DNODE_OBJECT && 1085b24ab676SJeff Bonwick db->db_state != DB_NOFILL) 1086c717a561Smaybee arc_buf_thaw(db->db_buf); 1087c717a561Smaybee } 1088fa9e4066Sahrens mutex_exit(&db->db_mtx); 10897e2186e3Sbonwick return (dr); 1090fa9e4066Sahrens } 1091fa9e4066Sahrens 1092fa9e4066Sahrens /* 1093fa9e4066Sahrens * Only valid if not already dirty. 1094fa9e4066Sahrens */ 109514843421SMatthew Ahrens ASSERT(dn->dn_object == 0 || 109614843421SMatthew Ahrens dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1097fa9e4066Sahrens (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1098fa9e4066Sahrens 1099fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, db->db_level); 1100fa9e4066Sahrens ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1101fa9e4066Sahrens dn->dn_phys->dn_nlevels > db->db_level || 1102fa9e4066Sahrens dn->dn_next_nlevels[txgoff] > db->db_level || 1103fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1104fa9e4066Sahrens dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1105fa9e4066Sahrens 1106fa9e4066Sahrens /* 1107fa9e4066Sahrens * We should only be dirtying in syncing context if it's the 110814843421SMatthew Ahrens * mos or we're initializing the os or it's a special object. 110914843421SMatthew Ahrens * However, we are allowed to dirty in syncing context provided 111014843421SMatthew Ahrens * we already dirtied it in open context. Hence we must make 111114843421SMatthew Ahrens * this assertion only if we're not already dirty. 1112fa9e4066Sahrens */ 1113744947dcSTom Erickson os = dn->dn_objset; 111414843421SMatthew Ahrens ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 111514843421SMatthew Ahrens os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1116fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1117fa9e4066Sahrens 1118fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1119fa9e4066Sahrens 11200a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 11211934e92fSmaybee /* 11221934e92fSmaybee * Update the accounting. 1123d3469faaSMark Maybee * Note: we delay "free accounting" until after we drop 1124d3469faaSMark Maybee * the db_mtx. This keeps us from grabbing other locks 1125b24ab676SJeff Bonwick * (and possibly deadlocking) in bp_get_dsize() while 1126d3469faaSMark Maybee * also holding the db_mtx. 11271934e92fSmaybee */ 11281934e92fSmaybee dnode_willuse_space(dn, db->db.db_size, tx); 1129d3469faaSMark Maybee do_free_accounting = dbuf_block_freeable(db); 11301934e92fSmaybee } 11311934e92fSmaybee 1132ea8dc4b6Seschrock /* 1133ea8dc4b6Seschrock * If this buffer is dirty in an old transaction group we need 1134ea8dc4b6Seschrock * to make a copy of it so that the changes we make in this 1135ea8dc4b6Seschrock * transaction group won't leak out when we sync the older txg. 1136ea8dc4b6Seschrock */ 1137c717a561Smaybee dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1138c717a561Smaybee if (db->db_level == 0) { 1139c717a561Smaybee void *data_old = db->db_buf; 1140c717a561Smaybee 114182c9918fSTim Haley if (db->db_state != DB_NOFILL) { 11420a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 114382c9918fSTim Haley dbuf_fix_old_data(db, tx->tx_txg); 114482c9918fSTim Haley data_old = db->db.db_data; 114582c9918fSTim Haley } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 114682c9918fSTim Haley /* 114782c9918fSTim Haley * Release the data buffer from the cache so 114882c9918fSTim Haley * that we can modify it without impacting 114982c9918fSTim Haley * possible other users of this cached data 115082c9918fSTim Haley * block. Note that indirect blocks and 115182c9918fSTim Haley * private objects are not released until the 115282c9918fSTim Haley * syncing state (since they are only modified 115382c9918fSTim Haley * then). 115482c9918fSTim Haley */ 115582c9918fSTim Haley arc_release(db->db_buf, db); 115682c9918fSTim Haley dbuf_fix_old_data(db, tx->tx_txg); 115782c9918fSTim Haley data_old = db->db_buf; 115882c9918fSTim Haley } 115982c9918fSTim Haley ASSERT(data_old != NULL); 1160fa9e4066Sahrens } 1161c717a561Smaybee dr->dt.dl.dr_data = data_old; 1162c717a561Smaybee } else { 1163c717a561Smaybee mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1164c717a561Smaybee list_create(&dr->dt.di.dr_children, 1165c717a561Smaybee sizeof (dbuf_dirty_record_t), 1166c717a561Smaybee offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1167fa9e4066Sahrens } 116869962b56SMatthew Ahrens if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 116969962b56SMatthew Ahrens dr->dr_accounted = db->db.db_size; 1170c717a561Smaybee dr->dr_dbuf = db; 1171c717a561Smaybee dr->dr_txg = tx->tx_txg; 1172c717a561Smaybee dr->dr_next = *drp; 1173c717a561Smaybee *drp = dr; 1174fa9e4066Sahrens 1175fa9e4066Sahrens /* 1176fa9e4066Sahrens * We could have been freed_in_flight between the dbuf_noread 1177fa9e4066Sahrens * and dbuf_dirty. We win, as though the dbuf_noread() had 1178fa9e4066Sahrens * happened after the free. 1179fa9e4066Sahrens */ 11800a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 11810a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) { 1182c717a561Smaybee mutex_enter(&dn->dn_mtx); 1183bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] != NULL) { 1184bf16b11eSMatthew Ahrens range_tree_clear(dn->dn_free_ranges[txgoff], 1185bf16b11eSMatthew Ahrens db->db_blkid, 1); 1186bf16b11eSMatthew Ahrens } 1187c717a561Smaybee mutex_exit(&dn->dn_mtx); 1188c717a561Smaybee db->db_freed_in_flight = FALSE; 1189fa9e4066Sahrens } 1190fa9e4066Sahrens 1191fa9e4066Sahrens /* 1192fa9e4066Sahrens * This buffer is now part of this txg 1193fa9e4066Sahrens */ 1194fa9e4066Sahrens dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1195fa9e4066Sahrens db->db_dirtycnt += 1; 1196fa9e4066Sahrens ASSERT3U(db->db_dirtycnt, <=, 3); 1197fa9e4066Sahrens 1198fa9e4066Sahrens mutex_exit(&db->db_mtx); 1199fa9e4066Sahrens 12000a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID || 12010a586ceaSMark Shellenbaum db->db_blkid == DMU_SPILL_BLKID) { 1202c717a561Smaybee mutex_enter(&dn->dn_mtx); 1203c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1204c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1205c717a561Smaybee mutex_exit(&dn->dn_mtx); 1206fa9e4066Sahrens dnode_setdirty(dn, tx); 1207744947dcSTom Erickson DB_DNODE_EXIT(db); 1208c717a561Smaybee return (dr); 1209d3469faaSMark Maybee } else if (do_free_accounting) { 1210d3469faaSMark Maybee blkptr_t *bp = db->db_blkptr; 1211d3469faaSMark Maybee int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1212b24ab676SJeff Bonwick bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1213d3469faaSMark Maybee /* 1214d3469faaSMark Maybee * This is only a guess -- if the dbuf is dirty 1215d3469faaSMark Maybee * in a previous txg, we don't know how much 1216d3469faaSMark Maybee * space it will use on disk yet. We should 1217d3469faaSMark Maybee * really have the struct_rwlock to access 1218d3469faaSMark Maybee * db_blkptr, but since this is just a guess, 1219d3469faaSMark Maybee * it's OK if we get an odd answer. 1220d3469faaSMark Maybee */ 1221837b568bSGeorge Wilson ddt_prefetch(os->os_spa, bp); 1222d3469faaSMark Maybee dnode_willuse_space(dn, -willfree, tx); 1223fa9e4066Sahrens } 1224fa9e4066Sahrens 1225fa9e4066Sahrens if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1226fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1227fa9e4066Sahrens drop_struct_lock = TRUE; 1228fa9e4066Sahrens } 1229fa9e4066Sahrens 12308346f03fSJonathan W Adams if (db->db_level == 0) { 12318346f03fSJonathan W Adams dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock); 12328346f03fSJonathan W Adams ASSERT(dn->dn_maxblkid >= db->db_blkid); 12338346f03fSJonathan W Adams } 12348346f03fSJonathan W Adams 123544eda4d7Smaybee if (db->db_level+1 < dn->dn_nlevels) { 1236c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 1237c717a561Smaybee dbuf_dirty_record_t *di; 1238c717a561Smaybee int parent_held = FALSE; 1239c717a561Smaybee 1240c717a561Smaybee if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1241c717a561Smaybee int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1242c717a561Smaybee 1243c717a561Smaybee parent = dbuf_hold_level(dn, db->db_level+1, 1244c717a561Smaybee db->db_blkid >> epbs, FTAG); 124501025c89SJohn Harres ASSERT(parent != NULL); 1246c717a561Smaybee parent_held = TRUE; 1247c717a561Smaybee } 1248fa9e4066Sahrens if (drop_struct_lock) 1249fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1250c717a561Smaybee ASSERT3U(db->db_level+1, ==, parent->db_level); 1251c717a561Smaybee di = dbuf_dirty(parent, tx); 1252c717a561Smaybee if (parent_held) 1253c717a561Smaybee dbuf_rele(parent, FTAG); 1254c717a561Smaybee 1255c717a561Smaybee mutex_enter(&db->db_mtx); 125669962b56SMatthew Ahrens /* 125769962b56SMatthew Ahrens * Since we've dropped the mutex, it's possible that 125869962b56SMatthew Ahrens * dbuf_undirty() might have changed this out from under us. 125969962b56SMatthew Ahrens */ 1260c717a561Smaybee if (db->db_last_dirty == dr || 1261c717a561Smaybee dn->dn_object == DMU_META_DNODE_OBJECT) { 1262c717a561Smaybee mutex_enter(&di->dt.di.dr_mtx); 1263c717a561Smaybee ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1264c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1265c717a561Smaybee list_insert_tail(&di->dt.di.dr_children, dr); 1266c717a561Smaybee mutex_exit(&di->dt.di.dr_mtx); 1267c717a561Smaybee dr->dr_parent = di; 1268c717a561Smaybee } 1269c717a561Smaybee mutex_exit(&db->db_mtx); 1270fa9e4066Sahrens } else { 1271c717a561Smaybee ASSERT(db->db_level+1 == dn->dn_nlevels); 1272c717a561Smaybee ASSERT(db->db_blkid < dn->dn_nblkptr); 1273744947dcSTom Erickson ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1274c717a561Smaybee mutex_enter(&dn->dn_mtx); 1275c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 1276c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1277c717a561Smaybee mutex_exit(&dn->dn_mtx); 1278fa9e4066Sahrens if (drop_struct_lock) 1279fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1280fa9e4066Sahrens } 1281fa9e4066Sahrens 1282fa9e4066Sahrens dnode_setdirty(dn, tx); 1283744947dcSTom Erickson DB_DNODE_EXIT(db); 1284c717a561Smaybee return (dr); 1285fa9e4066Sahrens } 1286fa9e4066Sahrens 12873b2aab18SMatthew Ahrens /* 12883e30c24aSWill Andrews * Undirty a buffer in the transaction group referenced by the given 12893e30c24aSWill Andrews * transaction. Return whether this evicted the dbuf. 12903b2aab18SMatthew Ahrens */ 12913b2aab18SMatthew Ahrens static boolean_t 1292fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1293fa9e4066Sahrens { 1294744947dcSTom Erickson dnode_t *dn; 1295c717a561Smaybee uint64_t txg = tx->tx_txg; 129617f17c2dSbonwick dbuf_dirty_record_t *dr, **drp; 1297fa9e4066Sahrens 1298c717a561Smaybee ASSERT(txg != 0); 12990a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 13003b2aab18SMatthew Ahrens ASSERT0(db->db_level); 13013b2aab18SMatthew Ahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1302fa9e4066Sahrens 1303fa9e4066Sahrens /* 1304fa9e4066Sahrens * If this buffer is not dirty, we're done. 1305fa9e4066Sahrens */ 130617f17c2dSbonwick for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1307c717a561Smaybee if (dr->dr_txg <= txg) 1308c717a561Smaybee break; 13093b2aab18SMatthew Ahrens if (dr == NULL || dr->dr_txg < txg) 13103b2aab18SMatthew Ahrens return (B_FALSE); 1311c717a561Smaybee ASSERT(dr->dr_txg == txg); 1312b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 1313fa9e4066Sahrens 1314744947dcSTom Erickson DB_DNODE_ENTER(db); 1315744947dcSTom Erickson dn = DB_DNODE(db); 1316744947dcSTom Erickson 1317fa9e4066Sahrens dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1318fa9e4066Sahrens 1319fa9e4066Sahrens ASSERT(db->db.db_size != 0); 1320fa9e4066Sahrens 132169962b56SMatthew Ahrens /* 132269962b56SMatthew Ahrens * Any space we accounted for in dp_dirty_* will be cleaned up by 132369962b56SMatthew Ahrens * dsl_pool_sync(). This is relatively rare so the discrepancy 132469962b56SMatthew Ahrens * is not a big deal. 132569962b56SMatthew Ahrens */ 1326fa9e4066Sahrens 132717f17c2dSbonwick *drp = dr->dr_next; 1328c717a561Smaybee 13293f2366c2SGordon Ross /* 13303f2366c2SGordon Ross * Note that there are three places in dbuf_dirty() 13313f2366c2SGordon Ross * where this dirty record may be put on a list. 13323f2366c2SGordon Ross * Make sure to do a list_remove corresponding to 13333f2366c2SGordon Ross * every one of those list_insert calls. 13343f2366c2SGordon Ross */ 1335c717a561Smaybee if (dr->dr_parent) { 1336c717a561Smaybee mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1337c717a561Smaybee list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1338c717a561Smaybee mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 13393f2366c2SGordon Ross } else if (db->db_blkid == DMU_SPILL_BLKID || 13403f2366c2SGordon Ross db->db_level+1 == dn->dn_nlevels) { 1341cdb0ab79Smaybee ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1342c717a561Smaybee mutex_enter(&dn->dn_mtx); 1343c717a561Smaybee list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1344c717a561Smaybee mutex_exit(&dn->dn_mtx); 1345c717a561Smaybee } 1346744947dcSTom Erickson DB_DNODE_EXIT(db); 1347c717a561Smaybee 13483b2aab18SMatthew Ahrens if (db->db_state != DB_NOFILL) { 13493b2aab18SMatthew Ahrens dbuf_unoverride(dr); 1350c717a561Smaybee 1351c717a561Smaybee ASSERT(db->db_buf != NULL); 13523b2aab18SMatthew Ahrens ASSERT(dr->dt.dl.dr_data != NULL); 13533b2aab18SMatthew Ahrens if (dr->dt.dl.dr_data != db->db_buf) 13543b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1355c717a561Smaybee } 1356c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1357fa9e4066Sahrens 1358fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 1359fa9e4066Sahrens db->db_dirtycnt -= 1; 1360fa9e4066Sahrens 1361c717a561Smaybee if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1362ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 1363fa9e4066Sahrens 1364b24ab676SJeff Bonwick ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1365ea8dc4b6Seschrock dbuf_set_data(db, NULL); 13663b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1367fa9e4066Sahrens dbuf_evict(db); 13683b2aab18SMatthew Ahrens return (B_TRUE); 1369fa9e4066Sahrens } 1370fa9e4066Sahrens 13713b2aab18SMatthew Ahrens return (B_FALSE); 1372fa9e4066Sahrens } 1373fa9e4066Sahrens 1374fa9e4066Sahrens void 137543466aaeSMax Grossman dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1376fa9e4066Sahrens { 137743466aaeSMax Grossman dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 13781ab7f2deSmaybee int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1379fa9e4066Sahrens 1380fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1381fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1382fa9e4066Sahrens 1383744947dcSTom Erickson DB_DNODE_ENTER(db); 1384744947dcSTom Erickson if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1385fa9e4066Sahrens rf |= DB_RF_HAVESTRUCT; 1386744947dcSTom Erickson DB_DNODE_EXIT(db); 1387ea8dc4b6Seschrock (void) dbuf_read(db, NULL, rf); 1388c717a561Smaybee (void) dbuf_dirty(db, tx); 1389fa9e4066Sahrens } 1390fa9e4066Sahrens 139182c9918fSTim Haley void 139282c9918fSTim Haley dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 139382c9918fSTim Haley { 139482c9918fSTim Haley dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 139582c9918fSTim Haley 139682c9918fSTim Haley db->db_state = DB_NOFILL; 139782c9918fSTim Haley 139882c9918fSTim Haley dmu_buf_will_fill(db_fake, tx); 139982c9918fSTim Haley } 140082c9918fSTim Haley 1401fa9e4066Sahrens void 1402ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1403fa9e4066Sahrens { 1404ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1405ea8dc4b6Seschrock 14060a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1407fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1408fa9e4066Sahrens ASSERT(db->db_level == 0); 1409fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 1410fa9e4066Sahrens 1411ea8dc4b6Seschrock ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1412fa9e4066Sahrens dmu_tx_private_ok(tx)); 1413fa9e4066Sahrens 1414fa9e4066Sahrens dbuf_noread(db); 1415c717a561Smaybee (void) dbuf_dirty(db, tx); 1416fa9e4066Sahrens } 1417fa9e4066Sahrens 1418fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done 1419fa9e4066Sahrens /* ARGSUSED */ 1420fa9e4066Sahrens void 1421fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1422fa9e4066Sahrens { 1423fa9e4066Sahrens mutex_enter(&db->db_mtx); 14249c9dc39aSek DBUF_VERIFY(db); 1425fa9e4066Sahrens 1426fa9e4066Sahrens if (db->db_state == DB_FILL) { 1427c717a561Smaybee if (db->db_level == 0 && db->db_freed_in_flight) { 14280a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1429fa9e4066Sahrens /* we were freed while filling */ 1430fa9e4066Sahrens /* XXX dbuf_undirty? */ 1431fa9e4066Sahrens bzero(db->db.db_data, db->db.db_size); 1432c717a561Smaybee db->db_freed_in_flight = FALSE; 1433fa9e4066Sahrens } 1434fa9e4066Sahrens db->db_state = DB_CACHED; 1435fa9e4066Sahrens cv_broadcast(&db->db_changed); 1436fa9e4066Sahrens } 1437fa9e4066Sahrens mutex_exit(&db->db_mtx); 1438fa9e4066Sahrens } 1439fa9e4066Sahrens 14405d7b4d43SMatthew Ahrens void 14415d7b4d43SMatthew Ahrens dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 14425d7b4d43SMatthew Ahrens bp_embedded_type_t etype, enum zio_compress comp, 14435d7b4d43SMatthew Ahrens int uncompressed_size, int compressed_size, int byteorder, 14445d7b4d43SMatthew Ahrens dmu_tx_t *tx) 14455d7b4d43SMatthew Ahrens { 14465d7b4d43SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 14475d7b4d43SMatthew Ahrens struct dirty_leaf *dl; 14485d7b4d43SMatthew Ahrens dmu_object_type_t type; 14495d7b4d43SMatthew Ahrens 14505d7b4d43SMatthew Ahrens DB_DNODE_ENTER(db); 14515d7b4d43SMatthew Ahrens type = DB_DNODE(db)->dn_type; 14525d7b4d43SMatthew Ahrens DB_DNODE_EXIT(db); 14535d7b4d43SMatthew Ahrens 14545d7b4d43SMatthew Ahrens ASSERT0(db->db_level); 14555d7b4d43SMatthew Ahrens ASSERT(db->db_blkid != DMU_BONUS_BLKID); 14565d7b4d43SMatthew Ahrens 14575d7b4d43SMatthew Ahrens dmu_buf_will_not_fill(dbuf, tx); 14585d7b4d43SMatthew Ahrens 14595d7b4d43SMatthew Ahrens ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 14605d7b4d43SMatthew Ahrens dl = &db->db_last_dirty->dt.dl; 14615d7b4d43SMatthew Ahrens encode_embedded_bp_compressed(&dl->dr_overridden_by, 14625d7b4d43SMatthew Ahrens data, comp, uncompressed_size, compressed_size); 14635d7b4d43SMatthew Ahrens BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 14645d7b4d43SMatthew Ahrens BP_SET_TYPE(&dl->dr_overridden_by, type); 14655d7b4d43SMatthew Ahrens BP_SET_LEVEL(&dl->dr_overridden_by, 0); 14665d7b4d43SMatthew Ahrens BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 14675d7b4d43SMatthew Ahrens 14685d7b4d43SMatthew Ahrens dl->dr_override_state = DR_OVERRIDDEN; 14695d7b4d43SMatthew Ahrens dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 14705d7b4d43SMatthew Ahrens } 14715d7b4d43SMatthew Ahrens 14722fdbea25SAleksandr Guzovskiy /* 14732fdbea25SAleksandr Guzovskiy * Directly assign a provided arc buf to a given dbuf if it's not referenced 14742fdbea25SAleksandr Guzovskiy * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 14752fdbea25SAleksandr Guzovskiy */ 14762fdbea25SAleksandr Guzovskiy void 14772fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 14782fdbea25SAleksandr Guzovskiy { 14792fdbea25SAleksandr Guzovskiy ASSERT(!refcount_is_zero(&db->db_holds)); 14800a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 14812fdbea25SAleksandr Guzovskiy ASSERT(db->db_level == 0); 14822fdbea25SAleksandr Guzovskiy ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 14832fdbea25SAleksandr Guzovskiy ASSERT(buf != NULL); 14842fdbea25SAleksandr Guzovskiy ASSERT(arc_buf_size(buf) == db->db.db_size); 14852fdbea25SAleksandr Guzovskiy ASSERT(tx->tx_txg != 0); 14862fdbea25SAleksandr Guzovskiy 14872fdbea25SAleksandr Guzovskiy arc_return_buf(buf, db); 14882fdbea25SAleksandr Guzovskiy ASSERT(arc_released(buf)); 14892fdbea25SAleksandr Guzovskiy 14902fdbea25SAleksandr Guzovskiy mutex_enter(&db->db_mtx); 14912fdbea25SAleksandr Guzovskiy 14922fdbea25SAleksandr Guzovskiy while (db->db_state == DB_READ || db->db_state == DB_FILL) 14932fdbea25SAleksandr Guzovskiy cv_wait(&db->db_changed, &db->db_mtx); 14942fdbea25SAleksandr Guzovskiy 14952fdbea25SAleksandr Guzovskiy ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 14962fdbea25SAleksandr Guzovskiy 14972fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED && 14982fdbea25SAleksandr Guzovskiy refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 14992fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 15002fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 15012fdbea25SAleksandr Guzovskiy bcopy(buf->b_data, db->db.db_data, db->db.db_size); 15023b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 1503c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied(); 15042fdbea25SAleksandr Guzovskiy return; 15052fdbea25SAleksandr Guzovskiy } 15062fdbea25SAleksandr Guzovskiy 1507c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy(); 15082fdbea25SAleksandr Guzovskiy if (db->db_state == DB_CACHED) { 15092fdbea25SAleksandr Guzovskiy dbuf_dirty_record_t *dr = db->db_last_dirty; 15102fdbea25SAleksandr Guzovskiy 15112fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf != NULL); 15122fdbea25SAleksandr Guzovskiy if (dr != NULL && dr->dr_txg == tx->tx_txg) { 15132fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_data == db->db_buf); 15142fdbea25SAleksandr Guzovskiy if (!arc_released(db->db_buf)) { 15152fdbea25SAleksandr Guzovskiy ASSERT(dr->dt.dl.dr_override_state == 15162fdbea25SAleksandr Guzovskiy DR_OVERRIDDEN); 15172fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 15182fdbea25SAleksandr Guzovskiy } 15192fdbea25SAleksandr Guzovskiy dr->dt.dl.dr_data = buf; 15203b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 15212fdbea25SAleksandr Guzovskiy } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 15222fdbea25SAleksandr Guzovskiy arc_release(db->db_buf, db); 15233b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(db->db_buf, db)); 15242fdbea25SAleksandr Guzovskiy } 15252fdbea25SAleksandr Guzovskiy db->db_buf = NULL; 15262fdbea25SAleksandr Guzovskiy } 15272fdbea25SAleksandr Guzovskiy ASSERT(db->db_buf == NULL); 15282fdbea25SAleksandr Guzovskiy dbuf_set_data(db, buf); 15292fdbea25SAleksandr Guzovskiy db->db_state = DB_FILL; 15302fdbea25SAleksandr Guzovskiy mutex_exit(&db->db_mtx); 15312fdbea25SAleksandr Guzovskiy (void) dbuf_dirty(db, tx); 153243466aaeSMax Grossman dmu_buf_fill_done(&db->db, tx); 15332fdbea25SAleksandr Guzovskiy } 15342fdbea25SAleksandr Guzovskiy 1535ea8dc4b6Seschrock /* 1536ea8dc4b6Seschrock * "Clear" the contents of this dbuf. This will mark the dbuf 153769962b56SMatthew Ahrens * EVICTING and clear *most* of its references. Unfortunately, 1538ea8dc4b6Seschrock * when we are not holding the dn_dbufs_mtx, we can't clear the 1539ea8dc4b6Seschrock * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1540ea8dc4b6Seschrock * in this case. For callers from the DMU we will usually see: 1541ea8dc4b6Seschrock * dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy() 1542ea8dc4b6Seschrock * For the arc callback, we will usually see: 1543744947dcSTom Erickson * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1544ea8dc4b6Seschrock * Sometimes, though, we will get a mix of these two: 1545ea8dc4b6Seschrock * DMU: dbuf_clear()->arc_buf_evict() 1546ea8dc4b6Seschrock * ARC: dbuf_do_evict()->dbuf_destroy() 1547ea8dc4b6Seschrock */ 1548ea8dc4b6Seschrock void 1549fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db) 1550fa9e4066Sahrens { 1551744947dcSTom Erickson dnode_t *dn; 1552ea8dc4b6Seschrock dmu_buf_impl_t *parent = db->db_parent; 1553744947dcSTom Erickson dmu_buf_impl_t *dndb; 1554ea8dc4b6Seschrock int dbuf_gone = FALSE; 1555fa9e4066Sahrens 1556fa9e4066Sahrens ASSERT(MUTEX_HELD(&db->db_mtx)); 1557fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1558fa9e4066Sahrens 1559ea8dc4b6Seschrock dbuf_evict_user(db); 1560ea8dc4b6Seschrock 1561fa9e4066Sahrens if (db->db_state == DB_CACHED) { 1562ea8dc4b6Seschrock ASSERT(db->db.db_data != NULL); 15630a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 1564ea8dc4b6Seschrock zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 15655a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 15660e8c6158Smaybee } 1567fa9e4066Sahrens db->db.db_data = NULL; 1568fa9e4066Sahrens db->db_state = DB_UNCACHED; 1569fa9e4066Sahrens } 1570fa9e4066Sahrens 157182c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1572fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1573fa9e4066Sahrens 1574ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1575ea8dc4b6Seschrock db->db_blkptr = NULL; 1576ea8dc4b6Seschrock 1577744947dcSTom Erickson DB_DNODE_ENTER(db); 1578744947dcSTom Erickson dn = DB_DNODE(db); 1579744947dcSTom Erickson dndb = dn->dn_dbuf; 15800a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1581ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1582744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1583744947dcSTom Erickson membar_producer(); 1584744947dcSTom Erickson DB_DNODE_EXIT(db); 1585744947dcSTom Erickson /* 1586744947dcSTom Erickson * Decrementing the dbuf count means that the hold corresponding 1587744947dcSTom Erickson * to the removed dbuf is no longer discounted in dnode_move(), 1588744947dcSTom Erickson * so the dnode cannot be moved until after we release the hold. 1589744947dcSTom Erickson * The membar_producer() ensures visibility of the decremented 1590744947dcSTom Erickson * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1591744947dcSTom Erickson * release any lock. 1592744947dcSTom Erickson */ 1593ea8dc4b6Seschrock dnode_rele(dn, db); 1594744947dcSTom Erickson db->db_dnode_handle = NULL; 1595744947dcSTom Erickson } else { 1596744947dcSTom Erickson DB_DNODE_EXIT(db); 1597ea8dc4b6Seschrock } 1598ea8dc4b6Seschrock 1599ea8dc4b6Seschrock if (db->db_buf) 1600ea8dc4b6Seschrock dbuf_gone = arc_buf_evict(db->db_buf); 1601ea8dc4b6Seschrock 1602ea8dc4b6Seschrock if (!dbuf_gone) 1603ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1604fa9e4066Sahrens 1605fa9e4066Sahrens /* 1606744947dcSTom Erickson * If this dbuf is referenced from an indirect dbuf, 1607fa9e4066Sahrens * decrement the ref count on the indirect dbuf. 1608fa9e4066Sahrens */ 1609c543ec06Sahrens if (parent && parent != dndb) 1610ea8dc4b6Seschrock dbuf_rele(parent, db); 1611fa9e4066Sahrens } 1612fa9e4066Sahrens 1613fa9e4066Sahrens static int 1614fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1615fa9e4066Sahrens dmu_buf_impl_t **parentp, blkptr_t **bpp) 1616fa9e4066Sahrens { 1617fa9e4066Sahrens int nlevels, epbs; 1618fa9e4066Sahrens 16190b69c2f0Sahrens *parentp = NULL; 16200b69c2f0Sahrens *bpp = NULL; 16210b69c2f0Sahrens 16220a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 16230a586ceaSMark Shellenbaum 16240a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) { 16250a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 162606e0070dSMark Shellenbaum if (dn->dn_have_spill && 162706e0070dSMark Shellenbaum (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 16280a586ceaSMark Shellenbaum *bpp = &dn->dn_phys->dn_spill; 16290a586ceaSMark Shellenbaum else 16300a586ceaSMark Shellenbaum *bpp = NULL; 16310a586ceaSMark Shellenbaum dbuf_add_ref(dn->dn_dbuf, NULL); 16320a586ceaSMark Shellenbaum *parentp = dn->dn_dbuf; 16330a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 16340a586ceaSMark Shellenbaum return (0); 16350a586ceaSMark Shellenbaum } 1636ea8dc4b6Seschrock 1637fa9e4066Sahrens if (dn->dn_phys->dn_nlevels == 0) 1638fa9e4066Sahrens nlevels = 1; 1639fa9e4066Sahrens else 1640fa9e4066Sahrens nlevels = dn->dn_phys->dn_nlevels; 1641fa9e4066Sahrens 1642fa9e4066Sahrens epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1643fa9e4066Sahrens 1644fa9e4066Sahrens ASSERT3U(level * epbs, <, 64); 1645fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1646ea8dc4b6Seschrock if (level >= nlevels || 1647fa9e4066Sahrens (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1648fa9e4066Sahrens /* the buffer has no parent yet */ 1649be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 1650fa9e4066Sahrens } else if (level < nlevels-1) { 1651fa9e4066Sahrens /* this block is referenced from an indirect block */ 1652fa9e4066Sahrens int err = dbuf_hold_impl(dn, level+1, 1653fa9e4066Sahrens blkid >> epbs, fail_sparse, NULL, parentp); 1654fa9e4066Sahrens if (err) 1655fa9e4066Sahrens return (err); 1656ea8dc4b6Seschrock err = dbuf_read(*parentp, NULL, 1657ea8dc4b6Seschrock (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1658c543ec06Sahrens if (err) { 1659c543ec06Sahrens dbuf_rele(*parentp, NULL); 1660c543ec06Sahrens *parentp = NULL; 1661c543ec06Sahrens return (err); 1662ea8dc4b6Seschrock } 1663c543ec06Sahrens *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1664c543ec06Sahrens (blkid & ((1ULL << epbs) - 1)); 1665c543ec06Sahrens return (0); 1666fa9e4066Sahrens } else { 1667fa9e4066Sahrens /* the block is referenced from the dnode */ 1668fa9e4066Sahrens ASSERT3U(level, ==, nlevels-1); 1669fa9e4066Sahrens ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1670fa9e4066Sahrens blkid < dn->dn_phys->dn_nblkptr); 1671c543ec06Sahrens if (dn->dn_dbuf) { 1672c543ec06Sahrens dbuf_add_ref(dn->dn_dbuf, NULL); 1673c543ec06Sahrens *parentp = dn->dn_dbuf; 1674c543ec06Sahrens } 1675fa9e4066Sahrens *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1676fa9e4066Sahrens return (0); 1677fa9e4066Sahrens } 1678fa9e4066Sahrens } 1679fa9e4066Sahrens 1680fa9e4066Sahrens static dmu_buf_impl_t * 1681fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1682fa9e4066Sahrens dmu_buf_impl_t *parent, blkptr_t *blkptr) 1683fa9e4066Sahrens { 1684503ad85cSMatthew Ahrens objset_t *os = dn->dn_objset; 1685fa9e4066Sahrens dmu_buf_impl_t *db, *odb; 1686fa9e4066Sahrens 1687fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1688fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE); 1689fa9e4066Sahrens 1690fa9e4066Sahrens db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1691fa9e4066Sahrens 1692fa9e4066Sahrens db->db_objset = os; 1693fa9e4066Sahrens db->db.db_object = dn->dn_object; 1694fa9e4066Sahrens db->db_level = level; 1695fa9e4066Sahrens db->db_blkid = blkid; 1696c717a561Smaybee db->db_last_dirty = NULL; 1697ea8dc4b6Seschrock db->db_dirtycnt = 0; 1698744947dcSTom Erickson db->db_dnode_handle = dn->dn_handle; 1699ea8dc4b6Seschrock db->db_parent = parent; 1700ea8dc4b6Seschrock db->db_blkptr = blkptr; 1701fa9e4066Sahrens 1702c717a561Smaybee db->db_user_ptr = NULL; 1703c717a561Smaybee db->db_user_data_ptr_ptr = NULL; 1704c717a561Smaybee db->db_evict_func = NULL; 1705c717a561Smaybee db->db_immediate_evict = 0; 1706c717a561Smaybee db->db_freed_in_flight = 0; 1707ea8dc4b6Seschrock 17080a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) { 1709ea8dc4b6Seschrock ASSERT3P(parent, ==, dn->dn_dbuf); 17101934e92fSmaybee db->db.db_size = DN_MAX_BONUSLEN - 17111934e92fSmaybee (dn->dn_nblkptr-1) * sizeof (blkptr_t); 17121934e92fSmaybee ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 17130a586ceaSMark Shellenbaum db->db.db_offset = DMU_BONUS_BLKID; 1714ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1715ea8dc4b6Seschrock /* the bonus dbuf is not placed in the hash table */ 17165a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1717ea8dc4b6Seschrock return (db); 17180a586ceaSMark Shellenbaum } else if (blkid == DMU_SPILL_BLKID) { 17190a586ceaSMark Shellenbaum db->db.db_size = (blkptr != NULL) ? 17200a586ceaSMark Shellenbaum BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 17210a586ceaSMark Shellenbaum db->db.db_offset = 0; 1722fa9e4066Sahrens } else { 1723fa9e4066Sahrens int blocksize = 172469962b56SMatthew Ahrens db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 1725fa9e4066Sahrens db->db.db_size = blocksize; 1726fa9e4066Sahrens db->db.db_offset = db->db_blkid * blocksize; 1727fa9e4066Sahrens } 1728fa9e4066Sahrens 1729fa9e4066Sahrens /* 1730fa9e4066Sahrens * Hold the dn_dbufs_mtx while we get the new dbuf 1731fa9e4066Sahrens * in the hash table *and* added to the dbufs list. 1732fa9e4066Sahrens * This prevents a possible deadlock with someone 1733fa9e4066Sahrens * trying to look up this dbuf before its added to the 1734fa9e4066Sahrens * dn_dbufs list. 1735fa9e4066Sahrens */ 1736fa9e4066Sahrens mutex_enter(&dn->dn_dbufs_mtx); 1737ea8dc4b6Seschrock db->db_state = DB_EVICTING; 1738fa9e4066Sahrens if ((odb = dbuf_hash_insert(db)) != NULL) { 1739fa9e4066Sahrens /* someone else inserted it first */ 1740fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 1741fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1742fa9e4066Sahrens return (odb); 1743fa9e4066Sahrens } 1744fa9e4066Sahrens list_insert_head(&dn->dn_dbufs, db); 1745713d6c20SMatthew Ahrens if (db->db_level == 0 && db->db_blkid >= 1746713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid) 1747713d6c20SMatthew Ahrens dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1748ea8dc4b6Seschrock db->db_state = DB_UNCACHED; 1749fa9e4066Sahrens mutex_exit(&dn->dn_dbufs_mtx); 17505a98e54bSBrendan Gregg - Sun Microsystems arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1751fa9e4066Sahrens 1752fa9e4066Sahrens if (parent && parent != dn->dn_dbuf) 1753fa9e4066Sahrens dbuf_add_ref(parent, db); 1754fa9e4066Sahrens 1755ea8dc4b6Seschrock ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1756ea8dc4b6Seschrock refcount_count(&dn->dn_holds) > 0); 1757fa9e4066Sahrens (void) refcount_add(&dn->dn_holds, db); 1758744947dcSTom Erickson (void) atomic_inc_32_nv(&dn->dn_dbufs_count); 1759fa9e4066Sahrens 1760fa9e4066Sahrens dprintf_dbuf(db, "db=%p\n", db); 1761fa9e4066Sahrens 1762fa9e4066Sahrens return (db); 1763fa9e4066Sahrens } 1764fa9e4066Sahrens 1765fa9e4066Sahrens static int 1766ea8dc4b6Seschrock dbuf_do_evict(void *private) 1767fa9e4066Sahrens { 1768ea8dc4b6Seschrock arc_buf_t *buf = private; 1769ea8dc4b6Seschrock dmu_buf_impl_t *db = buf->b_private; 1770fa9e4066Sahrens 1771ea8dc4b6Seschrock if (!MUTEX_HELD(&db->db_mtx)) 1772ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 1773fa9e4066Sahrens 1774ea8dc4b6Seschrock ASSERT(refcount_is_zero(&db->db_holds)); 1775fa9e4066Sahrens 1776ea8dc4b6Seschrock if (db->db_state != DB_EVICTING) { 1777ea8dc4b6Seschrock ASSERT(db->db_state == DB_CACHED); 1778ea8dc4b6Seschrock DBUF_VERIFY(db); 1779ea8dc4b6Seschrock db->db_buf = NULL; 1780ea8dc4b6Seschrock dbuf_evict(db); 1781ea8dc4b6Seschrock } else { 1782ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 1783ea8dc4b6Seschrock dbuf_destroy(db); 1784fa9e4066Sahrens } 1785ea8dc4b6Seschrock return (0); 1786fa9e4066Sahrens } 1787fa9e4066Sahrens 1788fa9e4066Sahrens static void 1789fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db) 1790fa9e4066Sahrens { 1791fa9e4066Sahrens ASSERT(refcount_is_zero(&db->db_holds)); 1792fa9e4066Sahrens 17930a586ceaSMark Shellenbaum if (db->db_blkid != DMU_BONUS_BLKID) { 1794ea8dc4b6Seschrock /* 1795ea8dc4b6Seschrock * If this dbuf is still on the dn_dbufs list, 1796ea8dc4b6Seschrock * remove it from that list. 1797ea8dc4b6Seschrock */ 1798744947dcSTom Erickson if (db->db_dnode_handle != NULL) { 1799744947dcSTom Erickson dnode_t *dn; 18001934e92fSmaybee 1801744947dcSTom Erickson DB_DNODE_ENTER(db); 1802744947dcSTom Erickson dn = DB_DNODE(db); 18031934e92fSmaybee mutex_enter(&dn->dn_dbufs_mtx); 1804ea8dc4b6Seschrock list_remove(&dn->dn_dbufs, db); 1805744947dcSTom Erickson (void) atomic_dec_32_nv(&dn->dn_dbufs_count); 1806c543ec06Sahrens mutex_exit(&dn->dn_dbufs_mtx); 1807744947dcSTom Erickson DB_DNODE_EXIT(db); 1808744947dcSTom Erickson /* 1809744947dcSTom Erickson * Decrementing the dbuf count means that the hold 1810744947dcSTom Erickson * corresponding to the removed dbuf is no longer 1811744947dcSTom Erickson * discounted in dnode_move(), so the dnode cannot be 1812744947dcSTom Erickson * moved until after we release the hold. 1813744947dcSTom Erickson */ 1814ea8dc4b6Seschrock dnode_rele(dn, db); 1815744947dcSTom Erickson db->db_dnode_handle = NULL; 1816ea8dc4b6Seschrock } 1817ea8dc4b6Seschrock dbuf_hash_remove(db); 1818ea8dc4b6Seschrock } 1819ea8dc4b6Seschrock db->db_parent = NULL; 1820ea8dc4b6Seschrock db->db_buf = NULL; 1821ea8dc4b6Seschrock 1822f82bfe17Sgw ASSERT(!list_link_active(&db->db_link)); 1823fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 1824fa9e4066Sahrens ASSERT(db->db_hash_next == NULL); 1825fa9e4066Sahrens ASSERT(db->db_blkptr == NULL); 1826fa9e4066Sahrens ASSERT(db->db_data_pending == NULL); 1827fa9e4066Sahrens 1828fa9e4066Sahrens kmem_cache_free(dbuf_cache, db); 18295a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1830fa9e4066Sahrens } 1831fa9e4066Sahrens 1832fa9e4066Sahrens void 183369962b56SMatthew Ahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio) 1834fa9e4066Sahrens { 183513506d1eSmaybee dmu_buf_impl_t *db = NULL; 1836fa9e4066Sahrens blkptr_t *bp = NULL; 1837fa9e4066Sahrens 18380a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1839fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1840fa9e4066Sahrens 1841fa9e4066Sahrens if (dnode_block_freed(dn, blkid)) 1842fa9e4066Sahrens return; 1843fa9e4066Sahrens 1844fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1845fa9e4066Sahrens if (db = dbuf_find(dn, 0, blkid)) { 18469e9c486fSGeorge Wilson /* 18479e9c486fSGeorge Wilson * This dbuf is already in the cache. We assume that 18489e9c486fSGeorge Wilson * it is already CACHED, or else about to be either 18499e9c486fSGeorge Wilson * read or filled. 18509e9c486fSGeorge Wilson */ 1851fa9e4066Sahrens mutex_exit(&db->db_mtx); 18529e9c486fSGeorge Wilson return; 1853fa9e4066Sahrens } 1854fa9e4066Sahrens 185513506d1eSmaybee if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 18565d7b4d43SMatthew Ahrens if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) { 1857b24ab676SJeff Bonwick dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 185813506d1eSmaybee uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1859*7802d7bfSMatthew Ahrens zbookmark_phys_t zb; 1860b24ab676SJeff Bonwick 1861b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1862b24ab676SJeff Bonwick dn->dn_object, 0, blkid); 1863ea8dc4b6Seschrock 18641b912ec7SGeorge Wilson (void) arc_read(NULL, dn->dn_objset->os_spa, 186569962b56SMatthew Ahrens bp, NULL, NULL, prio, 1866fa9e4066Sahrens ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 186713506d1eSmaybee &aflags, &zb); 1868fa9e4066Sahrens } 186913506d1eSmaybee if (db) 187013506d1eSmaybee dbuf_rele(db, NULL); 1871fa9e4066Sahrens } 1872fa9e4066Sahrens } 1873fa9e4066Sahrens 1874fa9e4066Sahrens /* 1875fa9e4066Sahrens * Returns with db_holds incremented, and db_mtx not held. 1876fa9e4066Sahrens * Note: dn_struct_rwlock must be held. 1877fa9e4066Sahrens */ 1878fa9e4066Sahrens int 1879fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1880fa9e4066Sahrens void *tag, dmu_buf_impl_t **dbp) 1881fa9e4066Sahrens { 1882fa9e4066Sahrens dmu_buf_impl_t *db, *parent = NULL; 1883fa9e4066Sahrens 18840a586ceaSMark Shellenbaum ASSERT(blkid != DMU_BONUS_BLKID); 1885fa9e4066Sahrens ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1886fa9e4066Sahrens ASSERT3U(dn->dn_nlevels, >, level); 1887fa9e4066Sahrens 1888fa9e4066Sahrens *dbp = NULL; 1889ea8dc4b6Seschrock top: 1890fa9e4066Sahrens /* dbuf_find() returns with db_mtx held */ 1891fa9e4066Sahrens db = dbuf_find(dn, level, blkid); 1892fa9e4066Sahrens 1893fa9e4066Sahrens if (db == NULL) { 1894fa9e4066Sahrens blkptr_t *bp = NULL; 1895fa9e4066Sahrens int err; 1896fa9e4066Sahrens 1897c543ec06Sahrens ASSERT3P(parent, ==, NULL); 1898fa9e4066Sahrens err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1899fa9e4066Sahrens if (fail_sparse) { 1900fa9e4066Sahrens if (err == 0 && bp && BP_IS_HOLE(bp)) 1901be6fd75aSMatthew Ahrens err = SET_ERROR(ENOENT); 1902fa9e4066Sahrens if (err) { 1903c543ec06Sahrens if (parent) 1904ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1905fa9e4066Sahrens return (err); 1906fa9e4066Sahrens } 1907fa9e4066Sahrens } 1908ea8dc4b6Seschrock if (err && err != ENOENT) 1909ea8dc4b6Seschrock return (err); 1910fa9e4066Sahrens db = dbuf_create(dn, level, blkid, parent, bp); 1911fa9e4066Sahrens } 1912fa9e4066Sahrens 1913ea8dc4b6Seschrock if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1914ea8dc4b6Seschrock arc_buf_add_ref(db->db_buf, db); 1915ea8dc4b6Seschrock if (db->db_buf->b_data == NULL) { 1916ea8dc4b6Seschrock dbuf_clear(db); 1917c543ec06Sahrens if (parent) { 1918c543ec06Sahrens dbuf_rele(parent, NULL); 1919c543ec06Sahrens parent = NULL; 1920c543ec06Sahrens } 1921ea8dc4b6Seschrock goto top; 1922ea8dc4b6Seschrock } 1923ea8dc4b6Seschrock ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1924ea8dc4b6Seschrock } 1925ea8dc4b6Seschrock 1926ea8dc4b6Seschrock ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1927ea8dc4b6Seschrock 1928fa9e4066Sahrens /* 1929c717a561Smaybee * If this buffer is currently syncing out, and we are are 1930c717a561Smaybee * still referencing it from db_data, we need to make a copy 1931c717a561Smaybee * of it in case we decide we want to dirty it again in this txg. 1932fa9e4066Sahrens */ 19330a586ceaSMark Shellenbaum if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1934ea8dc4b6Seschrock dn->dn_object != DMU_META_DNODE_OBJECT && 1935c717a561Smaybee db->db_state == DB_CACHED && db->db_data_pending) { 1936c717a561Smaybee dbuf_dirty_record_t *dr = db->db_data_pending; 1937fa9e4066Sahrens 1938c717a561Smaybee if (dr->dt.dl.dr_data == db->db_buf) { 1939c717a561Smaybee arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1940c717a561Smaybee 1941c717a561Smaybee dbuf_set_data(db, 1942744947dcSTom Erickson arc_buf_alloc(dn->dn_objset->os_spa, 1943c717a561Smaybee db->db.db_size, db, type)); 1944c717a561Smaybee bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 1945c717a561Smaybee db->db.db_size); 1946c717a561Smaybee } 1947fa9e4066Sahrens } 1948fa9e4066Sahrens 1949ea8dc4b6Seschrock (void) refcount_add(&db->db_holds, tag); 1950fa9e4066Sahrens dbuf_update_data(db); 19519c9dc39aSek DBUF_VERIFY(db); 1952fa9e4066Sahrens mutex_exit(&db->db_mtx); 1953fa9e4066Sahrens 1954fa9e4066Sahrens /* NOTE: we can't rele the parent until after we drop the db_mtx */ 1955c543ec06Sahrens if (parent) 1956ea8dc4b6Seschrock dbuf_rele(parent, NULL); 1957fa9e4066Sahrens 1958744947dcSTom Erickson ASSERT3P(DB_DNODE(db), ==, dn); 1959fa9e4066Sahrens ASSERT3U(db->db_blkid, ==, blkid); 1960fa9e4066Sahrens ASSERT3U(db->db_level, ==, level); 1961fa9e4066Sahrens *dbp = db; 1962fa9e4066Sahrens 1963fa9e4066Sahrens return (0); 1964fa9e4066Sahrens } 1965fa9e4066Sahrens 1966fa9e4066Sahrens dmu_buf_impl_t * 1967ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 1968fa9e4066Sahrens { 1969fa9e4066Sahrens dmu_buf_impl_t *db; 1970ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 1971ea8dc4b6Seschrock return (err ? NULL : db); 1972fa9e4066Sahrens } 1973fa9e4066Sahrens 1974fa9e4066Sahrens dmu_buf_impl_t * 1975fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 1976fa9e4066Sahrens { 1977fa9e4066Sahrens dmu_buf_impl_t *db; 1978ea8dc4b6Seschrock int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 1979ea8dc4b6Seschrock return (err ? NULL : db); 1980fa9e4066Sahrens } 1981fa9e4066Sahrens 19821934e92fSmaybee void 1983ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn) 1984fa9e4066Sahrens { 1985ea8dc4b6Seschrock ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 1986ea8dc4b6Seschrock 1987ea8dc4b6Seschrock ASSERT(dn->dn_bonus == NULL); 19880a586ceaSMark Shellenbaum dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 19890a586ceaSMark Shellenbaum } 19900a586ceaSMark Shellenbaum 19910a586ceaSMark Shellenbaum int 19920a586ceaSMark Shellenbaum dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 19930a586ceaSMark Shellenbaum { 19940a586ceaSMark Shellenbaum dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1995744947dcSTom Erickson dnode_t *dn; 1996744947dcSTom Erickson 19970a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 1998be6fd75aSMatthew Ahrens return (SET_ERROR(ENOTSUP)); 19990a586ceaSMark Shellenbaum if (blksz == 0) 20000a586ceaSMark Shellenbaum blksz = SPA_MINBLOCKSIZE; 20010a586ceaSMark Shellenbaum if (blksz > SPA_MAXBLOCKSIZE) 20020a586ceaSMark Shellenbaum blksz = SPA_MAXBLOCKSIZE; 20030a586ceaSMark Shellenbaum else 20040a586ceaSMark Shellenbaum blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 20050a586ceaSMark Shellenbaum 2006744947dcSTom Erickson DB_DNODE_ENTER(db); 2007744947dcSTom Erickson dn = DB_DNODE(db); 2008744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 20090a586ceaSMark Shellenbaum dbuf_new_size(db, blksz, tx); 2010744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 2011744947dcSTom Erickson DB_DNODE_EXIT(db); 20120a586ceaSMark Shellenbaum 20130a586ceaSMark Shellenbaum return (0); 20140a586ceaSMark Shellenbaum } 20150a586ceaSMark Shellenbaum 20160a586ceaSMark Shellenbaum void 20170a586ceaSMark Shellenbaum dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 20180a586ceaSMark Shellenbaum { 20190a586ceaSMark Shellenbaum dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2020fa9e4066Sahrens } 2021fa9e4066Sahrens 2022ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref 2023fa9e4066Sahrens void 2024fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2025fa9e4066Sahrens { 2026ea8dc4b6Seschrock int64_t holds = refcount_add(&db->db_holds, tag); 2027ea8dc4b6Seschrock ASSERT(holds > 1); 2028fa9e4066Sahrens } 2029fa9e4066Sahrens 2030744947dcSTom Erickson /* 2031744947dcSTom Erickson * If you call dbuf_rele() you had better not be referencing the dnode handle 2032744947dcSTom Erickson * unless you have some other direct or indirect hold on the dnode. (An indirect 2033744947dcSTom Erickson * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2034744947dcSTom Erickson * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2035744947dcSTom Erickson * dnode's parent dbuf evicting its dnode handles. 2036744947dcSTom Erickson */ 2037fa9e4066Sahrens void 2038ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag) 2039b24ab676SJeff Bonwick { 2040b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2041b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, tag); 2042b24ab676SJeff Bonwick } 2043b24ab676SJeff Bonwick 204443466aaeSMax Grossman void 204543466aaeSMax Grossman dmu_buf_rele(dmu_buf_t *db, void *tag) 204643466aaeSMax Grossman { 204743466aaeSMax Grossman dbuf_rele((dmu_buf_impl_t *)db, tag); 204843466aaeSMax Grossman } 204943466aaeSMax Grossman 2050b24ab676SJeff Bonwick /* 2051b24ab676SJeff Bonwick * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2052b24ab676SJeff Bonwick * db_dirtycnt and db_holds to be updated atomically. 2053b24ab676SJeff Bonwick */ 2054b24ab676SJeff Bonwick void 2055b24ab676SJeff Bonwick dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2056fa9e4066Sahrens { 2057fa9e4066Sahrens int64_t holds; 2058fa9e4066Sahrens 2059b24ab676SJeff Bonwick ASSERT(MUTEX_HELD(&db->db_mtx)); 20609c9dc39aSek DBUF_VERIFY(db); 2061fa9e4066Sahrens 2062744947dcSTom Erickson /* 2063744947dcSTom Erickson * Remove the reference to the dbuf before removing its hold on the 2064744947dcSTom Erickson * dnode so we can guarantee in dnode_move() that a referenced bonus 2065744947dcSTom Erickson * buffer has a corresponding dnode hold. 2066744947dcSTom Erickson */ 2067fa9e4066Sahrens holds = refcount_remove(&db->db_holds, tag); 2068ea8dc4b6Seschrock ASSERT(holds >= 0); 2069ea8dc4b6Seschrock 2070c717a561Smaybee /* 2071c717a561Smaybee * We can't freeze indirects if there is a possibility that they 2072c717a561Smaybee * may be modified in the current syncing context. 2073c717a561Smaybee */ 2074c717a561Smaybee if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 20756b4acc8bSahrens arc_buf_freeze(db->db_buf); 20766b4acc8bSahrens 2077ea8dc4b6Seschrock if (holds == db->db_dirtycnt && 2078c717a561Smaybee db->db_level == 0 && db->db_immediate_evict) 2079ea8dc4b6Seschrock dbuf_evict_user(db); 2080fa9e4066Sahrens 2081fa9e4066Sahrens if (holds == 0) { 20820a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2083ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 2084744947dcSTom Erickson 2085744947dcSTom Erickson /* 2086744947dcSTom Erickson * If the dnode moves here, we cannot cross this barrier 2087744947dcSTom Erickson * until the move completes. 2088744947dcSTom Erickson */ 2089744947dcSTom Erickson DB_DNODE_ENTER(db); 2090744947dcSTom Erickson (void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count); 2091744947dcSTom Erickson DB_DNODE_EXIT(db); 2092744947dcSTom Erickson /* 2093744947dcSTom Erickson * The bonus buffer's dnode hold is no longer discounted 2094744947dcSTom Erickson * in dnode_move(). The dnode cannot move until after 2095744947dcSTom Erickson * the dnode_rele(). 2096744947dcSTom Erickson */ 2097744947dcSTom Erickson dnode_rele(DB_DNODE(db), db); 2098ea8dc4b6Seschrock } else if (db->db_buf == NULL) { 2099ea8dc4b6Seschrock /* 2100ea8dc4b6Seschrock * This is a special case: we never associated this 2101ea8dc4b6Seschrock * dbuf with any data allocated from the ARC. 2102ea8dc4b6Seschrock */ 210382c9918fSTim Haley ASSERT(db->db_state == DB_UNCACHED || 210482c9918fSTim Haley db->db_state == DB_NOFILL); 2105ea8dc4b6Seschrock dbuf_evict(db); 21066b4acc8bSahrens } else if (arc_released(db->db_buf)) { 2107ea8dc4b6Seschrock arc_buf_t *buf = db->db_buf; 2108ea8dc4b6Seschrock /* 2109ea8dc4b6Seschrock * This dbuf has anonymous data associated with it. 2110ea8dc4b6Seschrock */ 2111ea8dc4b6Seschrock dbuf_set_data(db, NULL); 21123b2aab18SMatthew Ahrens VERIFY(arc_buf_remove_ref(buf, db)); 2113ea8dc4b6Seschrock dbuf_evict(db); 2114ea8dc4b6Seschrock } else { 21153b2aab18SMatthew Ahrens VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 21169253d63dSGeorge Wilson 21179253d63dSGeorge Wilson /* 21189253d63dSGeorge Wilson * A dbuf will be eligible for eviction if either the 21199253d63dSGeorge Wilson * 'primarycache' property is set or a duplicate 21209253d63dSGeorge Wilson * copy of this buffer is already cached in the arc. 21219253d63dSGeorge Wilson * 21229253d63dSGeorge Wilson * In the case of the 'primarycache' a buffer 21239253d63dSGeorge Wilson * is considered for eviction if it matches the 21249253d63dSGeorge Wilson * criteria set in the property. 21259253d63dSGeorge Wilson * 21269253d63dSGeorge Wilson * To decide if our buffer is considered a 21279253d63dSGeorge Wilson * duplicate, we must call into the arc to determine 21289253d63dSGeorge Wilson * if multiple buffers are referencing the same 21299253d63dSGeorge Wilson * block on-disk. If so, then we simply evict 21309253d63dSGeorge Wilson * ourselves. 21319253d63dSGeorge Wilson */ 21329253d63dSGeorge Wilson if (!DBUF_IS_CACHEABLE(db) || 21339253d63dSGeorge Wilson arc_buf_eviction_needed(db->db_buf)) 21343baa08fcSek dbuf_clear(db); 21353baa08fcSek else 21363baa08fcSek mutex_exit(&db->db_mtx); 2137ea8dc4b6Seschrock } 2138fa9e4066Sahrens } else { 2139fa9e4066Sahrens mutex_exit(&db->db_mtx); 2140fa9e4066Sahrens } 2141fa9e4066Sahrens } 2142fa9e4066Sahrens 2143fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount 2144fa9e4066Sahrens uint64_t 2145fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db) 2146fa9e4066Sahrens { 2147fa9e4066Sahrens return (refcount_count(&db->db_holds)); 2148fa9e4066Sahrens } 2149fa9e4066Sahrens 2150fa9e4066Sahrens void * 2151fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2152fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2153fa9e4066Sahrens { 2154fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2155fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2156fa9e4066Sahrens } 2157fa9e4066Sahrens 2158fa9e4066Sahrens void * 2159fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2160fa9e4066Sahrens dmu_buf_evict_func_t *evict_func) 2161fa9e4066Sahrens { 2162fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2163fa9e4066Sahrens 2164c717a561Smaybee db->db_immediate_evict = TRUE; 2165fa9e4066Sahrens return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2166fa9e4066Sahrens user_data_ptr_ptr, evict_func)); 2167fa9e4066Sahrens } 2168fa9e4066Sahrens 2169fa9e4066Sahrens void * 2170fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2171fa9e4066Sahrens void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2172fa9e4066Sahrens { 2173fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2174fa9e4066Sahrens ASSERT(db->db_level == 0); 2175fa9e4066Sahrens 2176fa9e4066Sahrens ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2177fa9e4066Sahrens 2178fa9e4066Sahrens mutex_enter(&db->db_mtx); 2179fa9e4066Sahrens 2180c717a561Smaybee if (db->db_user_ptr == old_user_ptr) { 2181c717a561Smaybee db->db_user_ptr = user_ptr; 2182c717a561Smaybee db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2183c717a561Smaybee db->db_evict_func = evict_func; 2184fa9e4066Sahrens 2185fa9e4066Sahrens dbuf_update_data(db); 2186fa9e4066Sahrens } else { 2187c717a561Smaybee old_user_ptr = db->db_user_ptr; 2188fa9e4066Sahrens } 2189fa9e4066Sahrens 2190fa9e4066Sahrens mutex_exit(&db->db_mtx); 2191fa9e4066Sahrens return (old_user_ptr); 2192fa9e4066Sahrens } 2193fa9e4066Sahrens 2194fa9e4066Sahrens void * 2195fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake) 2196fa9e4066Sahrens { 2197fa9e4066Sahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2198fa9e4066Sahrens ASSERT(!refcount_is_zero(&db->db_holds)); 2199fa9e4066Sahrens 2200c717a561Smaybee return (db->db_user_ptr); 2201fa9e4066Sahrens } 2202fa9e4066Sahrens 22033d692628SSanjeev Bagewadi boolean_t 22043d692628SSanjeev Bagewadi dmu_buf_freeable(dmu_buf_t *dbuf) 22053d692628SSanjeev Bagewadi { 22063d692628SSanjeev Bagewadi boolean_t res = B_FALSE; 22073d692628SSanjeev Bagewadi dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 22083d692628SSanjeev Bagewadi 22093d692628SSanjeev Bagewadi if (db->db_blkptr) 22103d692628SSanjeev Bagewadi res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2211c7cd2421SGeorge Wilson db->db_blkptr, db->db_blkptr->blk_birth); 22123d692628SSanjeev Bagewadi 22133d692628SSanjeev Bagewadi return (res); 22143d692628SSanjeev Bagewadi } 22153d692628SSanjeev Bagewadi 221680901aeaSGeorge Wilson blkptr_t * 221780901aeaSGeorge Wilson dmu_buf_get_blkptr(dmu_buf_t *db) 221880901aeaSGeorge Wilson { 221980901aeaSGeorge Wilson dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 222080901aeaSGeorge Wilson return (dbi->db_blkptr); 222180901aeaSGeorge Wilson } 222280901aeaSGeorge Wilson 2223c717a561Smaybee static void 2224c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2225fa9e4066Sahrens { 2226c717a561Smaybee /* ASSERT(dmu_tx_is_syncing(tx) */ 2227c717a561Smaybee ASSERT(MUTEX_HELD(&db->db_mtx)); 2228c717a561Smaybee 2229c717a561Smaybee if (db->db_blkptr != NULL) 2230c717a561Smaybee return; 2231c717a561Smaybee 22320a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 22330a586ceaSMark Shellenbaum db->db_blkptr = &dn->dn_phys->dn_spill; 22340a586ceaSMark Shellenbaum BP_ZERO(db->db_blkptr); 22350a586ceaSMark Shellenbaum return; 22360a586ceaSMark Shellenbaum } 2237c717a561Smaybee if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2238c717a561Smaybee /* 2239c717a561Smaybee * This buffer was allocated at a time when there was 2240c717a561Smaybee * no available blkptrs from the dnode, or it was 2241c717a561Smaybee * inappropriate to hook it in (i.e., nlevels mis-match). 2242c717a561Smaybee */ 2243c717a561Smaybee ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2244c717a561Smaybee ASSERT(db->db_parent == NULL); 2245c717a561Smaybee db->db_parent = dn->dn_dbuf; 2246c717a561Smaybee db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2247c717a561Smaybee DBUF_VERIFY(db); 2248c717a561Smaybee } else { 2249c717a561Smaybee dmu_buf_impl_t *parent = db->db_parent; 2250c717a561Smaybee int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2251c717a561Smaybee 2252c717a561Smaybee ASSERT(dn->dn_phys->dn_nlevels > 1); 2253c717a561Smaybee if (parent == NULL) { 2254c717a561Smaybee mutex_exit(&db->db_mtx); 2255c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_READER); 2256c717a561Smaybee (void) dbuf_hold_impl(dn, db->db_level+1, 2257c717a561Smaybee db->db_blkid >> epbs, FALSE, db, &parent); 2258c717a561Smaybee rw_exit(&dn->dn_struct_rwlock); 2259c717a561Smaybee mutex_enter(&db->db_mtx); 2260c717a561Smaybee db->db_parent = parent; 2261c717a561Smaybee } 2262c717a561Smaybee db->db_blkptr = (blkptr_t *)parent->db.db_data + 2263c717a561Smaybee (db->db_blkid & ((1ULL << epbs) - 1)); 2264c717a561Smaybee DBUF_VERIFY(db); 2265c717a561Smaybee } 2266c717a561Smaybee } 2267c717a561Smaybee 2268c717a561Smaybee static void 2269c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2270c717a561Smaybee { 2271c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2272744947dcSTom Erickson dnode_t *dn; 2273c717a561Smaybee zio_t *zio; 2274c717a561Smaybee 2275c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx)); 2276c717a561Smaybee 2277c717a561Smaybee dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2278c717a561Smaybee 2279c717a561Smaybee mutex_enter(&db->db_mtx); 2280c717a561Smaybee 2281c717a561Smaybee ASSERT(db->db_level > 0); 2282c717a561Smaybee DBUF_VERIFY(db); 2283c717a561Smaybee 22843e30c24aSWill Andrews /* Read the block if it hasn't been read yet. */ 2285c717a561Smaybee if (db->db_buf == NULL) { 2286c717a561Smaybee mutex_exit(&db->db_mtx); 2287c717a561Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2288c717a561Smaybee mutex_enter(&db->db_mtx); 2289c717a561Smaybee } 2290c717a561Smaybee ASSERT3U(db->db_state, ==, DB_CACHED); 2291c717a561Smaybee ASSERT(db->db_buf != NULL); 2292c717a561Smaybee 2293744947dcSTom Erickson DB_DNODE_ENTER(db); 2294744947dcSTom Erickson dn = DB_DNODE(db); 22953e30c24aSWill Andrews /* Indirect block size must match what the dnode thinks it is. */ 2296744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2297c717a561Smaybee dbuf_check_blkptr(dn, db); 2298744947dcSTom Erickson DB_DNODE_EXIT(db); 2299c717a561Smaybee 23003e30c24aSWill Andrews /* Provide the pending dirty record to child dbufs */ 2301c717a561Smaybee db->db_data_pending = dr; 2302c717a561Smaybee 2303af2c4821Smaybee mutex_exit(&db->db_mtx); 2304088f3894Sahrens dbuf_write(dr, db->db_buf, tx); 2305c717a561Smaybee 2306c717a561Smaybee zio = dr->dr_zio; 2307c717a561Smaybee mutex_enter(&dr->dt.di.dr_mtx); 2308c717a561Smaybee dbuf_sync_list(&dr->dt.di.dr_children, tx); 2309c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2310c717a561Smaybee mutex_exit(&dr->dt.di.dr_mtx); 2311c717a561Smaybee zio_nowait(zio); 2312c717a561Smaybee } 2313c717a561Smaybee 2314c717a561Smaybee static void 2315c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2316c717a561Smaybee { 2317c717a561Smaybee arc_buf_t **datap = &dr->dt.dl.dr_data; 2318c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 2319744947dcSTom Erickson dnode_t *dn; 2320744947dcSTom Erickson objset_t *os; 2321c717a561Smaybee uint64_t txg = tx->tx_txg; 2322fa9e4066Sahrens 2323fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx)); 2324fa9e4066Sahrens 2325fa9e4066Sahrens dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2326fa9e4066Sahrens 2327fa9e4066Sahrens mutex_enter(&db->db_mtx); 2328fa9e4066Sahrens /* 2329fa9e4066Sahrens * To be synced, we must be dirtied. But we 2330fa9e4066Sahrens * might have been freed after the dirty. 2331fa9e4066Sahrens */ 2332fa9e4066Sahrens if (db->db_state == DB_UNCACHED) { 2333fa9e4066Sahrens /* This buffer has been freed since it was dirtied */ 2334fa9e4066Sahrens ASSERT(db->db.db_data == NULL); 2335fa9e4066Sahrens } else if (db->db_state == DB_FILL) { 2336fa9e4066Sahrens /* This buffer was freed and is now being re-filled */ 2337c717a561Smaybee ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2338fa9e4066Sahrens } else { 233982c9918fSTim Haley ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2340fa9e4066Sahrens } 23419c9dc39aSek DBUF_VERIFY(db); 2342fa9e4066Sahrens 2343744947dcSTom Erickson DB_DNODE_ENTER(db); 2344744947dcSTom Erickson dn = DB_DNODE(db); 2345744947dcSTom Erickson 23460a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 23470a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx); 23480a586ceaSMark Shellenbaum dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 23490a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx); 23500a586ceaSMark Shellenbaum } 23510a586ceaSMark Shellenbaum 2352fa9e4066Sahrens /* 2353c717a561Smaybee * If this is a bonus buffer, simply copy the bonus data into the 2354c717a561Smaybee * dnode. It will be written out when the dnode is synced (and it 2355c717a561Smaybee * will be synced, since it must have been dirty for dbuf_sync to 2356c717a561Smaybee * be called). 2357fa9e4066Sahrens */ 23580a586ceaSMark Shellenbaum if (db->db_blkid == DMU_BONUS_BLKID) { 2359c717a561Smaybee dbuf_dirty_record_t **drp; 23601934e92fSmaybee 2361ea8dc4b6Seschrock ASSERT(*datap != NULL); 2362fb09f5aaSMadhav Suresh ASSERT0(db->db_level); 2363ea8dc4b6Seschrock ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2364ea8dc4b6Seschrock bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2365744947dcSTom Erickson DB_DNODE_EXIT(db); 2366744947dcSTom Erickson 23670e8c6158Smaybee if (*datap != db->db.db_data) { 2368ea8dc4b6Seschrock zio_buf_free(*datap, DN_MAX_BONUSLEN); 23695a98e54bSBrendan Gregg - Sun Microsystems arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 23700e8c6158Smaybee } 2371ea8dc4b6Seschrock db->db_data_pending = NULL; 2372c717a561Smaybee drp = &db->db_last_dirty; 2373c717a561Smaybee while (*drp != dr) 2374c717a561Smaybee drp = &(*drp)->dr_next; 237517f17c2dSbonwick ASSERT(dr->dr_next == NULL); 2376b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 237717f17c2dSbonwick *drp = dr->dr_next; 2378c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2379ea8dc4b6Seschrock ASSERT(db->db_dirtycnt > 0); 2380ea8dc4b6Seschrock db->db_dirtycnt -= 1; 2381b24ab676SJeff Bonwick dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2382ea8dc4b6Seschrock return; 2383ea8dc4b6Seschrock } 2384ea8dc4b6Seschrock 2385744947dcSTom Erickson os = dn->dn_objset; 2386744947dcSTom Erickson 2387f82bfe17Sgw /* 2388f82bfe17Sgw * This function may have dropped the db_mtx lock allowing a dmu_sync 2389f82bfe17Sgw * operation to sneak in. As a result, we need to ensure that we 2390f82bfe17Sgw * don't check the dr_override_state until we have returned from 2391f82bfe17Sgw * dbuf_check_blkptr. 2392f82bfe17Sgw */ 2393f82bfe17Sgw dbuf_check_blkptr(dn, db); 2394f82bfe17Sgw 2395c717a561Smaybee /* 2396744947dcSTom Erickson * If this buffer is in the middle of an immediate write, 2397c717a561Smaybee * wait for the synchronous IO to complete. 2398c717a561Smaybee */ 2399c717a561Smaybee while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2400c717a561Smaybee ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2401c717a561Smaybee cv_wait(&db->db_changed, &db->db_mtx); 2402c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2403c717a561Smaybee } 2404c5c6ffa0Smaybee 2405ab69d62fSMatthew Ahrens if (db->db_state != DB_NOFILL && 2406ab69d62fSMatthew Ahrens dn->dn_object != DMU_META_DNODE_OBJECT && 2407ab69d62fSMatthew Ahrens refcount_count(&db->db_holds) > 1 && 2408b24ab676SJeff Bonwick dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2409ab69d62fSMatthew Ahrens *datap == db->db_buf) { 2410ab69d62fSMatthew Ahrens /* 2411ab69d62fSMatthew Ahrens * If this buffer is currently "in use" (i.e., there 2412ab69d62fSMatthew Ahrens * are active holds and db_data still references it), 2413ab69d62fSMatthew Ahrens * then make a copy before we start the write so that 2414ab69d62fSMatthew Ahrens * any modifications from the open txg will not leak 2415ab69d62fSMatthew Ahrens * into this write. 2416ab69d62fSMatthew Ahrens * 2417ab69d62fSMatthew Ahrens * NOTE: this copy does not need to be made for 2418ab69d62fSMatthew Ahrens * objects only modified in the syncing context (e.g. 2419ab69d62fSMatthew Ahrens * DNONE_DNODE blocks). 2420ab69d62fSMatthew Ahrens */ 2421ab69d62fSMatthew Ahrens int blksz = arc_buf_size(*datap); 2422ab69d62fSMatthew Ahrens arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2423ab69d62fSMatthew Ahrens *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2424ab69d62fSMatthew Ahrens bcopy(db->db.db_data, (*datap)->b_data, blksz); 242582c9918fSTim Haley } 2426c717a561Smaybee db->db_data_pending = dr; 2427fa9e4066Sahrens 2428c717a561Smaybee mutex_exit(&db->db_mtx); 2429fa9e4066Sahrens 2430088f3894Sahrens dbuf_write(dr, *datap, tx); 2431fa9e4066Sahrens 2432c717a561Smaybee ASSERT(!list_link_active(&dr->dr_dirty_node)); 2433744947dcSTom Erickson if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2434c717a561Smaybee list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2435744947dcSTom Erickson DB_DNODE_EXIT(db); 2436744947dcSTom Erickson } else { 2437744947dcSTom Erickson /* 2438744947dcSTom Erickson * Although zio_nowait() does not "wait for an IO", it does 2439744947dcSTom Erickson * initiate the IO. If this is an empty write it seems plausible 2440744947dcSTom Erickson * that the IO could actually be completed before the nowait 2441744947dcSTom Erickson * returns. We need to DB_DNODE_EXIT() first in case 2442744947dcSTom Erickson * zio_nowait() invalidates the dbuf. 2443744947dcSTom Erickson */ 2444744947dcSTom Erickson DB_DNODE_EXIT(db); 2445c717a561Smaybee zio_nowait(dr->dr_zio); 2446744947dcSTom Erickson } 2447c717a561Smaybee } 244823b11526Smaybee 2449c717a561Smaybee void 2450c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2451c717a561Smaybee { 2452c717a561Smaybee dbuf_dirty_record_t *dr; 2453c717a561Smaybee 2454c717a561Smaybee while (dr = list_head(list)) { 2455c717a561Smaybee if (dr->dr_zio != NULL) { 2456c717a561Smaybee /* 2457c717a561Smaybee * If we find an already initialized zio then we 2458c717a561Smaybee * are processing the meta-dnode, and we have finished. 2459c717a561Smaybee * The dbufs for all dnodes are put back on the list 2460c717a561Smaybee * during processing, so that we can zio_wait() 2461c717a561Smaybee * these IOs after initiating all child IOs. 2462c717a561Smaybee */ 2463c717a561Smaybee ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2464c717a561Smaybee DMU_META_DNODE_OBJECT); 2465c717a561Smaybee break; 246623b11526Smaybee } 2467c717a561Smaybee list_remove(list, dr); 2468c717a561Smaybee if (dr->dr_dbuf->db_level > 0) 2469c717a561Smaybee dbuf_sync_indirect(dr, tx); 2470c717a561Smaybee else 2471c717a561Smaybee dbuf_sync_leaf(dr, tx); 247223b11526Smaybee } 2473c717a561Smaybee } 247423b11526Smaybee 2475fa9e4066Sahrens /* ARGSUSED */ 2476fa9e4066Sahrens static void 2477c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2478fa9e4066Sahrens { 2479fa9e4066Sahrens dmu_buf_impl_t *db = vdb; 2480744947dcSTom Erickson dnode_t *dn; 2481e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 2482c717a561Smaybee blkptr_t *bp_orig = &zio->io_bp_orig; 2483b24ab676SJeff Bonwick spa_t *spa = zio->io_spa; 2484b24ab676SJeff Bonwick int64_t delta; 2485fa9e4066Sahrens uint64_t fill = 0; 2486b24ab676SJeff Bonwick int i; 2487fa9e4066Sahrens 24885d7b4d43SMatthew Ahrens ASSERT3P(db->db_blkptr, ==, bp); 2489e14bb325SJeff Bonwick 2490744947dcSTom Erickson DB_DNODE_ENTER(db); 2491744947dcSTom Erickson dn = DB_DNODE(db); 2492b24ab676SJeff Bonwick delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2493b24ab676SJeff Bonwick dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2494b24ab676SJeff Bonwick zio->io_prev_space_delta = delta; 2495fa9e4066Sahrens 249643466aaeSMax Grossman if (bp->blk_birth != 0) { 249743466aaeSMax Grossman ASSERT((db->db_blkid != DMU_SPILL_BLKID && 249843466aaeSMax Grossman BP_GET_TYPE(bp) == dn->dn_type) || 249943466aaeSMax Grossman (db->db_blkid == DMU_SPILL_BLKID && 25005d7b4d43SMatthew Ahrens BP_GET_TYPE(bp) == dn->dn_bonustype) || 25015d7b4d43SMatthew Ahrens BP_IS_EMBEDDED(bp)); 250243466aaeSMax Grossman ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2503c717a561Smaybee } 2504c5c6ffa0Smaybee 2505c717a561Smaybee mutex_enter(&db->db_mtx); 2506fa9e4066Sahrens 25070a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 25080a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 25090a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 25100a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 25110a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 25120a586ceaSMark Shellenbaum } 25130a586ceaSMark Shellenbaum #endif 25140a586ceaSMark Shellenbaum 2515fa9e4066Sahrens if (db->db_level == 0) { 2516fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 25170a586ceaSMark Shellenbaum if (db->db_blkid > dn->dn_phys->dn_maxblkid && 25180a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) 2519fa9e4066Sahrens dn->dn_phys->dn_maxblkid = db->db_blkid; 2520fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 2521fa9e4066Sahrens 2522fa9e4066Sahrens if (dn->dn_type == DMU_OT_DNODE) { 2523fa9e4066Sahrens dnode_phys_t *dnp = db->db.db_data; 2524fa9e4066Sahrens for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2525fa9e4066Sahrens i--, dnp++) { 2526fa9e4066Sahrens if (dnp->dn_type != DMU_OT_NONE) 2527fa9e4066Sahrens fill++; 2528fa9e4066Sahrens } 2529fa9e4066Sahrens } else { 253043466aaeSMax Grossman if (BP_IS_HOLE(bp)) { 253143466aaeSMax Grossman fill = 0; 253243466aaeSMax Grossman } else { 253343466aaeSMax Grossman fill = 1; 253443466aaeSMax Grossman } 2535fa9e4066Sahrens } 2536fa9e4066Sahrens } else { 2537e14bb325SJeff Bonwick blkptr_t *ibp = db->db.db_data; 2538fa9e4066Sahrens ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2539e14bb325SJeff Bonwick for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2540e14bb325SJeff Bonwick if (BP_IS_HOLE(ibp)) 2541fa9e4066Sahrens continue; 25425d7b4d43SMatthew Ahrens fill += BP_GET_FILL(ibp); 2543fa9e4066Sahrens } 2544fa9e4066Sahrens } 2545744947dcSTom Erickson DB_DNODE_EXIT(db); 2546fa9e4066Sahrens 25475d7b4d43SMatthew Ahrens if (!BP_IS_EMBEDDED(bp)) 25485d7b4d43SMatthew Ahrens bp->blk_fill = fill; 2549c717a561Smaybee 2550c717a561Smaybee mutex_exit(&db->db_mtx); 2551c717a561Smaybee } 2552fa9e4066Sahrens 255369962b56SMatthew Ahrens /* 255469962b56SMatthew Ahrens * The SPA will call this callback several times for each zio - once 255569962b56SMatthew Ahrens * for every physical child i/o (zio->io_phys_children times). This 255669962b56SMatthew Ahrens * allows the DMU to monitor the progress of each logical i/o. For example, 255769962b56SMatthew Ahrens * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 255869962b56SMatthew Ahrens * block. There may be a long delay before all copies/fragments are completed, 255969962b56SMatthew Ahrens * so this callback allows us to retire dirty space gradually, as the physical 256069962b56SMatthew Ahrens * i/os complete. 256169962b56SMatthew Ahrens */ 256269962b56SMatthew Ahrens /* ARGSUSED */ 256369962b56SMatthew Ahrens static void 256469962b56SMatthew Ahrens dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 256569962b56SMatthew Ahrens { 256669962b56SMatthew Ahrens dmu_buf_impl_t *db = arg; 256769962b56SMatthew Ahrens objset_t *os = db->db_objset; 256869962b56SMatthew Ahrens dsl_pool_t *dp = dmu_objset_pool(os); 256969962b56SMatthew Ahrens dbuf_dirty_record_t *dr; 257069962b56SMatthew Ahrens int delta = 0; 257169962b56SMatthew Ahrens 257269962b56SMatthew Ahrens dr = db->db_data_pending; 257369962b56SMatthew Ahrens ASSERT3U(dr->dr_txg, ==, zio->io_txg); 257469962b56SMatthew Ahrens 257569962b56SMatthew Ahrens /* 257669962b56SMatthew Ahrens * The callback will be called io_phys_children times. Retire one 257769962b56SMatthew Ahrens * portion of our dirty space each time we are called. Any rounding 257869962b56SMatthew Ahrens * error will be cleaned up by dsl_pool_sync()'s call to 257969962b56SMatthew Ahrens * dsl_pool_undirty_space(). 258069962b56SMatthew Ahrens */ 258169962b56SMatthew Ahrens delta = dr->dr_accounted / zio->io_phys_children; 258269962b56SMatthew Ahrens dsl_pool_undirty_space(dp, delta, zio->io_txg); 258369962b56SMatthew Ahrens } 258469962b56SMatthew Ahrens 2585c717a561Smaybee /* ARGSUSED */ 2586c717a561Smaybee static void 2587c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2588c717a561Smaybee { 2589c717a561Smaybee dmu_buf_impl_t *db = vdb; 2590b24ab676SJeff Bonwick blkptr_t *bp_orig = &zio->io_bp_orig; 259143466aaeSMax Grossman blkptr_t *bp = db->db_blkptr; 259243466aaeSMax Grossman objset_t *os = db->db_objset; 259343466aaeSMax Grossman dmu_tx_t *tx = os->os_synctx; 2594c717a561Smaybee dbuf_dirty_record_t **drp, *dr; 2595c717a561Smaybee 2596fb09f5aaSMadhav Suresh ASSERT0(zio->io_error); 2597b24ab676SJeff Bonwick ASSERT(db->db_blkptr == bp); 2598b24ab676SJeff Bonwick 259980901aeaSGeorge Wilson /* 260080901aeaSGeorge Wilson * For nopwrites and rewrites we ensure that the bp matches our 260180901aeaSGeorge Wilson * original and bypass all the accounting. 260280901aeaSGeorge Wilson */ 260380901aeaSGeorge Wilson if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2604b24ab676SJeff Bonwick ASSERT(BP_EQUAL(bp, bp_orig)); 2605b24ab676SJeff Bonwick } else { 260643466aaeSMax Grossman dsl_dataset_t *ds = os->os_dsl_dataset; 2607b24ab676SJeff Bonwick (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2608b24ab676SJeff Bonwick dsl_dataset_block_born(ds, bp, tx); 2609b24ab676SJeff Bonwick } 2610c717a561Smaybee 2611c717a561Smaybee mutex_enter(&db->db_mtx); 2612c717a561Smaybee 2613b24ab676SJeff Bonwick DBUF_VERIFY(db); 2614b24ab676SJeff Bonwick 2615c717a561Smaybee drp = &db->db_last_dirty; 261617f17c2dSbonwick while ((dr = *drp) != db->db_data_pending) 261717f17c2dSbonwick drp = &dr->dr_next; 261817f17c2dSbonwick ASSERT(!list_link_active(&dr->dr_dirty_node)); 2619b24ab676SJeff Bonwick ASSERT(dr->dr_dbuf == db); 262017f17c2dSbonwick ASSERT(dr->dr_next == NULL); 262117f17c2dSbonwick *drp = dr->dr_next; 2622c717a561Smaybee 26230a586ceaSMark Shellenbaum #ifdef ZFS_DEBUG 26240a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) { 2625744947dcSTom Erickson dnode_t *dn; 2626744947dcSTom Erickson 2627744947dcSTom Erickson DB_DNODE_ENTER(db); 2628744947dcSTom Erickson dn = DB_DNODE(db); 26290a586ceaSMark Shellenbaum ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 26300a586ceaSMark Shellenbaum ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 26310a586ceaSMark Shellenbaum db->db_blkptr == &dn->dn_phys->dn_spill); 2632744947dcSTom Erickson DB_DNODE_EXIT(db); 26330a586ceaSMark Shellenbaum } 26340a586ceaSMark Shellenbaum #endif 26350a586ceaSMark Shellenbaum 2636c717a561Smaybee if (db->db_level == 0) { 26370a586ceaSMark Shellenbaum ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2638c717a561Smaybee ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 263982c9918fSTim Haley if (db->db_state != DB_NOFILL) { 264082c9918fSTim Haley if (dr->dt.dl.dr_data != db->db_buf) 264182c9918fSTim Haley VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 26423b2aab18SMatthew Ahrens db)); 2643b24ab676SJeff Bonwick else if (!arc_released(db->db_buf)) 264482c9918fSTim Haley arc_set_callback(db->db_buf, dbuf_do_evict, db); 264582c9918fSTim Haley } 2646c717a561Smaybee } else { 2647744947dcSTom Erickson dnode_t *dn; 2648744947dcSTom Erickson 2649744947dcSTom Erickson DB_DNODE_ENTER(db); 2650744947dcSTom Erickson dn = DB_DNODE(db); 2651c717a561Smaybee ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 265243466aaeSMax Grossman ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 2653c717a561Smaybee if (!BP_IS_HOLE(db->db_blkptr)) { 2654c717a561Smaybee int epbs = 2655c717a561Smaybee dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 265643466aaeSMax Grossman ASSERT3U(db->db_blkid, <=, 265743466aaeSMax Grossman dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 2658c717a561Smaybee ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2659c717a561Smaybee db->db.db_size); 26605d7b4d43SMatthew Ahrens if (!arc_released(db->db_buf)) 26615d7b4d43SMatthew Ahrens arc_set_callback(db->db_buf, dbuf_do_evict, db); 2662c717a561Smaybee } 2663744947dcSTom Erickson DB_DNODE_EXIT(db); 2664c25056deSgw mutex_destroy(&dr->dt.di.dr_mtx); 2665c25056deSgw list_destroy(&dr->dt.di.dr_children); 2666c717a561Smaybee } 2667c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2668fa9e4066Sahrens 2669fa9e4066Sahrens cv_broadcast(&db->db_changed); 2670fa9e4066Sahrens ASSERT(db->db_dirtycnt > 0); 2671fa9e4066Sahrens db->db_dirtycnt -= 1; 2672c717a561Smaybee db->db_data_pending = NULL; 267343466aaeSMax Grossman dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 2674b24ab676SJeff Bonwick } 2675b24ab676SJeff Bonwick 2676b24ab676SJeff Bonwick static void 2677b24ab676SJeff Bonwick dbuf_write_nofill_ready(zio_t *zio) 2678b24ab676SJeff Bonwick { 2679b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, zio->io_private); 2680b24ab676SJeff Bonwick } 2681b24ab676SJeff Bonwick 2682b24ab676SJeff Bonwick static void 2683b24ab676SJeff Bonwick dbuf_write_nofill_done(zio_t *zio) 2684b24ab676SJeff Bonwick { 2685b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, zio->io_private); 2686b24ab676SJeff Bonwick } 2687b24ab676SJeff Bonwick 2688b24ab676SJeff Bonwick static void 2689b24ab676SJeff Bonwick dbuf_write_override_ready(zio_t *zio) 2690b24ab676SJeff Bonwick { 2691b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2692b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2693b24ab676SJeff Bonwick 2694b24ab676SJeff Bonwick dbuf_write_ready(zio, NULL, db); 2695b24ab676SJeff Bonwick } 2696b24ab676SJeff Bonwick 2697b24ab676SJeff Bonwick static void 2698b24ab676SJeff Bonwick dbuf_write_override_done(zio_t *zio) 2699b24ab676SJeff Bonwick { 2700b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = zio->io_private; 2701b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2702b24ab676SJeff Bonwick blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2703b24ab676SJeff Bonwick 2704b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2705b24ab676SJeff Bonwick if (!BP_EQUAL(zio->io_bp, obp)) { 2706b24ab676SJeff Bonwick if (!BP_IS_HOLE(obp)) 2707b24ab676SJeff Bonwick dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2708b24ab676SJeff Bonwick arc_release(dr->dt.dl.dr_data, db); 2709b24ab676SJeff Bonwick } 2710fa9e4066Sahrens mutex_exit(&db->db_mtx); 2711fa9e4066Sahrens 2712b24ab676SJeff Bonwick dbuf_write_done(zio, NULL, db); 2713b24ab676SJeff Bonwick } 2714b24ab676SJeff Bonwick 27153e30c24aSWill Andrews /* Issue I/O to commit a dirty buffer to disk. */ 2716b24ab676SJeff Bonwick static void 2717b24ab676SJeff Bonwick dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2718b24ab676SJeff Bonwick { 2719b24ab676SJeff Bonwick dmu_buf_impl_t *db = dr->dr_dbuf; 2720744947dcSTom Erickson dnode_t *dn; 2721744947dcSTom Erickson objset_t *os; 2722b24ab676SJeff Bonwick dmu_buf_impl_t *parent = db->db_parent; 2723b24ab676SJeff Bonwick uint64_t txg = tx->tx_txg; 2724*7802d7bfSMatthew Ahrens zbookmark_phys_t zb; 2725b24ab676SJeff Bonwick zio_prop_t zp; 2726b24ab676SJeff Bonwick zio_t *zio; 27270a586ceaSMark Shellenbaum int wp_flag = 0; 2728b24ab676SJeff Bonwick 2729744947dcSTom Erickson DB_DNODE_ENTER(db); 2730744947dcSTom Erickson dn = DB_DNODE(db); 2731744947dcSTom Erickson os = dn->dn_objset; 2732744947dcSTom Erickson 2733b24ab676SJeff Bonwick if (db->db_state != DB_NOFILL) { 2734b24ab676SJeff Bonwick if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2735b24ab676SJeff Bonwick /* 2736b24ab676SJeff Bonwick * Private object buffers are released here rather 2737b24ab676SJeff Bonwick * than in dbuf_dirty() since they are only modified 2738b24ab676SJeff Bonwick * in the syncing context and we don't want the 2739b24ab676SJeff Bonwick * overhead of making multiple copies of the data. 2740b24ab676SJeff Bonwick */ 2741b24ab676SJeff Bonwick if (BP_IS_HOLE(db->db_blkptr)) { 2742b24ab676SJeff Bonwick arc_buf_thaw(data); 2743b24ab676SJeff Bonwick } else { 27443f9d6ad7SLin Ling dbuf_release_bp(db); 2745b24ab676SJeff Bonwick } 2746b24ab676SJeff Bonwick } 2747b24ab676SJeff Bonwick } 2748b24ab676SJeff Bonwick 2749b24ab676SJeff Bonwick if (parent != dn->dn_dbuf) { 27503e30c24aSWill Andrews /* Our parent is an indirect block. */ 27513e30c24aSWill Andrews /* We have a dirty parent that has been scheduled for write. */ 2752b24ab676SJeff Bonwick ASSERT(parent && parent->db_data_pending); 27533e30c24aSWill Andrews /* Our parent's buffer is one level closer to the dnode. */ 2754b24ab676SJeff Bonwick ASSERT(db->db_level == parent->db_level-1); 27553e30c24aSWill Andrews /* 27563e30c24aSWill Andrews * We're about to modify our parent's db_data by modifying 27573e30c24aSWill Andrews * our block pointer, so the parent must be released. 27583e30c24aSWill Andrews */ 2759b24ab676SJeff Bonwick ASSERT(arc_released(parent->db_buf)); 2760b24ab676SJeff Bonwick zio = parent->db_data_pending->dr_zio; 2761b24ab676SJeff Bonwick } else { 27623e30c24aSWill Andrews /* Our parent is the dnode itself. */ 27630a586ceaSMark Shellenbaum ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 27640a586ceaSMark Shellenbaum db->db_blkid != DMU_SPILL_BLKID) || 27650a586ceaSMark Shellenbaum (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 27660a586ceaSMark Shellenbaum if (db->db_blkid != DMU_SPILL_BLKID) 27670a586ceaSMark Shellenbaum ASSERT3P(db->db_blkptr, ==, 27680a586ceaSMark Shellenbaum &dn->dn_phys->dn_blkptr[db->db_blkid]); 2769b24ab676SJeff Bonwick zio = dn->dn_zio; 2770b24ab676SJeff Bonwick } 2771b24ab676SJeff Bonwick 2772b24ab676SJeff Bonwick ASSERT(db->db_level == 0 || data == db->db_buf); 2773b24ab676SJeff Bonwick ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2774b24ab676SJeff Bonwick ASSERT(zio); 2775fa9e4066Sahrens 2776b24ab676SJeff Bonwick SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2777b24ab676SJeff Bonwick os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2778b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 2779b24ab676SJeff Bonwick 27800a586ceaSMark Shellenbaum if (db->db_blkid == DMU_SPILL_BLKID) 27810a586ceaSMark Shellenbaum wp_flag = WP_SPILL; 27820a586ceaSMark Shellenbaum wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 27830a586ceaSMark Shellenbaum 27840a586ceaSMark Shellenbaum dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2785744947dcSTom Erickson DB_DNODE_EXIT(db); 2786b24ab676SJeff Bonwick 27875d7b4d43SMatthew Ahrens if (db->db_level == 0 && 27885d7b4d43SMatthew Ahrens dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 27895d7b4d43SMatthew Ahrens /* 27905d7b4d43SMatthew Ahrens * The BP for this block has been provided by open context 27915d7b4d43SMatthew Ahrens * (by dmu_sync() or dmu_buf_write_embedded()). 27925d7b4d43SMatthew Ahrens */ 27935d7b4d43SMatthew Ahrens void *contents = (data != NULL) ? data->b_data : NULL; 27945d7b4d43SMatthew Ahrens 2795b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 27965d7b4d43SMatthew Ahrens db->db_blkptr, contents, db->db.db_size, &zp, 279769962b56SMatthew Ahrens dbuf_write_override_ready, NULL, dbuf_write_override_done, 279869962b56SMatthew Ahrens dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2799b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 2800b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2801b24ab676SJeff Bonwick zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 280280901aeaSGeorge Wilson dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2803b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 2804b24ab676SJeff Bonwick } else if (db->db_state == DB_NOFILL) { 2805810e43b2SBill Pijewski ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 2806810e43b2SBill Pijewski zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 2807b24ab676SJeff Bonwick dr->dr_zio = zio_write(zio, os->os_spa, txg, 2808b24ab676SJeff Bonwick db->db_blkptr, NULL, db->db.db_size, &zp, 280969962b56SMatthew Ahrens dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, 2810b24ab676SJeff Bonwick ZIO_PRIORITY_ASYNC_WRITE, 2811b24ab676SJeff Bonwick ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2812b24ab676SJeff Bonwick } else { 2813b24ab676SJeff Bonwick ASSERT(arc_released(data)); 2814b24ab676SJeff Bonwick dr->dr_zio = arc_write(zio, os->os_spa, txg, 2815aad02571SSaso Kiselkov db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 2816aad02571SSaso Kiselkov DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 281769962b56SMatthew Ahrens dbuf_write_physdone, dbuf_write_done, db, 281869962b56SMatthew Ahrens ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2819b24ab676SJeff Bonwick } 2820fa9e4066Sahrens } 2821