Lines Matching refs:db
54 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
175 dmu_buf_impl_t *db = vdb; in dbuf_cons() local
176 bzero(db, sizeof (dmu_buf_impl_t)); in dbuf_cons()
178 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); in dbuf_cons()
179 rw_init(&db->db_rwlock, NULL, RW_DEFAULT, NULL); in dbuf_cons()
180 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); in dbuf_cons()
181 multilist_link_init(&db->db_cache_link); in dbuf_cons()
182 zfs_refcount_create(&db->db_holds); in dbuf_cons()
191 dmu_buf_impl_t *db = vdb; in dbuf_dest() local
192 mutex_destroy(&db->db_mtx); in dbuf_dest()
193 rw_destroy(&db->db_rwlock); in dbuf_dest()
194 cv_destroy(&db->db_changed); in dbuf_dest()
195 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_dest()
196 zfs_refcount_destroy(&db->db_holds); in dbuf_dest()
217 ((dbuf)->db.db_object == (obj) && \
228 dmu_buf_impl_t *db; in dbuf_find() local
231 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { in dbuf_find()
232 if (DBUF_EQUAL(db, os, obj, level, blkid)) { in dbuf_find()
233 mutex_enter(&db->db_mtx); in dbuf_find()
234 if (db->db_state != DB_EVICTING) { in dbuf_find()
236 return (db); in dbuf_find()
238 mutex_exit(&db->db_mtx); in dbuf_find()
249 dmu_buf_impl_t *db = NULL; in dbuf_find_bonus() local
254 db = dn->dn_bonus; in dbuf_find_bonus()
255 mutex_enter(&db->db_mtx); in dbuf_find_bonus()
260 return (db); in dbuf_find_bonus()
270 dbuf_hash_insert(dmu_buf_impl_t *db) in dbuf_hash_insert() argument
273 objset_t *os = db->db_objset; in dbuf_hash_insert()
274 uint64_t obj = db->db.db_object; in dbuf_hash_insert()
275 int level = db->db_level; in dbuf_hash_insert()
276 uint64_t blkid = db->db_blkid; in dbuf_hash_insert()
293 mutex_enter(&db->db_mtx); in dbuf_hash_insert()
294 db->db_hash_next = h->hash_table[idx]; in dbuf_hash_insert()
295 h->hash_table[idx] = db; in dbuf_hash_insert()
306 dbuf_hash_remove(dmu_buf_impl_t *db) in dbuf_hash_remove() argument
309 uint64_t hv = dbuf_hash(db->db_objset, db->db.db_object, in dbuf_hash_remove()
310 db->db_level, db->db_blkid); in dbuf_hash_remove()
318 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hash_remove()
319 ASSERT(db->db_state == DB_EVICTING); in dbuf_hash_remove()
320 ASSERT(!MUTEX_HELD(&db->db_mtx)); in dbuf_hash_remove()
324 while ((dbf = *dbp) != db) { in dbuf_hash_remove()
328 *dbp = db->db_hash_next; in dbuf_hash_remove()
329 db->db_hash_next = NULL; in dbuf_hash_remove()
340 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type) in dbuf_verify_user() argument
345 if (db->db_user == NULL) in dbuf_verify_user()
349 ASSERT(db->db_level == 0); in dbuf_verify_user()
352 ASSERT(db->db.db_data != NULL); in dbuf_verify_user()
353 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_verify_user()
355 holds = zfs_refcount_count(&db->db_holds); in dbuf_verify_user()
366 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
368 if (db->db_user_immediate_evict == TRUE) in dbuf_verify_user()
369 ASSERT3U(holds, >=, db->db_dirtycnt); in dbuf_verify_user()
377 dbuf_evict_user(dmu_buf_impl_t *db) in dbuf_evict_user() argument
379 dmu_buf_user_t *dbu = db->db_user; in dbuf_evict_user()
381 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_evict_user()
386 dbuf_verify_user(db, DBVU_EVICTING); in dbuf_evict_user()
387 db->db_user = NULL; in dbuf_evict_user()
416 dbuf_is_metadata(dmu_buf_impl_t *db) in dbuf_is_metadata() argument
418 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) { in dbuf_is_metadata()
423 DB_DNODE_ENTER(db); in dbuf_is_metadata()
424 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); in dbuf_is_metadata()
425 DB_DNODE_EXIT(db); in dbuf_is_metadata()
437 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db) in dbuf_include_in_metadata_cache() argument
439 DB_DNODE_ENTER(db); in dbuf_include_in_metadata_cache()
440 dmu_object_type_t type = DB_DNODE(db)->dn_type; in dbuf_include_in_metadata_cache()
441 DB_DNODE_EXIT(db); in dbuf_include_in_metadata_cache()
457 dmu_buf_impl_t *, db); in dbuf_include_in_metadata_cache()
477 dmu_buf_impl_t *db = obj; in dbuf_cache_multilist_index_func() local
491 return (dbuf_hash(db->db_objset, db->db.db_object, in dbuf_cache_multilist_index_func()
492 db->db_level, db->db_blkid) % in dbuf_cache_multilist_index_func()
528 dmu_buf_impl_t *db = multilist_sublist_tail(mls); in dbuf_evict_one() local
529 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) { in dbuf_evict_one()
530 db = multilist_sublist_prev(mls, db); in dbuf_evict_one()
533 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db, in dbuf_evict_one()
536 if (db != NULL) { in dbuf_evict_one()
537 multilist_sublist_remove(mls, db); in dbuf_evict_one()
541 db->db.db_size, db); in dbuf_evict_one()
542 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE); in dbuf_evict_one()
543 db->db_caching_status = DB_NO_CACHE; in dbuf_evict_one()
544 dbuf_destroy(db); in dbuf_evict_one()
719 dbuf_verify(dmu_buf_impl_t *db) in dbuf_verify() argument
724 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_verify()
729 ASSERT(db->db_objset != NULL); in dbuf_verify()
730 DB_DNODE_ENTER(db); in dbuf_verify()
731 dn = DB_DNODE(db); in dbuf_verify()
733 ASSERT(db->db_parent == NULL); in dbuf_verify()
734 ASSERT(db->db_blkptr == NULL); in dbuf_verify()
736 ASSERT3U(db->db.db_object, ==, dn->dn_object); in dbuf_verify()
737 ASSERT3P(db->db_objset, ==, dn->dn_objset); in dbuf_verify()
738 ASSERT3U(db->db_level, <, dn->dn_nlevels); in dbuf_verify()
739 ASSERT(db->db_blkid == DMU_BONUS_BLKID || in dbuf_verify()
740 db->db_blkid == DMU_SPILL_BLKID || in dbuf_verify()
743 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_verify()
745 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_verify()
746 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); in dbuf_verify()
747 } else if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_verify()
749 ASSERT0(db->db.db_offset); in dbuf_verify()
751 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); in dbuf_verify()
754 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) in dbuf_verify()
755 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
757 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) in dbuf_verify()
758 ASSERT(dr->dr_dbuf == db); in dbuf_verify()
765 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_verify()
766 dr = db->db_data_pending; in dbuf_verify()
771 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); in dbuf_verify()
775 if (db->db_blkptr) { in dbuf_verify()
776 if (db->db_parent == dn->dn_dbuf) { in dbuf_verify()
779 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) in dbuf_verify()
780 ASSERT(db->db_parent == NULL); in dbuf_verify()
782 ASSERT(db->db_parent != NULL); in dbuf_verify()
783 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_verify()
784 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
785 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_verify()
788 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; in dbuf_verify()
789 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); in dbuf_verify()
790 ASSERT3U(db->db_parent->db.db_object, ==, in dbuf_verify()
791 db->db.db_object); in dbuf_verify()
797 if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) { in dbuf_verify()
798 ASSERT3P(db->db_blkptr, ==, in dbuf_verify()
799 ((blkptr_t *)db->db_parent->db.db_data + in dbuf_verify()
800 db->db_blkid % epb)); in dbuf_verify()
804 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && in dbuf_verify()
805 (db->db_buf == NULL || db->db_buf->b_data) && in dbuf_verify()
806 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && in dbuf_verify()
807 db->db_state != DB_FILL && !dn->dn_free_txg) { in dbuf_verify()
819 if (db->db_dirtycnt == 0) { in dbuf_verify()
820 if (db->db_level == 0) { in dbuf_verify()
821 uint64_t *buf = db->db.db_data; in dbuf_verify()
824 for (i = 0; i < db->db.db_size >> 3; i++) { in dbuf_verify()
828 blkptr_t *bps = db->db.db_data; in dbuf_verify()
829 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, in dbuf_verify()
830 db->db.db_size); in dbuf_verify()
839 i < db->db.db_size / sizeof (blkptr_t); in dbuf_verify()
858 DB_DNODE_EXIT(db); in dbuf_verify()
863 dbuf_clear_data(dmu_buf_impl_t *db) in dbuf_clear_data() argument
865 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_clear_data()
866 dbuf_evict_user(db); in dbuf_clear_data()
867 ASSERT3P(db->db_buf, ==, NULL); in dbuf_clear_data()
868 db->db.db_data = NULL; in dbuf_clear_data()
869 if (db->db_state != DB_NOFILL) in dbuf_clear_data()
870 db->db_state = DB_UNCACHED; in dbuf_clear_data()
878 dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, void *tag) in dmu_buf_lock_parent() argument
881 if (db->db_parent != NULL) { in dmu_buf_lock_parent()
882 rw_enter(&db->db_parent->db_rwlock, rw); in dmu_buf_lock_parent()
884 } else if (dmu_objset_ds(db->db_objset) != NULL) { in dmu_buf_lock_parent()
885 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw, in dmu_buf_lock_parent()
903 dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, void *tag) in dmu_buf_unlock_parent() argument
906 rw_exit(&db->db_parent->db_rwlock); in dmu_buf_unlock_parent()
908 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag); in dmu_buf_unlock_parent()
912 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) in dbuf_set_data() argument
914 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_set_data()
917 db->db_buf = buf; in dbuf_set_data()
919 db->db.db_data = buf->b_data; in dbuf_set_data()
926 dbuf_loan_arcbuf(dmu_buf_impl_t *db) in dbuf_loan_arcbuf() argument
930 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_loan_arcbuf()
931 mutex_enter(&db->db_mtx); in dbuf_loan_arcbuf()
932 if (arc_released(db->db_buf) || zfs_refcount_count(&db->db_holds) > 1) { in dbuf_loan_arcbuf()
933 int blksz = db->db.db_size; in dbuf_loan_arcbuf()
934 spa_t *spa = db->db_objset->os_spa; in dbuf_loan_arcbuf()
936 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
938 bcopy(db->db.db_data, abuf->b_data, blksz); in dbuf_loan_arcbuf()
940 abuf = db->db_buf; in dbuf_loan_arcbuf()
941 arc_loan_inuse_buf(abuf, db); in dbuf_loan_arcbuf()
942 db->db_buf = NULL; in dbuf_loan_arcbuf()
943 dbuf_clear_data(db); in dbuf_loan_arcbuf()
944 mutex_exit(&db->db_mtx); in dbuf_loan_arcbuf()
989 dmu_buf_impl_t *db = vdb; in dbuf_read_done() local
991 mutex_enter(&db->db_mtx); in dbuf_read_done()
992 ASSERT3U(db->db_state, ==, DB_READ); in dbuf_read_done()
996 ASSERT(zfs_refcount_count(&db->db_holds) > 0); in dbuf_read_done()
997 ASSERT(db->db_buf == NULL); in dbuf_read_done()
998 ASSERT(db->db.db_data == NULL); in dbuf_read_done()
1002 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_read_done()
1003 ASSERT3P(db->db_buf, ==, NULL); in dbuf_read_done()
1004 db->db_state = DB_UNCACHED; in dbuf_read_done()
1005 } else if (db->db_level == 0 && db->db_freed_in_flight) { in dbuf_read_done()
1009 buf = arc_alloc_buf(db->db_objset->os_spa, in dbuf_read_done()
1010 db, DBUF_GET_BUFC_TYPE(db), db->db.db_size); in dbuf_read_done()
1012 arc_release(buf, db); in dbuf_read_done()
1013 bzero(buf->b_data, db->db.db_size); in dbuf_read_done()
1015 db->db_freed_in_flight = FALSE; in dbuf_read_done()
1016 dbuf_set_data(db, buf); in dbuf_read_done()
1017 db->db_state = DB_CACHED; in dbuf_read_done()
1021 dbuf_set_data(db, buf); in dbuf_read_done()
1022 db->db_state = DB_CACHED; in dbuf_read_done()
1024 cv_broadcast(&db->db_changed); in dbuf_read_done()
1025 dbuf_rele_and_unlock(db, NULL, B_FALSE); in dbuf_read_done()
1042 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, uint32_t flags) in dbuf_read_verify_dnode_crypt() argument
1045 objset_t *os = db->db_objset; in dbuf_read_verify_dnode_crypt()
1050 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_verify_dnode_crypt()
1056 DB_DNODE_ENTER(db); in dbuf_read_verify_dnode_crypt()
1057 dn = DB_DNODE(db); in dbuf_read_verify_dnode_crypt()
1061 DB_DNODE_EXIT(db); in dbuf_read_verify_dnode_crypt()
1074 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1076 (db->db_blkid == DMU_BONUS_BLKID && in dbuf_read_verify_dnode_crypt()
1080 DB_DNODE_EXIT(db); in dbuf_read_verify_dnode_crypt()
1090 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags, in dbuf_read_impl() argument
1098 DB_DNODE_ENTER(db); in dbuf_read_impl()
1099 dn = DB_DNODE(db); in dbuf_read_impl()
1100 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read_impl()
1101 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_read_impl()
1102 ASSERT(db->db_state == DB_UNCACHED); in dbuf_read_impl()
1103 ASSERT(db->db_buf == NULL); in dbuf_read_impl()
1104 ASSERT(db->db_parent == NULL || in dbuf_read_impl()
1105 RW_LOCK_HELD(&db->db_parent->db_rwlock)); in dbuf_read_impl()
1107 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_read_impl()
1116 err = dbuf_read_verify_dnode_crypt(db, flags); in dbuf_read_impl()
1118 DB_DNODE_EXIT(db); in dbuf_read_impl()
1119 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1123 ASSERT3U(bonuslen, <=, db->db.db_size); in dbuf_read_impl()
1124 db->db.db_data = zio_buf_alloc(max_bonuslen); in dbuf_read_impl()
1127 bzero(db->db.db_data, max_bonuslen); in dbuf_read_impl()
1129 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); in dbuf_read_impl()
1130 DB_DNODE_EXIT(db); in dbuf_read_impl()
1131 db->db_state = DB_CACHED; in dbuf_read_impl()
1132 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1133 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1142 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || in dbuf_read_impl()
1143 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || in dbuf_read_impl()
1144 BP_IS_HOLE(db->db_blkptr)))) { in dbuf_read_impl()
1145 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_read_impl()
1147 dbuf_set_data(db, arc_alloc_buf(db->db_objset->os_spa, db, type, in dbuf_read_impl()
1148 db->db.db_size)); in dbuf_read_impl()
1149 bzero(db->db.db_data, db->db.db_size); in dbuf_read_impl()
1151 if (db->db_blkptr != NULL && db->db_level > 0 && in dbuf_read_impl()
1152 BP_IS_HOLE(db->db_blkptr) && in dbuf_read_impl()
1153 db->db_blkptr->blk_birth != 0) { in dbuf_read_impl()
1154 blkptr_t *bps = db->db.db_data; in dbuf_read_impl()
1156 DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t)); in dbuf_read_impl()
1159 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, in dbuf_read_impl()
1162 BP_GET_LEVEL(db->db_blkptr) == 1 ? in dbuf_read_impl()
1164 BP_GET_LSIZE(db->db_blkptr)); in dbuf_read_impl()
1165 BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); in dbuf_read_impl()
1167 BP_GET_LEVEL(db->db_blkptr) - 1); in dbuf_read_impl()
1168 BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); in dbuf_read_impl()
1171 DB_DNODE_EXIT(db); in dbuf_read_impl()
1172 db->db_state = DB_CACHED; in dbuf_read_impl()
1173 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1174 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1178 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read_impl()
1179 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read_impl()
1185 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(db->db_blkptr)) { in dbuf_read_impl()
1186 spa_log_error(db->db_objset->os_spa, &zb); in dbuf_read_impl()
1188 "object set %llu", dmu_objset_id(db->db_objset)); in dbuf_read_impl()
1189 DB_DNODE_EXIT(db); in dbuf_read_impl()
1190 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1191 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1195 err = dbuf_read_verify_dnode_crypt(db, flags); in dbuf_read_impl()
1197 DB_DNODE_EXIT(db); in dbuf_read_impl()
1198 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1199 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1203 DB_DNODE_EXIT(db); in dbuf_read_impl()
1205 db->db_state = DB_READ; in dbuf_read_impl()
1206 mutex_exit(&db->db_mtx); in dbuf_read_impl()
1208 if (DBUF_IS_L2CACHEABLE(db)) in dbuf_read_impl()
1211 dbuf_add_ref(db, NULL); in dbuf_read_impl()
1216 if ((flags & DB_RF_NO_DECRYPT) && BP_IS_PROTECTED(db->db_blkptr)) in dbuf_read_impl()
1225 blkptr_t bp = *db->db_blkptr; in dbuf_read_impl()
1226 dmu_buf_unlock_parent(db, dblt, tag); in dbuf_read_impl()
1227 (void) arc_read(zio, db->db_objset->os_spa, &bp, in dbuf_read_impl()
1228 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags, in dbuf_read_impl()
1247 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) in dbuf_fix_old_data() argument
1249 dbuf_dirty_record_t *dr = db->db_last_dirty; in dbuf_fix_old_data()
1251 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_fix_old_data()
1252 ASSERT(db->db.db_data != NULL); in dbuf_fix_old_data()
1253 ASSERT(db->db_level == 0); in dbuf_fix_old_data()
1254 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); in dbuf_fix_old_data()
1258 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) in dbuf_fix_old_data()
1269 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_fix_old_data()
1271 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1275 bcopy(db->db.db_data, dr->dt.dl.dr_data, bonuslen); in dbuf_fix_old_data()
1276 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) { in dbuf_fix_old_data()
1277 dnode_t *dn = DB_DNODE(db); in dbuf_fix_old_data()
1278 int size = arc_buf_size(db->db_buf); in dbuf_fix_old_data()
1279 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_fix_old_data()
1280 spa_t *spa = db->db_objset->os_spa; in dbuf_fix_old_data()
1282 arc_get_compression(db->db_buf); in dbuf_fix_old_data()
1284 if (arc_is_encrypted(db->db_buf)) { in dbuf_fix_old_data()
1290 arc_get_raw_params(db->db_buf, &byteorder, salt, in dbuf_fix_old_data()
1292 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db, in dbuf_fix_old_data()
1294 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf), in dbuf_fix_old_data()
1298 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db, in dbuf_fix_old_data()
1299 size, arc_buf_lsize(db->db_buf), compress_type); in dbuf_fix_old_data()
1301 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size); in dbuf_fix_old_data()
1303 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); in dbuf_fix_old_data()
1305 db->db_buf = NULL; in dbuf_fix_old_data()
1306 dbuf_clear_data(db); in dbuf_fix_old_data()
1311 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) in dbuf_read() argument
1321 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_read()
1323 if (db->db_state == DB_NOFILL) in dbuf_read()
1326 DB_DNODE_ENTER(db); in dbuf_read()
1327 dn = DB_DNODE(db); in dbuf_read()
1329 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_read()
1331 DBUF_IS_CACHEABLE(db); in dbuf_read()
1333 mutex_enter(&db->db_mtx); in dbuf_read()
1334 if (db->db_state == DB_CACHED) { in dbuf_read()
1341 err = dbuf_read_verify_dnode_crypt(db, flags); in dbuf_read()
1350 if (err == 0 && db->db_buf != NULL && in dbuf_read()
1352 (arc_is_encrypted(db->db_buf) || in dbuf_read()
1353 arc_is_unauthenticated(db->db_buf) || in dbuf_read()
1354 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) { in dbuf_read()
1357 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_read()
1358 db->db.db_object, db->db_level, db->db_blkid); in dbuf_read()
1359 dbuf_fix_old_data(db, spa_syncing_txg(spa)); in dbuf_read()
1360 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE); in dbuf_read()
1361 dbuf_set_data(db, db->db_buf); in dbuf_read()
1363 mutex_exit(&db->db_mtx); in dbuf_read()
1365 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, in dbuf_read()
1368 DB_DNODE_EXIT(db); in dbuf_read()
1369 } else if (db->db_state == DB_UNCACHED) { in dbuf_read()
1373 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_read()
1376 db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { in dbuf_read()
1380 err = dbuf_read_impl(db, zio, flags, dblt, FTAG); in dbuf_read()
1386 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, in dbuf_read()
1390 DB_DNODE_EXIT(db); in dbuf_read()
1403 mutex_exit(&db->db_mtx); in dbuf_read()
1405 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, in dbuf_read()
1408 DB_DNODE_EXIT(db); in dbuf_read()
1411 mutex_enter(&db->db_mtx); in dbuf_read()
1413 while (db->db_state == DB_READ || in dbuf_read()
1414 db->db_state == DB_FILL) { in dbuf_read()
1415 ASSERT(db->db_state == DB_READ || in dbuf_read()
1418 db, zio_t *, zio); in dbuf_read()
1419 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_read()
1421 if (db->db_state == DB_UNCACHED) in dbuf_read()
1424 mutex_exit(&db->db_mtx); in dbuf_read()
1431 dbuf_noread(dmu_buf_impl_t *db) in dbuf_noread() argument
1433 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_noread()
1434 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_noread()
1435 mutex_enter(&db->db_mtx); in dbuf_noread()
1436 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_noread()
1437 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_noread()
1438 if (db->db_state == DB_UNCACHED) { in dbuf_noread()
1439 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_noread()
1440 spa_t *spa = db->db_objset->os_spa; in dbuf_noread()
1442 ASSERT(db->db_buf == NULL); in dbuf_noread()
1443 ASSERT(db->db.db_data == NULL); in dbuf_noread()
1444 dbuf_set_data(db, arc_alloc_buf(spa, db, type, db->db.db_size)); in dbuf_noread()
1445 db->db_state = DB_FILL; in dbuf_noread()
1446 } else if (db->db_state == DB_NOFILL) { in dbuf_noread()
1447 dbuf_clear_data(db); in dbuf_noread()
1449 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_noread()
1451 mutex_exit(&db->db_mtx); in dbuf_noread()
1457 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_unoverride() local
1461 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_unoverride()
1468 ASSERT(db->db_level == 0); in dbuf_unoverride()
1470 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_unoverride()
1474 ASSERT(db->db_data_pending != dr); in dbuf_unoverride()
1478 zio_free(db->db_objset->os_spa, txg, bp); in dbuf_unoverride()
1492 arc_release(dr->dt.dl.dr_data, db); in dbuf_unoverride()
1505 dmu_buf_impl_t *db, *db_next; in dbuf_free_range() local
1519 db = avl_find(&dn->dn_dbufs, &db_search, &where); in dbuf_free_range()
1520 ASSERT3P(db, ==, NULL); in dbuf_free_range()
1522 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); in dbuf_free_range()
1524 for (; db != NULL; db = db_next) { in dbuf_free_range()
1525 db_next = AVL_NEXT(&dn->dn_dbufs, db); in dbuf_free_range()
1526 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_free_range()
1528 if (db->db_level != 0 || db->db_blkid > end_blkid) { in dbuf_free_range()
1531 ASSERT3U(db->db_blkid, >=, start_blkid); in dbuf_free_range()
1534 mutex_enter(&db->db_mtx); in dbuf_free_range()
1535 if (dbuf_undirty(db, tx)) { in dbuf_free_range()
1540 if (db->db_state == DB_UNCACHED || in dbuf_free_range()
1541 db->db_state == DB_NOFILL || in dbuf_free_range()
1542 db->db_state == DB_EVICTING) { in dbuf_free_range()
1543 ASSERT(db->db.db_data == NULL); in dbuf_free_range()
1544 mutex_exit(&db->db_mtx); in dbuf_free_range()
1547 if (db->db_state == DB_READ || db->db_state == DB_FILL) { in dbuf_free_range()
1549 db->db_freed_in_flight = TRUE; in dbuf_free_range()
1550 mutex_exit(&db->db_mtx); in dbuf_free_range()
1553 if (zfs_refcount_count(&db->db_holds) == 0) { in dbuf_free_range()
1554 ASSERT(db->db_buf); in dbuf_free_range()
1555 dbuf_destroy(db); in dbuf_free_range()
1560 if (db->db_last_dirty != NULL) { in dbuf_free_range()
1561 dbuf_dirty_record_t *dr = db->db_last_dirty; in dbuf_free_range()
1569 if (db->db_blkid != DMU_SPILL_BLKID && in dbuf_free_range()
1570 db->db_blkid > dn->dn_maxblkid) in dbuf_free_range()
1571 dn->dn_maxblkid = db->db_blkid; in dbuf_free_range()
1580 dbuf_fix_old_data(db, txg); in dbuf_free_range()
1584 if (db->db_state == DB_CACHED) { in dbuf_free_range()
1585 ASSERT(db->db.db_data != NULL); in dbuf_free_range()
1586 arc_release(db->db_buf, db); in dbuf_free_range()
1587 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_free_range()
1588 bzero(db->db.db_data, db->db.db_size); in dbuf_free_range()
1589 rw_exit(&db->db_rwlock); in dbuf_free_range()
1590 arc_buf_freeze(db->db_buf); in dbuf_free_range()
1593 mutex_exit(&db->db_mtx); in dbuf_free_range()
1599 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) in dbuf_new_size() argument
1602 int osize = db->db.db_size; in dbuf_new_size()
1603 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_new_size()
1606 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_new_size()
1608 DB_DNODE_ENTER(db); in dbuf_new_size()
1609 dn = DB_DNODE(db); in dbuf_new_size()
1615 dmu_buf_will_dirty(&db->db, tx); in dbuf_new_size()
1618 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size); in dbuf_new_size()
1621 obuf = db->db_buf; in dbuf_new_size()
1627 mutex_enter(&db->db_mtx); in dbuf_new_size()
1628 dbuf_set_data(db, buf); in dbuf_new_size()
1629 arc_buf_destroy(obuf, db); in dbuf_new_size()
1630 db->db.db_size = size; in dbuf_new_size()
1632 if (db->db_level == 0) { in dbuf_new_size()
1633 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); in dbuf_new_size()
1634 db->db_last_dirty->dt.dl.dr_data = buf; in dbuf_new_size()
1636 mutex_exit(&db->db_mtx); in dbuf_new_size()
1639 DB_DNODE_EXIT(db); in dbuf_new_size()
1643 dbuf_release_bp(dmu_buf_impl_t *db) in dbuf_release_bp() argument
1645 objset_t *os = db->db_objset; in dbuf_release_bp()
1650 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); in dbuf_release_bp()
1652 (void) arc_release(db->db_buf, db); in dbuf_release_bp()
1662 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_redirty() local
1664 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_redirty()
1666 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { in dbuf_redirty()
1672 if (db->db.db_object != DMU_META_DNODE_OBJECT && in dbuf_redirty()
1673 db->db_state != DB_NOFILL) { in dbuf_redirty()
1675 ASSERT(arc_released(db->db_buf)); in dbuf_redirty()
1676 arc_buf_thaw(db->db_buf); in dbuf_redirty()
1682 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_dirty() argument
1691 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_dirty()
1692 DMU_TX_DIRTY_BUF(tx, db); in dbuf_dirty()
1694 DB_DNODE_ENTER(db); in dbuf_dirty()
1695 dn = DB_DNODE(db); in dbuf_dirty()
1722 mutex_enter(&db->db_mtx); in dbuf_dirty()
1728 ASSERT(db->db_level != 0 || in dbuf_dirty()
1729 db->db_state == DB_CACHED || db->db_state == DB_FILL || in dbuf_dirty()
1730 db->db_state == DB_NOFILL); in dbuf_dirty()
1758 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_dirty()
1764 drp = &db->db_last_dirty; in dbuf_dirty()
1766 db->db.db_object == DMU_META_DNODE_OBJECT); in dbuf_dirty()
1770 DB_DNODE_EXIT(db); in dbuf_dirty()
1773 mutex_exit(&db->db_mtx); in dbuf_dirty()
1784 ASSERT3U(dn->dn_nlevels, >, db->db_level); in dbuf_dirty()
1803 ASSERT(db->db.db_size != 0); in dbuf_dirty()
1805 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_dirty()
1807 if (db->db_blkid != DMU_BONUS_BLKID) { in dbuf_dirty()
1808 dmu_objset_willuse_space(os, db->db.db_size, tx); in dbuf_dirty()
1817 if (db->db_level == 0) { in dbuf_dirty()
1818 void *data_old = db->db_buf; in dbuf_dirty()
1820 if (db->db_state != DB_NOFILL) { in dbuf_dirty()
1821 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_dirty()
1822 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
1823 data_old = db->db.db_data; in dbuf_dirty()
1824 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { in dbuf_dirty()
1834 arc_release(db->db_buf, db); in dbuf_dirty()
1835 dbuf_fix_old_data(db, tx->tx_txg); in dbuf_dirty()
1836 data_old = db->db_buf; in dbuf_dirty()
1847 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) in dbuf_dirty()
1848 dr->dr_accounted = db->db.db_size; in dbuf_dirty()
1849 dr->dr_dbuf = db; in dbuf_dirty()
1859 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_dirty()
1860 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_dirty()
1864 db->db_blkid, 1); in dbuf_dirty()
1867 db->db_freed_in_flight = FALSE; in dbuf_dirty()
1873 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); in dbuf_dirty()
1874 db->db_dirtycnt += 1; in dbuf_dirty()
1875 ASSERT3U(db->db_dirtycnt, <=, 3); in dbuf_dirty()
1877 mutex_exit(&db->db_mtx); in dbuf_dirty()
1879 if (db->db_blkid == DMU_BONUS_BLKID || in dbuf_dirty()
1880 db->db_blkid == DMU_SPILL_BLKID) { in dbuf_dirty()
1886 DB_DNODE_EXIT(db); in dbuf_dirty()
1901 if (db->db_blkptr != NULL) { in dbuf_dirty()
1902 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG); in dbuf_dirty()
1903 ddt_prefetch(os->os_spa, db->db_blkptr); in dbuf_dirty()
1904 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_dirty()
1911 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || in dbuf_dirty()
1912 dn->dn_phys->dn_nlevels > db->db_level || in dbuf_dirty()
1913 dn->dn_next_nlevels[txgoff] > db->db_level || in dbuf_dirty()
1914 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || in dbuf_dirty()
1915 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); in dbuf_dirty()
1918 if (db->db_level == 0) { in dbuf_dirty()
1919 ASSERT(!db->db_objset->os_raw_receive || in dbuf_dirty()
1920 dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
1921 dnode_new_blkid(dn, db->db_blkid, tx, in dbuf_dirty()
1923 ASSERT(dn->dn_maxblkid >= db->db_blkid); in dbuf_dirty()
1926 if (db->db_level+1 < dn->dn_nlevels) { in dbuf_dirty()
1927 dmu_buf_impl_t *parent = db->db_parent; in dbuf_dirty()
1931 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { in dbuf_dirty()
1933 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_dirty()
1934 db->db_blkid >> epbs, FTAG); in dbuf_dirty()
1940 ASSERT3U(db->db_level + 1, ==, parent->db_level); in dbuf_dirty()
1945 mutex_enter(&db->db_mtx); in dbuf_dirty()
1950 if (db->db_last_dirty == dr || in dbuf_dirty()
1959 mutex_exit(&db->db_mtx); in dbuf_dirty()
1961 ASSERT(db->db_level + 1 == dn->dn_nlevels); in dbuf_dirty()
1962 ASSERT(db->db_blkid < dn->dn_nblkptr); in dbuf_dirty()
1963 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); in dbuf_dirty()
1973 DB_DNODE_EXIT(db); in dbuf_dirty()
1982 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_undirty() argument
1996 ASSERT(db->db_objset == in dbuf_undirty()
1997 dmu_objset_pool(db->db_objset)->dp_meta_objset || in dbuf_undirty()
1998 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset))); in dbuf_undirty()
1999 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_undirty()
2000 ASSERT0(db->db_level); in dbuf_undirty()
2001 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_undirty()
2006 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) in dbuf_undirty()
2012 ASSERT(dr->dr_dbuf == db); in dbuf_undirty()
2014 DB_DNODE_ENTER(db); in dbuf_undirty()
2015 dn = DB_DNODE(db); in dbuf_undirty()
2017 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); in dbuf_undirty()
2019 ASSERT(db->db.db_size != 0); in dbuf_undirty()
2036 } else if (db->db_blkid == DMU_SPILL_BLKID || in dbuf_undirty()
2037 db->db_level + 1 == dn->dn_nlevels) { in dbuf_undirty()
2038 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); in dbuf_undirty()
2043 DB_DNODE_EXIT(db); in dbuf_undirty()
2045 if (db->db_state != DB_NOFILL) { in dbuf_undirty()
2048 ASSERT(db->db_buf != NULL); in dbuf_undirty()
2050 if (dr->dt.dl.dr_data != db->db_buf) in dbuf_undirty()
2051 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_undirty()
2056 ASSERT(db->db_dirtycnt > 0); in dbuf_undirty()
2057 db->db_dirtycnt -= 1; in dbuf_undirty()
2059 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { in dbuf_undirty()
2060 ASSERT(db->db_state == DB_NOFILL || arc_released(db->db_buf)); in dbuf_undirty()
2061 dbuf_destroy(db); in dbuf_undirty()
2071 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_dirty_impl() local
2074 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_dirty_impl()
2082 mutex_enter(&db->db_mtx); in dmu_buf_will_dirty_impl()
2084 for (dr = db->db_last_dirty; in dmu_buf_will_dirty_impl()
2091 if (dr->dr_txg == tx->tx_txg && db->db_state == DB_CACHED) { in dmu_buf_will_dirty_impl()
2094 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2098 mutex_exit(&db->db_mtx); in dmu_buf_will_dirty_impl()
2100 DB_DNODE_ENTER(db); in dmu_buf_will_dirty_impl()
2101 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) in dmu_buf_will_dirty_impl()
2103 DB_DNODE_EXIT(db); in dmu_buf_will_dirty_impl()
2104 (void) dbuf_read(db, NULL, flags); in dmu_buf_will_dirty_impl()
2105 (void) dbuf_dirty(db, tx); in dmu_buf_will_dirty_impl()
2118 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_not_fill() local
2120 db->db_state = DB_NOFILL; in dmu_buf_will_not_fill()
2128 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_will_fill() local
2130 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_will_fill()
2132 ASSERT(db->db_level == 0); in dmu_buf_will_fill()
2133 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dmu_buf_will_fill()
2135 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || in dmu_buf_will_fill()
2138 dbuf_noread(db); in dmu_buf_will_fill()
2139 (void) dbuf_dirty(db, tx); in dmu_buf_will_fill()
2153 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_crypt_params() local
2160 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dmu_buf_set_crypt_params()
2161 ASSERT3U(db->db_level, ==, 0); in dmu_buf_set_crypt_params()
2166 dr = db->db_last_dirty; in dmu_buf_set_crypt_params()
2183 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_fill_done() argument
2185 mutex_enter(&db->db_mtx); in dbuf_fill_done()
2186 DBUF_VERIFY(db); in dbuf_fill_done()
2188 if (db->db_state == DB_FILL) { in dbuf_fill_done()
2189 if (db->db_level == 0 && db->db_freed_in_flight) { in dbuf_fill_done()
2190 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_fill_done()
2193 bzero(db->db.db_data, db->db.db_size); in dbuf_fill_done()
2194 db->db_freed_in_flight = FALSE; in dbuf_fill_done()
2196 db->db_state = DB_CACHED; in dbuf_fill_done()
2197 cv_broadcast(&db->db_changed); in dbuf_fill_done()
2199 mutex_exit(&db->db_mtx); in dbuf_fill_done()
2208 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; in dmu_buf_write_embedded() local
2213 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset), in dmu_buf_write_embedded()
2217 DB_DNODE_ENTER(db); in dmu_buf_write_embedded()
2218 type = DB_DNODE(db)->dn_type; in dmu_buf_write_embedded()
2219 DB_DNODE_EXIT(db); in dmu_buf_write_embedded()
2221 ASSERT0(db->db_level); in dmu_buf_write_embedded()
2222 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dmu_buf_write_embedded()
2226 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); in dmu_buf_write_embedded()
2227 dl = &db->db_last_dirty->dt.dl; in dmu_buf_write_embedded()
2236 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; in dmu_buf_write_embedded()
2244 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) in dbuf_assign_arcbuf() argument
2246 ASSERT(!zfs_refcount_is_zero(&db->db_holds)); in dbuf_assign_arcbuf()
2247 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_assign_arcbuf()
2248 ASSERT(db->db_level == 0); in dbuf_assign_arcbuf()
2249 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf)); in dbuf_assign_arcbuf()
2251 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size); in dbuf_assign_arcbuf()
2254 arc_return_buf(buf, db); in dbuf_assign_arcbuf()
2257 mutex_enter(&db->db_mtx); in dbuf_assign_arcbuf()
2259 while (db->db_state == DB_READ || db->db_state == DB_FILL) in dbuf_assign_arcbuf()
2260 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_assign_arcbuf()
2262 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); in dbuf_assign_arcbuf()
2264 if (db->db_state == DB_CACHED && in dbuf_assign_arcbuf()
2265 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { in dbuf_assign_arcbuf()
2273 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
2274 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
2275 bcopy(buf->b_data, db->db.db_data, db->db.db_size); in dbuf_assign_arcbuf()
2276 arc_buf_destroy(buf, db); in dbuf_assign_arcbuf()
2282 if (db->db_state == DB_CACHED) { in dbuf_assign_arcbuf()
2283 dbuf_dirty_record_t *dr = db->db_last_dirty; in dbuf_assign_arcbuf()
2285 ASSERT(db->db_buf != NULL); in dbuf_assign_arcbuf()
2287 ASSERT(dr->dt.dl.dr_data == db->db_buf); in dbuf_assign_arcbuf()
2289 if (!arc_released(db->db_buf)) { in dbuf_assign_arcbuf()
2292 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
2295 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
2296 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { in dbuf_assign_arcbuf()
2297 arc_release(db->db_buf, db); in dbuf_assign_arcbuf()
2298 arc_buf_destroy(db->db_buf, db); in dbuf_assign_arcbuf()
2300 db->db_buf = NULL; in dbuf_assign_arcbuf()
2302 ASSERT(db->db_buf == NULL); in dbuf_assign_arcbuf()
2303 dbuf_set_data(db, buf); in dbuf_assign_arcbuf()
2304 db->db_state = DB_FILL; in dbuf_assign_arcbuf()
2305 mutex_exit(&db->db_mtx); in dbuf_assign_arcbuf()
2306 (void) dbuf_dirty(db, tx); in dbuf_assign_arcbuf()
2307 dmu_buf_fill_done(&db->db, tx); in dbuf_assign_arcbuf()
2311 dbuf_destroy(dmu_buf_impl_t *db) in dbuf_destroy() argument
2314 dmu_buf_impl_t *parent = db->db_parent; in dbuf_destroy()
2317 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_destroy()
2318 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
2320 if (db->db_buf != NULL) { in dbuf_destroy()
2321 arc_buf_destroy(db->db_buf, db); in dbuf_destroy()
2322 db->db_buf = NULL; in dbuf_destroy()
2325 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_destroy()
2326 int slots = DB_DNODE(db)->dn_num_slots; in dbuf_destroy()
2328 if (db->db.db_data != NULL) { in dbuf_destroy()
2329 zio_buf_free(db->db.db_data, bonuslen); in dbuf_destroy()
2331 db->db_state = DB_UNCACHED; in dbuf_destroy()
2335 dbuf_clear_data(db); in dbuf_destroy()
2337 if (multilist_link_active(&db->db_cache_link)) { in dbuf_destroy()
2338 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_destroy()
2339 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_destroy()
2341 multilist_remove(dbuf_caches[db->db_caching_status].cache, db); in dbuf_destroy()
2343 &dbuf_caches[db->db_caching_status].size, in dbuf_destroy()
2344 db->db.db_size, db); in dbuf_destroy()
2346 db->db_caching_status = DB_NO_CACHE; in dbuf_destroy()
2349 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); in dbuf_destroy()
2350 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
2352 db->db_state = DB_EVICTING; in dbuf_destroy()
2353 db->db_blkptr = NULL; in dbuf_destroy()
2360 mutex_exit(&db->db_mtx); in dbuf_destroy()
2362 DB_DNODE_ENTER(db); in dbuf_destroy()
2363 dn = DB_DNODE(db); in dbuf_destroy()
2365 if (db->db_blkid != DMU_BONUS_BLKID) { in dbuf_destroy()
2369 avl_remove(&dn->dn_dbufs, db); in dbuf_destroy()
2372 DB_DNODE_EXIT(db); in dbuf_destroy()
2384 dnode_rele_and_unlock(dn, db, B_TRUE); in dbuf_destroy()
2385 db->db_dnode_handle = NULL; in dbuf_destroy()
2387 dbuf_hash_remove(db); in dbuf_destroy()
2389 DB_DNODE_EXIT(db); in dbuf_destroy()
2392 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_destroy()
2394 db->db_parent = NULL; in dbuf_destroy()
2396 ASSERT(db->db_buf == NULL); in dbuf_destroy()
2397 ASSERT(db->db.db_data == NULL); in dbuf_destroy()
2398 ASSERT(db->db_hash_next == NULL); in dbuf_destroy()
2399 ASSERT(db->db_blkptr == NULL); in dbuf_destroy()
2400 ASSERT(db->db_data_pending == NULL); in dbuf_destroy()
2401 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE); in dbuf_destroy()
2402 ASSERT(!multilist_link_active(&db->db_cache_link)); in dbuf_destroy()
2404 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_destroy()
2413 dbuf_rele_and_unlock(parent, db, B_TRUE); in dbuf_destroy()
2490 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + in dbuf_findbp()
2515 dmu_buf_impl_t *db, *odb; in dbuf_create() local
2520 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP); in dbuf_create()
2522 db->db_objset = os; in dbuf_create()
2523 db->db.db_object = dn->dn_object; in dbuf_create()
2524 db->db_level = level; in dbuf_create()
2525 db->db_blkid = blkid; in dbuf_create()
2526 db->db_last_dirty = NULL; in dbuf_create()
2527 db->db_dirtycnt = 0; in dbuf_create()
2528 db->db_dnode_handle = dn->dn_handle; in dbuf_create()
2529 db->db_parent = parent; in dbuf_create()
2530 db->db_blkptr = blkptr; in dbuf_create()
2532 db->db_user = NULL; in dbuf_create()
2533 db->db_user_immediate_evict = FALSE; in dbuf_create()
2534 db->db_freed_in_flight = FALSE; in dbuf_create()
2535 db->db_pending_evict = FALSE; in dbuf_create()
2539 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - in dbuf_create()
2541 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); in dbuf_create()
2542 db->db.db_offset = DMU_BONUS_BLKID; in dbuf_create()
2543 db->db_state = DB_UNCACHED; in dbuf_create()
2544 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
2547 return (db); in dbuf_create()
2549 db->db.db_size = (blkptr != NULL) ? in dbuf_create()
2551 db->db.db_offset = 0; in dbuf_create()
2554 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; in dbuf_create()
2555 db->db.db_size = blocksize; in dbuf_create()
2556 db->db.db_offset = db->db_blkid * blocksize; in dbuf_create()
2567 db->db_state = DB_EVICTING; in dbuf_create()
2568 if ((odb = dbuf_hash_insert(db)) != NULL) { in dbuf_create()
2570 kmem_cache_free(dbuf_kmem_cache, db); in dbuf_create()
2574 avl_add(&dn->dn_dbufs, db); in dbuf_create()
2576 db->db_state = DB_UNCACHED; in dbuf_create()
2577 db->db_caching_status = DB_NO_CACHE; in dbuf_create()
2582 dbuf_add_ref(parent, db); in dbuf_create()
2586 (void) zfs_refcount_add(&dn->dn_holds, db); in dbuf_create()
2589 dprintf_dbuf(db, "db=%p\n", db); in dbuf_create()
2591 return (db); in dbuf_create()
2698 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode, in dbuf_prefetch_indirect_done() local
2700 if (db == NULL) { in dbuf_prefetch_indirect_done()
2704 (void) dbuf_read(db, NULL, in dbuf_prefetch_indirect_done()
2706 dbuf_rele(db, FTAG); in dbuf_prefetch_indirect_done()
2779 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object, in dbuf_prefetch_impl() local
2781 if (db != NULL) { in dbuf_prefetch_impl()
2782 mutex_exit(&db->db_mtx); in dbuf_prefetch_impl()
2800 dmu_buf_impl_t *db; in dbuf_prefetch_impl() local
2803 FALSE, TRUE, FTAG, &db) == 0) { in dbuf_prefetch_impl()
2804 blkptr_t *bpp = db->db_buf->b_data; in dbuf_prefetch_impl()
2806 dbuf_rele(db, FTAG); in dbuf_prefetch_impl()
2899 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db, dbuf_dirty_record_t *dr) in dbuf_hold_copy() argument
2911 dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
2916 dbuf_set_data(db, arc_alloc_compressed_buf( in dbuf_hold_copy()
2917 dn->dn_objset->os_spa, db, arc_buf_size(data), in dbuf_hold_copy()
2920 dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db, in dbuf_hold_copy()
2921 DBUF_GET_BUFC_TYPE(db), db->db.db_size)); in dbuf_hold_copy()
2924 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_hold_copy()
2925 bcopy(data->b_data, db->db.db_data, arc_buf_size(data)); in dbuf_hold_copy()
2926 rw_exit(&db->db_rwlock); in dbuf_hold_copy()
2938 dmu_buf_impl_t *db, *parent = NULL; in dbuf_hold_impl() local
2947 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid); in dbuf_hold_impl()
2949 if (db == NULL) { in dbuf_hold_impl()
2969 db = dbuf_create(dn, level, blkid, parent, bp); in dbuf_hold_impl()
2972 if (fail_uncached && db->db_state != DB_CACHED) { in dbuf_hold_impl()
2973 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
2977 if (db->db_buf != NULL) { in dbuf_hold_impl()
2978 arc_buf_access(db->db_buf); in dbuf_hold_impl()
2979 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); in dbuf_hold_impl()
2982 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); in dbuf_hold_impl()
2989 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && in dbuf_hold_impl()
2991 db->db_state == DB_CACHED && db->db_data_pending) { in dbuf_hold_impl()
2992 dbuf_dirty_record_t *dr = db->db_data_pending; in dbuf_hold_impl()
2993 if (dr->dt.dl.dr_data == db->db_buf) in dbuf_hold_impl()
2994 dbuf_hold_copy(dn, db, dr); in dbuf_hold_impl()
2997 if (multilist_link_active(&db->db_cache_link)) { in dbuf_hold_impl()
2998 ASSERT(zfs_refcount_is_zero(&db->db_holds)); in dbuf_hold_impl()
2999 ASSERT(db->db_caching_status == DB_DBUF_CACHE || in dbuf_hold_impl()
3000 db->db_caching_status == DB_DBUF_METADATA_CACHE); in dbuf_hold_impl()
3002 multilist_remove(dbuf_caches[db->db_caching_status].cache, db); in dbuf_hold_impl()
3004 &dbuf_caches[db->db_caching_status].size, in dbuf_hold_impl()
3005 db->db.db_size, db); in dbuf_hold_impl()
3007 db->db_caching_status = DB_NO_CACHE; in dbuf_hold_impl()
3009 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_hold_impl()
3010 DBUF_VERIFY(db); in dbuf_hold_impl()
3011 mutex_exit(&db->db_mtx); in dbuf_hold_impl()
3017 ASSERT3P(DB_DNODE(db), ==, dn); in dbuf_hold_impl()
3018 ASSERT3U(db->db_blkid, ==, blkid); in dbuf_hold_impl()
3019 ASSERT3U(db->db_level, ==, level); in dbuf_hold_impl()
3020 *dbp = db; in dbuf_hold_impl()
3034 dmu_buf_impl_t *db; in dbuf_hold_level() local
3035 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db); in dbuf_hold_level()
3036 return (err ? NULL : db); in dbuf_hold_level()
3051 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_spill_set_blksz() local
3053 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_spill_set_blksz()
3057 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset))); in dbuf_spill_set_blksz()
3060 dbuf_new_size(db, blksz, tx); in dbuf_spill_set_blksz()
3073 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) in dbuf_add_ref() argument
3075 int64_t holds = zfs_refcount_add(&db->db_holds, tag); in dbuf_add_ref()
3084 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dbuf_try_add_ref() local
3094 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) { in dbuf_try_add_ref()
3095 (void) zfs_refcount_add(&db->db_holds, tag); in dbuf_try_add_ref()
3111 dbuf_rele(dmu_buf_impl_t *db, void *tag) in dbuf_rele() argument
3113 mutex_enter(&db->db_mtx); in dbuf_rele()
3114 dbuf_rele_and_unlock(db, tag, B_FALSE); in dbuf_rele()
3118 dmu_buf_rele(dmu_buf_t *db, void *tag) in dmu_buf_rele() argument
3120 dbuf_rele((dmu_buf_impl_t *)db, tag); in dmu_buf_rele()
3137 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag, boolean_t evicting) in dbuf_rele_and_unlock() argument
3141 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_rele_and_unlock()
3142 DBUF_VERIFY(db); in dbuf_rele_and_unlock()
3149 holds = zfs_refcount_remove(&db->db_holds, tag); in dbuf_rele_and_unlock()
3156 if (db->db_buf != NULL && in dbuf_rele_and_unlock()
3157 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) { in dbuf_rele_and_unlock()
3158 arc_buf_freeze(db->db_buf); in dbuf_rele_and_unlock()
3161 if (holds == db->db_dirtycnt && in dbuf_rele_and_unlock()
3162 db->db_level == 0 && db->db_user_immediate_evict) in dbuf_rele_and_unlock()
3163 dbuf_evict_user(db); in dbuf_rele_and_unlock()
3166 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_rele_and_unlock()
3168 boolean_t evict_dbuf = db->db_pending_evict; in dbuf_rele_and_unlock()
3174 DB_DNODE_ENTER(db); in dbuf_rele_and_unlock()
3176 dn = DB_DNODE(db); in dbuf_rele_and_unlock()
3185 DB_DNODE_EXIT(db); in dbuf_rele_and_unlock()
3191 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
3196 dnode_rele(dn, db); in dbuf_rele_and_unlock()
3197 } else if (db->db_buf == NULL) { in dbuf_rele_and_unlock()
3202 ASSERT(db->db_state == DB_UNCACHED || in dbuf_rele_and_unlock()
3203 db->db_state == DB_NOFILL); in dbuf_rele_and_unlock()
3204 dbuf_destroy(db); in dbuf_rele_and_unlock()
3205 } else if (arc_released(db->db_buf)) { in dbuf_rele_and_unlock()
3209 dbuf_destroy(db); in dbuf_rele_and_unlock()
3213 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_rele_and_unlock()
3215 if (!DBUF_IS_CACHEABLE(db) && in dbuf_rele_and_unlock()
3216 db->db_blkptr != NULL && in dbuf_rele_and_unlock()
3217 !BP_IS_HOLE(db->db_blkptr) && in dbuf_rele_and_unlock()
3218 !BP_IS_EMBEDDED(db->db_blkptr)) { in dbuf_rele_and_unlock()
3220 bp = *db->db_blkptr; in dbuf_rele_and_unlock()
3223 if (!DBUF_IS_CACHEABLE(db) || in dbuf_rele_and_unlock()
3224 db->db_pending_evict) { in dbuf_rele_and_unlock()
3225 dbuf_destroy(db); in dbuf_rele_and_unlock()
3226 } else if (!multilist_link_active(&db->db_cache_link)) { in dbuf_rele_and_unlock()
3227 ASSERT3U(db->db_caching_status, ==, in dbuf_rele_and_unlock()
3231 dbuf_include_in_metadata_cache(db) ? in dbuf_rele_and_unlock()
3233 db->db_caching_status = dcs; in dbuf_rele_and_unlock()
3235 multilist_insert(dbuf_caches[dcs].cache, db); in dbuf_rele_and_unlock()
3237 &dbuf_caches[dcs].size, db->db.db_size, db); in dbuf_rele_and_unlock()
3238 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
3240 if (db->db_caching_status == DB_DBUF_CACHE && in dbuf_rele_and_unlock()
3250 mutex_exit(&db->db_mtx); in dbuf_rele_and_unlock()
3257 dbuf_refcount(dmu_buf_impl_t *db) in dbuf_refcount() argument
3259 return (zfs_refcount_count(&db->db_holds)); in dbuf_refcount()
3266 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_user_refcount() local
3268 mutex_enter(&db->db_mtx); in dmu_buf_user_refcount()
3269 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt); in dmu_buf_user_refcount()
3270 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt; in dmu_buf_user_refcount()
3271 mutex_exit(&db->db_mtx); in dmu_buf_user_refcount()
3280 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_replace_user() local
3282 mutex_enter(&db->db_mtx); in dmu_buf_replace_user()
3283 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
3284 if (db->db_user == old_user) in dmu_buf_replace_user()
3285 db->db_user = new_user; in dmu_buf_replace_user()
3287 old_user = db->db_user; in dmu_buf_replace_user()
3288 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_replace_user()
3289 mutex_exit(&db->db_mtx); in dmu_buf_replace_user()
3303 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_set_user_ie() local
3305 db->db_user_immediate_evict = TRUE; in dmu_buf_set_user_ie()
3318 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; in dmu_buf_get_user() local
3320 dbuf_verify_user(db, DBVU_NOT_EVICTING); in dmu_buf_get_user()
3321 return (db->db_user); in dmu_buf_get_user()
3331 dmu_buf_get_blkptr(dmu_buf_t *db) in dmu_buf_get_blkptr() argument
3333 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_blkptr()
3338 dmu_buf_get_objset(dmu_buf_t *db) in dmu_buf_get_objset() argument
3340 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_get_objset()
3345 dmu_buf_dnode_enter(dmu_buf_t *db) in dmu_buf_dnode_enter() argument
3347 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_dnode_enter()
3353 dmu_buf_dnode_exit(dmu_buf_t *db) in dmu_buf_dnode_exit() argument
3355 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; in dmu_buf_dnode_exit()
3360 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) in dbuf_check_blkptr() argument
3363 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_check_blkptr()
3365 if (db->db_blkptr != NULL) in dbuf_check_blkptr()
3368 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_check_blkptr()
3369 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); in dbuf_check_blkptr()
3370 BP_ZERO(db->db_blkptr); in dbuf_check_blkptr()
3373 if (db->db_level == dn->dn_phys->dn_nlevels-1) { in dbuf_check_blkptr()
3379 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); in dbuf_check_blkptr()
3380 ASSERT(db->db_parent == NULL); in dbuf_check_blkptr()
3381 db->db_parent = dn->dn_dbuf; in dbuf_check_blkptr()
3382 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; in dbuf_check_blkptr()
3383 DBUF_VERIFY(db); in dbuf_check_blkptr()
3385 dmu_buf_impl_t *parent = db->db_parent; in dbuf_check_blkptr()
3390 mutex_exit(&db->db_mtx); in dbuf_check_blkptr()
3392 parent = dbuf_hold_level(dn, db->db_level + 1, in dbuf_check_blkptr()
3393 db->db_blkid >> epbs, db); in dbuf_check_blkptr()
3395 mutex_enter(&db->db_mtx); in dbuf_check_blkptr()
3396 db->db_parent = parent; in dbuf_check_blkptr()
3398 db->db_blkptr = (blkptr_t *)parent->db.db_data + in dbuf_check_blkptr()
3399 (db->db_blkid & ((1ULL << epbs) - 1)); in dbuf_check_blkptr()
3400 DBUF_VERIFY(db); in dbuf_check_blkptr()
3417 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_prepare_encrypted_dnode_leaf() local
3419 ASSERT(MUTEX_HELD(&db->db_mtx)); in dbuf_prepare_encrypted_dnode_leaf()
3420 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT); in dbuf_prepare_encrypted_dnode_leaf()
3421 ASSERT3U(db->db_level, ==, 0); in dbuf_prepare_encrypted_dnode_leaf()
3423 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) { in dbuf_prepare_encrypted_dnode_leaf()
3433 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
3434 db->db.db_object, db->db_level, db->db_blkid); in dbuf_prepare_encrypted_dnode_leaf()
3435 err = arc_untransform(db->db_buf, db->db_objset->os_spa, in dbuf_prepare_encrypted_dnode_leaf()
3440 (void) arc_release(dr->dt.dl.dr_data, db); in dbuf_prepare_encrypted_dnode_leaf()
3442 dmu_objset_id(db->db_objset), in dbuf_prepare_encrypted_dnode_leaf()
3451 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_indirect() local
3457 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_indirect()
3459 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
3461 ASSERT(db->db_level > 0); in dbuf_sync_indirect()
3462 DBUF_VERIFY(db); in dbuf_sync_indirect()
3465 if (db->db_buf == NULL) { in dbuf_sync_indirect()
3466 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
3467 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); in dbuf_sync_indirect()
3468 mutex_enter(&db->db_mtx); in dbuf_sync_indirect()
3470 ASSERT3U(db->db_state, ==, DB_CACHED); in dbuf_sync_indirect()
3471 ASSERT(db->db_buf != NULL); in dbuf_sync_indirect()
3473 DB_DNODE_ENTER(db); in dbuf_sync_indirect()
3474 dn = DB_DNODE(db); in dbuf_sync_indirect()
3476 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_sync_indirect()
3477 dbuf_check_blkptr(dn, db); in dbuf_sync_indirect()
3478 DB_DNODE_EXIT(db); in dbuf_sync_indirect()
3481 db->db_data_pending = dr; in dbuf_sync_indirect()
3483 mutex_exit(&db->db_mtx); in dbuf_sync_indirect()
3485 dbuf_write(dr, db->db_buf, tx); in dbuf_sync_indirect()
3489 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx); in dbuf_sync_indirect()
3499 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_sync_leaf() local
3506 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); in dbuf_sync_leaf()
3508 mutex_enter(&db->db_mtx); in dbuf_sync_leaf()
3513 if (db->db_state == DB_UNCACHED) { in dbuf_sync_leaf()
3515 ASSERT(db->db.db_data == NULL); in dbuf_sync_leaf()
3516 } else if (db->db_state == DB_FILL) { in dbuf_sync_leaf()
3518 ASSERT(db->db.db_data != dr->dt.dl.dr_data); in dbuf_sync_leaf()
3520 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); in dbuf_sync_leaf()
3522 DBUF_VERIFY(db); in dbuf_sync_leaf()
3524 DB_DNODE_ENTER(db); in dbuf_sync_leaf()
3525 dn = DB_DNODE(db); in dbuf_sync_leaf()
3527 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_sync_leaf()
3539 if (db->db_blkid == DMU_BONUS_BLKID) { in dbuf_sync_leaf()
3543 ASSERT0(db->db_level); in dbuf_sync_leaf()
3548 DB_DNODE_EXIT(db); in dbuf_sync_leaf()
3550 if (*datap != db->db.db_data) { in dbuf_sync_leaf()
3551 int slots = DB_DNODE(db)->dn_num_slots; in dbuf_sync_leaf()
3556 db->db_data_pending = NULL; in dbuf_sync_leaf()
3557 drp = &db->db_last_dirty; in dbuf_sync_leaf()
3561 ASSERT(dr->dr_dbuf == db); in dbuf_sync_leaf()
3564 ASSERT(db->db_dirtycnt > 0); in dbuf_sync_leaf()
3565 db->db_dirtycnt -= 1; in dbuf_sync_leaf()
3566 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg, B_FALSE); in dbuf_sync_leaf()
3578 dbuf_check_blkptr(dn, db); in dbuf_sync_leaf()
3586 cv_wait(&db->db_changed, &db->db_mtx); in dbuf_sync_leaf()
3597 if (db->db_state != DB_NOFILL && in dbuf_sync_leaf()
3599 zfs_refcount_count(&db->db_holds) > 1 && in dbuf_sync_leaf()
3601 *datap == db->db_buf) { in dbuf_sync_leaf()
3615 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); in dbuf_sync_leaf()
3625 *datap = arc_alloc_raw_buf(os->os_spa, db, in dbuf_sync_leaf()
3630 *datap = arc_alloc_compressed_buf(os->os_spa, db, in dbuf_sync_leaf()
3633 *datap = arc_alloc_buf(os->os_spa, db, type, psize); in dbuf_sync_leaf()
3635 bcopy(db->db.db_data, (*datap)->b_data, psize); in dbuf_sync_leaf()
3637 db->db_data_pending = dr; in dbuf_sync_leaf()
3639 mutex_exit(&db->db_mtx); in dbuf_sync_leaf()
3646 DB_DNODE_EXIT(db); in dbuf_sync_leaf()
3655 DB_DNODE_EXIT(db); in dbuf_sync_leaf()
3674 ASSERT3U(dr->dr_dbuf->db.db_object, ==, in dbuf_sync_list()
3694 dmu_buf_impl_t *db = vdb; in dbuf_write_ready() local
3703 ASSERT3P(db->db_blkptr, !=, NULL); in dbuf_write_ready()
3704 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp); in dbuf_write_ready()
3706 DB_DNODE_ENTER(db); in dbuf_write_ready()
3707 dn = DB_DNODE(db); in dbuf_write_ready()
3713 ASSERT((db->db_blkid != DMU_SPILL_BLKID && in dbuf_write_ready()
3715 (db->db_blkid == DMU_SPILL_BLKID && in dbuf_write_ready()
3718 ASSERT(BP_GET_LEVEL(bp) == db->db_level); in dbuf_write_ready()
3721 mutex_enter(&db->db_mtx); in dbuf_write_ready()
3724 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_ready()
3727 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_ready()
3731 if (db->db_level == 0) { in dbuf_write_ready()
3733 if (db->db_blkid > dn->dn_phys->dn_maxblkid && in dbuf_write_ready()
3734 db->db_blkid != DMU_SPILL_BLKID) { in dbuf_write_ready()
3735 ASSERT0(db->db_objset->os_raw_receive); in dbuf_write_ready()
3736 dn->dn_phys->dn_maxblkid = db->db_blkid; in dbuf_write_ready()
3742 while (i < db->db.db_size) { in dbuf_write_ready()
3744 (void *)(((char *)db->db.db_data) + i); in dbuf_write_ready()
3761 blkptr_t *ibp = db->db.db_data; in dbuf_write_ready()
3762 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); in dbuf_write_ready()
3763 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { in dbuf_write_ready()
3769 DB_DNODE_EXIT(db); in dbuf_write_ready()
3774 mutex_exit(&db->db_mtx); in dbuf_write_ready()
3776 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG); in dbuf_write_ready()
3777 *db->db_blkptr = *bp; in dbuf_write_ready()
3778 dmu_buf_unlock_parent(db, dblt, FTAG); in dbuf_write_ready()
3792 dmu_buf_impl_t *db = vdb; in dbuf_write_children_ready() local
3797 ASSERT3U(db->db_level, >, 0); in dbuf_write_children_ready()
3798 DB_DNODE_ENTER(db); in dbuf_write_children_ready()
3799 dn = DB_DNODE(db); in dbuf_write_children_ready()
3804 for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { in dbuf_write_children_ready()
3819 rw_enter(&db->db_rwlock, RW_WRITER); in dbuf_write_children_ready()
3820 bzero(db->db.db_data, db->db.db_size); in dbuf_write_children_ready()
3821 rw_exit(&db->db_rwlock); in dbuf_write_children_ready()
3823 DB_DNODE_EXIT(db); in dbuf_write_children_ready()
3839 dmu_buf_impl_t *db = arg; in dbuf_write_physdone() local
3840 objset_t *os = db->db_objset; in dbuf_write_physdone()
3845 dr = db->db_data_pending; in dbuf_write_physdone()
3862 dmu_buf_impl_t *db = vdb; in dbuf_write_done() local
3864 blkptr_t *bp = db->db_blkptr; in dbuf_write_done()
3865 objset_t *os = db->db_objset; in dbuf_write_done()
3870 ASSERT(db->db_blkptr == bp); in dbuf_write_done()
3884 mutex_enter(&db->db_mtx); in dbuf_write_done()
3886 DBUF_VERIFY(db); in dbuf_write_done()
3888 drp = &db->db_last_dirty; in dbuf_write_done()
3889 while ((dr = *drp) != db->db_data_pending) in dbuf_write_done()
3892 ASSERT(dr->dr_dbuf == db); in dbuf_write_done()
3897 if (db->db_blkid == DMU_SPILL_BLKID) { in dbuf_write_done()
3900 DB_DNODE_ENTER(db); in dbuf_write_done()
3901 dn = DB_DNODE(db); in dbuf_write_done()
3903 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && in dbuf_write_done()
3904 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); in dbuf_write_done()
3905 DB_DNODE_EXIT(db); in dbuf_write_done()
3909 if (db->db_level == 0) { in dbuf_write_done()
3910 ASSERT(db->db_blkid != DMU_BONUS_BLKID); in dbuf_write_done()
3912 if (db->db_state != DB_NOFILL) { in dbuf_write_done()
3913 if (dr->dt.dl.dr_data != db->db_buf) in dbuf_write_done()
3914 arc_buf_destroy(dr->dt.dl.dr_data, db); in dbuf_write_done()
3919 DB_DNODE_ENTER(db); in dbuf_write_done()
3920 dn = DB_DNODE(db); in dbuf_write_done()
3922 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); in dbuf_write_done()
3923 if (!BP_IS_HOLE(db->db_blkptr)) { in dbuf_write_done()
3926 ASSERT3U(db->db_blkid, <=, in dbuf_write_done()
3927 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); in dbuf_write_done()
3928 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, in dbuf_write_done()
3929 db->db.db_size); in dbuf_write_done()
3931 DB_DNODE_EXIT(db); in dbuf_write_done()
3937 cv_broadcast(&db->db_changed); in dbuf_write_done()
3938 ASSERT(db->db_dirtycnt > 0); in dbuf_write_done()
3939 db->db_dirtycnt -= 1; in dbuf_write_done()
3940 db->db_data_pending = NULL; in dbuf_write_done()
3941 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE); in dbuf_write_done()
3960 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_ready() local
3962 dbuf_write_ready(zio, NULL, db); in dbuf_write_override_ready()
3969 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write_override_done() local
3972 mutex_enter(&db->db_mtx); in dbuf_write_override_done()
3976 arc_release(dr->dt.dl.dr_data, db); in dbuf_write_override_done()
3978 mutex_exit(&db->db_mtx); in dbuf_write_override_done()
3979 dbuf_write_done(zio, NULL, db); in dbuf_write_override_done()
4043 dbuf_can_remap(const dmu_buf_impl_t *db) in dbuf_can_remap() argument
4045 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_can_remap()
4046 blkptr_t *bp = db->db.db_data; in dbuf_can_remap()
4049 ASSERT3U(db->db_level, >, 0); in dbuf_can_remap()
4050 ASSERT3S(db->db_state, ==, DB_CACHED); in dbuf_can_remap()
4055 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { in dbuf_can_remap()
4096 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx) in dbuf_remap() argument
4098 spa_t *spa = dmu_objset_spa(db->db_objset); in dbuf_remap()
4104 if (db->db_level > 0) { in dbuf_remap()
4105 blkptr_t *bp = db->db.db_data; in dbuf_remap()
4106 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) { in dbuf_remap()
4107 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx); in dbuf_remap()
4109 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) { in dbuf_remap()
4110 dnode_phys_t *dnp = db->db.db_data; in dbuf_remap()
4111 ASSERT3U(db->db_dnode_handle->dnh_dnode->dn_type, ==, in dbuf_remap()
4113 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT; i++) { in dbuf_remap()
4129 dmu_buf_impl_t *db = dr->dr_dbuf; in dbuf_write() local
4132 dmu_buf_impl_t *parent = db->db_parent; in dbuf_write()
4141 DB_DNODE_ENTER(db); in dbuf_write()
4142 dn = DB_DNODE(db); in dbuf_write()
4145 if (db->db_state != DB_NOFILL) { in dbuf_write()
4146 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { in dbuf_write()
4153 if (BP_IS_HOLE(db->db_blkptr)) { in dbuf_write()
4156 dbuf_release_bp(db); in dbuf_write()
4158 dbuf_remap(dn, db, tx); in dbuf_write()
4167 ASSERT(db->db_level == parent->db_level-1); in dbuf_write()
4176 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && in dbuf_write()
4177 db->db_blkid != DMU_SPILL_BLKID) || in dbuf_write()
4178 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); in dbuf_write()
4179 if (db->db_blkid != DMU_SPILL_BLKID) in dbuf_write()
4180 ASSERT3P(db->db_blkptr, ==, in dbuf_write()
4181 &dn->dn_phys->dn_blkptr[db->db_blkid]); in dbuf_write()
4185 ASSERT(db->db_level == 0 || data == db->db_buf); in dbuf_write()
4186 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); in dbuf_write()
4191 db->db.db_object, db->db_level, db->db_blkid); in dbuf_write()
4193 if (db->db_blkid == DMU_SPILL_BLKID) in dbuf_write()
4195 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; in dbuf_write()
4197 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); in dbuf_write()
4199 DB_DNODE_EXIT(db); in dbuf_write()
4207 dr->dr_bp_copy = *db->db_blkptr; in dbuf_write()
4209 if (db->db_level == 0 && in dbuf_write()
4219 contents, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
4223 mutex_enter(&db->db_mtx); in dbuf_write()
4227 mutex_exit(&db->db_mtx); in dbuf_write()
4228 } else if (db->db_state == DB_NOFILL) { in dbuf_write()
4232 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp, in dbuf_write()
4234 dbuf_write_nofill_done, db, in dbuf_write()
4246 if (db->db_level != 0) in dbuf_write()
4250 &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db), in dbuf_write()
4252 dbuf_write_physdone, dbuf_write_done, db, in dbuf_write()