xref: /illumos-gate/usr/src/uts/common/fs/zfs/dbuf.c (revision 1c8564a7573482b45fcc6f9bc0c2de70f92c193c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/zfs_context.h>
27 #include <sys/dmu.h>
28 #include <sys/dmu_impl.h>
29 #include <sys/dbuf.h>
30 #include <sys/dmu_objset.h>
31 #include <sys/dsl_dataset.h>
32 #include <sys/dsl_dir.h>
33 #include <sys/dmu_tx.h>
34 #include <sys/spa.h>
35 #include <sys/zio.h>
36 #include <sys/dmu_zfetch.h>
37 
38 static void dbuf_destroy(dmu_buf_impl_t *db);
39 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
40 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
41 static arc_done_func_t dbuf_write_ready;
42 static arc_done_func_t dbuf_write_done;
43 
44 /*
45  * Global data structures and functions for the dbuf cache.
46  */
47 static kmem_cache_t *dbuf_cache;
48 
49 /* ARGSUSED */
50 static int
51 dbuf_cons(void *vdb, void *unused, int kmflag)
52 {
53 	dmu_buf_impl_t *db = vdb;
54 	bzero(db, sizeof (dmu_buf_impl_t));
55 
56 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
57 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
58 	refcount_create(&db->db_holds);
59 	return (0);
60 }
61 
62 /* ARGSUSED */
63 static void
64 dbuf_dest(void *vdb, void *unused)
65 {
66 	dmu_buf_impl_t *db = vdb;
67 	mutex_destroy(&db->db_mtx);
68 	cv_destroy(&db->db_changed);
69 	refcount_destroy(&db->db_holds);
70 }
71 
72 /*
73  * dbuf hash table routines
74  */
75 static dbuf_hash_table_t dbuf_hash_table;
76 
77 static uint64_t dbuf_hash_count;
78 
79 static uint64_t
80 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
81 {
82 	uintptr_t osv = (uintptr_t)os;
83 	uint64_t crc = -1ULL;
84 
85 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
86 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
87 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
88 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
89 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
90 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
91 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
92 
93 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
94 
95 	return (crc);
96 }
97 
98 #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
99 
100 #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
101 	((dbuf)->db.db_object == (obj) &&		\
102 	(dbuf)->db_objset == (os) &&			\
103 	(dbuf)->db_level == (level) &&			\
104 	(dbuf)->db_blkid == (blkid))
105 
106 dmu_buf_impl_t *
107 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
108 {
109 	dbuf_hash_table_t *h = &dbuf_hash_table;
110 	objset_impl_t *os = dn->dn_objset;
111 	uint64_t obj = dn->dn_object;
112 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
113 	uint64_t idx = hv & h->hash_table_mask;
114 	dmu_buf_impl_t *db;
115 
116 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
117 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
118 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
119 			mutex_enter(&db->db_mtx);
120 			if (db->db_state != DB_EVICTING) {
121 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
122 				return (db);
123 			}
124 			mutex_exit(&db->db_mtx);
125 		}
126 	}
127 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
128 	return (NULL);
129 }
130 
131 /*
132  * Insert an entry into the hash table.  If there is already an element
133  * equal to elem in the hash table, then the already existing element
134  * will be returned and the new element will not be inserted.
135  * Otherwise returns NULL.
136  */
137 static dmu_buf_impl_t *
138 dbuf_hash_insert(dmu_buf_impl_t *db)
139 {
140 	dbuf_hash_table_t *h = &dbuf_hash_table;
141 	objset_impl_t *os = db->db_objset;
142 	uint64_t obj = db->db.db_object;
143 	int level = db->db_level;
144 	uint64_t blkid = db->db_blkid;
145 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
146 	uint64_t idx = hv & h->hash_table_mask;
147 	dmu_buf_impl_t *dbf;
148 
149 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
150 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
151 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
152 			mutex_enter(&dbf->db_mtx);
153 			if (dbf->db_state != DB_EVICTING) {
154 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
155 				return (dbf);
156 			}
157 			mutex_exit(&dbf->db_mtx);
158 		}
159 	}
160 
161 	mutex_enter(&db->db_mtx);
162 	db->db_hash_next = h->hash_table[idx];
163 	h->hash_table[idx] = db;
164 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
165 	atomic_add_64(&dbuf_hash_count, 1);
166 
167 	return (NULL);
168 }
169 
170 /*
171  * Remove an entry from the hash table.  This operation will
172  * fail if there are any existing holds on the db.
173  */
174 static void
175 dbuf_hash_remove(dmu_buf_impl_t *db)
176 {
177 	dbuf_hash_table_t *h = &dbuf_hash_table;
178 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
179 	    db->db_level, db->db_blkid);
180 	uint64_t idx = hv & h->hash_table_mask;
181 	dmu_buf_impl_t *dbf, **dbp;
182 
183 	/*
184 	 * We musn't hold db_mtx to maintin lock ordering:
185 	 * DBUF_HASH_MUTEX > db_mtx.
186 	 */
187 	ASSERT(refcount_is_zero(&db->db_holds));
188 	ASSERT(db->db_state == DB_EVICTING);
189 	ASSERT(!MUTEX_HELD(&db->db_mtx));
190 
191 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
192 	dbp = &h->hash_table[idx];
193 	while ((dbf = *dbp) != db) {
194 		dbp = &dbf->db_hash_next;
195 		ASSERT(dbf != NULL);
196 	}
197 	*dbp = db->db_hash_next;
198 	db->db_hash_next = NULL;
199 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
200 	atomic_add_64(&dbuf_hash_count, -1);
201 }
202 
203 static arc_evict_func_t dbuf_do_evict;
204 
205 static void
206 dbuf_evict_user(dmu_buf_impl_t *db)
207 {
208 	ASSERT(MUTEX_HELD(&db->db_mtx));
209 
210 	if (db->db_level != 0 || db->db_evict_func == NULL)
211 		return;
212 
213 	if (db->db_user_data_ptr_ptr)
214 		*db->db_user_data_ptr_ptr = db->db.db_data;
215 	db->db_evict_func(&db->db, db->db_user_ptr);
216 	db->db_user_ptr = NULL;
217 	db->db_user_data_ptr_ptr = NULL;
218 	db->db_evict_func = NULL;
219 }
220 
221 void
222 dbuf_evict(dmu_buf_impl_t *db)
223 {
224 	ASSERT(MUTEX_HELD(&db->db_mtx));
225 	ASSERT(db->db_buf == NULL);
226 	ASSERT(db->db_data_pending == NULL);
227 
228 	dbuf_clear(db);
229 	dbuf_destroy(db);
230 }
231 
232 void
233 dbuf_init(void)
234 {
235 	uint64_t hsize = 1ULL << 16;
236 	dbuf_hash_table_t *h = &dbuf_hash_table;
237 	int i;
238 
239 	/*
240 	 * The hash table is big enough to fill all of physical memory
241 	 * with an average 4K block size.  The table will take up
242 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
243 	 */
244 	while (hsize * 4096 < physmem * PAGESIZE)
245 		hsize <<= 1;
246 
247 retry:
248 	h->hash_table_mask = hsize - 1;
249 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
250 	if (h->hash_table == NULL) {
251 		/* XXX - we should really return an error instead of assert */
252 		ASSERT(hsize > (1ULL << 10));
253 		hsize >>= 1;
254 		goto retry;
255 	}
256 
257 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
258 	    sizeof (dmu_buf_impl_t),
259 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
260 
261 	for (i = 0; i < DBUF_MUTEXES; i++)
262 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
263 }
264 
265 void
266 dbuf_fini(void)
267 {
268 	dbuf_hash_table_t *h = &dbuf_hash_table;
269 	int i;
270 
271 	for (i = 0; i < DBUF_MUTEXES; i++)
272 		mutex_destroy(&h->hash_mutexes[i]);
273 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
274 	kmem_cache_destroy(dbuf_cache);
275 }
276 
277 /*
278  * Other stuff.
279  */
280 
281 #ifdef ZFS_DEBUG
282 static void
283 dbuf_verify(dmu_buf_impl_t *db)
284 {
285 	dnode_t *dn = db->db_dnode;
286 
287 	ASSERT(MUTEX_HELD(&db->db_mtx));
288 
289 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
290 		return;
291 
292 	ASSERT(db->db_objset != NULL);
293 	if (dn == NULL) {
294 		ASSERT(db->db_parent == NULL);
295 		ASSERT(db->db_blkptr == NULL);
296 	} else {
297 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
298 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
299 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
300 		ASSERT(db->db_blkid == DB_BONUS_BLKID ||
301 		    list_head(&dn->dn_dbufs));
302 	}
303 	if (db->db_blkid == DB_BONUS_BLKID) {
304 		ASSERT(dn != NULL);
305 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
306 		ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID);
307 	} else {
308 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
309 	}
310 
311 	if (db->db_level == 0) {
312 		/* we can be momentarily larger in dnode_set_blksz() */
313 		if (db->db_blkid != DB_BONUS_BLKID && dn) {
314 			ASSERT3U(db->db.db_size, >=, dn->dn_datablksz);
315 		}
316 		if (db->db.db_object == DMU_META_DNODE_OBJECT) {
317 			dbuf_dirty_record_t *dr = db->db_data_pending;
318 			/*
319 			 * it should only be modified in syncing
320 			 * context, so make sure we only have
321 			 * one copy of the data.
322 			 */
323 			ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
324 		}
325 	}
326 
327 	/* verify db->db_blkptr */
328 	if (db->db_blkptr) {
329 		if (db->db_parent == dn->dn_dbuf) {
330 			/* db is pointed to by the dnode */
331 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
332 			if (db->db.db_object == DMU_META_DNODE_OBJECT)
333 				ASSERT(db->db_parent == NULL);
334 			else
335 				ASSERT(db->db_parent != NULL);
336 			ASSERT3P(db->db_blkptr, ==,
337 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
338 		} else {
339 			/* db is pointed to by an indirect block */
340 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
341 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
342 			ASSERT3U(db->db_parent->db.db_object, ==,
343 			    db->db.db_object);
344 			/*
345 			 * dnode_grow_indblksz() can make this fail if we don't
346 			 * have the struct_rwlock.  XXX indblksz no longer
347 			 * grows.  safe to do this now?
348 			 */
349 			if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) {
350 				ASSERT3P(db->db_blkptr, ==,
351 				    ((blkptr_t *)db->db_parent->db.db_data +
352 				    db->db_blkid % epb));
353 			}
354 		}
355 	}
356 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
357 	    db->db.db_data && db->db_blkid != DB_BONUS_BLKID &&
358 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
359 		/*
360 		 * If the blkptr isn't set but they have nonzero data,
361 		 * it had better be dirty, otherwise we'll lose that
362 		 * data when we evict this buffer.
363 		 */
364 		if (db->db_dirtycnt == 0) {
365 			uint64_t *buf = db->db.db_data;
366 			int i;
367 
368 			for (i = 0; i < db->db.db_size >> 3; i++) {
369 				ASSERT(buf[i] == 0);
370 			}
371 		}
372 	}
373 }
374 #endif
375 
376 static void
377 dbuf_update_data(dmu_buf_impl_t *db)
378 {
379 	ASSERT(MUTEX_HELD(&db->db_mtx));
380 	if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
381 		ASSERT(!refcount_is_zero(&db->db_holds));
382 		*db->db_user_data_ptr_ptr = db->db.db_data;
383 	}
384 }
385 
386 static void
387 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
388 {
389 	ASSERT(MUTEX_HELD(&db->db_mtx));
390 	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
391 	db->db_buf = buf;
392 	if (buf != NULL) {
393 		ASSERT(buf->b_data != NULL);
394 		db->db.db_data = buf->b_data;
395 		if (!arc_released(buf))
396 			arc_set_callback(buf, dbuf_do_evict, db);
397 		dbuf_update_data(db);
398 	} else {
399 		dbuf_evict_user(db);
400 		db->db.db_data = NULL;
401 		db->db_state = DB_UNCACHED;
402 	}
403 }
404 
405 uint64_t
406 dbuf_whichblock(dnode_t *dn, uint64_t offset)
407 {
408 	if (dn->dn_datablkshift) {
409 		return (offset >> dn->dn_datablkshift);
410 	} else {
411 		ASSERT3U(offset, <, dn->dn_datablksz);
412 		return (0);
413 	}
414 }
415 
416 static void
417 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
418 {
419 	dmu_buf_impl_t *db = vdb;
420 
421 	mutex_enter(&db->db_mtx);
422 	ASSERT3U(db->db_state, ==, DB_READ);
423 	/*
424 	 * All reads are synchronous, so we must have a hold on the dbuf
425 	 */
426 	ASSERT(refcount_count(&db->db_holds) > 0);
427 	ASSERT(db->db_buf == NULL);
428 	ASSERT(db->db.db_data == NULL);
429 	if (db->db_level == 0 && db->db_freed_in_flight) {
430 		/* we were freed in flight; disregard any error */
431 		arc_release(buf, db);
432 		bzero(buf->b_data, db->db.db_size);
433 		arc_buf_freeze(buf);
434 		db->db_freed_in_flight = FALSE;
435 		dbuf_set_data(db, buf);
436 		db->db_state = DB_CACHED;
437 	} else if (zio == NULL || zio->io_error == 0) {
438 		dbuf_set_data(db, buf);
439 		db->db_state = DB_CACHED;
440 	} else {
441 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
442 		ASSERT3P(db->db_buf, ==, NULL);
443 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
444 		db->db_state = DB_UNCACHED;
445 	}
446 	cv_broadcast(&db->db_changed);
447 	mutex_exit(&db->db_mtx);
448 	dbuf_rele(db, NULL);
449 }
450 
451 static void
452 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
453 {
454 	dnode_t *dn = db->db_dnode;
455 	zbookmark_t zb;
456 	uint32_t aflags = ARC_NOWAIT;
457 	arc_buf_t *pbuf;
458 
459 	ASSERT(!refcount_is_zero(&db->db_holds));
460 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
461 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
462 	ASSERT(MUTEX_HELD(&db->db_mtx));
463 	ASSERT(db->db_state == DB_UNCACHED);
464 	ASSERT(db->db_buf == NULL);
465 
466 	if (db->db_blkid == DB_BONUS_BLKID) {
467 		int bonuslen = dn->dn_bonuslen;
468 
469 		ASSERT3U(bonuslen, <=, db->db.db_size);
470 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
471 		arc_space_consume(DN_MAX_BONUSLEN);
472 		if (bonuslen < DN_MAX_BONUSLEN)
473 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
474 		bcopy(DN_BONUS(dn->dn_phys), db->db.db_data,
475 		    bonuslen);
476 		dbuf_update_data(db);
477 		db->db_state = DB_CACHED;
478 		mutex_exit(&db->db_mtx);
479 		return;
480 	}
481 
482 	/*
483 	 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
484 	 * processes the delete record and clears the bp while we are waiting
485 	 * for the dn_mtx (resulting in a "no" from block_freed).
486 	 */
487 	if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
488 	    (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
489 	    BP_IS_HOLE(db->db_blkptr)))) {
490 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
491 
492 		dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa,
493 		    db->db.db_size, db, type));
494 		bzero(db->db.db_data, db->db.db_size);
495 		db->db_state = DB_CACHED;
496 		*flags |= DB_RF_CACHED;
497 		mutex_exit(&db->db_mtx);
498 		return;
499 	}
500 
501 	db->db_state = DB_READ;
502 	mutex_exit(&db->db_mtx);
503 
504 	if (DBUF_IS_L2CACHEABLE(db))
505 		aflags |= ARC_L2CACHE;
506 
507 	zb.zb_objset = db->db_objset->os_dsl_dataset ?
508 	    db->db_objset->os_dsl_dataset->ds_object : 0;
509 	zb.zb_object = db->db.db_object;
510 	zb.zb_level = db->db_level;
511 	zb.zb_blkid = db->db_blkid;
512 
513 	dbuf_add_ref(db, NULL);
514 	/* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */
515 
516 	if (db->db_parent)
517 		pbuf = db->db_parent->db_buf;
518 	else
519 		pbuf = db->db_objset->os_phys_buf;
520 
521 	(void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf,
522 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
523 	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
524 	    &aflags, &zb);
525 	if (aflags & ARC_CACHED)
526 		*flags |= DB_RF_CACHED;
527 }
528 
529 int
530 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
531 {
532 	int err = 0;
533 	int havepzio = (zio != NULL);
534 	int prefetch;
535 
536 	/*
537 	 * We don't have to hold the mutex to check db_state because it
538 	 * can't be freed while we have a hold on the buffer.
539 	 */
540 	ASSERT(!refcount_is_zero(&db->db_holds));
541 
542 	if ((flags & DB_RF_HAVESTRUCT) == 0)
543 		rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER);
544 
545 	prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
546 	    (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL &&
547 	    DBUF_IS_CACHEABLE(db);
548 
549 	mutex_enter(&db->db_mtx);
550 	if (db->db_state == DB_CACHED) {
551 		mutex_exit(&db->db_mtx);
552 		if (prefetch)
553 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
554 			    db->db.db_size, TRUE);
555 		if ((flags & DB_RF_HAVESTRUCT) == 0)
556 			rw_exit(&db->db_dnode->dn_struct_rwlock);
557 	} else if (db->db_state == DB_UNCACHED) {
558 		if (zio == NULL) {
559 			zio = zio_root(db->db_dnode->dn_objset->os_spa,
560 			    NULL, NULL, ZIO_FLAG_CANFAIL);
561 		}
562 		dbuf_read_impl(db, zio, &flags);
563 
564 		/* dbuf_read_impl has dropped db_mtx for us */
565 
566 		if (prefetch)
567 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
568 			    db->db.db_size, flags & DB_RF_CACHED);
569 
570 		if ((flags & DB_RF_HAVESTRUCT) == 0)
571 			rw_exit(&db->db_dnode->dn_struct_rwlock);
572 
573 		if (!havepzio)
574 			err = zio_wait(zio);
575 	} else {
576 		mutex_exit(&db->db_mtx);
577 		if (prefetch)
578 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
579 			    db->db.db_size, TRUE);
580 		if ((flags & DB_RF_HAVESTRUCT) == 0)
581 			rw_exit(&db->db_dnode->dn_struct_rwlock);
582 
583 		mutex_enter(&db->db_mtx);
584 		if ((flags & DB_RF_NEVERWAIT) == 0) {
585 			while (db->db_state == DB_READ ||
586 			    db->db_state == DB_FILL) {
587 				ASSERT(db->db_state == DB_READ ||
588 				    (flags & DB_RF_HAVESTRUCT) == 0);
589 				cv_wait(&db->db_changed, &db->db_mtx);
590 			}
591 			if (db->db_state == DB_UNCACHED)
592 				err = EIO;
593 		}
594 		mutex_exit(&db->db_mtx);
595 	}
596 
597 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
598 	return (err);
599 }
600 
601 static void
602 dbuf_noread(dmu_buf_impl_t *db)
603 {
604 	ASSERT(!refcount_is_zero(&db->db_holds));
605 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
606 	mutex_enter(&db->db_mtx);
607 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
608 		cv_wait(&db->db_changed, &db->db_mtx);
609 	if (db->db_state == DB_UNCACHED) {
610 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
611 
612 		ASSERT(db->db_buf == NULL);
613 		ASSERT(db->db.db_data == NULL);
614 		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
615 		    db->db.db_size, db, type));
616 		db->db_state = DB_FILL;
617 	} else {
618 		ASSERT3U(db->db_state, ==, DB_CACHED);
619 	}
620 	mutex_exit(&db->db_mtx);
621 }
622 
623 /*
624  * This is our just-in-time copy function.  It makes a copy of
625  * buffers, that have been modified in a previous transaction
626  * group, before we modify them in the current active group.
627  *
628  * This function is used in two places: when we are dirtying a
629  * buffer for the first time in a txg, and when we are freeing
630  * a range in a dnode that includes this buffer.
631  *
632  * Note that when we are called from dbuf_free_range() we do
633  * not put a hold on the buffer, we just traverse the active
634  * dbuf list for the dnode.
635  */
636 static void
637 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
638 {
639 	dbuf_dirty_record_t *dr = db->db_last_dirty;
640 
641 	ASSERT(MUTEX_HELD(&db->db_mtx));
642 	ASSERT(db->db.db_data != NULL);
643 	ASSERT(db->db_level == 0);
644 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
645 
646 	if (dr == NULL ||
647 	    (dr->dt.dl.dr_data !=
648 	    ((db->db_blkid  == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
649 		return;
650 
651 	/*
652 	 * If the last dirty record for this dbuf has not yet synced
653 	 * and its referencing the dbuf data, either:
654 	 * 	reset the reference to point to a new copy,
655 	 * or (if there a no active holders)
656 	 *	just null out the current db_data pointer.
657 	 */
658 	ASSERT(dr->dr_txg >= txg - 2);
659 	if (db->db_blkid == DB_BONUS_BLKID) {
660 		/* Note that the data bufs here are zio_bufs */
661 		dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
662 		arc_space_consume(DN_MAX_BONUSLEN);
663 		bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
664 	} else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
665 		int size = db->db.db_size;
666 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
667 		dr->dt.dl.dr_data = arc_buf_alloc(
668 		    db->db_dnode->dn_objset->os_spa, size, db, type);
669 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
670 	} else {
671 		dbuf_set_data(db, NULL);
672 	}
673 }
674 
675 void
676 dbuf_unoverride(dbuf_dirty_record_t *dr)
677 {
678 	dmu_buf_impl_t *db = dr->dr_dbuf;
679 	uint64_t txg = dr->dr_txg;
680 
681 	ASSERT(MUTEX_HELD(&db->db_mtx));
682 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
683 	ASSERT(db->db_level == 0);
684 
685 	if (db->db_blkid == DB_BONUS_BLKID ||
686 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
687 		return;
688 
689 	/* free this block */
690 	if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) {
691 		/* XXX can get silent EIO here */
692 		(void) dsl_free(NULL,
693 		    spa_get_dsl(db->db_dnode->dn_objset->os_spa),
694 		    txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT);
695 	}
696 	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
697 	/*
698 	 * Release the already-written buffer, so we leave it in
699 	 * a consistent dirty state.  Note that all callers are
700 	 * modifying the buffer, so they will immediately do
701 	 * another (redundant) arc_release().  Therefore, leave
702 	 * the buf thawed to save the effort of freezing &
703 	 * immediately re-thawing it.
704 	 */
705 	arc_release(dr->dt.dl.dr_data, db);
706 }
707 
708 /*
709  * Evict (if its unreferenced) or clear (if its referenced) any level-0
710  * data blocks in the free range, so that any future readers will find
711  * empty blocks.  Also, if we happen accross any level-1 dbufs in the
712  * range that have not already been marked dirty, mark them dirty so
713  * they stay in memory.
714  */
715 void
716 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
717 {
718 	dmu_buf_impl_t *db, *db_next;
719 	uint64_t txg = tx->tx_txg;
720 	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
721 	uint64_t first_l1 = start >> epbs;
722 	uint64_t last_l1 = end >> epbs;
723 
724 	if (end > dn->dn_maxblkid) {
725 		end = dn->dn_maxblkid;
726 		last_l1 = end >> epbs;
727 	}
728 	dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
729 	mutex_enter(&dn->dn_dbufs_mtx);
730 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
731 		db_next = list_next(&dn->dn_dbufs, db);
732 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
733 
734 		if (db->db_level == 1 &&
735 		    db->db_blkid >= first_l1 && db->db_blkid <= last_l1) {
736 			mutex_enter(&db->db_mtx);
737 			if (db->db_last_dirty &&
738 			    db->db_last_dirty->dr_txg < txg) {
739 				dbuf_add_ref(db, FTAG);
740 				mutex_exit(&db->db_mtx);
741 				dbuf_will_dirty(db, tx);
742 				dbuf_rele(db, FTAG);
743 			} else {
744 				mutex_exit(&db->db_mtx);
745 			}
746 		}
747 
748 		if (db->db_level != 0)
749 			continue;
750 		dprintf_dbuf(db, "found buf %s\n", "");
751 		if (db->db_blkid < start || db->db_blkid > end)
752 			continue;
753 
754 		/* found a level 0 buffer in the range */
755 		if (dbuf_undirty(db, tx))
756 			continue;
757 
758 		mutex_enter(&db->db_mtx);
759 		if (db->db_state == DB_UNCACHED ||
760 		    db->db_state == DB_EVICTING) {
761 			ASSERT(db->db.db_data == NULL);
762 			mutex_exit(&db->db_mtx);
763 			continue;
764 		}
765 		if (db->db_state == DB_READ || db->db_state == DB_FILL) {
766 			/* will be handled in dbuf_read_done or dbuf_rele */
767 			db->db_freed_in_flight = TRUE;
768 			mutex_exit(&db->db_mtx);
769 			continue;
770 		}
771 		if (refcount_count(&db->db_holds) == 0) {
772 			ASSERT(db->db_buf);
773 			dbuf_clear(db);
774 			continue;
775 		}
776 		/* The dbuf is referenced */
777 
778 		if (db->db_last_dirty != NULL) {
779 			dbuf_dirty_record_t *dr = db->db_last_dirty;
780 
781 			if (dr->dr_txg == txg) {
782 				/*
783 				 * This buffer is "in-use", re-adjust the file
784 				 * size to reflect that this buffer may
785 				 * contain new data when we sync.
786 				 */
787 				if (db->db_blkid > dn->dn_maxblkid)
788 					dn->dn_maxblkid = db->db_blkid;
789 				dbuf_unoverride(dr);
790 			} else {
791 				/*
792 				 * This dbuf is not dirty in the open context.
793 				 * Either uncache it (if its not referenced in
794 				 * the open context) or reset its contents to
795 				 * empty.
796 				 */
797 				dbuf_fix_old_data(db, txg);
798 			}
799 		}
800 		/* clear the contents if its cached */
801 		if (db->db_state == DB_CACHED) {
802 			ASSERT(db->db.db_data != NULL);
803 			arc_release(db->db_buf, db);
804 			bzero(db->db.db_data, db->db.db_size);
805 			arc_buf_freeze(db->db_buf);
806 		}
807 
808 		mutex_exit(&db->db_mtx);
809 	}
810 	mutex_exit(&dn->dn_dbufs_mtx);
811 }
812 
813 static int
814 dbuf_block_freeable(dmu_buf_impl_t *db)
815 {
816 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
817 	uint64_t birth_txg = 0;
818 
819 	/*
820 	 * We don't need any locking to protect db_blkptr:
821 	 * If it's syncing, then db_last_dirty will be set
822 	 * so we'll ignore db_blkptr.
823 	 */
824 	ASSERT(MUTEX_HELD(&db->db_mtx));
825 	if (db->db_last_dirty)
826 		birth_txg = db->db_last_dirty->dr_txg;
827 	else if (db->db_blkptr)
828 		birth_txg = db->db_blkptr->blk_birth;
829 
830 	/* If we don't exist or are in a snapshot, we can't be freed */
831 	if (birth_txg)
832 		return (ds == NULL ||
833 		    dsl_dataset_block_freeable(ds, birth_txg));
834 	else
835 		return (FALSE);
836 }
837 
838 void
839 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
840 {
841 	arc_buf_t *buf, *obuf;
842 	int osize = db->db.db_size;
843 	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
844 
845 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
846 
847 	/* XXX does *this* func really need the lock? */
848 	ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock));
849 
850 	/*
851 	 * This call to dbuf_will_dirty() with the dn_struct_rwlock held
852 	 * is OK, because there can be no other references to the db
853 	 * when we are changing its size, so no concurrent DB_FILL can
854 	 * be happening.
855 	 */
856 	/*
857 	 * XXX we should be doing a dbuf_read, checking the return
858 	 * value and returning that up to our callers
859 	 */
860 	dbuf_will_dirty(db, tx);
861 
862 	/* create the data buffer for the new block */
863 	buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type);
864 
865 	/* copy old block data to the new block */
866 	obuf = db->db_buf;
867 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
868 	/* zero the remainder */
869 	if (size > osize)
870 		bzero((uint8_t *)buf->b_data + osize, size - osize);
871 
872 	mutex_enter(&db->db_mtx);
873 	dbuf_set_data(db, buf);
874 	VERIFY(arc_buf_remove_ref(obuf, db) == 1);
875 	db->db.db_size = size;
876 
877 	if (db->db_level == 0) {
878 		ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
879 		db->db_last_dirty->dt.dl.dr_data = buf;
880 	}
881 	mutex_exit(&db->db_mtx);
882 
883 	dnode_willuse_space(db->db_dnode, size-osize, tx);
884 }
885 
886 dbuf_dirty_record_t *
887 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
888 {
889 	dnode_t *dn = db->db_dnode;
890 	objset_impl_t *os = dn->dn_objset;
891 	dbuf_dirty_record_t **drp, *dr;
892 	int drop_struct_lock = FALSE;
893 	int txgoff = tx->tx_txg & TXG_MASK;
894 
895 	ASSERT(tx->tx_txg != 0);
896 	ASSERT(!refcount_is_zero(&db->db_holds));
897 	DMU_TX_DIRTY_BUF(tx, db);
898 
899 	/*
900 	 * Shouldn't dirty a regular buffer in syncing context.  Private
901 	 * objects may be dirtied in syncing context, but only if they
902 	 * were already pre-dirtied in open context.
903 	 * XXX We may want to prohibit dirtying in syncing context even
904 	 * if they did pre-dirty.
905 	 */
906 	ASSERT(!dmu_tx_is_syncing(tx) ||
907 	    BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
908 	    dn->dn_object == DMU_META_DNODE_OBJECT ||
909 	    dn->dn_objset->os_dsl_dataset == NULL ||
910 	    dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir));
911 
912 	/*
913 	 * We make this assert for private objects as well, but after we
914 	 * check if we're already dirty.  They are allowed to re-dirty
915 	 * in syncing context.
916 	 */
917 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
918 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
919 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
920 
921 	mutex_enter(&db->db_mtx);
922 	/*
923 	 * XXX make this true for indirects too?  The problem is that
924 	 * transactions created with dmu_tx_create_assigned() from
925 	 * syncing context don't bother holding ahead.
926 	 */
927 	ASSERT(db->db_level != 0 ||
928 	    db->db_state == DB_CACHED || db->db_state == DB_FILL);
929 
930 	mutex_enter(&dn->dn_mtx);
931 	/*
932 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
933 	 * initialize the objset.
934 	 */
935 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
936 	    !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
937 		dn->dn_dirtyctx =
938 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
939 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
940 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
941 	}
942 	mutex_exit(&dn->dn_mtx);
943 
944 	/*
945 	 * If this buffer is already dirty, we're done.
946 	 */
947 	drp = &db->db_last_dirty;
948 	ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
949 	    db->db.db_object == DMU_META_DNODE_OBJECT);
950 	while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
951 		drp = &dr->dr_next;
952 	if (dr && dr->dr_txg == tx->tx_txg) {
953 		if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) {
954 			/*
955 			 * If this buffer has already been written out,
956 			 * we now need to reset its state.
957 			 */
958 			dbuf_unoverride(dr);
959 			if (db->db.db_object != DMU_META_DNODE_OBJECT)
960 				arc_buf_thaw(db->db_buf);
961 		}
962 		mutex_exit(&db->db_mtx);
963 		return (dr);
964 	}
965 
966 	/*
967 	 * Only valid if not already dirty.
968 	 */
969 	ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
970 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
971 
972 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
973 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
974 	    dn->dn_phys->dn_nlevels > db->db_level ||
975 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
976 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
977 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
978 
979 	/*
980 	 * We should only be dirtying in syncing context if it's the
981 	 * mos, a spa os, or we're initializing the os.  However, we are
982 	 * allowed to dirty in syncing context provided we already
983 	 * dirtied it in open context.  Hence we must make this
984 	 * assertion only if we're not already dirty.
985 	 */
986 	ASSERT(!dmu_tx_is_syncing(tx) ||
987 	    os->os_dsl_dataset == NULL ||
988 	    !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) ||
989 	    !BP_IS_HOLE(os->os_rootbp));
990 	ASSERT(db->db.db_size != 0);
991 
992 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
993 
994 	if (db->db_blkid != DB_BONUS_BLKID) {
995 		/*
996 		 * Update the accounting.
997 		 */
998 		if (dbuf_block_freeable(db)) {
999 			blkptr_t *bp = db->db_blkptr;
1000 			int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1001 			    bp_get_dasize(os->os_spa, bp) : db->db.db_size;
1002 			/*
1003 			 * This is only a guess -- if the dbuf is dirty
1004 			 * in a previous txg, we don't know how much
1005 			 * space it will use on disk yet.  We should
1006 			 * really have the struct_rwlock to access
1007 			 * db_blkptr, but since this is just a guess,
1008 			 * it's OK if we get an odd answer.
1009 			 */
1010 			dnode_willuse_space(dn, -willfree, tx);
1011 		}
1012 		dnode_willuse_space(dn, db->db.db_size, tx);
1013 	}
1014 
1015 	/*
1016 	 * If this buffer is dirty in an old transaction group we need
1017 	 * to make a copy of it so that the changes we make in this
1018 	 * transaction group won't leak out when we sync the older txg.
1019 	 */
1020 	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1021 	if (db->db_level == 0) {
1022 		void *data_old = db->db_buf;
1023 
1024 		if (db->db_blkid == DB_BONUS_BLKID) {
1025 			dbuf_fix_old_data(db, tx->tx_txg);
1026 			data_old = db->db.db_data;
1027 		} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1028 			/*
1029 			 * Release the data buffer from the cache so that we
1030 			 * can modify it without impacting possible other users
1031 			 * of this cached data block.  Note that indirect
1032 			 * blocks and private objects are not released until the
1033 			 * syncing state (since they are only modified then).
1034 			 */
1035 			arc_release(db->db_buf, db);
1036 			dbuf_fix_old_data(db, tx->tx_txg);
1037 			data_old = db->db_buf;
1038 		}
1039 		ASSERT(data_old != NULL);
1040 		dr->dt.dl.dr_data = data_old;
1041 	} else {
1042 		mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1043 		list_create(&dr->dt.di.dr_children,
1044 		    sizeof (dbuf_dirty_record_t),
1045 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
1046 	}
1047 	dr->dr_dbuf = db;
1048 	dr->dr_txg = tx->tx_txg;
1049 	dr->dr_next = *drp;
1050 	*drp = dr;
1051 
1052 	/*
1053 	 * We could have been freed_in_flight between the dbuf_noread
1054 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1055 	 * happened after the free.
1056 	 */
1057 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) {
1058 		mutex_enter(&dn->dn_mtx);
1059 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1060 		mutex_exit(&dn->dn_mtx);
1061 		db->db_freed_in_flight = FALSE;
1062 	}
1063 
1064 	/*
1065 	 * This buffer is now part of this txg
1066 	 */
1067 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1068 	db->db_dirtycnt += 1;
1069 	ASSERT3U(db->db_dirtycnt, <=, 3);
1070 
1071 	mutex_exit(&db->db_mtx);
1072 
1073 	if (db->db_blkid == DB_BONUS_BLKID) {
1074 		mutex_enter(&dn->dn_mtx);
1075 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1076 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1077 		mutex_exit(&dn->dn_mtx);
1078 		dnode_setdirty(dn, tx);
1079 		return (dr);
1080 	}
1081 
1082 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1083 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1084 		drop_struct_lock = TRUE;
1085 	}
1086 
1087 	if (db->db_level == 0) {
1088 		dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1089 		ASSERT(dn->dn_maxblkid >= db->db_blkid);
1090 	}
1091 
1092 	if (db->db_level+1 < dn->dn_nlevels) {
1093 		dmu_buf_impl_t *parent = db->db_parent;
1094 		dbuf_dirty_record_t *di;
1095 		int parent_held = FALSE;
1096 
1097 		if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1098 			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1099 
1100 			parent = dbuf_hold_level(dn, db->db_level+1,
1101 			    db->db_blkid >> epbs, FTAG);
1102 			parent_held = TRUE;
1103 		}
1104 		if (drop_struct_lock)
1105 			rw_exit(&dn->dn_struct_rwlock);
1106 		ASSERT3U(db->db_level+1, ==, parent->db_level);
1107 		di = dbuf_dirty(parent, tx);
1108 		if (parent_held)
1109 			dbuf_rele(parent, FTAG);
1110 
1111 		mutex_enter(&db->db_mtx);
1112 		/*  possible race with dbuf_undirty() */
1113 		if (db->db_last_dirty == dr ||
1114 		    dn->dn_object == DMU_META_DNODE_OBJECT) {
1115 			mutex_enter(&di->dt.di.dr_mtx);
1116 			ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1117 			ASSERT(!list_link_active(&dr->dr_dirty_node));
1118 			list_insert_tail(&di->dt.di.dr_children, dr);
1119 			mutex_exit(&di->dt.di.dr_mtx);
1120 			dr->dr_parent = di;
1121 		}
1122 		mutex_exit(&db->db_mtx);
1123 	} else {
1124 		ASSERT(db->db_level+1 == dn->dn_nlevels);
1125 		ASSERT(db->db_blkid < dn->dn_nblkptr);
1126 		ASSERT(db->db_parent == NULL ||
1127 		    db->db_parent == db->db_dnode->dn_dbuf);
1128 		mutex_enter(&dn->dn_mtx);
1129 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1130 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1131 		mutex_exit(&dn->dn_mtx);
1132 		if (drop_struct_lock)
1133 			rw_exit(&dn->dn_struct_rwlock);
1134 	}
1135 
1136 	dnode_setdirty(dn, tx);
1137 	return (dr);
1138 }
1139 
1140 static int
1141 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1142 {
1143 	dnode_t *dn = db->db_dnode;
1144 	uint64_t txg = tx->tx_txg;
1145 	dbuf_dirty_record_t *dr, **drp;
1146 
1147 	ASSERT(txg != 0);
1148 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1149 
1150 	mutex_enter(&db->db_mtx);
1151 
1152 	/*
1153 	 * If this buffer is not dirty, we're done.
1154 	 */
1155 	for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1156 		if (dr->dr_txg <= txg)
1157 			break;
1158 	if (dr == NULL || dr->dr_txg < txg) {
1159 		mutex_exit(&db->db_mtx);
1160 		return (0);
1161 	}
1162 	ASSERT(dr->dr_txg == txg);
1163 
1164 	/*
1165 	 * If this buffer is currently held, we cannot undirty
1166 	 * it, since one of the current holders may be in the
1167 	 * middle of an update.  Note that users of dbuf_undirty()
1168 	 * should not place a hold on the dbuf before the call.
1169 	 */
1170 	if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1171 		mutex_exit(&db->db_mtx);
1172 		/* Make sure we don't toss this buffer at sync phase */
1173 		mutex_enter(&dn->dn_mtx);
1174 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1175 		mutex_exit(&dn->dn_mtx);
1176 		return (0);
1177 	}
1178 
1179 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1180 
1181 	ASSERT(db->db.db_size != 0);
1182 
1183 	/* XXX would be nice to fix up dn_towrite_space[] */
1184 
1185 	*drp = dr->dr_next;
1186 
1187 	if (dr->dr_parent) {
1188 		mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1189 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1190 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1191 	} else if (db->db_level+1 == dn->dn_nlevels) {
1192 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1193 		mutex_enter(&dn->dn_mtx);
1194 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1195 		mutex_exit(&dn->dn_mtx);
1196 	}
1197 
1198 	if (db->db_level == 0) {
1199 		dbuf_unoverride(dr);
1200 
1201 		ASSERT(db->db_buf != NULL);
1202 		ASSERT(dr->dt.dl.dr_data != NULL);
1203 		if (dr->dt.dl.dr_data != db->db_buf)
1204 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1);
1205 	} else {
1206 		ASSERT(db->db_buf != NULL);
1207 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1208 		mutex_destroy(&dr->dt.di.dr_mtx);
1209 		list_destroy(&dr->dt.di.dr_children);
1210 	}
1211 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
1212 
1213 	ASSERT(db->db_dirtycnt > 0);
1214 	db->db_dirtycnt -= 1;
1215 
1216 	if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1217 		arc_buf_t *buf = db->db_buf;
1218 
1219 		ASSERT(arc_released(buf));
1220 		dbuf_set_data(db, NULL);
1221 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
1222 		dbuf_evict(db);
1223 		return (1);
1224 	}
1225 
1226 	mutex_exit(&db->db_mtx);
1227 	return (0);
1228 }
1229 
1230 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1231 void
1232 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1233 {
1234 	int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1235 
1236 	ASSERT(tx->tx_txg != 0);
1237 	ASSERT(!refcount_is_zero(&db->db_holds));
1238 
1239 	if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock))
1240 		rf |= DB_RF_HAVESTRUCT;
1241 	(void) dbuf_read(db, NULL, rf);
1242 	(void) dbuf_dirty(db, tx);
1243 }
1244 
1245 void
1246 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1247 {
1248 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1249 
1250 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1251 	ASSERT(tx->tx_txg != 0);
1252 	ASSERT(db->db_level == 0);
1253 	ASSERT(!refcount_is_zero(&db->db_holds));
1254 
1255 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1256 	    dmu_tx_private_ok(tx));
1257 
1258 	dbuf_noread(db);
1259 	(void) dbuf_dirty(db, tx);
1260 }
1261 
1262 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1263 /* ARGSUSED */
1264 void
1265 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1266 {
1267 	mutex_enter(&db->db_mtx);
1268 	DBUF_VERIFY(db);
1269 
1270 	if (db->db_state == DB_FILL) {
1271 		if (db->db_level == 0 && db->db_freed_in_flight) {
1272 			ASSERT(db->db_blkid != DB_BONUS_BLKID);
1273 			/* we were freed while filling */
1274 			/* XXX dbuf_undirty? */
1275 			bzero(db->db.db_data, db->db.db_size);
1276 			db->db_freed_in_flight = FALSE;
1277 		}
1278 		db->db_state = DB_CACHED;
1279 		cv_broadcast(&db->db_changed);
1280 	}
1281 	mutex_exit(&db->db_mtx);
1282 }
1283 
1284 /*
1285  * "Clear" the contents of this dbuf.  This will mark the dbuf
1286  * EVICTING and clear *most* of its references.  Unfortunetely,
1287  * when we are not holding the dn_dbufs_mtx, we can't clear the
1288  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1289  * in this case.  For callers from the DMU we will usually see:
1290  *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1291  * For the arc callback, we will usually see:
1292  * 	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1293  * Sometimes, though, we will get a mix of these two:
1294  *	DMU: dbuf_clear()->arc_buf_evict()
1295  *	ARC: dbuf_do_evict()->dbuf_destroy()
1296  */
1297 void
1298 dbuf_clear(dmu_buf_impl_t *db)
1299 {
1300 	dnode_t *dn = db->db_dnode;
1301 	dmu_buf_impl_t *parent = db->db_parent;
1302 	dmu_buf_impl_t *dndb = dn->dn_dbuf;
1303 	int dbuf_gone = FALSE;
1304 
1305 	ASSERT(MUTEX_HELD(&db->db_mtx));
1306 	ASSERT(refcount_is_zero(&db->db_holds));
1307 
1308 	dbuf_evict_user(db);
1309 
1310 	if (db->db_state == DB_CACHED) {
1311 		ASSERT(db->db.db_data != NULL);
1312 		if (db->db_blkid == DB_BONUS_BLKID) {
1313 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1314 			arc_space_return(DN_MAX_BONUSLEN);
1315 		}
1316 		db->db.db_data = NULL;
1317 		db->db_state = DB_UNCACHED;
1318 	}
1319 
1320 	ASSERT3U(db->db_state, ==, DB_UNCACHED);
1321 	ASSERT(db->db_data_pending == NULL);
1322 
1323 	db->db_state = DB_EVICTING;
1324 	db->db_blkptr = NULL;
1325 
1326 	if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1327 		list_remove(&dn->dn_dbufs, db);
1328 		dnode_rele(dn, db);
1329 		db->db_dnode = NULL;
1330 	}
1331 
1332 	if (db->db_buf)
1333 		dbuf_gone = arc_buf_evict(db->db_buf);
1334 
1335 	if (!dbuf_gone)
1336 		mutex_exit(&db->db_mtx);
1337 
1338 	/*
1339 	 * If this dbuf is referened from an indirect dbuf,
1340 	 * decrement the ref count on the indirect dbuf.
1341 	 */
1342 	if (parent && parent != dndb)
1343 		dbuf_rele(parent, db);
1344 }
1345 
1346 static int
1347 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1348     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1349 {
1350 	int nlevels, epbs;
1351 
1352 	*parentp = NULL;
1353 	*bpp = NULL;
1354 
1355 	ASSERT(blkid != DB_BONUS_BLKID);
1356 
1357 	if (dn->dn_phys->dn_nlevels == 0)
1358 		nlevels = 1;
1359 	else
1360 		nlevels = dn->dn_phys->dn_nlevels;
1361 
1362 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1363 
1364 	ASSERT3U(level * epbs, <, 64);
1365 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1366 	if (level >= nlevels ||
1367 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1368 		/* the buffer has no parent yet */
1369 		return (ENOENT);
1370 	} else if (level < nlevels-1) {
1371 		/* this block is referenced from an indirect block */
1372 		int err = dbuf_hold_impl(dn, level+1,
1373 		    blkid >> epbs, fail_sparse, NULL, parentp);
1374 		if (err)
1375 			return (err);
1376 		err = dbuf_read(*parentp, NULL,
1377 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1378 		if (err) {
1379 			dbuf_rele(*parentp, NULL);
1380 			*parentp = NULL;
1381 			return (err);
1382 		}
1383 		*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1384 		    (blkid & ((1ULL << epbs) - 1));
1385 		return (0);
1386 	} else {
1387 		/* the block is referenced from the dnode */
1388 		ASSERT3U(level, ==, nlevels-1);
1389 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1390 		    blkid < dn->dn_phys->dn_nblkptr);
1391 		if (dn->dn_dbuf) {
1392 			dbuf_add_ref(dn->dn_dbuf, NULL);
1393 			*parentp = dn->dn_dbuf;
1394 		}
1395 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1396 		return (0);
1397 	}
1398 }
1399 
1400 static dmu_buf_impl_t *
1401 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1402     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1403 {
1404 	objset_impl_t *os = dn->dn_objset;
1405 	dmu_buf_impl_t *db, *odb;
1406 
1407 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1408 	ASSERT(dn->dn_type != DMU_OT_NONE);
1409 
1410 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1411 
1412 	db->db_objset = os;
1413 	db->db.db_object = dn->dn_object;
1414 	db->db_level = level;
1415 	db->db_blkid = blkid;
1416 	db->db_last_dirty = NULL;
1417 	db->db_dirtycnt = 0;
1418 	db->db_dnode = dn;
1419 	db->db_parent = parent;
1420 	db->db_blkptr = blkptr;
1421 
1422 	db->db_user_ptr = NULL;
1423 	db->db_user_data_ptr_ptr = NULL;
1424 	db->db_evict_func = NULL;
1425 	db->db_immediate_evict = 0;
1426 	db->db_freed_in_flight = 0;
1427 
1428 	if (blkid == DB_BONUS_BLKID) {
1429 		ASSERT3P(parent, ==, dn->dn_dbuf);
1430 		db->db.db_size = DN_MAX_BONUSLEN -
1431 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
1432 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1433 		db->db.db_offset = DB_BONUS_BLKID;
1434 		db->db_state = DB_UNCACHED;
1435 		/* the bonus dbuf is not placed in the hash table */
1436 		arc_space_consume(sizeof (dmu_buf_impl_t));
1437 		return (db);
1438 	} else {
1439 		int blocksize =
1440 		    db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1441 		db->db.db_size = blocksize;
1442 		db->db.db_offset = db->db_blkid * blocksize;
1443 	}
1444 
1445 	/*
1446 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1447 	 * in the hash table *and* added to the dbufs list.
1448 	 * This prevents a possible deadlock with someone
1449 	 * trying to look up this dbuf before its added to the
1450 	 * dn_dbufs list.
1451 	 */
1452 	mutex_enter(&dn->dn_dbufs_mtx);
1453 	db->db_state = DB_EVICTING;
1454 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1455 		/* someone else inserted it first */
1456 		kmem_cache_free(dbuf_cache, db);
1457 		mutex_exit(&dn->dn_dbufs_mtx);
1458 		return (odb);
1459 	}
1460 	list_insert_head(&dn->dn_dbufs, db);
1461 	db->db_state = DB_UNCACHED;
1462 	mutex_exit(&dn->dn_dbufs_mtx);
1463 	arc_space_consume(sizeof (dmu_buf_impl_t));
1464 
1465 	if (parent && parent != dn->dn_dbuf)
1466 		dbuf_add_ref(parent, db);
1467 
1468 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1469 	    refcount_count(&dn->dn_holds) > 0);
1470 	(void) refcount_add(&dn->dn_holds, db);
1471 
1472 	dprintf_dbuf(db, "db=%p\n", db);
1473 
1474 	return (db);
1475 }
1476 
1477 static int
1478 dbuf_do_evict(void *private)
1479 {
1480 	arc_buf_t *buf = private;
1481 	dmu_buf_impl_t *db = buf->b_private;
1482 
1483 	if (!MUTEX_HELD(&db->db_mtx))
1484 		mutex_enter(&db->db_mtx);
1485 
1486 	ASSERT(refcount_is_zero(&db->db_holds));
1487 
1488 	if (db->db_state != DB_EVICTING) {
1489 		ASSERT(db->db_state == DB_CACHED);
1490 		DBUF_VERIFY(db);
1491 		db->db_buf = NULL;
1492 		dbuf_evict(db);
1493 	} else {
1494 		mutex_exit(&db->db_mtx);
1495 		dbuf_destroy(db);
1496 	}
1497 	return (0);
1498 }
1499 
1500 static void
1501 dbuf_destroy(dmu_buf_impl_t *db)
1502 {
1503 	ASSERT(refcount_is_zero(&db->db_holds));
1504 
1505 	if (db->db_blkid != DB_BONUS_BLKID) {
1506 		/*
1507 		 * If this dbuf is still on the dn_dbufs list,
1508 		 * remove it from that list.
1509 		 */
1510 		if (db->db_dnode) {
1511 			dnode_t *dn = db->db_dnode;
1512 
1513 			mutex_enter(&dn->dn_dbufs_mtx);
1514 			list_remove(&dn->dn_dbufs, db);
1515 			mutex_exit(&dn->dn_dbufs_mtx);
1516 
1517 			dnode_rele(dn, db);
1518 			db->db_dnode = NULL;
1519 		}
1520 		dbuf_hash_remove(db);
1521 	}
1522 	db->db_parent = NULL;
1523 	db->db_buf = NULL;
1524 
1525 	ASSERT(!list_link_active(&db->db_link));
1526 	ASSERT(db->db.db_data == NULL);
1527 	ASSERT(db->db_hash_next == NULL);
1528 	ASSERT(db->db_blkptr == NULL);
1529 	ASSERT(db->db_data_pending == NULL);
1530 
1531 	kmem_cache_free(dbuf_cache, db);
1532 	arc_space_return(sizeof (dmu_buf_impl_t));
1533 }
1534 
1535 void
1536 dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1537 {
1538 	dmu_buf_impl_t *db = NULL;
1539 	blkptr_t *bp = NULL;
1540 
1541 	ASSERT(blkid != DB_BONUS_BLKID);
1542 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1543 
1544 	if (dnode_block_freed(dn, blkid))
1545 		return;
1546 
1547 	/* dbuf_find() returns with db_mtx held */
1548 	if (db = dbuf_find(dn, 0, blkid)) {
1549 		if (refcount_count(&db->db_holds) > 0) {
1550 			/*
1551 			 * This dbuf is active.  We assume that it is
1552 			 * already CACHED, or else about to be either
1553 			 * read or filled.
1554 			 */
1555 			mutex_exit(&db->db_mtx);
1556 			return;
1557 		}
1558 		mutex_exit(&db->db_mtx);
1559 		db = NULL;
1560 	}
1561 
1562 	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
1563 		if (bp && !BP_IS_HOLE(bp)) {
1564 			arc_buf_t *pbuf;
1565 			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
1566 			zbookmark_t zb;
1567 			zb.zb_objset = dn->dn_objset->os_dsl_dataset ?
1568 			    dn->dn_objset->os_dsl_dataset->ds_object : 0;
1569 			zb.zb_object = dn->dn_object;
1570 			zb.zb_level = 0;
1571 			zb.zb_blkid = blkid;
1572 
1573 			if (db)
1574 				pbuf = db->db_buf;
1575 			else
1576 				pbuf = dn->dn_objset->os_phys_buf;
1577 
1578 			(void) arc_read(NULL, dn->dn_objset->os_spa,
1579 			    bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
1580 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
1581 			    &aflags, &zb);
1582 		}
1583 		if (db)
1584 			dbuf_rele(db, NULL);
1585 	}
1586 }
1587 
1588 /*
1589  * Returns with db_holds incremented, and db_mtx not held.
1590  * Note: dn_struct_rwlock must be held.
1591  */
1592 int
1593 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1594     void *tag, dmu_buf_impl_t **dbp)
1595 {
1596 	dmu_buf_impl_t *db, *parent = NULL;
1597 
1598 	ASSERT(blkid != DB_BONUS_BLKID);
1599 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1600 	ASSERT3U(dn->dn_nlevels, >, level);
1601 
1602 	*dbp = NULL;
1603 top:
1604 	/* dbuf_find() returns with db_mtx held */
1605 	db = dbuf_find(dn, level, blkid);
1606 
1607 	if (db == NULL) {
1608 		blkptr_t *bp = NULL;
1609 		int err;
1610 
1611 		ASSERT3P(parent, ==, NULL);
1612 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1613 		if (fail_sparse) {
1614 			if (err == 0 && bp && BP_IS_HOLE(bp))
1615 				err = ENOENT;
1616 			if (err) {
1617 				if (parent)
1618 					dbuf_rele(parent, NULL);
1619 				return (err);
1620 			}
1621 		}
1622 		if (err && err != ENOENT)
1623 			return (err);
1624 		db = dbuf_create(dn, level, blkid, parent, bp);
1625 	}
1626 
1627 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1628 		arc_buf_add_ref(db->db_buf, db);
1629 		if (db->db_buf->b_data == NULL) {
1630 			dbuf_clear(db);
1631 			if (parent) {
1632 				dbuf_rele(parent, NULL);
1633 				parent = NULL;
1634 			}
1635 			goto top;
1636 		}
1637 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1638 	}
1639 
1640 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1641 
1642 	/*
1643 	 * If this buffer is currently syncing out, and we are are
1644 	 * still referencing it from db_data, we need to make a copy
1645 	 * of it in case we decide we want to dirty it again in this txg.
1646 	 */
1647 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
1648 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
1649 	    db->db_state == DB_CACHED && db->db_data_pending) {
1650 		dbuf_dirty_record_t *dr = db->db_data_pending;
1651 
1652 		if (dr->dt.dl.dr_data == db->db_buf) {
1653 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1654 
1655 			dbuf_set_data(db,
1656 			    arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
1657 			    db->db.db_size, db, type));
1658 			bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
1659 			    db->db.db_size);
1660 		}
1661 	}
1662 
1663 	(void) refcount_add(&db->db_holds, tag);
1664 	dbuf_update_data(db);
1665 	DBUF_VERIFY(db);
1666 	mutex_exit(&db->db_mtx);
1667 
1668 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
1669 	if (parent)
1670 		dbuf_rele(parent, NULL);
1671 
1672 	ASSERT3P(db->db_dnode, ==, dn);
1673 	ASSERT3U(db->db_blkid, ==, blkid);
1674 	ASSERT3U(db->db_level, ==, level);
1675 	*dbp = db;
1676 
1677 	return (0);
1678 }
1679 
1680 dmu_buf_impl_t *
1681 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1682 {
1683 	dmu_buf_impl_t *db;
1684 	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1685 	return (err ? NULL : db);
1686 }
1687 
1688 dmu_buf_impl_t *
1689 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1690 {
1691 	dmu_buf_impl_t *db;
1692 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1693 	return (err ? NULL : db);
1694 }
1695 
1696 void
1697 dbuf_create_bonus(dnode_t *dn)
1698 {
1699 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1700 
1701 	ASSERT(dn->dn_bonus == NULL);
1702 	dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL);
1703 }
1704 
1705 #pragma weak dmu_buf_add_ref = dbuf_add_ref
1706 void
1707 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
1708 {
1709 	int64_t holds = refcount_add(&db->db_holds, tag);
1710 	ASSERT(holds > 1);
1711 }
1712 
1713 #pragma weak dmu_buf_rele = dbuf_rele
1714 void
1715 dbuf_rele(dmu_buf_impl_t *db, void *tag)
1716 {
1717 	int64_t holds;
1718 
1719 	mutex_enter(&db->db_mtx);
1720 	DBUF_VERIFY(db);
1721 
1722 	holds = refcount_remove(&db->db_holds, tag);
1723 	ASSERT(holds >= 0);
1724 
1725 	/*
1726 	 * We can't freeze indirects if there is a possibility that they
1727 	 * may be modified in the current syncing context.
1728 	 */
1729 	if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
1730 		arc_buf_freeze(db->db_buf);
1731 
1732 	if (holds == db->db_dirtycnt &&
1733 	    db->db_level == 0 && db->db_immediate_evict)
1734 		dbuf_evict_user(db);
1735 
1736 	if (holds == 0) {
1737 		if (db->db_blkid == DB_BONUS_BLKID) {
1738 			mutex_exit(&db->db_mtx);
1739 			dnode_rele(db->db_dnode, db);
1740 		} else if (db->db_buf == NULL) {
1741 			/*
1742 			 * This is a special case: we never associated this
1743 			 * dbuf with any data allocated from the ARC.
1744 			 */
1745 			ASSERT3U(db->db_state, ==, DB_UNCACHED);
1746 			dbuf_evict(db);
1747 		} else if (arc_released(db->db_buf)) {
1748 			arc_buf_t *buf = db->db_buf;
1749 			/*
1750 			 * This dbuf has anonymous data associated with it.
1751 			 */
1752 			dbuf_set_data(db, NULL);
1753 			VERIFY(arc_buf_remove_ref(buf, db) == 1);
1754 			dbuf_evict(db);
1755 		} else {
1756 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
1757 			if (!DBUF_IS_CACHEABLE(db))
1758 				dbuf_clear(db);
1759 			else
1760 				mutex_exit(&db->db_mtx);
1761 		}
1762 	} else {
1763 		mutex_exit(&db->db_mtx);
1764 	}
1765 }
1766 
1767 #pragma weak dmu_buf_refcount = dbuf_refcount
1768 uint64_t
1769 dbuf_refcount(dmu_buf_impl_t *db)
1770 {
1771 	return (refcount_count(&db->db_holds));
1772 }
1773 
1774 void *
1775 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1776     dmu_buf_evict_func_t *evict_func)
1777 {
1778 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1779 	    user_data_ptr_ptr, evict_func));
1780 }
1781 
1782 void *
1783 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1784     dmu_buf_evict_func_t *evict_func)
1785 {
1786 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1787 
1788 	db->db_immediate_evict = TRUE;
1789 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1790 	    user_data_ptr_ptr, evict_func));
1791 }
1792 
1793 void *
1794 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
1795     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
1796 {
1797 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1798 	ASSERT(db->db_level == 0);
1799 
1800 	ASSERT((user_ptr == NULL) == (evict_func == NULL));
1801 
1802 	mutex_enter(&db->db_mtx);
1803 
1804 	if (db->db_user_ptr == old_user_ptr) {
1805 		db->db_user_ptr = user_ptr;
1806 		db->db_user_data_ptr_ptr = user_data_ptr_ptr;
1807 		db->db_evict_func = evict_func;
1808 
1809 		dbuf_update_data(db);
1810 	} else {
1811 		old_user_ptr = db->db_user_ptr;
1812 	}
1813 
1814 	mutex_exit(&db->db_mtx);
1815 	return (old_user_ptr);
1816 }
1817 
1818 void *
1819 dmu_buf_get_user(dmu_buf_t *db_fake)
1820 {
1821 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1822 	ASSERT(!refcount_is_zero(&db->db_holds));
1823 
1824 	return (db->db_user_ptr);
1825 }
1826 
1827 static void
1828 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
1829 {
1830 	/* ASSERT(dmu_tx_is_syncing(tx) */
1831 	ASSERT(MUTEX_HELD(&db->db_mtx));
1832 
1833 	if (db->db_blkptr != NULL)
1834 		return;
1835 
1836 	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
1837 		/*
1838 		 * This buffer was allocated at a time when there was
1839 		 * no available blkptrs from the dnode, or it was
1840 		 * inappropriate to hook it in (i.e., nlevels mis-match).
1841 		 */
1842 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
1843 		ASSERT(db->db_parent == NULL);
1844 		db->db_parent = dn->dn_dbuf;
1845 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
1846 		DBUF_VERIFY(db);
1847 	} else {
1848 		dmu_buf_impl_t *parent = db->db_parent;
1849 		int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
1850 
1851 		ASSERT(dn->dn_phys->dn_nlevels > 1);
1852 		if (parent == NULL) {
1853 			mutex_exit(&db->db_mtx);
1854 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
1855 			(void) dbuf_hold_impl(dn, db->db_level+1,
1856 			    db->db_blkid >> epbs, FALSE, db, &parent);
1857 			rw_exit(&dn->dn_struct_rwlock);
1858 			mutex_enter(&db->db_mtx);
1859 			db->db_parent = parent;
1860 		}
1861 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
1862 		    (db->db_blkid & ((1ULL << epbs) - 1));
1863 		DBUF_VERIFY(db);
1864 	}
1865 }
1866 
1867 static void
1868 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
1869 {
1870 	dmu_buf_impl_t *db = dr->dr_dbuf;
1871 	dnode_t *dn = db->db_dnode;
1872 	zio_t *zio;
1873 
1874 	ASSERT(dmu_tx_is_syncing(tx));
1875 
1876 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
1877 
1878 	mutex_enter(&db->db_mtx);
1879 
1880 	ASSERT(db->db_level > 0);
1881 	DBUF_VERIFY(db);
1882 
1883 	if (db->db_buf == NULL) {
1884 		mutex_exit(&db->db_mtx);
1885 		(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
1886 		mutex_enter(&db->db_mtx);
1887 	}
1888 	ASSERT3U(db->db_state, ==, DB_CACHED);
1889 	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
1890 	ASSERT(db->db_buf != NULL);
1891 
1892 	dbuf_check_blkptr(dn, db);
1893 
1894 	db->db_data_pending = dr;
1895 
1896 	mutex_exit(&db->db_mtx);
1897 	dbuf_write(dr, db->db_buf, tx);
1898 
1899 	zio = dr->dr_zio;
1900 	mutex_enter(&dr->dt.di.dr_mtx);
1901 	dbuf_sync_list(&dr->dt.di.dr_children, tx);
1902 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1903 	mutex_exit(&dr->dt.di.dr_mtx);
1904 	zio_nowait(zio);
1905 }
1906 
1907 static void
1908 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
1909 {
1910 	arc_buf_t **datap = &dr->dt.dl.dr_data;
1911 	dmu_buf_impl_t *db = dr->dr_dbuf;
1912 	dnode_t *dn = db->db_dnode;
1913 	objset_impl_t *os = dn->dn_objset;
1914 	uint64_t txg = tx->tx_txg;
1915 	int blksz;
1916 
1917 	ASSERT(dmu_tx_is_syncing(tx));
1918 
1919 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
1920 
1921 	mutex_enter(&db->db_mtx);
1922 	/*
1923 	 * To be synced, we must be dirtied.  But we
1924 	 * might have been freed after the dirty.
1925 	 */
1926 	if (db->db_state == DB_UNCACHED) {
1927 		/* This buffer has been freed since it was dirtied */
1928 		ASSERT(db->db.db_data == NULL);
1929 	} else if (db->db_state == DB_FILL) {
1930 		/* This buffer was freed and is now being re-filled */
1931 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
1932 	} else {
1933 		ASSERT3U(db->db_state, ==, DB_CACHED);
1934 	}
1935 	DBUF_VERIFY(db);
1936 
1937 	/*
1938 	 * If this is a bonus buffer, simply copy the bonus data into the
1939 	 * dnode.  It will be written out when the dnode is synced (and it
1940 	 * will be synced, since it must have been dirty for dbuf_sync to
1941 	 * be called).
1942 	 */
1943 	if (db->db_blkid == DB_BONUS_BLKID) {
1944 		dbuf_dirty_record_t **drp;
1945 
1946 		ASSERT(*datap != NULL);
1947 		ASSERT3U(db->db_level, ==, 0);
1948 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
1949 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
1950 		if (*datap != db->db.db_data) {
1951 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
1952 			arc_space_return(DN_MAX_BONUSLEN);
1953 		}
1954 		db->db_data_pending = NULL;
1955 		drp = &db->db_last_dirty;
1956 		while (*drp != dr)
1957 			drp = &(*drp)->dr_next;
1958 		ASSERT(dr->dr_next == NULL);
1959 		*drp = dr->dr_next;
1960 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
1961 		ASSERT(db->db_dirtycnt > 0);
1962 		db->db_dirtycnt -= 1;
1963 		mutex_exit(&db->db_mtx);
1964 		dbuf_rele(db, (void *)(uintptr_t)txg);
1965 		return;
1966 	}
1967 
1968 	/*
1969 	 * This function may have dropped the db_mtx lock allowing a dmu_sync
1970 	 * operation to sneak in. As a result, we need to ensure that we
1971 	 * don't check the dr_override_state until we have returned from
1972 	 * dbuf_check_blkptr.
1973 	 */
1974 	dbuf_check_blkptr(dn, db);
1975 
1976 	/*
1977 	 * If this buffer is in the middle of an immdiate write,
1978 	 * wait for the synchronous IO to complete.
1979 	 */
1980 	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
1981 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
1982 		cv_wait(&db->db_changed, &db->db_mtx);
1983 		ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
1984 	}
1985 
1986 	/*
1987 	 * If this dbuf has already been written out via an immediate write,
1988 	 * just complete the write by copying over the new block pointer and
1989 	 * updating the accounting via the write-completion functions.
1990 	 */
1991 	if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1992 		zio_t zio_fake;
1993 
1994 		zio_fake.io_private = &db;
1995 		zio_fake.io_error = 0;
1996 		zio_fake.io_bp = db->db_blkptr;
1997 		zio_fake.io_bp_orig = *db->db_blkptr;
1998 		zio_fake.io_txg = txg;
1999 
2000 		*db->db_blkptr = dr->dt.dl.dr_overridden_by;
2001 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
2002 		db->db_data_pending = dr;
2003 		dr->dr_zio = &zio_fake;
2004 		mutex_exit(&db->db_mtx);
2005 
2006 		if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg))
2007 			(void) dsl_dataset_block_kill(os->os_dsl_dataset,
2008 			    &zio_fake.io_bp_orig, dn->dn_zio, tx);
2009 
2010 		dbuf_write_ready(&zio_fake, db->db_buf, db);
2011 		dbuf_write_done(&zio_fake, db->db_buf, db);
2012 
2013 		return;
2014 	}
2015 
2016 	blksz = arc_buf_size(*datap);
2017 
2018 	if (dn->dn_object != DMU_META_DNODE_OBJECT) {
2019 		/*
2020 		 * If this buffer is currently "in use" (i.e., there are
2021 		 * active holds and db_data still references it), then make
2022 		 * a copy before we start the write so that any modifications
2023 		 * from the open txg will not leak into this write.
2024 		 *
2025 		 * NOTE: this copy does not need to be made for objects only
2026 		 * modified in the syncing context (e.g. DNONE_DNODE blocks).
2027 		 */
2028 		if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) {
2029 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2030 			*datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2031 			bcopy(db->db.db_data, (*datap)->b_data, blksz);
2032 		}
2033 	}
2034 
2035 	ASSERT(*datap != NULL);
2036 	db->db_data_pending = dr;
2037 
2038 	mutex_exit(&db->db_mtx);
2039 
2040 	dbuf_write(dr, *datap, tx);
2041 
2042 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2043 	if (dn->dn_object == DMU_META_DNODE_OBJECT)
2044 		list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2045 	else
2046 		zio_nowait(dr->dr_zio);
2047 }
2048 
2049 void
2050 dbuf_sync_list(list_t *list, dmu_tx_t *tx)
2051 {
2052 	dbuf_dirty_record_t *dr;
2053 
2054 	while (dr = list_head(list)) {
2055 		if (dr->dr_zio != NULL) {
2056 			/*
2057 			 * If we find an already initialized zio then we
2058 			 * are processing the meta-dnode, and we have finished.
2059 			 * The dbufs for all dnodes are put back on the list
2060 			 * during processing, so that we can zio_wait()
2061 			 * these IOs after initiating all child IOs.
2062 			 */
2063 			ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2064 			    DMU_META_DNODE_OBJECT);
2065 			break;
2066 		}
2067 		list_remove(list, dr);
2068 		if (dr->dr_dbuf->db_level > 0)
2069 			dbuf_sync_indirect(dr, tx);
2070 		else
2071 			dbuf_sync_leaf(dr, tx);
2072 	}
2073 }
2074 
2075 static void
2076 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
2077 {
2078 	dmu_buf_impl_t *db = dr->dr_dbuf;
2079 	dnode_t *dn = db->db_dnode;
2080 	objset_impl_t *os = dn->dn_objset;
2081 	dmu_buf_impl_t *parent = db->db_parent;
2082 	uint64_t txg = tx->tx_txg;
2083 	zbookmark_t zb;
2084 	writeprops_t wp = { 0 };
2085 	zio_t *zio;
2086 	int zio_flags;
2087 
2088 	if (!BP_IS_HOLE(db->db_blkptr) &&
2089 	    (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) {
2090 		/*
2091 		 * Private object buffers are released here rather
2092 		 * than in dbuf_dirty() since they are only modified
2093 		 * in the syncing context and we don't want the
2094 		 * overhead of making multiple copies of the data.
2095 		 */
2096 		arc_release(data, db);
2097 	} else {
2098 		ASSERT(arc_released(data));
2099 		/* XXX why do we need to thaw here? */
2100 		arc_buf_thaw(data);
2101 	}
2102 
2103 	if (parent != dn->dn_dbuf) {
2104 		ASSERT(parent && parent->db_data_pending);
2105 		ASSERT(db->db_level == parent->db_level-1);
2106 		ASSERT(arc_released(parent->db_buf));
2107 		zio = parent->db_data_pending->dr_zio;
2108 	} else {
2109 		ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1);
2110 		ASSERT3P(db->db_blkptr, ==,
2111 		    &dn->dn_phys->dn_blkptr[db->db_blkid]);
2112 		zio = dn->dn_zio;
2113 	}
2114 
2115 	ASSERT(db->db_level == 0 || data == db->db_buf);
2116 	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
2117 	ASSERT(zio);
2118 
2119 	zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0;
2120 	zb.zb_object = db->db.db_object;
2121 	zb.zb_level = db->db_level;
2122 	zb.zb_blkid = db->db_blkid;
2123 
2124 	zio_flags = ZIO_FLAG_MUSTSUCCEED;
2125 	if (dmu_ot[dn->dn_type].ot_metadata || zb.zb_level != 0)
2126 		zio_flags |= ZIO_FLAG_METADATA;
2127 	wp.wp_type = dn->dn_type;
2128 	wp.wp_level = db->db_level;
2129 	wp.wp_copies = os->os_copies;
2130 	wp.wp_dncompress = dn->dn_compress;
2131 	wp.wp_oscompress = os->os_compress;
2132 	wp.wp_dnchecksum = dn->dn_checksum;
2133 	wp.wp_oschecksum = os->os_checksum;
2134 
2135 	if (BP_IS_OLDER(db->db_blkptr, txg))
2136 		(void) dsl_dataset_block_kill(
2137 		    os->os_dsl_dataset, db->db_blkptr, zio, tx);
2138 
2139 	dr->dr_zio = arc_write(zio, os->os_spa, &wp,
2140 	    DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr,
2141 	    data, dbuf_write_ready, dbuf_write_done, db,
2142 	    ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb);
2143 }
2144 
2145 /* ARGSUSED */
2146 static void
2147 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2148 {
2149 	dmu_buf_impl_t *db = vdb;
2150 	dnode_t *dn = db->db_dnode;
2151 	objset_impl_t *os = dn->dn_objset;
2152 	blkptr_t *bp_orig = &zio->io_bp_orig;
2153 	uint64_t fill = 0;
2154 	int old_size, new_size, i;
2155 
2156 	dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", "");
2157 
2158 	old_size = bp_get_dasize(os->os_spa, bp_orig);
2159 	new_size = bp_get_dasize(os->os_spa, zio->io_bp);
2160 
2161 	dnode_diduse_space(dn, new_size-old_size);
2162 
2163 	if (BP_IS_HOLE(zio->io_bp)) {
2164 		dsl_dataset_t *ds = os->os_dsl_dataset;
2165 		dmu_tx_t *tx = os->os_synctx;
2166 
2167 		if (bp_orig->blk_birth == tx->tx_txg)
2168 			(void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx);
2169 		ASSERT3U(db->db_blkptr->blk_fill, ==, 0);
2170 		return;
2171 	}
2172 
2173 	mutex_enter(&db->db_mtx);
2174 
2175 	if (db->db_level == 0) {
2176 		mutex_enter(&dn->dn_mtx);
2177 		if (db->db_blkid > dn->dn_phys->dn_maxblkid)
2178 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2179 		mutex_exit(&dn->dn_mtx);
2180 
2181 		if (dn->dn_type == DMU_OT_DNODE) {
2182 			dnode_phys_t *dnp = db->db.db_data;
2183 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2184 			    i--, dnp++) {
2185 				if (dnp->dn_type != DMU_OT_NONE)
2186 					fill++;
2187 			}
2188 		} else {
2189 			fill = 1;
2190 		}
2191 	} else {
2192 		blkptr_t *bp = db->db.db_data;
2193 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2194 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) {
2195 			if (BP_IS_HOLE(bp))
2196 				continue;
2197 			ASSERT3U(BP_GET_LSIZE(bp), ==,
2198 			    db->db_level == 1 ? dn->dn_datablksz :
2199 			    (1<<dn->dn_phys->dn_indblkshift));
2200 			fill += bp->blk_fill;
2201 		}
2202 	}
2203 
2204 	db->db_blkptr->blk_fill = fill;
2205 	BP_SET_TYPE(db->db_blkptr, dn->dn_type);
2206 	BP_SET_LEVEL(db->db_blkptr, db->db_level);
2207 
2208 	mutex_exit(&db->db_mtx);
2209 
2210 	/* We must do this after we've set the bp's type and level */
2211 	if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp), BP_IDENTITY(bp_orig))) {
2212 		dsl_dataset_t *ds = os->os_dsl_dataset;
2213 		dmu_tx_t *tx = os->os_synctx;
2214 
2215 		if (bp_orig->blk_birth == tx->tx_txg)
2216 			(void) dsl_dataset_block_kill(ds, bp_orig, NULL, tx);
2217 		dsl_dataset_block_born(ds, zio->io_bp, tx);
2218 	}
2219 }
2220 
2221 /* ARGSUSED */
2222 static void
2223 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2224 {
2225 	dmu_buf_impl_t *db = vdb;
2226 	uint64_t txg = zio->io_txg;
2227 	dbuf_dirty_record_t **drp, *dr;
2228 
2229 	ASSERT3U(zio->io_error, ==, 0);
2230 
2231 	mutex_enter(&db->db_mtx);
2232 
2233 	drp = &db->db_last_dirty;
2234 	while ((dr = *drp) != db->db_data_pending)
2235 		drp = &dr->dr_next;
2236 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2237 	ASSERT(dr->dr_txg == txg);
2238 	ASSERT(dr->dr_next == NULL);
2239 	*drp = dr->dr_next;
2240 
2241 	if (db->db_level == 0) {
2242 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
2243 		ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2244 
2245 		if (dr->dt.dl.dr_data != db->db_buf)
2246 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1);
2247 		else if (!BP_IS_HOLE(db->db_blkptr))
2248 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2249 		else
2250 			ASSERT(arc_released(db->db_buf));
2251 	} else {
2252 		dnode_t *dn = db->db_dnode;
2253 
2254 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2255 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2256 		if (!BP_IS_HOLE(db->db_blkptr)) {
2257 			int epbs =
2258 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2259 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2260 			    db->db.db_size);
2261 			ASSERT3U(dn->dn_phys->dn_maxblkid
2262 			    >> (db->db_level * epbs), >=, db->db_blkid);
2263 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2264 		}
2265 		mutex_destroy(&dr->dt.di.dr_mtx);
2266 		list_destroy(&dr->dt.di.dr_children);
2267 	}
2268 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
2269 
2270 	cv_broadcast(&db->db_changed);
2271 	ASSERT(db->db_dirtycnt > 0);
2272 	db->db_dirtycnt -= 1;
2273 	db->db_data_pending = NULL;
2274 	mutex_exit(&db->db_mtx);
2275 
2276 	dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", "");
2277 
2278 	dbuf_rele(db, (void *)(uintptr_t)txg);
2279 }
2280