xref: /illumos-gate/usr/src/uts/common/fs/zfs/dbuf.c (revision ea8dc4b6d2251b437950c0056bc626b311c73c27)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/zfs_context.h>
29 #include <sys/dmu.h>
30 #include <sys/dmu_impl.h>
31 #include <sys/dbuf.h>
32 #include <sys/dmu_objset.h>
33 #include <sys/dsl_dataset.h>
34 #include <sys/dsl_dir.h>
35 #include <sys/dmu_tx.h>
36 #include <sys/spa.h>
37 #include <sys/zio.h>
38 #include <sys/dmu_zfetch.h>
39 
40 static void dbuf_destroy(dmu_buf_impl_t *db);
41 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
42 static arc_done_func_t dbuf_write_done;
43 
44 /*
45  * Global data structures and functions for the dbuf cache.
46  */
47 taskq_t *dbuf_tq;
48 static kmem_cache_t *dbuf_cache;
49 
50 /* ARGSUSED */
51 static int
52 dbuf_cons(void *vdb, void *unused, int kmflag)
53 {
54 	dmu_buf_impl_t *db = vdb;
55 	bzero(db, sizeof (dmu_buf_impl_t));
56 
57 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
58 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
59 	refcount_create(&db->db_holds);
60 	return (0);
61 }
62 
63 /* ARGSUSED */
64 static void
65 dbuf_dest(void *vdb, void *unused)
66 {
67 	dmu_buf_impl_t *db = vdb;
68 	mutex_destroy(&db->db_mtx);
69 	cv_destroy(&db->db_changed);
70 	refcount_destroy(&db->db_holds);
71 }
72 
73 /*
74  * dbuf hash table routines
75  */
76 static dbuf_hash_table_t dbuf_hash_table;
77 
78 static uint64_t dbuf_hash_count;
79 
80 static uint64_t
81 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
82 {
83 	uintptr_t osv = (uintptr_t)os;
84 	uint64_t crc = -1ULL;
85 
86 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
87 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
88 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
89 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
90 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
91 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
92 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
93 
94 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
95 
96 	return (crc);
97 }
98 
99 #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
100 
101 #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
102 	((dbuf)->db.db_object == (obj) &&		\
103 	(dbuf)->db_objset == (os) &&			\
104 	(dbuf)->db_level == (level) &&			\
105 	(dbuf)->db_blkid == (blkid))
106 
107 dmu_buf_impl_t *
108 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
109 {
110 	dbuf_hash_table_t *h = &dbuf_hash_table;
111 	objset_impl_t *os = dn->dn_objset;
112 	uint64_t obj = dn->dn_object;
113 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
114 	uint64_t idx = hv & h->hash_table_mask;
115 	dmu_buf_impl_t *db;
116 
117 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
118 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
119 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
120 			mutex_enter(&db->db_mtx);
121 			if (db->db_state != DB_EVICTING) {
122 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
123 				return (db);
124 			}
125 			mutex_exit(&db->db_mtx);
126 		}
127 	}
128 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
129 	return (NULL);
130 }
131 
132 /*
133  * Insert an entry into the hash table.  If there is already an element
134  * equal to elem in the hash table, then the already existing element
135  * will be returned and the new element will not be inserted.
136  * Otherwise returns NULL.
137  */
138 static dmu_buf_impl_t *
139 dbuf_hash_insert(dmu_buf_impl_t *db)
140 {
141 	dbuf_hash_table_t *h = &dbuf_hash_table;
142 	objset_impl_t *os = db->db_objset;
143 	uint64_t obj = db->db.db_object;
144 	int level = db->db_level;
145 	uint64_t blkid = db->db_blkid;
146 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
147 	uint64_t idx = hv & h->hash_table_mask;
148 	dmu_buf_impl_t *dbf;
149 
150 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
151 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
152 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
153 			mutex_enter(&dbf->db_mtx);
154 			if (dbf->db_state != DB_EVICTING) {
155 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
156 				return (dbf);
157 			}
158 			mutex_exit(&dbf->db_mtx);
159 		}
160 	}
161 
162 	mutex_enter(&db->db_mtx);
163 	db->db_hash_next = h->hash_table[idx];
164 	h->hash_table[idx] = db;
165 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
166 	atomic_add_64(&dbuf_hash_count, 1);
167 
168 	return (NULL);
169 }
170 
171 /*
172  * Remove an entry from the hash table.  This operation will
173  * fail if there are any existing holds on the db.
174  */
175 static void
176 dbuf_hash_remove(dmu_buf_impl_t *db)
177 {
178 	dbuf_hash_table_t *h = &dbuf_hash_table;
179 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
180 	    db->db_level, db->db_blkid);
181 	uint64_t idx = hv & h->hash_table_mask;
182 	dmu_buf_impl_t *dbf, **dbp;
183 
184 	/*
185 	 * We musn't hold db_mtx to maintin lock ordering:
186 	 * DBUF_HASH_MUTEX > db_mtx.
187 	 */
188 	ASSERT(refcount_is_zero(&db->db_holds));
189 	ASSERT(db->db_state == DB_EVICTING);
190 	ASSERT(!MUTEX_HELD(&db->db_mtx));
191 
192 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
193 	dbp = &h->hash_table[idx];
194 	while ((dbf = *dbp) != db) {
195 		dbp = &dbf->db_hash_next;
196 		ASSERT(dbf != NULL);
197 	}
198 	*dbp = db->db_hash_next;
199 	db->db_hash_next = NULL;
200 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
201 	atomic_add_64(&dbuf_hash_count, -1);
202 }
203 
204 static arc_evict_func_t dbuf_do_evict;
205 
206 static void
207 dbuf_evict_user(dmu_buf_impl_t *db)
208 {
209 	ASSERT(MUTEX_HELD(&db->db_mtx));
210 
211 	if (db->db_level != 0 || db->db_d.db_evict_func == NULL)
212 		return;
213 
214 	if (db->db_d.db_user_data_ptr_ptr)
215 		*db->db_d.db_user_data_ptr_ptr = db->db.db_data;
216 	db->db_d.db_evict_func(&db->db, db->db_d.db_user_ptr);
217 	db->db_d.db_user_ptr = NULL;
218 	db->db_d.db_user_data_ptr_ptr = NULL;
219 	db->db_d.db_evict_func = NULL;
220 }
221 
222 void
223 dbuf_evict(dmu_buf_impl_t *db)
224 {
225 	int i;
226 
227 	ASSERT(MUTEX_HELD(&db->db_mtx));
228 	ASSERT(db->db_buf == NULL);
229 
230 #ifdef ZFS_DEBUG
231 	for (i = 0; i < TXG_SIZE; i++) {
232 		ASSERT(!list_link_active(&db->db_dirty_node[i]));
233 		ASSERT(db->db_level != 0 || db->db_d.db_data_old[i] == NULL);
234 	}
235 #endif
236 	dbuf_clear(db);
237 	dbuf_destroy(db);
238 }
239 
240 void
241 dbuf_init(void)
242 {
243 	uint64_t hsize = 1ULL << 16;
244 	dbuf_hash_table_t *h = &dbuf_hash_table;
245 	int i;
246 
247 	/*
248 	 * The hash table is big enough to fill all of physical memory
249 	 * with an average 4K block size.  The table will take up
250 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
251 	 */
252 	while (hsize * 4096 < physmem * PAGESIZE)
253 		hsize <<= 1;
254 
255 retry:
256 	h->hash_table_mask = hsize - 1;
257 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
258 	if (h->hash_table == NULL) {
259 		/* XXX - we should really return an error instead of assert */
260 		ASSERT(hsize > (1ULL << 10));
261 		hsize >>= 1;
262 		goto retry;
263 	}
264 
265 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
266 	    sizeof (dmu_buf_impl_t),
267 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
268 	dbuf_tq = taskq_create("dbuf_tq", 8, maxclsyspri, 50, INT_MAX,
269 	    TASKQ_PREPOPULATE);
270 
271 	for (i = 0; i < DBUF_MUTEXES; i++)
272 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
273 }
274 
275 void
276 dbuf_fini(void)
277 {
278 	dbuf_hash_table_t *h = &dbuf_hash_table;
279 	int i;
280 
281 	taskq_destroy(dbuf_tq);
282 	dbuf_tq = NULL;
283 
284 	for (i = 0; i < DBUF_MUTEXES; i++)
285 		mutex_destroy(&h->hash_mutexes[i]);
286 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
287 	kmem_cache_destroy(dbuf_cache);
288 }
289 
290 /*
291  * Other stuff.
292  */
293 
294 #ifdef ZFS_DEBUG
295 static void
296 dbuf_verify(dmu_buf_impl_t *db)
297 {
298 	int i;
299 	dnode_t *dn = db->db_dnode;
300 
301 	ASSERT(MUTEX_HELD(&db->db_mtx));
302 
303 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
304 		return;
305 
306 	ASSERT(db->db_objset != NULL);
307 	if (dn == NULL) {
308 		ASSERT(db->db_parent == NULL);
309 		ASSERT(db->db_blkptr == NULL);
310 	} else {
311 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
312 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
313 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
314 		ASSERT(db->db_blkid == DB_BONUS_BLKID ||
315 		    list_head(&dn->dn_dbufs));
316 	}
317 	if (db->db_blkid == DB_BONUS_BLKID) {
318 		ASSERT(dn != NULL);
319 		ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen);
320 		ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID);
321 	} else {
322 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
323 	}
324 
325 	if (db->db_level == 0) {
326 		/* we can be momentarily larger in dnode_set_blksz() */
327 		if (db->db_blkid != DB_BONUS_BLKID && dn) {
328 			ASSERT3U(db->db.db_size, >=, dn->dn_datablksz);
329 		}
330 		if (db->db.db_object == DMU_META_DNODE_OBJECT) {
331 			for (i = 0; i < TXG_SIZE; i++) {
332 				/*
333 				 * it should only be modified in syncing
334 				 * context, so make sure we only have
335 				 * one copy of the data.
336 				 */
337 				ASSERT(db->db_d.db_data_old[i] == NULL ||
338 				    db->db_d.db_data_old[i] == db->db_buf);
339 			}
340 		}
341 	}
342 
343 	/* verify db->db_blkptr */
344 	if (db->db_blkptr) {
345 		if (db->db_parent == dn->dn_dbuf) {
346 			/* db is pointed to by the dnode */
347 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
348 			if (db->db.db_object == DMU_META_DNODE_OBJECT)
349 				ASSERT(db->db_parent == NULL);
350 			else
351 				ASSERT(db->db_parent != NULL);
352 			ASSERT3P(db->db_blkptr, ==,
353 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
354 		} else {
355 			/* db is pointed to by an indirect block */
356 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
357 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
358 			ASSERT3U(db->db_parent->db.db_object, ==,
359 			    db->db.db_object);
360 			/*
361 			 * dnode_grow_indblksz() can make this fail if we don't
362 			 * have the struct_rwlock.  XXX indblksz no longer
363 			 * grows.  safe to do this now?
364 			 */
365 			if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) {
366 				ASSERT3P(db->db_blkptr, ==,
367 				    ((blkptr_t *)db->db_parent->db.db_data +
368 				    db->db_blkid % epb));
369 			}
370 		}
371 	}
372 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
373 	    db->db.db_data && db->db_blkid != DB_BONUS_BLKID &&
374 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
375 		/*
376 		 * If the blkptr isn't set but they have nonzero data,
377 		 * it had better be dirty, otherwise we'll lose that
378 		 * data when we evict this buffer.
379 		 */
380 		if (db->db_dirtycnt == 0) {
381 			uint64_t *buf = db->db.db_data;
382 			int i;
383 
384 			for (i = 0; i < db->db.db_size >> 3; i++) {
385 				ASSERT(buf[i] == 0);
386 			}
387 		}
388 	}
389 }
390 #endif
391 
392 static void
393 dbuf_update_data(dmu_buf_impl_t *db)
394 {
395 	ASSERT(MUTEX_HELD(&db->db_mtx));
396 	if (db->db_level == 0 && db->db_d.db_user_data_ptr_ptr) {
397 		ASSERT(!refcount_is_zero(&db->db_holds));
398 		*db->db_d.db_user_data_ptr_ptr = db->db.db_data;
399 	}
400 }
401 
402 static void
403 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
404 {
405 	ASSERT(MUTEX_HELD(&db->db_mtx));
406 	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
407 	db->db_buf = buf;
408 	if (buf != NULL) {
409 		ASSERT(buf->b_data != NULL);
410 		db->db.db_data = buf->b_data;
411 		if (!arc_released(buf))
412 			arc_set_callback(buf, dbuf_do_evict, db);
413 		dbuf_update_data(db);
414 	} else {
415 		dbuf_evict_user(db);
416 		db->db.db_data = NULL;
417 		db->db_state = DB_UNCACHED;
418 	}
419 }
420 
421 uint64_t
422 dbuf_whichblock(dnode_t *dn, uint64_t offset)
423 {
424 	if (dn->dn_datablkshift) {
425 		return (offset >> dn->dn_datablkshift);
426 	} else {
427 		ASSERT3U(offset, <, dn->dn_datablksz);
428 		return (0);
429 	}
430 }
431 
432 static void
433 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
434 {
435 	dmu_buf_impl_t *db = vdb;
436 
437 	mutex_enter(&db->db_mtx);
438 	ASSERT3U(db->db_state, ==, DB_READ);
439 	/*
440 	 * All reads are synchronous, so we must have a hold on the dbuf
441 	 */
442 	ASSERT(refcount_count(&db->db_holds) > 0);
443 	ASSERT(db->db_buf == NULL);
444 	ASSERT(db->db.db_data == NULL);
445 	if (db->db_level == 0 && db->db_d.db_freed_in_flight) {
446 		/* we were freed in flight; disregard any error */
447 		arc_release(buf, db);
448 		bzero(buf->b_data, db->db.db_size);
449 		db->db_d.db_freed_in_flight = FALSE;
450 		dbuf_set_data(db, buf);
451 		db->db_state = DB_CACHED;
452 	} else if (zio == NULL || zio->io_error == 0) {
453 		dbuf_set_data(db, buf);
454 		db->db_state = DB_CACHED;
455 	} else {
456 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
457 		ASSERT3P(db->db_buf, ==, NULL);
458 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
459 		db->db_state = DB_UNCACHED;
460 	}
461 	cv_broadcast(&db->db_changed);
462 	mutex_exit(&db->db_mtx);
463 	dbuf_rele(db, NULL);
464 }
465 
466 static void
467 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
468 {
469 	blkptr_t *bp;
470 	zbookmark_t zb;
471 
472 	ASSERT(!refcount_is_zero(&db->db_holds));
473 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
474 	ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock));
475 	ASSERT(MUTEX_HELD(&db->db_mtx));
476 	ASSERT(db->db_state == DB_UNCACHED);
477 	ASSERT(db->db_buf == NULL);
478 
479 	if (db->db_blkid == DB_BONUS_BLKID) {
480 		ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size);
481 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
482 		if (db->db.db_size < DN_MAX_BONUSLEN)
483 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
484 		bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data,
485 		    db->db.db_size);
486 		dbuf_update_data(db);
487 		db->db_state = DB_CACHED;
488 		mutex_exit(&db->db_mtx);
489 		return;
490 	}
491 
492 	if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid))
493 		bp = NULL;
494 	else
495 		bp = db->db_blkptr;
496 
497 	if (bp == NULL)
498 		dprintf_dbuf(db, "blkptr: %s\n", "NULL");
499 	else
500 		dprintf_dbuf_bp(db, bp, "%s", "blkptr:");
501 
502 	if (bp == NULL || BP_IS_HOLE(bp)) {
503 		ASSERT(bp == NULL || BP_IS_HOLE(bp));
504 		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
505 		    db->db.db_size, db));
506 		bzero(db->db.db_data, db->db.db_size);
507 		db->db_state = DB_CACHED;
508 		mutex_exit(&db->db_mtx);
509 		return;
510 	}
511 
512 	db->db_state = DB_READ;
513 	mutex_exit(&db->db_mtx);
514 
515 	zb.zb_objset = db->db_objset->os_dsl_dataset ?
516 	    db->db_objset->os_dsl_dataset->ds_object : 0;
517 	zb.zb_object = db->db.db_object;
518 	zb.zb_level = db->db_level;
519 	zb.zb_blkid = db->db_blkid;
520 
521 	dbuf_add_ref(db, NULL);
522 	/* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */
523 	(void) arc_read(zio, db->db_dnode->dn_objset->os_spa, bp,
524 	    db->db_level > 0 ? byteswap_uint64_array :
525 	    dmu_ot[db->db_dnode->dn_type].ot_byteswap,
526 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
527 	    (flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
528 	    ARC_NOWAIT, &zb);
529 }
530 
531 int
532 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
533 {
534 	int err = 0;
535 	int havepzio = (zio != NULL);
536 
537 	/*
538 	 * We don't have to hold the mutex to check db_state because it
539 	 * can't be freed while we have a hold on the buffer.
540 	 */
541 	ASSERT(!refcount_is_zero(&db->db_holds));
542 	if (db->db_state == DB_CACHED)
543 		return (0);
544 
545 	if ((flags & DB_RF_HAVESTRUCT) == 0)
546 		rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER);
547 
548 	mutex_enter(&db->db_mtx);
549 	if (db->db_state == DB_CACHED) {
550 		mutex_exit(&db->db_mtx);
551 		if ((flags & DB_RF_HAVESTRUCT) == 0)
552 			rw_exit(&db->db_dnode->dn_struct_rwlock);
553 	} else if (db->db_state == DB_UNCACHED) {
554 		if (zio == NULL) {
555 			zio = zio_root(db->db_dnode->dn_objset->os_spa,
556 			    NULL, NULL, ZIO_FLAG_CANFAIL);
557 		}
558 		dbuf_read_impl(db, zio, flags);
559 		/* dbuf_read_impl has dropped db_mtx for us */
560 
561 		if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
562 		    (flags & DB_RF_NOPREFETCH) == 0 &&
563 		    db->db_dnode != NULL) {
564 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
565 			    db->db.db_size);
566 		}
567 
568 		if ((flags & DB_RF_HAVESTRUCT) == 0)
569 			rw_exit(&db->db_dnode->dn_struct_rwlock);
570 
571 		if (!havepzio)
572 			err = zio_wait(zio);
573 	} else {
574 		if ((flags & DB_RF_HAVESTRUCT) == 0)
575 			rw_exit(&db->db_dnode->dn_struct_rwlock);
576 		if ((flags & DB_RF_NEVERWAIT) == 0) {
577 			while (db->db_state == DB_READ ||
578 			    db->db_state == DB_FILL) {
579 				ASSERT(db->db_state == DB_READ ||
580 				    (flags & DB_RF_HAVESTRUCT) == 0);
581 				cv_wait(&db->db_changed, &db->db_mtx);
582 			}
583 			if (db->db_state == DB_UNCACHED)
584 				err = EIO;
585 		}
586 		mutex_exit(&db->db_mtx);
587 	}
588 
589 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
590 	return (err);
591 }
592 
593 static void
594 dbuf_noread(dmu_buf_impl_t *db)
595 {
596 	ASSERT(!refcount_is_zero(&db->db_holds));
597 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
598 	mutex_enter(&db->db_mtx);
599 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
600 		cv_wait(&db->db_changed, &db->db_mtx);
601 	if (db->db_state == DB_UNCACHED) {
602 		ASSERT(db->db_buf == NULL);
603 		ASSERT(db->db.db_data == NULL);
604 		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
605 		    db->db.db_size, db));
606 		db->db_state = DB_FILL;
607 	} else {
608 		ASSERT3U(db->db_state, ==, DB_CACHED);
609 	}
610 	mutex_exit(&db->db_mtx);
611 }
612 
613 /*
614  * This is our just-in-time copy function.  It makes a copy of
615  * buffers, that have been modified in a previous transaction
616  * group, before we modify them in the current active group.
617  *
618  * This function is used in two places: when we are dirtying a
619  * buffer for the first time in a txg, and when we are freeing
620  * a range in a dnode that includes this buffer.
621  *
622  * Note that when we are called from dbuf_free_range() we do
623  * not put a hold on the buffer, we just traverse the active
624  * dbuf list for the dnode.
625  */
626 static void
627 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
628 {
629 	arc_buf_t **quiescing, **syncing;
630 
631 	ASSERT(MUTEX_HELD(&db->db_mtx));
632 	ASSERT(db->db.db_data != NULL);
633 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
634 
635 	quiescing = (arc_buf_t **)&db->db_d.db_data_old[(txg-1)&TXG_MASK];
636 	syncing = (arc_buf_t **)&db->db_d.db_data_old[(txg-2)&TXG_MASK];
637 
638 	/*
639 	 * If this buffer is referenced from the current quiescing
640 	 * transaction group: either make a copy and reset the reference
641 	 * to point to the copy, or (if there a no active holders) just
642 	 * null out the current db_data pointer.
643 	 */
644 	if (*quiescing == db->db_buf) {
645 		/*
646 		 * If the quiescing txg is "dirty", then we better not
647 		 * be referencing the same buffer from the syncing txg.
648 		 */
649 		ASSERT(*syncing != db->db_buf);
650 		if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
651 			int size = db->db.db_size;
652 			*quiescing = arc_buf_alloc(
653 			    db->db_dnode->dn_objset->os_spa, size, db);
654 			bcopy(db->db.db_data, (*quiescing)->b_data, size);
655 		} else {
656 			dbuf_set_data(db, NULL);
657 		}
658 		return;
659 	}
660 
661 	/*
662 	 * If this buffer is referenced from the current syncing
663 	 * transaction group: either
664 	 *	1 - make a copy and reset the reference, or
665 	 *	2 - if there are no holders, just null the current db_data.
666 	 */
667 	if (*syncing == db->db_buf) {
668 		ASSERT3P(*quiescing, ==, NULL);
669 		ASSERT3U(db->db_dirtycnt, ==, 1);
670 		if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
671 			int size = db->db.db_size;
672 			/* we can't copy if we have already started a write */
673 			ASSERT(*syncing != db->db_data_pending);
674 			*syncing = arc_buf_alloc(
675 			    db->db_dnode->dn_objset->os_spa, size, db);
676 			bcopy(db->db.db_data, (*syncing)->b_data, size);
677 		} else {
678 			dbuf_set_data(db, NULL);
679 		}
680 	}
681 }
682 
683 /*
684  * This is the "bonus buffer" version of the above routine
685  */
686 static void
687 dbuf_fix_old_bonus_data(dmu_buf_impl_t *db, uint64_t txg)
688 {
689 	void **quiescing, **syncing;
690 
691 	ASSERT(MUTEX_HELD(&db->db_mtx));
692 	ASSERT(db->db.db_data != NULL);
693 	ASSERT(db->db_blkid == DB_BONUS_BLKID);
694 
695 	quiescing = &db->db_d.db_data_old[(txg-1)&TXG_MASK];
696 	syncing = &db->db_d.db_data_old[(txg-2)&TXG_MASK];
697 
698 	if (*quiescing == db->db.db_data) {
699 		ASSERT(*syncing != db->db.db_data);
700 		*quiescing = zio_buf_alloc(DN_MAX_BONUSLEN);
701 		bcopy(db->db.db_data, *quiescing, DN_MAX_BONUSLEN);
702 	} else if (*syncing == db->db.db_data) {
703 		ASSERT3P(*quiescing, ==, NULL);
704 		ASSERT3U(db->db_dirtycnt, ==, 1);
705 		*syncing = zio_buf_alloc(DN_MAX_BONUSLEN);
706 		bcopy(db->db.db_data, *syncing, DN_MAX_BONUSLEN);
707 	}
708 }
709 
710 void
711 dbuf_unoverride(dmu_buf_impl_t *db, uint64_t txg)
712 {
713 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
714 	ASSERT(MUTEX_HELD(&db->db_mtx));
715 	if (db->db_d.db_overridden_by[txg&TXG_MASK] == IN_DMU_SYNC) {
716 		db->db_d.db_overridden_by[txg&TXG_MASK] = NULL;
717 	} else if (db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) {
718 		/* free this block */
719 		ASSERT(list_link_active(&db->db_dirty_node[txg&TXG_MASK]) ||
720 		    db->db_dnode->dn_free_txg == txg);
721 		if (!BP_IS_HOLE(db->db_d.db_overridden_by[txg&TXG_MASK])) {
722 			/* XXX can get silent EIO here */
723 			(void) arc_free(NULL, db->db_dnode->dn_objset->os_spa,
724 			    txg, db->db_d.db_overridden_by[txg&TXG_MASK],
725 			    NULL, NULL, ARC_WAIT);
726 		}
727 		kmem_free(db->db_d.db_overridden_by[txg&TXG_MASK],
728 		    sizeof (blkptr_t));
729 		db->db_d.db_overridden_by[txg&TXG_MASK] = NULL;
730 		/* release the already-written buffer */
731 		arc_release(db->db_d.db_data_old[txg&TXG_MASK], db);
732 	}
733 }
734 
735 void
736 dbuf_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx)
737 {
738 	dmu_buf_impl_t *db, *db_next;
739 	uint64_t txg = tx->tx_txg;
740 
741 	dprintf_dnode(dn, "blkid=%llu nblks=%llu\n", blkid, nblks);
742 	mutex_enter(&dn->dn_dbufs_mtx);
743 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
744 		db_next = list_next(&dn->dn_dbufs, db);
745 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
746 		if (db->db_level != 0)
747 			continue;
748 		dprintf_dbuf(db, "found buf %s\n", "");
749 		if (db->db_blkid < blkid ||
750 		    db->db_blkid >= blkid+nblks)
751 			continue;
752 
753 		/* found a level 0 buffer in the range */
754 		if (dbuf_undirty(db, tx))
755 			continue;
756 
757 		mutex_enter(&db->db_mtx);
758 		if (db->db_state == DB_UNCACHED ||
759 		    db->db_state == DB_EVICTING) {
760 			ASSERT(db->db.db_data == NULL);
761 			mutex_exit(&db->db_mtx);
762 			continue;
763 		}
764 		if (db->db_state == DB_READ) {
765 			/* this will be handled in dbuf_read_done() */
766 			db->db_d.db_freed_in_flight = TRUE;
767 			mutex_exit(&db->db_mtx);
768 			continue;
769 		}
770 		if (db->db_state == DB_FILL) {
771 			/* this will be handled in dbuf_rele() */
772 			db->db_d.db_freed_in_flight = TRUE;
773 			mutex_exit(&db->db_mtx);
774 			continue;
775 		}
776 		if (refcount_count(&db->db_holds) == 0) {
777 			ASSERT(db->db_buf);
778 			dbuf_clear(db);
779 			continue;
780 		}
781 		/* The dbuf is CACHED and referenced */
782 
783 		if (!list_link_active(&db->db_dirty_node[txg & TXG_MASK])) {
784 			/*
785 			 * This dbuf is not currently dirty.  We will either
786 			 * uncache it (if its not referenced in the open
787 			 * context) or reset its contents to empty.
788 			 */
789 			dbuf_fix_old_data(db, txg);
790 		} else if (db->db_d.db_overridden_by[txg & TXG_MASK] != NULL) {
791 			/*
792 			 * This dbuf is overridden.  Clear that state.
793 			 */
794 			dbuf_unoverride(db, txg);
795 		}
796 		/* fill in with appropriate data */
797 		if (db->db_state == DB_CACHED) {
798 			ASSERT(db->db.db_data != NULL);
799 			arc_release(db->db_buf, db);
800 			bzero(db->db.db_data, db->db.db_size);
801 		}
802 
803 		mutex_exit(&db->db_mtx);
804 	}
805 	mutex_exit(&dn->dn_dbufs_mtx);
806 }
807 
808 static int
809 dbuf_new_block(dmu_buf_impl_t *db)
810 {
811 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
812 	uint64_t birth_txg = 0;
813 
814 	/* Don't count meta-objects */
815 	if (ds == NULL)
816 		return (FALSE);
817 
818 	/*
819 	 * We don't need any locking to protect db_blkptr:
820 	 * If it's syncing, then db_dirtied will be set so we'll
821 	 * ignore db_blkptr.
822 	 */
823 	ASSERT(MUTEX_HELD(&db->db_mtx)); /* XXX strictly necessary? */
824 	/* If we have been dirtied since the last snapshot, its not new */
825 	if (db->db_dirtied)
826 		birth_txg = db->db_dirtied;
827 	else if (db->db_blkptr)
828 		birth_txg = db->db_blkptr->blk_birth;
829 
830 	if (birth_txg)
831 		return (!dsl_dataset_block_freeable(ds, birth_txg));
832 	else
833 		return (TRUE);
834 }
835 
836 void
837 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
838 {
839 	arc_buf_t *buf, *obuf;
840 	int osize = db->db.db_size;
841 
842 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
843 
844 	/* XXX does *this* func really need the lock? */
845 	ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock));
846 
847 	if (osize == size)
848 		return;
849 
850 	/*
851 	 * This call to dbuf_will_dirty() with the dn_struct_rwlock held
852 	 * is OK, because there can be no other references to the db
853 	 * when we are changing its size, so no concurrent DB_FILL can
854 	 * be happening.
855 	 */
856 	/* Make a copy of the data if necessary */
857 	/*
858 	 * XXX we should be doing a dbuf_read, checking the return
859 	 * value and returning that up to our callers
860 	 */
861 	dbuf_will_dirty(db, tx);
862 
863 	/* create the data buffer for the new block */
864 	buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db);
865 
866 	/* copy old block data to the new block */
867 	obuf = db->db_buf;
868 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
869 	/* zero the remainder */
870 	if (size > osize)
871 		bzero((uint8_t *)buf->b_data + osize, size - osize);
872 
873 	mutex_enter(&db->db_mtx);
874 	/* ASSERT3U(refcount_count(&db->db_holds), ==, 1); */
875 	dbuf_set_data(db, buf);
876 	VERIFY(arc_buf_remove_ref(obuf, db) == 1);
877 	db->db.db_size = size;
878 
879 	/* fix up the dirty info */
880 	if (db->db_level == 0)
881 		db->db_d.db_data_old[tx->tx_txg&TXG_MASK] = buf;
882 	mutex_exit(&db->db_mtx);
883 
884 	dnode_willuse_space(db->db_dnode, size-osize, tx);
885 }
886 
887 void
888 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
889 {
890 	dnode_t *dn = db->db_dnode;
891 	objset_impl_t *os = dn->dn_objset;
892 	int drop_struct_lock = FALSE;
893 	int txgoff = tx->tx_txg & TXG_MASK;
894 
895 	ASSERT(tx->tx_txg != 0);
896 	ASSERT(!refcount_is_zero(&db->db_holds));
897 	DMU_TX_DIRTY_BUF(tx, db);
898 
899 	/*
900 	 * Shouldn't dirty a regular buffer in syncing context.  Private
901 	 * objects may be dirtied in syncing context, but only if they
902 	 * were already pre-dirtied in open context.
903 	 * XXX We may want to prohibit dirtying in syncing context even
904 	 * if they did pre-dirty.
905 	 */
906 	ASSERT(!(dmu_tx_is_syncing(tx) &&
907 	    !BP_IS_HOLE(&dn->dn_objset->os_rootbp) &&
908 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
909 	    dn->dn_objset->os_dsl_dataset != NULL &&
910 	    !dsl_dir_is_private(
911 	    dn->dn_objset->os_dsl_dataset->ds_dir)));
912 
913 	/*
914 	 * We make this assert for private objects as well, but after we
915 	 * check if we're already dirty.  They are allowed to re-dirty
916 	 * in syncing context.
917 	 */
918 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
919 	    dn->dn_dirtyctx == DN_UNDIRTIED ||
920 	    dn->dn_dirtyctx ==
921 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
922 
923 	mutex_enter(&db->db_mtx);
924 	/* XXX make this true for indirects too? */
925 	ASSERT(db->db_level != 0 || db->db_state == DB_CACHED ||
926 	    db->db_state == DB_FILL);
927 
928 	/*
929 	 * If this buffer is currently part of an "overridden" region,
930 	 * we now need to remove it from that region.
931 	 */
932 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
933 	    db->db_d.db_overridden_by[txgoff] != NULL) {
934 		dbuf_unoverride(db, tx->tx_txg);
935 	}
936 
937 	mutex_enter(&dn->dn_mtx);
938 	/*
939 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
940 	 * initialize the objset.
941 	 */
942 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
943 	    !BP_IS_HOLE(&dn->dn_objset->os_rootbp)) {
944 		dn->dn_dirtyctx =
945 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
946 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
947 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
948 	}
949 	mutex_exit(&dn->dn_mtx);
950 
951 	/*
952 	 * If this buffer is already dirty, we're done.
953 	 */
954 	if (list_link_active(&db->db_dirty_node[txgoff])) {
955 		mutex_exit(&db->db_mtx);
956 		return;
957 	}
958 
959 	/*
960 	 * Only valid if not already dirty.
961 	 */
962 	ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
963 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
964 
965 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
966 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
967 	    dn->dn_phys->dn_nlevels > db->db_level ||
968 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
969 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
970 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
971 
972 	/*
973 	 * We should only be dirtying in syncing context if it's the
974 	 * mos, a spa os, or we're initializing the os.  However, we are
975 	 * allowed to dirty in syncing context provided we already
976 	 * dirtied it in open context.  Hence we must make this
977 	 * assertion only if we're not already dirty.
978 	 */
979 	ASSERT(!dmu_tx_is_syncing(tx) ||
980 	    os->os_dsl_dataset == NULL ||
981 	    !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) ||
982 	    !BP_IS_HOLE(&os->os_rootbp));
983 	ASSERT(db->db.db_size != 0);
984 
985 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
986 
987 	/*
988 	 * If this buffer is dirty in an old transaction group we need
989 	 * to make a copy of it so that the changes we make in this
990 	 * transaction group won't leak out when we sync the older txg.
991 	 */
992 	if (db->db_blkid == DB_BONUS_BLKID) {
993 		ASSERT(db->db.db_data != NULL);
994 		ASSERT(db->db_d.db_data_old[txgoff] == NULL);
995 		dbuf_fix_old_bonus_data(db, tx->tx_txg);
996 		db->db_d.db_data_old[txgoff] = db->db.db_data;
997 	} else if (db->db_level == 0) {
998 		/*
999 		 * Release the data buffer from the cache so that we
1000 		 * can modify it without impacting possible other users
1001 		 * of this cached data block.  Note that indirect blocks
1002 		 * and private objects are not released until the syncing
1003 		 * state (since they are only modified then).
1004 		 */
1005 		ASSERT(db->db_buf != NULL);
1006 		ASSERT(db->db_d.db_data_old[txgoff] == NULL);
1007 		if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1008 			arc_release(db->db_buf, db);
1009 			dbuf_fix_old_data(db, tx->tx_txg);
1010 			ASSERT(db->db_buf != NULL);
1011 		}
1012 		db->db_d.db_data_old[txgoff] = db->db_buf;
1013 	}
1014 
1015 	mutex_enter(&dn->dn_mtx);
1016 	/*
1017 	 * We could have been freed_in_flight between the dbuf_noread
1018 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1019 	 * happened after the free.
1020 	 */
1021 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) {
1022 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1023 		db->db_d.db_freed_in_flight = FALSE;
1024 	}
1025 
1026 	db->db_dirtied = tx->tx_txg;
1027 	list_insert_tail(&dn->dn_dirty_dbufs[txgoff], db);
1028 	mutex_exit(&dn->dn_mtx);
1029 
1030 	if (db->db_blkid != DB_BONUS_BLKID) {
1031 		/*
1032 		 * Update the accounting.
1033 		 */
1034 		if (!dbuf_new_block(db) && db->db_blkptr) {
1035 			/*
1036 			 * This is only a guess -- if the dbuf is dirty
1037 			 * in a previous txg, we don't know how much
1038 			 * space it will use on disk yet.  We should
1039 			 * really have the struct_rwlock to access
1040 			 * db_blkptr, but since this is just a guess,
1041 			 * it's OK if we get an odd answer.
1042 			 */
1043 			dnode_willuse_space(dn,
1044 			    -BP_GET_ASIZE(db->db_blkptr), tx);
1045 		}
1046 		dnode_willuse_space(dn, db->db.db_size, tx);
1047 	}
1048 
1049 	/*
1050 	 * This buffer is now part of this txg
1051 	 */
1052 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1053 	db->db_dirtycnt += 1;
1054 	ASSERT3U(db->db_dirtycnt, <=, 3);
1055 
1056 	mutex_exit(&db->db_mtx);
1057 
1058 	if (db->db_blkid == DB_BONUS_BLKID) {
1059 		dnode_setdirty(dn, tx);
1060 		return;
1061 	}
1062 
1063 	if (db->db_level == 0)
1064 		dnode_new_blkid(dn, db->db_blkid, tx);
1065 
1066 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1067 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1068 		drop_struct_lock = TRUE;
1069 	}
1070 
1071 	if (db->db_level < dn->dn_nlevels-1) {
1072 		int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1073 		dmu_buf_impl_t *parent;
1074 		parent = dbuf_hold_level(dn, db->db_level+1,
1075 		    db->db_blkid >> epbs, FTAG);
1076 		if (drop_struct_lock)
1077 			rw_exit(&dn->dn_struct_rwlock);
1078 		dbuf_dirty(parent, tx);
1079 		dbuf_rele(parent, FTAG);
1080 	} else {
1081 		if (drop_struct_lock)
1082 			rw_exit(&dn->dn_struct_rwlock);
1083 	}
1084 
1085 	dnode_setdirty(dn, tx);
1086 }
1087 
1088 static int
1089 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1090 {
1091 	dnode_t *dn = db->db_dnode;
1092 	int txgoff = tx->tx_txg & TXG_MASK;
1093 	int64_t holds;
1094 
1095 	ASSERT(tx->tx_txg != 0);
1096 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1097 
1098 	mutex_enter(&db->db_mtx);
1099 
1100 	/*
1101 	 * If this buffer is not dirty, we're done.
1102 	 */
1103 	if (!list_link_active(&db->db_dirty_node[txgoff])) {
1104 		mutex_exit(&db->db_mtx);
1105 		return (0);
1106 	}
1107 
1108 	/*
1109 	 * If this buffer is currently held, we cannot undirty
1110 	 * it, since one of the current holders may be in the
1111 	 * middle of an update.  Note that users of dbuf_undirty()
1112 	 * should not place a hold on the dbuf before the call.
1113 	 * XXX - this check assumes we are being called from
1114 	 * dbuf_free_range(), perhaps we should move it there?
1115 	 */
1116 	if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1117 		mutex_exit(&db->db_mtx);
1118 		mutex_enter(&dn->dn_mtx);
1119 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1120 		mutex_exit(&dn->dn_mtx);
1121 		return (0);
1122 	}
1123 
1124 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1125 
1126 	dbuf_unoverride(db, tx->tx_txg);
1127 
1128 	ASSERT(db->db.db_size != 0);
1129 	if (db->db_level == 0) {
1130 		ASSERT(db->db_buf != NULL);
1131 		ASSERT(db->db_d.db_data_old[txgoff] != NULL);
1132 		if (db->db_d.db_data_old[txgoff] != db->db_buf)
1133 			VERIFY(arc_buf_remove_ref(
1134 			    db->db_d.db_data_old[txgoff], db) == 1);
1135 		db->db_d.db_data_old[txgoff] = NULL;
1136 	}
1137 
1138 	/* XXX would be nice to fix up dn_towrite_space[] */
1139 	/* XXX undo db_dirtied? but how? */
1140 	/* db->db_dirtied = tx->tx_txg; */
1141 
1142 	mutex_enter(&dn->dn_mtx);
1143 	list_remove(&dn->dn_dirty_dbufs[txgoff], db);
1144 	mutex_exit(&dn->dn_mtx);
1145 
1146 	ASSERT(db->db_dirtycnt > 0);
1147 	db->db_dirtycnt -= 1;
1148 
1149 	if ((holds = refcount_remove(&db->db_holds,
1150 	    (void *)(uintptr_t)tx->tx_txg)) == 0) {
1151 		arc_buf_t *buf = db->db_buf;
1152 
1153 		ASSERT(arc_released(buf));
1154 		dbuf_set_data(db, NULL);
1155 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
1156 		dbuf_evict(db);
1157 		return (1);
1158 	}
1159 	ASSERT(holds > 0);
1160 
1161 	mutex_exit(&db->db_mtx);
1162 	return (0);
1163 }
1164 
1165 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1166 void
1167 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1168 {
1169 	int rf = DB_RF_MUST_SUCCEED;
1170 
1171 	ASSERT(tx->tx_txg != 0);
1172 	ASSERT(!refcount_is_zero(&db->db_holds));
1173 
1174 	if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock))
1175 		rf |= DB_RF_HAVESTRUCT;
1176 	(void) dbuf_read(db, NULL, rf);
1177 	dbuf_dirty(db, tx);
1178 }
1179 
1180 void
1181 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1182 {
1183 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1184 
1185 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1186 	ASSERT(tx->tx_txg != 0);
1187 	ASSERT(db->db_level == 0);
1188 	ASSERT(!refcount_is_zero(&db->db_holds));
1189 
1190 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1191 	    dmu_tx_private_ok(tx));
1192 
1193 	dbuf_noread(db);
1194 	dbuf_dirty(db, tx);
1195 }
1196 
1197 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1198 /* ARGSUSED */
1199 void
1200 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1201 {
1202 	mutex_enter(&db->db_mtx);
1203 	DBUF_VERIFY(db);
1204 
1205 	if (db->db_state == DB_FILL) {
1206 		if (db->db_level == 0 && db->db_d.db_freed_in_flight) {
1207 			ASSERT(db->db_blkid != DB_BONUS_BLKID);
1208 			/* we were freed while filling */
1209 			/* XXX dbuf_undirty? */
1210 			bzero(db->db.db_data, db->db.db_size);
1211 			db->db_d.db_freed_in_flight = FALSE;
1212 		}
1213 		db->db_state = DB_CACHED;
1214 		cv_broadcast(&db->db_changed);
1215 	}
1216 	mutex_exit(&db->db_mtx);
1217 }
1218 
1219 /*
1220  * "Clear" the contents of this dbuf.  This will mark the dbuf
1221  * EVICTING and clear *most* of its references.  Unfortunetely,
1222  * when we are not holding the dn_dbufs_mtx, we can't clear the
1223  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1224  * in this case.  For callers from the DMU we will usually see:
1225  *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1226  * For the arc callback, we will usually see:
1227  * 	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1228  * Sometimes, though, we will get a mix of these two:
1229  *	DMU: dbuf_clear()->arc_buf_evict()
1230  *	ARC: dbuf_do_evict()->dbuf_destroy()
1231  */
1232 void
1233 dbuf_clear(dmu_buf_impl_t *db)
1234 {
1235 	dnode_t *dn = db->db_dnode;
1236 	dmu_buf_impl_t *parent = db->db_parent;
1237 	int dbuf_gone = FALSE;
1238 
1239 	ASSERT(MUTEX_HELD(&db->db_mtx));
1240 	ASSERT(refcount_is_zero(&db->db_holds));
1241 
1242 	dbuf_evict_user(db);
1243 
1244 	if (db->db_state == DB_CACHED) {
1245 		ASSERT(db->db.db_data != NULL);
1246 		if (db->db_blkid == DB_BONUS_BLKID)
1247 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1248 		db->db.db_data = NULL;
1249 		db->db_state = DB_UNCACHED;
1250 	}
1251 
1252 	ASSERT3U(db->db_state, ==, DB_UNCACHED);
1253 	ASSERT(db->db_data_pending == NULL);
1254 
1255 	db->db_state = DB_EVICTING;
1256 	db->db_blkptr = NULL;
1257 
1258 	if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1259 		list_remove(&dn->dn_dbufs, db);
1260 		dnode_rele(dn, db);
1261 	}
1262 
1263 	if (db->db_buf)
1264 		dbuf_gone = arc_buf_evict(db->db_buf);
1265 
1266 	if (!dbuf_gone)
1267 		mutex_exit(&db->db_mtx);
1268 
1269 	/*
1270 	 * If this dbuf is referened from an indirect dbuf,
1271 	 * decrement the ref count on the indirect dbuf.
1272 	 */
1273 	if (parent && parent != dn->dn_dbuf)
1274 		dbuf_rele(parent, db);
1275 }
1276 
1277 static int
1278 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1279     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1280 {
1281 	int nlevels, epbs;
1282 
1283 	ASSERT(blkid != DB_BONUS_BLKID);
1284 
1285 	if (dn->dn_phys->dn_nlevels == 0)
1286 		nlevels = 1;
1287 	else
1288 		nlevels = dn->dn_phys->dn_nlevels;
1289 
1290 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1291 
1292 	ASSERT3U(level * epbs, <, 64);
1293 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1294 	if (level >= nlevels ||
1295 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1296 		/* the buffer has no parent yet */
1297 		*parentp = NULL;
1298 		*bpp = NULL;
1299 		return (ENOENT);
1300 	} else if (level < nlevels-1) {
1301 		/* this block is referenced from an indirect block */
1302 		int err = dbuf_hold_impl(dn, level+1,
1303 		    blkid >> epbs, fail_sparse, NULL, parentp);
1304 		if (err)
1305 			return (err);
1306 		err = dbuf_read(*parentp, NULL,
1307 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1308 		if (err == 0) {
1309 			*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1310 			    (blkid & ((1ULL << epbs) - 1));
1311 		}
1312 		return (err);
1313 	} else {
1314 		/* the block is referenced from the dnode */
1315 		ASSERT3U(level, ==, nlevels-1);
1316 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1317 		    blkid < dn->dn_phys->dn_nblkptr);
1318 		*parentp = dn->dn_dbuf;
1319 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1320 		return (0);
1321 	}
1322 }
1323 
1324 static dmu_buf_impl_t *
1325 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1326     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1327 {
1328 	objset_impl_t *os = dn->dn_objset;
1329 	dmu_buf_impl_t *db, *odb;
1330 
1331 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1332 	ASSERT(dn->dn_type != DMU_OT_NONE);
1333 
1334 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1335 
1336 	db->db_objset = os;
1337 	db->db.db_object = dn->dn_object;
1338 	db->db_level = level;
1339 	db->db_blkid = blkid;
1340 	db->db_dirtied = 0;
1341 	db->db_dirtycnt = 0;
1342 	db->db_dnode = dn;
1343 	db->db_parent = parent;
1344 	db->db_blkptr = blkptr;
1345 
1346 	bzero(&db->db_d, sizeof (db->db_d));
1347 
1348 	if (blkid == DB_BONUS_BLKID) {
1349 		ASSERT3P(parent, ==, dn->dn_dbuf);
1350 		db->db.db_size = dn->dn_bonuslen;
1351 		db->db.db_offset = DB_BONUS_BLKID;
1352 		db->db_state = DB_UNCACHED;
1353 		/* the bonus dbuf is not placed in the hash table */
1354 		return (db);
1355 	} else {
1356 		int blocksize =
1357 		    db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1358 		db->db.db_size = blocksize;
1359 		db->db.db_offset = db->db_blkid * blocksize;
1360 	}
1361 
1362 	/*
1363 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1364 	 * in the hash table *and* added to the dbufs list.
1365 	 * This prevents a possible deadlock with someone
1366 	 * trying to look up this dbuf before its added to the
1367 	 * dn_dbufs list.
1368 	 */
1369 	mutex_enter(&dn->dn_dbufs_mtx);
1370 	db->db_state = DB_EVICTING;
1371 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1372 		/* someone else inserted it first */
1373 		kmem_cache_free(dbuf_cache, db);
1374 		mutex_exit(&dn->dn_dbufs_mtx);
1375 		return (odb);
1376 	}
1377 	list_insert_head(&dn->dn_dbufs, db);
1378 	db->db_state = DB_UNCACHED;
1379 	mutex_exit(&dn->dn_dbufs_mtx);
1380 
1381 	if (parent && parent != dn->dn_dbuf)
1382 		dbuf_add_ref(parent, db);
1383 
1384 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1385 	    refcount_count(&dn->dn_holds) > 0);
1386 	(void) refcount_add(&dn->dn_holds, db);
1387 
1388 	dprintf_dbuf(db, "db=%p\n", db);
1389 
1390 	return (db);
1391 }
1392 
1393 static int
1394 dbuf_do_evict(void *private)
1395 {
1396 	arc_buf_t *buf = private;
1397 	dmu_buf_impl_t *db = buf->b_private;
1398 
1399 	if (!MUTEX_HELD(&db->db_mtx))
1400 		mutex_enter(&db->db_mtx);
1401 
1402 	ASSERT(db->db_buf == buf);
1403 	ASSERT(refcount_is_zero(&db->db_holds));
1404 
1405 	if (db->db_state != DB_EVICTING) {
1406 		ASSERT(db->db_state == DB_CACHED);
1407 		DBUF_VERIFY(db);
1408 		db->db_buf = NULL;
1409 		dbuf_evict(db);
1410 	} else {
1411 		mutex_exit(&db->db_mtx);
1412 		dbuf_destroy(db);
1413 	}
1414 	return (0);
1415 }
1416 
1417 static void
1418 dbuf_destroy(dmu_buf_impl_t *db)
1419 {
1420 	ASSERT(refcount_is_zero(&db->db_holds));
1421 
1422 	if (db->db_blkid != DB_BONUS_BLKID) {
1423 		dnode_t *dn = db->db_dnode;
1424 
1425 		/*
1426 		 * If this dbuf is still on the dn_dbufs list,
1427 		 * remove it from that list.
1428 		 */
1429 		if (list_link_active(&db->db_link)) {
1430 			int need_mutex;
1431 
1432 			ASSERT(!MUTEX_HELD(&dn->dn_dbufs_mtx));
1433 			need_mutex = !MUTEX_HELD(&dn->dn_dbufs_mtx);
1434 			if (need_mutex)
1435 				mutex_enter(&dn->dn_dbufs_mtx);
1436 
1437 			/* remove from dn_dbufs */
1438 			list_remove(&dn->dn_dbufs, db);
1439 
1440 			if (need_mutex)
1441 				mutex_exit(&dn->dn_dbufs_mtx);
1442 
1443 			dnode_rele(dn, db);
1444 		}
1445 		dbuf_hash_remove(db);
1446 	}
1447 	db->db_parent = NULL;
1448 	db->db_dnode = NULL;
1449 	db->db_buf = NULL;
1450 
1451 	ASSERT(db->db.db_data == NULL);
1452 	ASSERT(db->db_hash_next == NULL);
1453 	ASSERT(db->db_blkptr == NULL);
1454 	ASSERT(db->db_data_pending == NULL);
1455 
1456 	kmem_cache_free(dbuf_cache, db);
1457 }
1458 
1459 void
1460 dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1461 {
1462 	dmu_buf_impl_t *db, *parent = NULL;
1463 	blkptr_t *bp = NULL;
1464 
1465 	ASSERT(blkid != DB_BONUS_BLKID);
1466 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1467 
1468 	if (dnode_block_freed(dn, blkid))
1469 		return;
1470 
1471 	/* dbuf_find() returns with db_mtx held */
1472 	if (db = dbuf_find(dn, 0, blkid)) {
1473 		/*
1474 		 * This dbuf is already in the cache.  We assume that
1475 		 * it is already CACHED, or else about to be either
1476 		 * read or filled.
1477 		 */
1478 		mutex_exit(&db->db_mtx);
1479 		return;
1480 	}
1481 
1482 	if (dbuf_findbp(dn, 0, blkid, TRUE, &parent, &bp) == 0) {
1483 		if (bp && !BP_IS_HOLE(bp)) {
1484 			zbookmark_t zb;
1485 			zb.zb_objset = dn->dn_objset->os_dsl_dataset ?
1486 			    dn->dn_objset->os_dsl_dataset->ds_object : 0;
1487 			zb.zb_object = dn->dn_object;
1488 			zb.zb_level = 0;
1489 			zb.zb_blkid = blkid;
1490 
1491 			(void) arc_read(NULL, dn->dn_objset->os_spa, bp,
1492 			    dmu_ot[dn->dn_type].ot_byteswap,
1493 			    NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
1494 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
1495 			    (ARC_NOWAIT | ARC_PREFETCH), &zb);
1496 		}
1497 		if (parent && parent != dn->dn_dbuf)
1498 			dbuf_rele(parent, NULL);
1499 	}
1500 }
1501 
1502 /*
1503  * Returns with db_holds incremented, and db_mtx not held.
1504  * Note: dn_struct_rwlock must be held.
1505  */
1506 int
1507 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1508     void *tag, dmu_buf_impl_t **dbp)
1509 {
1510 	dmu_buf_impl_t *db, *parent = NULL;
1511 
1512 	ASSERT(blkid != DB_BONUS_BLKID);
1513 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1514 	ASSERT3U(dn->dn_nlevels, >, level);
1515 
1516 	*dbp = NULL;
1517 top:
1518 	/* dbuf_find() returns with db_mtx held */
1519 	db = dbuf_find(dn, level, blkid);
1520 
1521 	if (db == NULL) {
1522 		blkptr_t *bp = NULL;
1523 		int err;
1524 
1525 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1526 		if (fail_sparse) {
1527 			if (err == 0 && bp && BP_IS_HOLE(bp))
1528 				err = ENOENT;
1529 			if (err) {
1530 				if (parent && parent != dn->dn_dbuf)
1531 					dbuf_rele(parent, NULL);
1532 				return (err);
1533 			}
1534 		}
1535 		if (err && err != ENOENT)
1536 			return (err);
1537 		db = dbuf_create(dn, level, blkid, parent, bp);
1538 	}
1539 
1540 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1541 		arc_buf_add_ref(db->db_buf, db);
1542 		if (db->db_buf->b_data == NULL) {
1543 			dbuf_clear(db);
1544 			goto top;
1545 		}
1546 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1547 	}
1548 
1549 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1550 
1551 	/*
1552 	 * If this buffer is currently syncing out, and we are
1553 	 * are still referencing it from db_data, we need to make
1554 	 * a copy of it in case we decide we want to dirty it
1555 	 * again in this txg.
1556 	 */
1557 	if (db->db_level == 0 && db->db_state == DB_CACHED &&
1558 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
1559 	    db->db_data_pending == db->db_buf) {
1560 		int size = (db->db_blkid == DB_BONUS_BLKID) ?
1561 		    DN_MAX_BONUSLEN : db->db.db_size;
1562 
1563 		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
1564 		    size, db));
1565 		bcopy(db->db_data_pending->b_data, db->db.db_data,
1566 		    db->db.db_size);
1567 	}
1568 
1569 	(void) refcount_add(&db->db_holds, tag);
1570 	dbuf_update_data(db);
1571 	DBUF_VERIFY(db);
1572 	mutex_exit(&db->db_mtx);
1573 
1574 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
1575 	if (parent && parent != dn->dn_dbuf)
1576 		dbuf_rele(parent, NULL);
1577 
1578 	ASSERT3P(db->db_dnode, ==, dn);
1579 	ASSERT3U(db->db_blkid, ==, blkid);
1580 	ASSERT3U(db->db_level, ==, level);
1581 	*dbp = db;
1582 
1583 	return (0);
1584 }
1585 
1586 dmu_buf_impl_t *
1587 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1588 {
1589 	dmu_buf_impl_t *db;
1590 	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1591 	return (err ? NULL : db);
1592 }
1593 
1594 dmu_buf_impl_t *
1595 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1596 {
1597 	dmu_buf_impl_t *db;
1598 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1599 	return (err ? NULL : db);
1600 }
1601 
1602 dmu_buf_impl_t *
1603 dbuf_create_bonus(dnode_t *dn)
1604 {
1605 	dmu_buf_impl_t *db = dn->dn_bonus;
1606 
1607 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1608 
1609 	ASSERT(dn->dn_bonus == NULL);
1610 	db = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL);
1611 	return (db);
1612 }
1613 
1614 #pragma weak dmu_buf_add_ref = dbuf_add_ref
1615 void
1616 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
1617 {
1618 	int64_t holds = refcount_add(&db->db_holds, tag);
1619 	ASSERT(holds > 1);
1620 }
1621 
1622 #pragma weak dmu_buf_rele = dbuf_rele
1623 void
1624 dbuf_rele(dmu_buf_impl_t *db, void *tag)
1625 {
1626 	int64_t holds;
1627 
1628 	mutex_enter(&db->db_mtx);
1629 	DBUF_VERIFY(db);
1630 
1631 	holds = refcount_remove(&db->db_holds, tag);
1632 	ASSERT(holds >= 0);
1633 
1634 	if (holds == db->db_dirtycnt &&
1635 	    db->db_level == 0 && db->db_d.db_immediate_evict)
1636 		dbuf_evict_user(db);
1637 
1638 	if (holds == 0) {
1639 		if (db->db_blkid == DB_BONUS_BLKID) {
1640 			mutex_exit(&db->db_mtx);
1641 			dnode_rele(db->db_dnode, db);
1642 		} else if (db->db_buf == NULL) {
1643 			/*
1644 			 * This is a special case: we never associated this
1645 			 * dbuf with any data allocated from the ARC.
1646 			 */
1647 			ASSERT3U(db->db_state, ==, DB_UNCACHED);
1648 			dbuf_evict(db);
1649 		} else  if (arc_released(db->db_buf)) {
1650 			arc_buf_t *buf = db->db_buf;
1651 			/*
1652 			 * This dbuf has anonymous data associated with it.
1653 			 */
1654 			dbuf_set_data(db, NULL);
1655 			VERIFY(arc_buf_remove_ref(buf, db) == 1);
1656 			dbuf_evict(db);
1657 		} else {
1658 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
1659 			mutex_exit(&db->db_mtx);
1660 		}
1661 	} else {
1662 		mutex_exit(&db->db_mtx);
1663 	}
1664 }
1665 
1666 #pragma weak dmu_buf_refcount = dbuf_refcount
1667 uint64_t
1668 dbuf_refcount(dmu_buf_impl_t *db)
1669 {
1670 	return (refcount_count(&db->db_holds));
1671 }
1672 
1673 void *
1674 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1675     dmu_buf_evict_func_t *evict_func)
1676 {
1677 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1678 	    user_data_ptr_ptr, evict_func));
1679 }
1680 
1681 void *
1682 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1683     dmu_buf_evict_func_t *evict_func)
1684 {
1685 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1686 
1687 	db->db_d.db_immediate_evict = TRUE;
1688 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1689 	    user_data_ptr_ptr, evict_func));
1690 }
1691 
1692 void *
1693 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
1694     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
1695 {
1696 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1697 	ASSERT(db->db_level == 0);
1698 
1699 	ASSERT((user_ptr == NULL) == (evict_func == NULL));
1700 
1701 	mutex_enter(&db->db_mtx);
1702 
1703 	if (db->db_d.db_user_ptr == old_user_ptr) {
1704 		db->db_d.db_user_ptr = user_ptr;
1705 		db->db_d.db_user_data_ptr_ptr = user_data_ptr_ptr;
1706 		db->db_d.db_evict_func = evict_func;
1707 
1708 		dbuf_update_data(db);
1709 	} else {
1710 		old_user_ptr = db->db_d.db_user_ptr;
1711 	}
1712 
1713 	mutex_exit(&db->db_mtx);
1714 	return (old_user_ptr);
1715 }
1716 
1717 void *
1718 dmu_buf_get_user(dmu_buf_t *db_fake)
1719 {
1720 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1721 	ASSERT(!refcount_is_zero(&db->db_holds));
1722 
1723 	return (db->db_d.db_user_ptr);
1724 }
1725 
1726 void
1727 dbuf_sync(dmu_buf_impl_t *db, zio_t *zio, dmu_tx_t *tx)
1728 {
1729 	arc_buf_t **data;
1730 	uint64_t txg = tx->tx_txg;
1731 	dnode_t *dn = db->db_dnode;
1732 	objset_impl_t *os = dn->dn_objset;
1733 	int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
1734 	int checksum, compress;
1735 	zbookmark_t zb;
1736 	int blksz;
1737 
1738 	ASSERT(dmu_tx_is_syncing(tx));
1739 
1740 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
1741 
1742 	mutex_enter(&db->db_mtx);
1743 	/*
1744 	 * To be synced, we must be dirtied.  But we
1745 	 * might have been freed after the dirty.
1746 	 */
1747 	if (db->db_state == DB_UNCACHED) {
1748 		/* This buffer has been freed since it was dirtied */
1749 		ASSERT(db->db.db_data == NULL);
1750 	} else if (db->db_state == DB_FILL) {
1751 		/* This buffer was freed and is now being re-filled */
1752 		ASSERT(db->db.db_data != db->db_d.db_data_old[txg&TXG_MASK]);
1753 	} else {
1754 		ASSERT3U(db->db_state, ==, DB_CACHED);
1755 	}
1756 	DBUF_VERIFY(db);
1757 
1758 	/*
1759 	 * Don't need a lock on db_dirty (dn_mtx), because it can't
1760 	 * be modified yet.
1761 	 */
1762 
1763 	if (db->db_blkid == DB_BONUS_BLKID) {
1764 		void **datap = &db->db_d.db_data_old[txg&TXG_MASK];
1765 		/*
1766 		 * Simply copy the bonus data into the dnode.  It will
1767 		 * be written out when the dnode is synced (and it will
1768 		 * be synced, since it must have been dirty for dbuf_sync
1769 		 * to be called).
1770 		 */
1771 		/*
1772 		 * Use dn_phys->dn_bonuslen since db.db_size is the length
1773 		 * of the bonus buffer in the open transaction rather than
1774 		 * the syncing transaction.
1775 		 */
1776 		ASSERT(*datap != NULL);
1777 		ASSERT3U(db->db_level, ==, 0);
1778 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
1779 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
1780 		if (*datap != db->db.db_data)
1781 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
1782 		db->db_d.db_data_old[txg&TXG_MASK] = NULL;
1783 		db->db_data_pending = NULL;
1784 		if (db->db_dirtied == txg)
1785 			db->db_dirtied = 0;
1786 		ASSERT(db->db_dirtycnt > 0);
1787 		db->db_dirtycnt -= 1;
1788 		mutex_exit(&db->db_mtx);
1789 		dbuf_rele(db, (void *)(uintptr_t)txg);
1790 		return;
1791 	}
1792 
1793 	if (db->db_level == 0) {
1794 		data = (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK];
1795 		blksz = arc_buf_size(*data);
1796 		/*
1797 		 * If this buffer is currently "in use" (i.e., there are
1798 		 * active holds and db_data still references it), then make
1799 		 * a copy before we start the write so that any modifications
1800 		 * from the open txg will not leak into this write.
1801 		 *
1802 		 * NOTE: this copy does not need to be made for objects only
1803 		 * modified in the syncing context (e.g. DNONE_DNODE blocks)
1804 		 * or if there is no actual write involved (bonus blocks).
1805 		 */
1806 		if (dn->dn_object != DMU_META_DNODE_OBJECT &&
1807 		    db->db_d.db_overridden_by[txg&TXG_MASK] == NULL) {
1808 			if (refcount_count(&db->db_holds) > 1 &&
1809 			    *data == db->db_buf) {
1810 				*data = arc_buf_alloc(os->os_spa, blksz, db);
1811 				bcopy(db->db.db_data, (*data)->b_data, blksz);
1812 			}
1813 			db->db_data_pending = *data;
1814 		} else if (dn->dn_object == DMU_META_DNODE_OBJECT) {
1815 			/*
1816 			 * Private object buffers are released here rather
1817 			 * than in dbuf_dirty() since they are only modified
1818 			 * in the syncing context and we don't want the
1819 			 * overhead of making multiple copies of the data.
1820 			 */
1821 			arc_release(db->db_buf, db);
1822 		}
1823 	} else {
1824 		data = &db->db_buf;
1825 		if (*data == NULL) {
1826 			/*
1827 			 * This can happen if we dirty and then free
1828 			 * the level-0 data blocks in the same txg. So
1829 			 * this indirect remains unchanged.
1830 			 */
1831 			if (db->db_dirtied == txg)
1832 				db->db_dirtied = 0;
1833 			ASSERT(db->db_dirtycnt > 0);
1834 			db->db_dirtycnt -= 1;
1835 			mutex_exit(&db->db_mtx);
1836 			dbuf_rele(db, (void *)(uintptr_t)txg);
1837 			return;
1838 		}
1839 		blksz = db->db.db_size;
1840 		ASSERT3U(blksz, ==, 1<<dn->dn_phys->dn_indblkshift);
1841 	}
1842 
1843 	ASSERT(*data != NULL);
1844 
1845 	if (db->db_level > 0 && !arc_released(db->db_buf)) {
1846 		/*
1847 		 * This indirect buffer was marked dirty, but
1848 		 * never modified (if it had been modified, then
1849 		 * we would have released the buffer).  There is
1850 		 * no reason to write anything.
1851 		 */
1852 		db->db_data_pending = NULL;
1853 		if (db->db_dirtied == txg)
1854 			db->db_dirtied = 0;
1855 		ASSERT(db->db_dirtycnt > 0);
1856 		db->db_dirtycnt -= 1;
1857 		mutex_exit(&db->db_mtx);
1858 		dbuf_rele(db, (void *)(uintptr_t)txg);
1859 		return;
1860 	} else if (db->db_blkptr == NULL &&
1861 	    db->db_level == dn->dn_phys->dn_nlevels-1 &&
1862 	    db->db_blkid < dn->dn_phys->dn_nblkptr) {
1863 		/*
1864 		 * This buffer was allocated at a time when there was
1865 		 * no available blkptrs from the dnode, or it was
1866 		 * inappropriate to hook it in (i.e., nlevels mis-match).
1867 		 */
1868 		ASSERT(db->db_blkptr == NULL);
1869 		ASSERT(db->db_parent == NULL);
1870 		db->db_parent = dn->dn_dbuf;
1871 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
1872 		DBUF_VERIFY(db);
1873 		mutex_exit(&db->db_mtx);
1874 	} else if (db->db_blkptr == NULL) {
1875 		dmu_buf_impl_t *parent = db->db_parent;
1876 
1877 		mutex_exit(&db->db_mtx);
1878 		ASSERT(dn->dn_phys->dn_nlevels > 1);
1879 		if (parent == NULL) {
1880 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
1881 			(void) dbuf_hold_impl(dn, db->db_level+1,
1882 			    db->db_blkid >> epbs, FALSE, FTAG, &parent);
1883 			rw_exit(&dn->dn_struct_rwlock);
1884 			dbuf_add_ref(parent, db);
1885 			db->db_parent = parent;
1886 			dbuf_rele(parent, FTAG);
1887 		}
1888 		(void) dbuf_read(parent, NULL, DB_RF_MUST_SUCCEED);
1889 	} else {
1890 		mutex_exit(&db->db_mtx);
1891 	}
1892 
1893 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || db->db_parent != NULL);
1894 
1895 	if (db->db_level > 0 &&
1896 	    db->db_blkid > dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)) {
1897 		/*
1898 		 * Don't write indirect blocks past EOF.
1899 		 * We get these when we truncate a file *after* dirtying
1900 		 * blocks in the truncate range (we undirty the level 0
1901 		 * blocks in dbuf_free_range(), but not the indirects).
1902 		 */
1903 #ifdef ZFS_DEBUG
1904 		/*
1905 		 * Verify that this indirect block is empty.
1906 		 */
1907 		blkptr_t *bplist;
1908 		int i;
1909 
1910 		mutex_enter(&db->db_mtx);
1911 		bplist = db->db.db_data;
1912 		for (i = 0; i < (1 << epbs); i++) {
1913 			if (!BP_IS_HOLE(&bplist[i])) {
1914 				panic("data past EOF: "
1915 				    "db=%p level=%d id=%llu i=%d\n",
1916 				    db, db->db_level,
1917 				    (u_longlong_t)db->db_blkid, i);
1918 			}
1919 		}
1920 		mutex_exit(&db->db_mtx);
1921 #endif
1922 		ASSERT(db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr));
1923 		mutex_enter(&db->db_mtx);
1924 		db->db_dirtycnt -= 1;
1925 		mutex_exit(&db->db_mtx);
1926 		dbuf_rele(db, (void *)(uintptr_t)txg);
1927 		return;
1928 	}
1929 
1930 	if (db->db_parent != dn->dn_dbuf) {
1931 		dmu_buf_impl_t *parent = db->db_parent;
1932 
1933 		mutex_enter(&db->db_mtx);
1934 		ASSERT(db->db_level == parent->db_level-1);
1935 		ASSERT(list_link_active(&parent->db_dirty_node[txg&TXG_MASK]));
1936 		/*
1937 		 * We may have read this indirect block after we dirtied it,
1938 		 * so never released it from the cache.
1939 		 */
1940 		arc_release(parent->db_buf, db->db_parent);
1941 
1942 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
1943 		    (db->db_blkid & ((1ULL << epbs) - 1));
1944 		DBUF_VERIFY(db);
1945 		mutex_exit(&db->db_mtx);
1946 #ifdef ZFS_DEBUG
1947 	} else {
1948 		/*
1949 		 * We don't need to dnode_setdirty(dn) because if we got
1950 		 * here then the parent is already dirty.
1951 		 */
1952 		ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1);
1953 		ASSERT3P(db->db_blkptr, ==,
1954 		    &dn->dn_phys->dn_blkptr[db->db_blkid]);
1955 #endif
1956 	}
1957 	ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
1958 
1959 	if (db->db_level == 0 &&
1960 	    db->db_d.db_overridden_by[txg&TXG_MASK] != NULL) {
1961 		arc_buf_t **old =
1962 		    (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK];
1963 		blkptr_t **bpp = &db->db_d.db_overridden_by[txg&TXG_MASK];
1964 		int old_size = BP_GET_ASIZE(db->db_blkptr);
1965 		int new_size = BP_GET_ASIZE(*bpp);
1966 
1967 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
1968 
1969 		dnode_diduse_space(dn, new_size-old_size);
1970 		mutex_enter(&dn->dn_mtx);
1971 		if (db->db_blkid > dn->dn_phys->dn_maxblkid)
1972 			dn->dn_phys->dn_maxblkid = db->db_blkid;
1973 		mutex_exit(&dn->dn_mtx);
1974 
1975 		dsl_dataset_block_born(os->os_dsl_dataset, *bpp, tx);
1976 		if (!BP_IS_HOLE(db->db_blkptr))
1977 			dsl_dataset_block_kill(os->os_dsl_dataset,
1978 			    db->db_blkptr, os->os_synctx);
1979 
1980 		mutex_enter(&db->db_mtx);
1981 		*db->db_blkptr = **bpp;
1982 		kmem_free(*bpp, sizeof (blkptr_t));
1983 		*bpp = NULL;
1984 
1985 		if (*old != db->db_buf)
1986 			VERIFY(arc_buf_remove_ref(*old, db) == 1);
1987 		else if (!BP_IS_HOLE(db->db_blkptr))
1988 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
1989 		else
1990 			ASSERT(arc_released(db->db_buf));
1991 		*old = NULL;
1992 		db->db_data_pending = NULL;
1993 
1994 		cv_broadcast(&db->db_changed);
1995 
1996 		ASSERT(db->db_dirtycnt > 0);
1997 		db->db_dirtycnt -= 1;
1998 		mutex_exit(&db->db_mtx);
1999 		dbuf_rele(db, (void *)(uintptr_t)txg);
2000 		return;
2001 	}
2002 
2003 	if (db->db_level > 0) {
2004 		/*
2005 		 * XXX -- we should design a compression algorithm
2006 		 * that specializes in arrays of bps.
2007 		 */
2008 		checksum = ZIO_CHECKSUM_FLETCHER_4;
2009 		compress = ZIO_COMPRESS_LZJB;
2010 	} else {
2011 		/*
2012 		 * Allow dnode settings to override objset settings,
2013 		 * except for metadata checksums.
2014 		 */
2015 		if (dmu_ot[dn->dn_type].ot_metadata) {
2016 			checksum = os->os_md_checksum;
2017 			compress = zio_compress_select(dn->dn_compress,
2018 			    os->os_md_compress);
2019 		} else {
2020 			checksum = zio_checksum_select(dn->dn_checksum,
2021 			    os->os_checksum);
2022 			compress = zio_compress_select(dn->dn_compress,
2023 			    os->os_compress);
2024 		}
2025 	}
2026 #ifdef ZFS_DEBUG
2027 	if (db->db_parent) {
2028 		ASSERT(list_link_active(
2029 		    &db->db_parent->db_dirty_node[txg&TXG_MASK]));
2030 		ASSERT(db->db_parent == dn->dn_dbuf ||
2031 		    db->db_parent->db_level > 0);
2032 		if (dn->dn_object == DMU_META_DNODE_OBJECT || db->db_level > 0)
2033 			ASSERT(*data == db->db_buf);
2034 	}
2035 #endif
2036 	ASSERT3U(db->db_blkptr->blk_birth, <=, tx->tx_txg);
2037 	zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0;
2038 	zb.zb_object = db->db.db_object;
2039 	zb.zb_level = db->db_level;
2040 	zb.zb_blkid = db->db_blkid;
2041 	(void) arc_write(zio, os->os_spa, checksum, compress, txg,
2042 	    db->db_blkptr, *data, dbuf_write_done, db,
2043 	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, ARC_NOWAIT, &zb);
2044 	/*
2045 	 * We can't access db after arc_write, since it could finish
2046 	 * and be freed, and we have no locks on it.
2047 	 */
2048 }
2049 
2050 struct dbuf_arg {
2051 	objset_impl_t *os;
2052 	blkptr_t bp;
2053 };
2054 
2055 static void
2056 dbuf_do_born(void *arg)
2057 {
2058 	struct dbuf_arg *da = arg;
2059 	dsl_dataset_block_born(da->os->os_dsl_dataset,
2060 	    &da->bp, da->os->os_synctx);
2061 	kmem_free(da, sizeof (struct dbuf_arg));
2062 }
2063 
2064 static void
2065 dbuf_do_kill(void *arg)
2066 {
2067 	struct dbuf_arg *da = arg;
2068 	dsl_dataset_block_kill(da->os->os_dsl_dataset,
2069 	    &da->bp, da->os->os_synctx);
2070 	kmem_free(da, sizeof (struct dbuf_arg));
2071 }
2072 
2073 /* ARGSUSED */
2074 static void
2075 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2076 {
2077 	dmu_buf_impl_t *db = vdb;
2078 	dnode_t *dn = db->db_dnode;
2079 	objset_impl_t *os = dn->dn_objset;
2080 	uint64_t txg = zio->io_txg;
2081 	uint64_t fill = 0;
2082 	int i;
2083 	int old_size, new_size;
2084 
2085 	ASSERT3U(zio->io_error, ==, 0);
2086 
2087 	dprintf_dbuf_bp(db, &zio->io_bp_orig, "bp_orig: %s", "");
2088 
2089 	old_size = BP_GET_ASIZE(&zio->io_bp_orig);
2090 	new_size = BP_GET_ASIZE(zio->io_bp);
2091 
2092 	dnode_diduse_space(dn, new_size-old_size);
2093 
2094 	mutex_enter(&db->db_mtx);
2095 
2096 	if (db->db_dirtied == txg)
2097 		db->db_dirtied = 0;
2098 
2099 	if (db->db_level == 0) {
2100 		arc_buf_t **old =
2101 		    (arc_buf_t **)&db->db_d.db_data_old[txg&TXG_MASK];
2102 
2103 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
2104 
2105 		if (*old != db->db_buf)
2106 			VERIFY(arc_buf_remove_ref(*old, db) == 1);
2107 		else if (!BP_IS_HOLE(db->db_blkptr))
2108 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2109 		else
2110 			ASSERT(arc_released(db->db_buf));
2111 		*old = NULL;
2112 		db->db_data_pending = NULL;
2113 
2114 		mutex_enter(&dn->dn_mtx);
2115 		if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2116 		    !BP_IS_HOLE(db->db_blkptr))
2117 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2118 		mutex_exit(&dn->dn_mtx);
2119 
2120 		if (dn->dn_type == DMU_OT_DNODE) {
2121 			dnode_phys_t *dnp = db->db.db_data;
2122 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2123 			    i--, dnp++) {
2124 				if (dnp->dn_type != DMU_OT_NONE)
2125 					fill++;
2126 			}
2127 		} else {
2128 			if (!BP_IS_HOLE(db->db_blkptr))
2129 				fill = 1;
2130 		}
2131 	} else {
2132 		blkptr_t *bp = db->db.db_data;
2133 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2134 		if (!BP_IS_HOLE(db->db_blkptr)) {
2135 			int epbs =
2136 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2137 			ASSERT3U(BP_GET_LSIZE(zio->io_bp), ==, db->db.db_size);
2138 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2139 			    db->db.db_size);
2140 			ASSERT3U(dn->dn_phys->dn_maxblkid
2141 			    >> (db->db_level * epbs), >=, db->db_blkid);
2142 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2143 		}
2144 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, bp++) {
2145 			if (BP_IS_HOLE(bp))
2146 				continue;
2147 			ASSERT3U(BP_GET_LSIZE(bp), ==,
2148 			    db->db_level == 1 ? dn->dn_datablksz :
2149 			    (1<<dn->dn_phys->dn_indblkshift));
2150 			fill += bp->blk_fill;
2151 		}
2152 	}
2153 
2154 	if (!BP_IS_HOLE(db->db_blkptr)) {
2155 		db->db_blkptr->blk_fill = fill;
2156 		BP_SET_TYPE(db->db_blkptr, dn->dn_type);
2157 		BP_SET_LEVEL(db->db_blkptr, db->db_level);
2158 	} else {
2159 		ASSERT3U(fill, ==, 0);
2160 		ASSERT3U(db->db_blkptr->blk_fill, ==, 0);
2161 	}
2162 
2163 	dprintf_dbuf_bp(db, db->db_blkptr,
2164 	    "wrote %llu bytes to blkptr:", zio->io_size);
2165 
2166 	ASSERT(db->db_parent == NULL ||
2167 	    list_link_active(&db->db_parent->db_dirty_node[txg&TXG_MASK]));
2168 	cv_broadcast(&db->db_changed);
2169 	ASSERT(db->db_dirtycnt > 0);
2170 	db->db_dirtycnt -= 1;
2171 	mutex_exit(&db->db_mtx);
2172 
2173 	/* We must do this after we've set the bp's type and level */
2174 	if (!DVA_EQUAL(BP_IDENTITY(zio->io_bp),
2175 	    BP_IDENTITY(&zio->io_bp_orig))) {
2176 		struct dbuf_arg *da;
2177 		da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP);
2178 		da->os = os;
2179 		da->bp = *zio->io_bp;
2180 		(void) taskq_dispatch(dbuf_tq, dbuf_do_born, da, 0);
2181 		if (!BP_IS_HOLE(&zio->io_bp_orig)) {
2182 			da = kmem_alloc(sizeof (struct dbuf_arg), KM_SLEEP);
2183 			da->os = os;
2184 			da->bp = zio->io_bp_orig;
2185 			(void) taskq_dispatch(dbuf_tq, dbuf_do_kill, da, 0);
2186 		}
2187 	}
2188 
2189 	dbuf_rele(db, (void *)(uintptr_t)txg);
2190 }
2191