xref: /illumos-gate/usr/src/uts/common/fs/zfs/dbuf.c (revision 3e30c24aeefdee1631958ecf17f18da671781956)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24  * Copyright (c) 2013 by Delphix. All rights reserved.
25  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
26  */
27 
28 #include <sys/zfs_context.h>
29 #include <sys/dmu.h>
30 #include <sys/dmu_impl.h>
31 #include <sys/dbuf.h>
32 #include <sys/dmu_objset.h>
33 #include <sys/dsl_dataset.h>
34 #include <sys/dsl_dir.h>
35 #include <sys/dmu_tx.h>
36 #include <sys/spa.h>
37 #include <sys/zio.h>
38 #include <sys/dmu_zfetch.h>
39 #include <sys/sa.h>
40 #include <sys/sa_impl.h>
41 
42 static void dbuf_destroy(dmu_buf_impl_t *db);
43 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
44 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
45 
46 /*
47  * Global data structures and functions for the dbuf cache.
48  */
49 static kmem_cache_t *dbuf_cache;
50 
51 /* ARGSUSED */
52 static int
53 dbuf_cons(void *vdb, void *unused, int kmflag)
54 {
55 	dmu_buf_impl_t *db = vdb;
56 	bzero(db, sizeof (dmu_buf_impl_t));
57 
58 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
59 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
60 	refcount_create(&db->db_holds);
61 	return (0);
62 }
63 
64 /* ARGSUSED */
65 static void
66 dbuf_dest(void *vdb, void *unused)
67 {
68 	dmu_buf_impl_t *db = vdb;
69 	mutex_destroy(&db->db_mtx);
70 	cv_destroy(&db->db_changed);
71 	refcount_destroy(&db->db_holds);
72 }
73 
74 /*
75  * dbuf hash table routines
76  */
77 static dbuf_hash_table_t dbuf_hash_table;
78 
79 static uint64_t dbuf_hash_count;
80 
81 static uint64_t
82 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
83 {
84 	uintptr_t osv = (uintptr_t)os;
85 	uint64_t crc = -1ULL;
86 
87 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
88 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
89 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
90 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
91 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
92 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
93 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
94 
95 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
96 
97 	return (crc);
98 }
99 
100 #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
101 
102 #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
103 	((dbuf)->db.db_object == (obj) &&		\
104 	(dbuf)->db_objset == (os) &&			\
105 	(dbuf)->db_level == (level) &&			\
106 	(dbuf)->db_blkid == (blkid))
107 
108 dmu_buf_impl_t *
109 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
110 {
111 	dbuf_hash_table_t *h = &dbuf_hash_table;
112 	objset_t *os = dn->dn_objset;
113 	uint64_t obj = dn->dn_object;
114 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
115 	uint64_t idx = hv & h->hash_table_mask;
116 	dmu_buf_impl_t *db;
117 
118 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
119 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
120 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
121 			mutex_enter(&db->db_mtx);
122 			if (db->db_state != DB_EVICTING) {
123 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
124 				return (db);
125 			}
126 			mutex_exit(&db->db_mtx);
127 		}
128 	}
129 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
130 	return (NULL);
131 }
132 
133 /*
134  * Insert an entry into the hash table.  If there is already an element
135  * equal to elem in the hash table, then the already existing element
136  * will be returned and the new element will not be inserted.
137  * Otherwise returns NULL.
138  */
139 static dmu_buf_impl_t *
140 dbuf_hash_insert(dmu_buf_impl_t *db)
141 {
142 	dbuf_hash_table_t *h = &dbuf_hash_table;
143 	objset_t *os = db->db_objset;
144 	uint64_t obj = db->db.db_object;
145 	int level = db->db_level;
146 	uint64_t blkid = db->db_blkid;
147 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
148 	uint64_t idx = hv & h->hash_table_mask;
149 	dmu_buf_impl_t *dbf;
150 
151 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
152 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
153 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
154 			mutex_enter(&dbf->db_mtx);
155 			if (dbf->db_state != DB_EVICTING) {
156 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
157 				return (dbf);
158 			}
159 			mutex_exit(&dbf->db_mtx);
160 		}
161 	}
162 
163 	mutex_enter(&db->db_mtx);
164 	db->db_hash_next = h->hash_table[idx];
165 	h->hash_table[idx] = db;
166 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
167 	atomic_add_64(&dbuf_hash_count, 1);
168 
169 	return (NULL);
170 }
171 
172 /*
173  * Remove an entry from the hash table.  This operation will
174  * fail if there are any existing holds on the db.
175  */
176 static void
177 dbuf_hash_remove(dmu_buf_impl_t *db)
178 {
179 	dbuf_hash_table_t *h = &dbuf_hash_table;
180 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
181 	    db->db_level, db->db_blkid);
182 	uint64_t idx = hv & h->hash_table_mask;
183 	dmu_buf_impl_t *dbf, **dbp;
184 
185 	/*
186 	 * We musn't hold db_mtx to maintin lock ordering:
187 	 * DBUF_HASH_MUTEX > db_mtx.
188 	 */
189 	ASSERT(refcount_is_zero(&db->db_holds));
190 	ASSERT(db->db_state == DB_EVICTING);
191 	ASSERT(!MUTEX_HELD(&db->db_mtx));
192 
193 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
194 	dbp = &h->hash_table[idx];
195 	while ((dbf = *dbp) != db) {
196 		dbp = &dbf->db_hash_next;
197 		ASSERT(dbf != NULL);
198 	}
199 	*dbp = db->db_hash_next;
200 	db->db_hash_next = NULL;
201 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
202 	atomic_add_64(&dbuf_hash_count, -1);
203 }
204 
205 static arc_evict_func_t dbuf_do_evict;
206 
207 static void
208 dbuf_evict_user(dmu_buf_impl_t *db)
209 {
210 	ASSERT(MUTEX_HELD(&db->db_mtx));
211 
212 	if (db->db_level != 0 || db->db_evict_func == NULL)
213 		return;
214 
215 	if (db->db_user_data_ptr_ptr)
216 		*db->db_user_data_ptr_ptr = db->db.db_data;
217 	db->db_evict_func(&db->db, db->db_user_ptr);
218 	db->db_user_ptr = NULL;
219 	db->db_user_data_ptr_ptr = NULL;
220 	db->db_evict_func = NULL;
221 }
222 
223 boolean_t
224 dbuf_is_metadata(dmu_buf_impl_t *db)
225 {
226 	if (db->db_level > 0) {
227 		return (B_TRUE);
228 	} else {
229 		boolean_t is_metadata;
230 
231 		DB_DNODE_ENTER(db);
232 		is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
233 		DB_DNODE_EXIT(db);
234 
235 		return (is_metadata);
236 	}
237 }
238 
239 void
240 dbuf_evict(dmu_buf_impl_t *db)
241 {
242 	ASSERT(MUTEX_HELD(&db->db_mtx));
243 	ASSERT(db->db_buf == NULL);
244 	ASSERT(db->db_data_pending == NULL);
245 
246 	dbuf_clear(db);
247 	dbuf_destroy(db);
248 }
249 
250 void
251 dbuf_init(void)
252 {
253 	uint64_t hsize = 1ULL << 16;
254 	dbuf_hash_table_t *h = &dbuf_hash_table;
255 	int i;
256 
257 	/*
258 	 * The hash table is big enough to fill all of physical memory
259 	 * with an average 4K block size.  The table will take up
260 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
261 	 */
262 	while (hsize * 4096 < physmem * PAGESIZE)
263 		hsize <<= 1;
264 
265 retry:
266 	h->hash_table_mask = hsize - 1;
267 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
268 	if (h->hash_table == NULL) {
269 		/* XXX - we should really return an error instead of assert */
270 		ASSERT(hsize > (1ULL << 10));
271 		hsize >>= 1;
272 		goto retry;
273 	}
274 
275 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
276 	    sizeof (dmu_buf_impl_t),
277 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
278 
279 	for (i = 0; i < DBUF_MUTEXES; i++)
280 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
281 }
282 
283 void
284 dbuf_fini(void)
285 {
286 	dbuf_hash_table_t *h = &dbuf_hash_table;
287 	int i;
288 
289 	for (i = 0; i < DBUF_MUTEXES; i++)
290 		mutex_destroy(&h->hash_mutexes[i]);
291 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
292 	kmem_cache_destroy(dbuf_cache);
293 }
294 
295 /*
296  * Other stuff.
297  */
298 
299 #ifdef ZFS_DEBUG
300 static void
301 dbuf_verify(dmu_buf_impl_t *db)
302 {
303 	dnode_t *dn;
304 	dbuf_dirty_record_t *dr;
305 
306 	ASSERT(MUTEX_HELD(&db->db_mtx));
307 
308 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
309 		return;
310 
311 	ASSERT(db->db_objset != NULL);
312 	DB_DNODE_ENTER(db);
313 	dn = DB_DNODE(db);
314 	if (dn == NULL) {
315 		ASSERT(db->db_parent == NULL);
316 		ASSERT(db->db_blkptr == NULL);
317 	} else {
318 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
319 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
320 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
321 		ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
322 		    db->db_blkid == DMU_SPILL_BLKID ||
323 		    !list_is_empty(&dn->dn_dbufs));
324 	}
325 	if (db->db_blkid == DMU_BONUS_BLKID) {
326 		ASSERT(dn != NULL);
327 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
328 		ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
329 	} else if (db->db_blkid == DMU_SPILL_BLKID) {
330 		ASSERT(dn != NULL);
331 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
332 		ASSERT0(db->db.db_offset);
333 	} else {
334 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
335 	}
336 
337 	for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
338 		ASSERT(dr->dr_dbuf == db);
339 
340 	for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
341 		ASSERT(dr->dr_dbuf == db);
342 
343 	/*
344 	 * We can't assert that db_size matches dn_datablksz because it
345 	 * can be momentarily different when another thread is doing
346 	 * dnode_set_blksz().
347 	 */
348 	if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
349 		dr = db->db_data_pending;
350 		/*
351 		 * It should only be modified in syncing context, so
352 		 * make sure we only have one copy of the data.
353 		 */
354 		ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
355 	}
356 
357 	/* verify db->db_blkptr */
358 	if (db->db_blkptr) {
359 		if (db->db_parent == dn->dn_dbuf) {
360 			/* db is pointed to by the dnode */
361 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
362 			if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
363 				ASSERT(db->db_parent == NULL);
364 			else
365 				ASSERT(db->db_parent != NULL);
366 			if (db->db_blkid != DMU_SPILL_BLKID)
367 				ASSERT3P(db->db_blkptr, ==,
368 				    &dn->dn_phys->dn_blkptr[db->db_blkid]);
369 		} else {
370 			/* db is pointed to by an indirect block */
371 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
372 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
373 			ASSERT3U(db->db_parent->db.db_object, ==,
374 			    db->db.db_object);
375 			/*
376 			 * dnode_grow_indblksz() can make this fail if we don't
377 			 * have the struct_rwlock.  XXX indblksz no longer
378 			 * grows.  safe to do this now?
379 			 */
380 			if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
381 				ASSERT3P(db->db_blkptr, ==,
382 				    ((blkptr_t *)db->db_parent->db.db_data +
383 				    db->db_blkid % epb));
384 			}
385 		}
386 	}
387 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
388 	    (db->db_buf == NULL || db->db_buf->b_data) &&
389 	    db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
390 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
391 		/*
392 		 * If the blkptr isn't set but they have nonzero data,
393 		 * it had better be dirty, otherwise we'll lose that
394 		 * data when we evict this buffer.
395 		 */
396 		if (db->db_dirtycnt == 0) {
397 			uint64_t *buf = db->db.db_data;
398 			int i;
399 
400 			for (i = 0; i < db->db.db_size >> 3; i++) {
401 				ASSERT(buf[i] == 0);
402 			}
403 		}
404 	}
405 	DB_DNODE_EXIT(db);
406 }
407 #endif
408 
409 static void
410 dbuf_update_data(dmu_buf_impl_t *db)
411 {
412 	ASSERT(MUTEX_HELD(&db->db_mtx));
413 	if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
414 		ASSERT(!refcount_is_zero(&db->db_holds));
415 		*db->db_user_data_ptr_ptr = db->db.db_data;
416 	}
417 }
418 
419 static void
420 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
421 {
422 	ASSERT(MUTEX_HELD(&db->db_mtx));
423 	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
424 	db->db_buf = buf;
425 	if (buf != NULL) {
426 		ASSERT(buf->b_data != NULL);
427 		db->db.db_data = buf->b_data;
428 		if (!arc_released(buf))
429 			arc_set_callback(buf, dbuf_do_evict, db);
430 		dbuf_update_data(db);
431 	} else {
432 		dbuf_evict_user(db);
433 		db->db.db_data = NULL;
434 		if (db->db_state != DB_NOFILL)
435 			db->db_state = DB_UNCACHED;
436 	}
437 }
438 
439 /*
440  * Loan out an arc_buf for read.  Return the loaned arc_buf.
441  */
442 arc_buf_t *
443 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
444 {
445 	arc_buf_t *abuf;
446 
447 	mutex_enter(&db->db_mtx);
448 	if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
449 		int blksz = db->db.db_size;
450 		spa_t *spa;
451 
452 		mutex_exit(&db->db_mtx);
453 		DB_GET_SPA(&spa, db);
454 		abuf = arc_loan_buf(spa, blksz);
455 		bcopy(db->db.db_data, abuf->b_data, blksz);
456 	} else {
457 		abuf = db->db_buf;
458 		arc_loan_inuse_buf(abuf, db);
459 		dbuf_set_data(db, NULL);
460 		mutex_exit(&db->db_mtx);
461 	}
462 	return (abuf);
463 }
464 
465 uint64_t
466 dbuf_whichblock(dnode_t *dn, uint64_t offset)
467 {
468 	if (dn->dn_datablkshift) {
469 		return (offset >> dn->dn_datablkshift);
470 	} else {
471 		ASSERT3U(offset, <, dn->dn_datablksz);
472 		return (0);
473 	}
474 }
475 
476 static void
477 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
478 {
479 	dmu_buf_impl_t *db = vdb;
480 
481 	mutex_enter(&db->db_mtx);
482 	ASSERT3U(db->db_state, ==, DB_READ);
483 	/*
484 	 * All reads are synchronous, so we must have a hold on the dbuf
485 	 */
486 	ASSERT(refcount_count(&db->db_holds) > 0);
487 	ASSERT(db->db_buf == NULL);
488 	ASSERT(db->db.db_data == NULL);
489 	if (db->db_level == 0 && db->db_freed_in_flight) {
490 		/* we were freed in flight; disregard any error */
491 		arc_release(buf, db);
492 		bzero(buf->b_data, db->db.db_size);
493 		arc_buf_freeze(buf);
494 		db->db_freed_in_flight = FALSE;
495 		dbuf_set_data(db, buf);
496 		db->db_state = DB_CACHED;
497 	} else if (zio == NULL || zio->io_error == 0) {
498 		dbuf_set_data(db, buf);
499 		db->db_state = DB_CACHED;
500 	} else {
501 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
502 		ASSERT3P(db->db_buf, ==, NULL);
503 		VERIFY(arc_buf_remove_ref(buf, db));
504 		db->db_state = DB_UNCACHED;
505 	}
506 	cv_broadcast(&db->db_changed);
507 	dbuf_rele_and_unlock(db, NULL);
508 }
509 
510 static void
511 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
512 {
513 	dnode_t *dn;
514 	spa_t *spa;
515 	zbookmark_t zb;
516 	uint32_t aflags = ARC_NOWAIT;
517 
518 	DB_DNODE_ENTER(db);
519 	dn = DB_DNODE(db);
520 	ASSERT(!refcount_is_zero(&db->db_holds));
521 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
522 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
523 	ASSERT(MUTEX_HELD(&db->db_mtx));
524 	ASSERT(db->db_state == DB_UNCACHED);
525 	ASSERT(db->db_buf == NULL);
526 
527 	if (db->db_blkid == DMU_BONUS_BLKID) {
528 		int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
529 
530 		ASSERT3U(bonuslen, <=, db->db.db_size);
531 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
532 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
533 		if (bonuslen < DN_MAX_BONUSLEN)
534 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
535 		if (bonuslen)
536 			bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
537 		DB_DNODE_EXIT(db);
538 		dbuf_update_data(db);
539 		db->db_state = DB_CACHED;
540 		mutex_exit(&db->db_mtx);
541 		return;
542 	}
543 
544 	/*
545 	 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
546 	 * processes the delete record and clears the bp while we are waiting
547 	 * for the dn_mtx (resulting in a "no" from block_freed).
548 	 */
549 	if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
550 	    (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
551 	    BP_IS_HOLE(db->db_blkptr)))) {
552 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
553 
554 		dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa,
555 		    db->db.db_size, db, type));
556 		DB_DNODE_EXIT(db);
557 		bzero(db->db.db_data, db->db.db_size);
558 		db->db_state = DB_CACHED;
559 		*flags |= DB_RF_CACHED;
560 		mutex_exit(&db->db_mtx);
561 		return;
562 	}
563 
564 	spa = dn->dn_objset->os_spa;
565 	DB_DNODE_EXIT(db);
566 
567 	db->db_state = DB_READ;
568 	mutex_exit(&db->db_mtx);
569 
570 	if (DBUF_IS_L2CACHEABLE(db))
571 		aflags |= ARC_L2CACHE;
572 	if (DBUF_IS_L2COMPRESSIBLE(db))
573 		aflags |= ARC_L2COMPRESS;
574 
575 	SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
576 	    db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
577 	    db->db.db_object, db->db_level, db->db_blkid);
578 
579 	dbuf_add_ref(db, NULL);
580 
581 	(void) arc_read(zio, spa, db->db_blkptr,
582 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
583 	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
584 	    &aflags, &zb);
585 	if (aflags & ARC_CACHED)
586 		*flags |= DB_RF_CACHED;
587 }
588 
589 int
590 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
591 {
592 	int err = 0;
593 	int havepzio = (zio != NULL);
594 	int prefetch;
595 	dnode_t *dn;
596 
597 	/*
598 	 * We don't have to hold the mutex to check db_state because it
599 	 * can't be freed while we have a hold on the buffer.
600 	 */
601 	ASSERT(!refcount_is_zero(&db->db_holds));
602 
603 	if (db->db_state == DB_NOFILL)
604 		return (SET_ERROR(EIO));
605 
606 	DB_DNODE_ENTER(db);
607 	dn = DB_DNODE(db);
608 	if ((flags & DB_RF_HAVESTRUCT) == 0)
609 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
610 
611 	prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
612 	    (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL &&
613 	    DBUF_IS_CACHEABLE(db);
614 
615 	mutex_enter(&db->db_mtx);
616 	if (db->db_state == DB_CACHED) {
617 		mutex_exit(&db->db_mtx);
618 		if (prefetch)
619 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
620 			    db->db.db_size, TRUE);
621 		if ((flags & DB_RF_HAVESTRUCT) == 0)
622 			rw_exit(&dn->dn_struct_rwlock);
623 		DB_DNODE_EXIT(db);
624 	} else if (db->db_state == DB_UNCACHED) {
625 		spa_t *spa = dn->dn_objset->os_spa;
626 
627 		if (zio == NULL)
628 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
629 		dbuf_read_impl(db, zio, &flags);
630 
631 		/* dbuf_read_impl has dropped db_mtx for us */
632 
633 		if (prefetch)
634 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
635 			    db->db.db_size, flags & DB_RF_CACHED);
636 
637 		if ((flags & DB_RF_HAVESTRUCT) == 0)
638 			rw_exit(&dn->dn_struct_rwlock);
639 		DB_DNODE_EXIT(db);
640 
641 		if (!havepzio)
642 			err = zio_wait(zio);
643 	} else {
644 		/*
645 		 * Another reader came in while the dbuf was in flight
646 		 * between UNCACHED and CACHED.  Either a writer will finish
647 		 * writing the buffer (sending the dbuf to CACHED) or the
648 		 * first reader's request will reach the read_done callback
649 		 * and send the dbuf to CACHED.  Otherwise, a failure
650 		 * occurred and the dbuf went to UNCACHED.
651 		 */
652 		mutex_exit(&db->db_mtx);
653 		if (prefetch)
654 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
655 			    db->db.db_size, TRUE);
656 		if ((flags & DB_RF_HAVESTRUCT) == 0)
657 			rw_exit(&dn->dn_struct_rwlock);
658 		DB_DNODE_EXIT(db);
659 
660 		/* Skip the wait per the caller's request. */
661 		mutex_enter(&db->db_mtx);
662 		if ((flags & DB_RF_NEVERWAIT) == 0) {
663 			while (db->db_state == DB_READ ||
664 			    db->db_state == DB_FILL) {
665 				ASSERT(db->db_state == DB_READ ||
666 				    (flags & DB_RF_HAVESTRUCT) == 0);
667 				cv_wait(&db->db_changed, &db->db_mtx);
668 			}
669 			if (db->db_state == DB_UNCACHED)
670 				err = SET_ERROR(EIO);
671 		}
672 		mutex_exit(&db->db_mtx);
673 	}
674 
675 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
676 	return (err);
677 }
678 
679 static void
680 dbuf_noread(dmu_buf_impl_t *db)
681 {
682 	ASSERT(!refcount_is_zero(&db->db_holds));
683 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
684 	mutex_enter(&db->db_mtx);
685 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
686 		cv_wait(&db->db_changed, &db->db_mtx);
687 	if (db->db_state == DB_UNCACHED) {
688 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
689 		spa_t *spa;
690 
691 		ASSERT(db->db_buf == NULL);
692 		ASSERT(db->db.db_data == NULL);
693 		DB_GET_SPA(&spa, db);
694 		dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
695 		db->db_state = DB_FILL;
696 	} else if (db->db_state == DB_NOFILL) {
697 		dbuf_set_data(db, NULL);
698 	} else {
699 		ASSERT3U(db->db_state, ==, DB_CACHED);
700 	}
701 	mutex_exit(&db->db_mtx);
702 }
703 
704 /*
705  * This is our just-in-time copy function.  It makes a copy of
706  * buffers, that have been modified in a previous transaction
707  * group, before we modify them in the current active group.
708  *
709  * This function is used in two places: when we are dirtying a
710  * buffer for the first time in a txg, and when we are freeing
711  * a range in a dnode that includes this buffer.
712  *
713  * Note that when we are called from dbuf_free_range() we do
714  * not put a hold on the buffer, we just traverse the active
715  * dbuf list for the dnode.
716  */
717 static void
718 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
719 {
720 	dbuf_dirty_record_t *dr = db->db_last_dirty;
721 
722 	ASSERT(MUTEX_HELD(&db->db_mtx));
723 	ASSERT(db->db.db_data != NULL);
724 	ASSERT(db->db_level == 0);
725 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
726 
727 	if (dr == NULL ||
728 	    (dr->dt.dl.dr_data !=
729 	    ((db->db_blkid  == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
730 		return;
731 
732 	/*
733 	 * If the last dirty record for this dbuf has not yet synced
734 	 * and its referencing the dbuf data, either:
735 	 *	reset the reference to point to a new copy,
736 	 * or (if there a no active holders)
737 	 *	just null out the current db_data pointer.
738 	 */
739 	ASSERT(dr->dr_txg >= txg - 2);
740 	if (db->db_blkid == DMU_BONUS_BLKID) {
741 		/* Note that the data bufs here are zio_bufs */
742 		dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
743 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
744 		bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
745 	} else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
746 		int size = db->db.db_size;
747 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
748 		spa_t *spa;
749 
750 		DB_GET_SPA(&spa, db);
751 		dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
752 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
753 	} else {
754 		dbuf_set_data(db, NULL);
755 	}
756 }
757 
758 void
759 dbuf_unoverride(dbuf_dirty_record_t *dr)
760 {
761 	dmu_buf_impl_t *db = dr->dr_dbuf;
762 	blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
763 	uint64_t txg = dr->dr_txg;
764 
765 	ASSERT(MUTEX_HELD(&db->db_mtx));
766 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
767 	ASSERT(db->db_level == 0);
768 
769 	if (db->db_blkid == DMU_BONUS_BLKID ||
770 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
771 		return;
772 
773 	ASSERT(db->db_data_pending != dr);
774 
775 	/* free this block */
776 	if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) {
777 		spa_t *spa;
778 
779 		DB_GET_SPA(&spa, db);
780 		zio_free(spa, txg, bp);
781 	}
782 	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
783 	dr->dt.dl.dr_nopwrite = B_FALSE;
784 
785 	/*
786 	 * Release the already-written buffer, so we leave it in
787 	 * a consistent dirty state.  Note that all callers are
788 	 * modifying the buffer, so they will immediately do
789 	 * another (redundant) arc_release().  Therefore, leave
790 	 * the buf thawed to save the effort of freezing &
791 	 * immediately re-thawing it.
792 	 */
793 	arc_release(dr->dt.dl.dr_data, db);
794 }
795 
796 /*
797  * Evict (if its unreferenced) or clear (if its referenced) any level-0
798  * data blocks in the free range, so that any future readers will find
799  * empty blocks.  Also, if we happen accross any level-1 dbufs in the
800  * range that have not already been marked dirty, mark them dirty so
801  * they stay in memory.
802  */
803 void
804 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
805 {
806 	dmu_buf_impl_t *db, *db_next;
807 	uint64_t txg = tx->tx_txg;
808 	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
809 	uint64_t first_l1 = start >> epbs;
810 	uint64_t last_l1 = end >> epbs;
811 
812 	if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) {
813 		end = dn->dn_maxblkid;
814 		last_l1 = end >> epbs;
815 	}
816 	dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
817 	mutex_enter(&dn->dn_dbufs_mtx);
818 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
819 		db_next = list_next(&dn->dn_dbufs, db);
820 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
821 
822 		if (db->db_level == 1 &&
823 		    db->db_blkid >= first_l1 && db->db_blkid <= last_l1) {
824 			mutex_enter(&db->db_mtx);
825 			if (db->db_last_dirty &&
826 			    db->db_last_dirty->dr_txg < txg) {
827 				dbuf_add_ref(db, FTAG);
828 				mutex_exit(&db->db_mtx);
829 				dbuf_will_dirty(db, tx);
830 				dbuf_rele(db, FTAG);
831 			} else {
832 				mutex_exit(&db->db_mtx);
833 			}
834 		}
835 
836 		if (db->db_level != 0)
837 			continue;
838 		dprintf_dbuf(db, "found buf %s\n", "");
839 		if (db->db_blkid < start || db->db_blkid > end)
840 			continue;
841 
842 		/* found a level 0 buffer in the range */
843 		mutex_enter(&db->db_mtx);
844 		if (dbuf_undirty(db, tx)) {
845 			/* mutex has been dropped and dbuf destroyed */
846 			continue;
847 		}
848 
849 		if (db->db_state == DB_UNCACHED ||
850 		    db->db_state == DB_NOFILL ||
851 		    db->db_state == DB_EVICTING) {
852 			ASSERT(db->db.db_data == NULL);
853 			mutex_exit(&db->db_mtx);
854 			continue;
855 		}
856 		if (db->db_state == DB_READ || db->db_state == DB_FILL) {
857 			/* will be handled in dbuf_read_done or dbuf_rele */
858 			db->db_freed_in_flight = TRUE;
859 			mutex_exit(&db->db_mtx);
860 			continue;
861 		}
862 		if (refcount_count(&db->db_holds) == 0) {
863 			ASSERT(db->db_buf);
864 			dbuf_clear(db);
865 			continue;
866 		}
867 		/* The dbuf is referenced */
868 
869 		if (db->db_last_dirty != NULL) {
870 			dbuf_dirty_record_t *dr = db->db_last_dirty;
871 
872 			if (dr->dr_txg == txg) {
873 				/*
874 				 * This buffer is "in-use", re-adjust the file
875 				 * size to reflect that this buffer may
876 				 * contain new data when we sync.
877 				 */
878 				if (db->db_blkid != DMU_SPILL_BLKID &&
879 				    db->db_blkid > dn->dn_maxblkid)
880 					dn->dn_maxblkid = db->db_blkid;
881 				dbuf_unoverride(dr);
882 			} else {
883 				/*
884 				 * This dbuf is not dirty in the open context.
885 				 * Either uncache it (if its not referenced in
886 				 * the open context) or reset its contents to
887 				 * empty.
888 				 */
889 				dbuf_fix_old_data(db, txg);
890 			}
891 		}
892 		/* clear the contents if its cached */
893 		if (db->db_state == DB_CACHED) {
894 			ASSERT(db->db.db_data != NULL);
895 			arc_release(db->db_buf, db);
896 			bzero(db->db.db_data, db->db.db_size);
897 			arc_buf_freeze(db->db_buf);
898 		}
899 
900 		mutex_exit(&db->db_mtx);
901 	}
902 	mutex_exit(&dn->dn_dbufs_mtx);
903 }
904 
905 static int
906 dbuf_block_freeable(dmu_buf_impl_t *db)
907 {
908 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
909 	uint64_t birth_txg = 0;
910 
911 	/*
912 	 * We don't need any locking to protect db_blkptr:
913 	 * If it's syncing, then db_last_dirty will be set
914 	 * so we'll ignore db_blkptr.
915 	 */
916 	ASSERT(MUTEX_HELD(&db->db_mtx));
917 	if (db->db_last_dirty)
918 		birth_txg = db->db_last_dirty->dr_txg;
919 	else if (db->db_blkptr)
920 		birth_txg = db->db_blkptr->blk_birth;
921 
922 	/*
923 	 * If we don't exist or are in a snapshot, we can't be freed.
924 	 * Don't pass the bp to dsl_dataset_block_freeable() since we
925 	 * are holding the db_mtx lock and might deadlock if we are
926 	 * prefetching a dedup-ed block.
927 	 */
928 	if (birth_txg)
929 		return (ds == NULL ||
930 		    dsl_dataset_block_freeable(ds, NULL, birth_txg));
931 	else
932 		return (FALSE);
933 }
934 
935 void
936 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
937 {
938 	arc_buf_t *buf, *obuf;
939 	int osize = db->db.db_size;
940 	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
941 	dnode_t *dn;
942 
943 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
944 
945 	DB_DNODE_ENTER(db);
946 	dn = DB_DNODE(db);
947 
948 	/* XXX does *this* func really need the lock? */
949 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
950 
951 	/*
952 	 * This call to dbuf_will_dirty() with the dn_struct_rwlock held
953 	 * is OK, because there can be no other references to the db
954 	 * when we are changing its size, so no concurrent DB_FILL can
955 	 * be happening.
956 	 */
957 	/*
958 	 * XXX we should be doing a dbuf_read, checking the return
959 	 * value and returning that up to our callers
960 	 */
961 	dbuf_will_dirty(db, tx);
962 
963 	/* create the data buffer for the new block */
964 	buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
965 
966 	/* copy old block data to the new block */
967 	obuf = db->db_buf;
968 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
969 	/* zero the remainder */
970 	if (size > osize)
971 		bzero((uint8_t *)buf->b_data + osize, size - osize);
972 
973 	mutex_enter(&db->db_mtx);
974 	dbuf_set_data(db, buf);
975 	VERIFY(arc_buf_remove_ref(obuf, db));
976 	db->db.db_size = size;
977 
978 	if (db->db_level == 0) {
979 		ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
980 		db->db_last_dirty->dt.dl.dr_data = buf;
981 	}
982 	mutex_exit(&db->db_mtx);
983 
984 	dnode_willuse_space(dn, size-osize, tx);
985 	DB_DNODE_EXIT(db);
986 }
987 
988 void
989 dbuf_release_bp(dmu_buf_impl_t *db)
990 {
991 	objset_t *os;
992 
993 	DB_GET_OBJSET(&os, db);
994 	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
995 	ASSERT(arc_released(os->os_phys_buf) ||
996 	    list_link_active(&os->os_dsl_dataset->ds_synced_link));
997 	ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
998 
999 	(void) arc_release(db->db_buf, db);
1000 }
1001 
1002 dbuf_dirty_record_t *
1003 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1004 {
1005 	dnode_t *dn;
1006 	objset_t *os;
1007 	dbuf_dirty_record_t **drp, *dr;
1008 	int drop_struct_lock = FALSE;
1009 	boolean_t do_free_accounting = B_FALSE;
1010 	int txgoff = tx->tx_txg & TXG_MASK;
1011 
1012 	ASSERT(tx->tx_txg != 0);
1013 	ASSERT(!refcount_is_zero(&db->db_holds));
1014 	DMU_TX_DIRTY_BUF(tx, db);
1015 
1016 	DB_DNODE_ENTER(db);
1017 	dn = DB_DNODE(db);
1018 	/*
1019 	 * Shouldn't dirty a regular buffer in syncing context.  Private
1020 	 * objects may be dirtied in syncing context, but only if they
1021 	 * were already pre-dirtied in open context.
1022 	 */
1023 	ASSERT(!dmu_tx_is_syncing(tx) ||
1024 	    BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
1025 	    DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1026 	    dn->dn_objset->os_dsl_dataset == NULL);
1027 	/*
1028 	 * We make this assert for private objects as well, but after we
1029 	 * check if we're already dirty.  They are allowed to re-dirty
1030 	 * in syncing context.
1031 	 */
1032 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1033 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1034 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1035 
1036 	mutex_enter(&db->db_mtx);
1037 	/*
1038 	 * XXX make this true for indirects too?  The problem is that
1039 	 * transactions created with dmu_tx_create_assigned() from
1040 	 * syncing context don't bother holding ahead.
1041 	 */
1042 	ASSERT(db->db_level != 0 ||
1043 	    db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1044 	    db->db_state == DB_NOFILL);
1045 
1046 	mutex_enter(&dn->dn_mtx);
1047 	/*
1048 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
1049 	 * initialize the objset.
1050 	 */
1051 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
1052 	    !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
1053 		dn->dn_dirtyctx =
1054 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
1055 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
1056 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
1057 	}
1058 	mutex_exit(&dn->dn_mtx);
1059 
1060 	if (db->db_blkid == DMU_SPILL_BLKID)
1061 		dn->dn_have_spill = B_TRUE;
1062 
1063 	/*
1064 	 * If this buffer is already dirty, we're done.
1065 	 */
1066 	drp = &db->db_last_dirty;
1067 	ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
1068 	    db->db.db_object == DMU_META_DNODE_OBJECT);
1069 	while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
1070 		drp = &dr->dr_next;
1071 	if (dr && dr->dr_txg == tx->tx_txg) {
1072 		DB_DNODE_EXIT(db);
1073 
1074 		if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1075 			/*
1076 			 * If this buffer has already been written out,
1077 			 * we now need to reset its state.
1078 			 */
1079 			dbuf_unoverride(dr);
1080 			if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1081 			    db->db_state != DB_NOFILL)
1082 				arc_buf_thaw(db->db_buf);
1083 		}
1084 		mutex_exit(&db->db_mtx);
1085 		return (dr);
1086 	}
1087 
1088 	/*
1089 	 * Only valid if not already dirty.
1090 	 */
1091 	ASSERT(dn->dn_object == 0 ||
1092 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1093 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1094 
1095 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
1096 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1097 	    dn->dn_phys->dn_nlevels > db->db_level ||
1098 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
1099 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1100 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1101 
1102 	/*
1103 	 * We should only be dirtying in syncing context if it's the
1104 	 * mos or we're initializing the os or it's a special object.
1105 	 * However, we are allowed to dirty in syncing context provided
1106 	 * we already dirtied it in open context.  Hence we must make
1107 	 * this assertion only if we're not already dirty.
1108 	 */
1109 	os = dn->dn_objset;
1110 	ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1111 	    os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp));
1112 	ASSERT(db->db.db_size != 0);
1113 
1114 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1115 
1116 	if (db->db_blkid != DMU_BONUS_BLKID) {
1117 		/*
1118 		 * Update the accounting.
1119 		 * Note: we delay "free accounting" until after we drop
1120 		 * the db_mtx.  This keeps us from grabbing other locks
1121 		 * (and possibly deadlocking) in bp_get_dsize() while
1122 		 * also holding the db_mtx.
1123 		 */
1124 		dnode_willuse_space(dn, db->db.db_size, tx);
1125 		do_free_accounting = dbuf_block_freeable(db);
1126 	}
1127 
1128 	/*
1129 	 * If this buffer is dirty in an old transaction group we need
1130 	 * to make a copy of it so that the changes we make in this
1131 	 * transaction group won't leak out when we sync the older txg.
1132 	 */
1133 	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1134 	if (db->db_level == 0) {
1135 		void *data_old = db->db_buf;
1136 
1137 		if (db->db_state != DB_NOFILL) {
1138 			if (db->db_blkid == DMU_BONUS_BLKID) {
1139 				dbuf_fix_old_data(db, tx->tx_txg);
1140 				data_old = db->db.db_data;
1141 			} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1142 				/*
1143 				 * Release the data buffer from the cache so
1144 				 * that we can modify it without impacting
1145 				 * possible other users of this cached data
1146 				 * block.  Note that indirect blocks and
1147 				 * private objects are not released until the
1148 				 * syncing state (since they are only modified
1149 				 * then).
1150 				 */
1151 				arc_release(db->db_buf, db);
1152 				dbuf_fix_old_data(db, tx->tx_txg);
1153 				data_old = db->db_buf;
1154 			}
1155 			ASSERT(data_old != NULL);
1156 		}
1157 		dr->dt.dl.dr_data = data_old;
1158 	} else {
1159 		mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1160 		list_create(&dr->dt.di.dr_children,
1161 		    sizeof (dbuf_dirty_record_t),
1162 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
1163 	}
1164 	dr->dr_dbuf = db;
1165 	dr->dr_txg = tx->tx_txg;
1166 	dr->dr_next = *drp;
1167 	*drp = dr;
1168 
1169 	/*
1170 	 * We could have been freed_in_flight between the dbuf_noread
1171 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1172 	 * happened after the free.
1173 	 */
1174 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1175 	    db->db_blkid != DMU_SPILL_BLKID) {
1176 		mutex_enter(&dn->dn_mtx);
1177 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1178 		mutex_exit(&dn->dn_mtx);
1179 		db->db_freed_in_flight = FALSE;
1180 	}
1181 
1182 	/*
1183 	 * This buffer is now part of this txg
1184 	 */
1185 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1186 	db->db_dirtycnt += 1;
1187 	ASSERT3U(db->db_dirtycnt, <=, 3);
1188 
1189 	mutex_exit(&db->db_mtx);
1190 
1191 	if (db->db_blkid == DMU_BONUS_BLKID ||
1192 	    db->db_blkid == DMU_SPILL_BLKID) {
1193 		mutex_enter(&dn->dn_mtx);
1194 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1195 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1196 		mutex_exit(&dn->dn_mtx);
1197 		dnode_setdirty(dn, tx);
1198 		DB_DNODE_EXIT(db);
1199 		return (dr);
1200 	} else if (do_free_accounting) {
1201 		blkptr_t *bp = db->db_blkptr;
1202 		int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1203 		    bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1204 		/*
1205 		 * This is only a guess -- if the dbuf is dirty
1206 		 * in a previous txg, we don't know how much
1207 		 * space it will use on disk yet.  We should
1208 		 * really have the struct_rwlock to access
1209 		 * db_blkptr, but since this is just a guess,
1210 		 * it's OK if we get an odd answer.
1211 		 */
1212 		ddt_prefetch(os->os_spa, bp);
1213 		dnode_willuse_space(dn, -willfree, tx);
1214 	}
1215 
1216 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1217 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1218 		drop_struct_lock = TRUE;
1219 	}
1220 
1221 	if (db->db_level == 0) {
1222 		dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1223 		ASSERT(dn->dn_maxblkid >= db->db_blkid);
1224 	}
1225 
1226 	if (db->db_level+1 < dn->dn_nlevels) {
1227 		dmu_buf_impl_t *parent = db->db_parent;
1228 		dbuf_dirty_record_t *di;
1229 		int parent_held = FALSE;
1230 
1231 		if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1232 			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1233 
1234 			parent = dbuf_hold_level(dn, db->db_level+1,
1235 			    db->db_blkid >> epbs, FTAG);
1236 			ASSERT(parent != NULL);
1237 			parent_held = TRUE;
1238 		}
1239 		if (drop_struct_lock)
1240 			rw_exit(&dn->dn_struct_rwlock);
1241 		ASSERT3U(db->db_level+1, ==, parent->db_level);
1242 		di = dbuf_dirty(parent, tx);
1243 		if (parent_held)
1244 			dbuf_rele(parent, FTAG);
1245 
1246 		mutex_enter(&db->db_mtx);
1247 		/*  possible race with dbuf_undirty() */
1248 		if (db->db_last_dirty == dr ||
1249 		    dn->dn_object == DMU_META_DNODE_OBJECT) {
1250 			mutex_enter(&di->dt.di.dr_mtx);
1251 			ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1252 			ASSERT(!list_link_active(&dr->dr_dirty_node));
1253 			list_insert_tail(&di->dt.di.dr_children, dr);
1254 			mutex_exit(&di->dt.di.dr_mtx);
1255 			dr->dr_parent = di;
1256 		}
1257 		mutex_exit(&db->db_mtx);
1258 	} else {
1259 		ASSERT(db->db_level+1 == dn->dn_nlevels);
1260 		ASSERT(db->db_blkid < dn->dn_nblkptr);
1261 		ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1262 		mutex_enter(&dn->dn_mtx);
1263 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1264 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1265 		mutex_exit(&dn->dn_mtx);
1266 		if (drop_struct_lock)
1267 			rw_exit(&dn->dn_struct_rwlock);
1268 	}
1269 
1270 	dnode_setdirty(dn, tx);
1271 	DB_DNODE_EXIT(db);
1272 	return (dr);
1273 }
1274 
1275 /*
1276  * Undirty a buffer in the transaction group referenced by the given
1277  * transaction.  Return whether this evicted the dbuf.
1278  */
1279 static boolean_t
1280 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1281 {
1282 	dnode_t *dn;
1283 	uint64_t txg = tx->tx_txg;
1284 	dbuf_dirty_record_t *dr, **drp;
1285 
1286 	ASSERT(txg != 0);
1287 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1288 	ASSERT0(db->db_level);
1289 	ASSERT(MUTEX_HELD(&db->db_mtx));
1290 
1291 	/*
1292 	 * If this buffer is not dirty, we're done.
1293 	 */
1294 	for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1295 		if (dr->dr_txg <= txg)
1296 			break;
1297 	if (dr == NULL || dr->dr_txg < txg)
1298 		return (B_FALSE);
1299 	ASSERT(dr->dr_txg == txg);
1300 	ASSERT(dr->dr_dbuf == db);
1301 
1302 	DB_DNODE_ENTER(db);
1303 	dn = DB_DNODE(db);
1304 
1305 	/*
1306 	 * Note:  This code will probably work even if there are concurrent
1307 	 * holders, but it is untested in that scenerio, as the ZPL and
1308 	 * ztest have additional locking (the range locks) that prevents
1309 	 * that type of concurrent access.
1310 	 */
1311 	ASSERT3U(refcount_count(&db->db_holds), ==, db->db_dirtycnt);
1312 
1313 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1314 
1315 	ASSERT(db->db.db_size != 0);
1316 
1317 	/* XXX would be nice to fix up dn_towrite_space[] */
1318 
1319 	*drp = dr->dr_next;
1320 
1321 	/*
1322 	 * Note that there are three places in dbuf_dirty()
1323 	 * where this dirty record may be put on a list.
1324 	 * Make sure to do a list_remove corresponding to
1325 	 * every one of those list_insert calls.
1326 	 */
1327 	if (dr->dr_parent) {
1328 		mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1329 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1330 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1331 	} else if (db->db_blkid == DMU_SPILL_BLKID ||
1332 	    db->db_level+1 == dn->dn_nlevels) {
1333 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1334 		mutex_enter(&dn->dn_mtx);
1335 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1336 		mutex_exit(&dn->dn_mtx);
1337 	}
1338 	DB_DNODE_EXIT(db);
1339 
1340 	if (db->db_state != DB_NOFILL) {
1341 		dbuf_unoverride(dr);
1342 
1343 		ASSERT(db->db_buf != NULL);
1344 		ASSERT(dr->dt.dl.dr_data != NULL);
1345 		if (dr->dt.dl.dr_data != db->db_buf)
1346 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db));
1347 	}
1348 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
1349 
1350 	ASSERT(db->db_dirtycnt > 0);
1351 	db->db_dirtycnt -= 1;
1352 
1353 	if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1354 		arc_buf_t *buf = db->db_buf;
1355 
1356 		ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1357 		dbuf_set_data(db, NULL);
1358 		VERIFY(arc_buf_remove_ref(buf, db));
1359 		dbuf_evict(db);
1360 		return (B_TRUE);
1361 	}
1362 
1363 	return (B_FALSE);
1364 }
1365 
1366 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1367 void
1368 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1369 {
1370 	int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1371 
1372 	ASSERT(tx->tx_txg != 0);
1373 	ASSERT(!refcount_is_zero(&db->db_holds));
1374 
1375 	DB_DNODE_ENTER(db);
1376 	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1377 		rf |= DB_RF_HAVESTRUCT;
1378 	DB_DNODE_EXIT(db);
1379 	(void) dbuf_read(db, NULL, rf);
1380 	(void) dbuf_dirty(db, tx);
1381 }
1382 
1383 void
1384 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1385 {
1386 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1387 
1388 	db->db_state = DB_NOFILL;
1389 
1390 	dmu_buf_will_fill(db_fake, tx);
1391 }
1392 
1393 void
1394 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1395 {
1396 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1397 
1398 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1399 	ASSERT(tx->tx_txg != 0);
1400 	ASSERT(db->db_level == 0);
1401 	ASSERT(!refcount_is_zero(&db->db_holds));
1402 
1403 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1404 	    dmu_tx_private_ok(tx));
1405 
1406 	dbuf_noread(db);
1407 	(void) dbuf_dirty(db, tx);
1408 }
1409 
1410 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1411 /* ARGSUSED */
1412 void
1413 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1414 {
1415 	mutex_enter(&db->db_mtx);
1416 	DBUF_VERIFY(db);
1417 
1418 	if (db->db_state == DB_FILL) {
1419 		if (db->db_level == 0 && db->db_freed_in_flight) {
1420 			ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1421 			/* we were freed while filling */
1422 			/* XXX dbuf_undirty? */
1423 			bzero(db->db.db_data, db->db.db_size);
1424 			db->db_freed_in_flight = FALSE;
1425 		}
1426 		db->db_state = DB_CACHED;
1427 		cv_broadcast(&db->db_changed);
1428 	}
1429 	mutex_exit(&db->db_mtx);
1430 }
1431 
1432 /*
1433  * Directly assign a provided arc buf to a given dbuf if it's not referenced
1434  * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
1435  */
1436 void
1437 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1438 {
1439 	ASSERT(!refcount_is_zero(&db->db_holds));
1440 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1441 	ASSERT(db->db_level == 0);
1442 	ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1443 	ASSERT(buf != NULL);
1444 	ASSERT(arc_buf_size(buf) == db->db.db_size);
1445 	ASSERT(tx->tx_txg != 0);
1446 
1447 	arc_return_buf(buf, db);
1448 	ASSERT(arc_released(buf));
1449 
1450 	mutex_enter(&db->db_mtx);
1451 
1452 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
1453 		cv_wait(&db->db_changed, &db->db_mtx);
1454 
1455 	ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1456 
1457 	if (db->db_state == DB_CACHED &&
1458 	    refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1459 		mutex_exit(&db->db_mtx);
1460 		(void) dbuf_dirty(db, tx);
1461 		bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1462 		VERIFY(arc_buf_remove_ref(buf, db));
1463 		xuio_stat_wbuf_copied();
1464 		return;
1465 	}
1466 
1467 	xuio_stat_wbuf_nocopy();
1468 	if (db->db_state == DB_CACHED) {
1469 		dbuf_dirty_record_t *dr = db->db_last_dirty;
1470 
1471 		ASSERT(db->db_buf != NULL);
1472 		if (dr != NULL && dr->dr_txg == tx->tx_txg) {
1473 			ASSERT(dr->dt.dl.dr_data == db->db_buf);
1474 			if (!arc_released(db->db_buf)) {
1475 				ASSERT(dr->dt.dl.dr_override_state ==
1476 				    DR_OVERRIDDEN);
1477 				arc_release(db->db_buf, db);
1478 			}
1479 			dr->dt.dl.dr_data = buf;
1480 			VERIFY(arc_buf_remove_ref(db->db_buf, db));
1481 		} else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1482 			arc_release(db->db_buf, db);
1483 			VERIFY(arc_buf_remove_ref(db->db_buf, db));
1484 		}
1485 		db->db_buf = NULL;
1486 	}
1487 	ASSERT(db->db_buf == NULL);
1488 	dbuf_set_data(db, buf);
1489 	db->db_state = DB_FILL;
1490 	mutex_exit(&db->db_mtx);
1491 	(void) dbuf_dirty(db, tx);
1492 	dbuf_fill_done(db, tx);
1493 }
1494 
1495 /*
1496  * "Clear" the contents of this dbuf.  This will mark the dbuf
1497  * EVICTING and clear *most* of its references.  Unfortunetely,
1498  * when we are not holding the dn_dbufs_mtx, we can't clear the
1499  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1500  * in this case.  For callers from the DMU we will usually see:
1501  *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1502  * For the arc callback, we will usually see:
1503  *	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1504  * Sometimes, though, we will get a mix of these two:
1505  *	DMU: dbuf_clear()->arc_buf_evict()
1506  *	ARC: dbuf_do_evict()->dbuf_destroy()
1507  */
1508 void
1509 dbuf_clear(dmu_buf_impl_t *db)
1510 {
1511 	dnode_t *dn;
1512 	dmu_buf_impl_t *parent = db->db_parent;
1513 	dmu_buf_impl_t *dndb;
1514 	int dbuf_gone = FALSE;
1515 
1516 	ASSERT(MUTEX_HELD(&db->db_mtx));
1517 	ASSERT(refcount_is_zero(&db->db_holds));
1518 
1519 	dbuf_evict_user(db);
1520 
1521 	if (db->db_state == DB_CACHED) {
1522 		ASSERT(db->db.db_data != NULL);
1523 		if (db->db_blkid == DMU_BONUS_BLKID) {
1524 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1525 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
1526 		}
1527 		db->db.db_data = NULL;
1528 		db->db_state = DB_UNCACHED;
1529 	}
1530 
1531 	ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1532 	ASSERT(db->db_data_pending == NULL);
1533 
1534 	db->db_state = DB_EVICTING;
1535 	db->db_blkptr = NULL;
1536 
1537 	DB_DNODE_ENTER(db);
1538 	dn = DB_DNODE(db);
1539 	dndb = dn->dn_dbuf;
1540 	if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1541 		list_remove(&dn->dn_dbufs, db);
1542 		(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1543 		membar_producer();
1544 		DB_DNODE_EXIT(db);
1545 		/*
1546 		 * Decrementing the dbuf count means that the hold corresponding
1547 		 * to the removed dbuf is no longer discounted in dnode_move(),
1548 		 * so the dnode cannot be moved until after we release the hold.
1549 		 * The membar_producer() ensures visibility of the decremented
1550 		 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually
1551 		 * release any lock.
1552 		 */
1553 		dnode_rele(dn, db);
1554 		db->db_dnode_handle = NULL;
1555 	} else {
1556 		DB_DNODE_EXIT(db);
1557 	}
1558 
1559 	if (db->db_buf)
1560 		dbuf_gone = arc_buf_evict(db->db_buf);
1561 
1562 	if (!dbuf_gone)
1563 		mutex_exit(&db->db_mtx);
1564 
1565 	/*
1566 	 * If this dbuf is referenced from an indirect dbuf,
1567 	 * decrement the ref count on the indirect dbuf.
1568 	 */
1569 	if (parent && parent != dndb)
1570 		dbuf_rele(parent, db);
1571 }
1572 
1573 static int
1574 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1575     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1576 {
1577 	int nlevels, epbs;
1578 
1579 	*parentp = NULL;
1580 	*bpp = NULL;
1581 
1582 	ASSERT(blkid != DMU_BONUS_BLKID);
1583 
1584 	if (blkid == DMU_SPILL_BLKID) {
1585 		mutex_enter(&dn->dn_mtx);
1586 		if (dn->dn_have_spill &&
1587 		    (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
1588 			*bpp = &dn->dn_phys->dn_spill;
1589 		else
1590 			*bpp = NULL;
1591 		dbuf_add_ref(dn->dn_dbuf, NULL);
1592 		*parentp = dn->dn_dbuf;
1593 		mutex_exit(&dn->dn_mtx);
1594 		return (0);
1595 	}
1596 
1597 	if (dn->dn_phys->dn_nlevels == 0)
1598 		nlevels = 1;
1599 	else
1600 		nlevels = dn->dn_phys->dn_nlevels;
1601 
1602 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1603 
1604 	ASSERT3U(level * epbs, <, 64);
1605 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1606 	if (level >= nlevels ||
1607 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1608 		/* the buffer has no parent yet */
1609 		return (SET_ERROR(ENOENT));
1610 	} else if (level < nlevels-1) {
1611 		/* this block is referenced from an indirect block */
1612 		int err = dbuf_hold_impl(dn, level+1,
1613 		    blkid >> epbs, fail_sparse, NULL, parentp);
1614 		if (err)
1615 			return (err);
1616 		err = dbuf_read(*parentp, NULL,
1617 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1618 		if (err) {
1619 			dbuf_rele(*parentp, NULL);
1620 			*parentp = NULL;
1621 			return (err);
1622 		}
1623 		*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1624 		    (blkid & ((1ULL << epbs) - 1));
1625 		return (0);
1626 	} else {
1627 		/* the block is referenced from the dnode */
1628 		ASSERT3U(level, ==, nlevels-1);
1629 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1630 		    blkid < dn->dn_phys->dn_nblkptr);
1631 		if (dn->dn_dbuf) {
1632 			dbuf_add_ref(dn->dn_dbuf, NULL);
1633 			*parentp = dn->dn_dbuf;
1634 		}
1635 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1636 		return (0);
1637 	}
1638 }
1639 
1640 static dmu_buf_impl_t *
1641 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1642     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1643 {
1644 	objset_t *os = dn->dn_objset;
1645 	dmu_buf_impl_t *db, *odb;
1646 
1647 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1648 	ASSERT(dn->dn_type != DMU_OT_NONE);
1649 
1650 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1651 
1652 	db->db_objset = os;
1653 	db->db.db_object = dn->dn_object;
1654 	db->db_level = level;
1655 	db->db_blkid = blkid;
1656 	db->db_last_dirty = NULL;
1657 	db->db_dirtycnt = 0;
1658 	db->db_dnode_handle = dn->dn_handle;
1659 	db->db_parent = parent;
1660 	db->db_blkptr = blkptr;
1661 
1662 	db->db_user_ptr = NULL;
1663 	db->db_user_data_ptr_ptr = NULL;
1664 	db->db_evict_func = NULL;
1665 	db->db_immediate_evict = 0;
1666 	db->db_freed_in_flight = 0;
1667 
1668 	if (blkid == DMU_BONUS_BLKID) {
1669 		ASSERT3P(parent, ==, dn->dn_dbuf);
1670 		db->db.db_size = DN_MAX_BONUSLEN -
1671 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
1672 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1673 		db->db.db_offset = DMU_BONUS_BLKID;
1674 		db->db_state = DB_UNCACHED;
1675 		/* the bonus dbuf is not placed in the hash table */
1676 		arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1677 		return (db);
1678 	} else if (blkid == DMU_SPILL_BLKID) {
1679 		db->db.db_size = (blkptr != NULL) ?
1680 		    BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
1681 		db->db.db_offset = 0;
1682 	} else {
1683 		int blocksize =
1684 		    db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1685 		db->db.db_size = blocksize;
1686 		db->db.db_offset = db->db_blkid * blocksize;
1687 	}
1688 
1689 	/*
1690 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1691 	 * in the hash table *and* added to the dbufs list.
1692 	 * This prevents a possible deadlock with someone
1693 	 * trying to look up this dbuf before its added to the
1694 	 * dn_dbufs list.
1695 	 */
1696 	mutex_enter(&dn->dn_dbufs_mtx);
1697 	db->db_state = DB_EVICTING;
1698 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1699 		/* someone else inserted it first */
1700 		kmem_cache_free(dbuf_cache, db);
1701 		mutex_exit(&dn->dn_dbufs_mtx);
1702 		return (odb);
1703 	}
1704 	list_insert_head(&dn->dn_dbufs, db);
1705 	db->db_state = DB_UNCACHED;
1706 	mutex_exit(&dn->dn_dbufs_mtx);
1707 	arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1708 
1709 	if (parent && parent != dn->dn_dbuf)
1710 		dbuf_add_ref(parent, db);
1711 
1712 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1713 	    refcount_count(&dn->dn_holds) > 0);
1714 	(void) refcount_add(&dn->dn_holds, db);
1715 	(void) atomic_inc_32_nv(&dn->dn_dbufs_count);
1716 
1717 	dprintf_dbuf(db, "db=%p\n", db);
1718 
1719 	return (db);
1720 }
1721 
1722 static int
1723 dbuf_do_evict(void *private)
1724 {
1725 	arc_buf_t *buf = private;
1726 	dmu_buf_impl_t *db = buf->b_private;
1727 
1728 	if (!MUTEX_HELD(&db->db_mtx))
1729 		mutex_enter(&db->db_mtx);
1730 
1731 	ASSERT(refcount_is_zero(&db->db_holds));
1732 
1733 	if (db->db_state != DB_EVICTING) {
1734 		ASSERT(db->db_state == DB_CACHED);
1735 		DBUF_VERIFY(db);
1736 		db->db_buf = NULL;
1737 		dbuf_evict(db);
1738 	} else {
1739 		mutex_exit(&db->db_mtx);
1740 		dbuf_destroy(db);
1741 	}
1742 	return (0);
1743 }
1744 
1745 static void
1746 dbuf_destroy(dmu_buf_impl_t *db)
1747 {
1748 	ASSERT(refcount_is_zero(&db->db_holds));
1749 
1750 	if (db->db_blkid != DMU_BONUS_BLKID) {
1751 		/*
1752 		 * If this dbuf is still on the dn_dbufs list,
1753 		 * remove it from that list.
1754 		 */
1755 		if (db->db_dnode_handle != NULL) {
1756 			dnode_t *dn;
1757 
1758 			DB_DNODE_ENTER(db);
1759 			dn = DB_DNODE(db);
1760 			mutex_enter(&dn->dn_dbufs_mtx);
1761 			list_remove(&dn->dn_dbufs, db);
1762 			(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1763 			mutex_exit(&dn->dn_dbufs_mtx);
1764 			DB_DNODE_EXIT(db);
1765 			/*
1766 			 * Decrementing the dbuf count means that the hold
1767 			 * corresponding to the removed dbuf is no longer
1768 			 * discounted in dnode_move(), so the dnode cannot be
1769 			 * moved until after we release the hold.
1770 			 */
1771 			dnode_rele(dn, db);
1772 			db->db_dnode_handle = NULL;
1773 		}
1774 		dbuf_hash_remove(db);
1775 	}
1776 	db->db_parent = NULL;
1777 	db->db_buf = NULL;
1778 
1779 	ASSERT(!list_link_active(&db->db_link));
1780 	ASSERT(db->db.db_data == NULL);
1781 	ASSERT(db->db_hash_next == NULL);
1782 	ASSERT(db->db_blkptr == NULL);
1783 	ASSERT(db->db_data_pending == NULL);
1784 
1785 	kmem_cache_free(dbuf_cache, db);
1786 	arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1787 }
1788 
1789 void
1790 dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1791 {
1792 	dmu_buf_impl_t *db = NULL;
1793 	blkptr_t *bp = NULL;
1794 
1795 	ASSERT(blkid != DMU_BONUS_BLKID);
1796 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1797 
1798 	if (dnode_block_freed(dn, blkid))
1799 		return;
1800 
1801 	/* dbuf_find() returns with db_mtx held */
1802 	if (db = dbuf_find(dn, 0, blkid)) {
1803 		/*
1804 		 * This dbuf is already in the cache.  We assume that
1805 		 * it is already CACHED, or else about to be either
1806 		 * read or filled.
1807 		 */
1808 		mutex_exit(&db->db_mtx);
1809 		return;
1810 	}
1811 
1812 	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
1813 		if (bp && !BP_IS_HOLE(bp)) {
1814 			int priority = dn->dn_type == DMU_OT_DDT_ZAP ?
1815 			    ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ;
1816 			dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
1817 			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
1818 			zbookmark_t zb;
1819 
1820 			SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
1821 			    dn->dn_object, 0, blkid);
1822 
1823 			(void) arc_read(NULL, dn->dn_objset->os_spa,
1824 			    bp, NULL, NULL, priority,
1825 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
1826 			    &aflags, &zb);
1827 		}
1828 		if (db)
1829 			dbuf_rele(db, NULL);
1830 	}
1831 }
1832 
1833 /*
1834  * Returns with db_holds incremented, and db_mtx not held.
1835  * Note: dn_struct_rwlock must be held.
1836  */
1837 int
1838 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1839     void *tag, dmu_buf_impl_t **dbp)
1840 {
1841 	dmu_buf_impl_t *db, *parent = NULL;
1842 
1843 	ASSERT(blkid != DMU_BONUS_BLKID);
1844 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1845 	ASSERT3U(dn->dn_nlevels, >, level);
1846 
1847 	*dbp = NULL;
1848 top:
1849 	/* dbuf_find() returns with db_mtx held */
1850 	db = dbuf_find(dn, level, blkid);
1851 
1852 	if (db == NULL) {
1853 		blkptr_t *bp = NULL;
1854 		int err;
1855 
1856 		ASSERT3P(parent, ==, NULL);
1857 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1858 		if (fail_sparse) {
1859 			if (err == 0 && bp && BP_IS_HOLE(bp))
1860 				err = SET_ERROR(ENOENT);
1861 			if (err) {
1862 				if (parent)
1863 					dbuf_rele(parent, NULL);
1864 				return (err);
1865 			}
1866 		}
1867 		if (err && err != ENOENT)
1868 			return (err);
1869 		db = dbuf_create(dn, level, blkid, parent, bp);
1870 	}
1871 
1872 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1873 		arc_buf_add_ref(db->db_buf, db);
1874 		if (db->db_buf->b_data == NULL) {
1875 			dbuf_clear(db);
1876 			if (parent) {
1877 				dbuf_rele(parent, NULL);
1878 				parent = NULL;
1879 			}
1880 			goto top;
1881 		}
1882 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1883 	}
1884 
1885 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1886 
1887 	/*
1888 	 * If this buffer is currently syncing out, and we are are
1889 	 * still referencing it from db_data, we need to make a copy
1890 	 * of it in case we decide we want to dirty it again in this txg.
1891 	 */
1892 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1893 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
1894 	    db->db_state == DB_CACHED && db->db_data_pending) {
1895 		dbuf_dirty_record_t *dr = db->db_data_pending;
1896 
1897 		if (dr->dt.dl.dr_data == db->db_buf) {
1898 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1899 
1900 			dbuf_set_data(db,
1901 			    arc_buf_alloc(dn->dn_objset->os_spa,
1902 			    db->db.db_size, db, type));
1903 			bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
1904 			    db->db.db_size);
1905 		}
1906 	}
1907 
1908 	(void) refcount_add(&db->db_holds, tag);
1909 	dbuf_update_data(db);
1910 	DBUF_VERIFY(db);
1911 	mutex_exit(&db->db_mtx);
1912 
1913 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
1914 	if (parent)
1915 		dbuf_rele(parent, NULL);
1916 
1917 	ASSERT3P(DB_DNODE(db), ==, dn);
1918 	ASSERT3U(db->db_blkid, ==, blkid);
1919 	ASSERT3U(db->db_level, ==, level);
1920 	*dbp = db;
1921 
1922 	return (0);
1923 }
1924 
1925 dmu_buf_impl_t *
1926 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1927 {
1928 	dmu_buf_impl_t *db;
1929 	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1930 	return (err ? NULL : db);
1931 }
1932 
1933 dmu_buf_impl_t *
1934 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1935 {
1936 	dmu_buf_impl_t *db;
1937 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1938 	return (err ? NULL : db);
1939 }
1940 
1941 void
1942 dbuf_create_bonus(dnode_t *dn)
1943 {
1944 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1945 
1946 	ASSERT(dn->dn_bonus == NULL);
1947 	dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL);
1948 }
1949 
1950 int
1951 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
1952 {
1953 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1954 	dnode_t *dn;
1955 
1956 	if (db->db_blkid != DMU_SPILL_BLKID)
1957 		return (SET_ERROR(ENOTSUP));
1958 	if (blksz == 0)
1959 		blksz = SPA_MINBLOCKSIZE;
1960 	if (blksz > SPA_MAXBLOCKSIZE)
1961 		blksz = SPA_MAXBLOCKSIZE;
1962 	else
1963 		blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
1964 
1965 	DB_DNODE_ENTER(db);
1966 	dn = DB_DNODE(db);
1967 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1968 	dbuf_new_size(db, blksz, tx);
1969 	rw_exit(&dn->dn_struct_rwlock);
1970 	DB_DNODE_EXIT(db);
1971 
1972 	return (0);
1973 }
1974 
1975 void
1976 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx)
1977 {
1978 	dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
1979 }
1980 
1981 #pragma weak dmu_buf_add_ref = dbuf_add_ref
1982 void
1983 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
1984 {
1985 	int64_t holds = refcount_add(&db->db_holds, tag);
1986 	ASSERT(holds > 1);
1987 }
1988 
1989 /*
1990  * If you call dbuf_rele() you had better not be referencing the dnode handle
1991  * unless you have some other direct or indirect hold on the dnode. (An indirect
1992  * hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
1993  * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
1994  * dnode's parent dbuf evicting its dnode handles.
1995  */
1996 #pragma weak dmu_buf_rele = dbuf_rele
1997 void
1998 dbuf_rele(dmu_buf_impl_t *db, void *tag)
1999 {
2000 	mutex_enter(&db->db_mtx);
2001 	dbuf_rele_and_unlock(db, tag);
2002 }
2003 
2004 /*
2005  * dbuf_rele() for an already-locked dbuf.  This is necessary to allow
2006  * db_dirtycnt and db_holds to be updated atomically.
2007  */
2008 void
2009 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2010 {
2011 	int64_t holds;
2012 
2013 	ASSERT(MUTEX_HELD(&db->db_mtx));
2014 	DBUF_VERIFY(db);
2015 
2016 	/*
2017 	 * Remove the reference to the dbuf before removing its hold on the
2018 	 * dnode so we can guarantee in dnode_move() that a referenced bonus
2019 	 * buffer has a corresponding dnode hold.
2020 	 */
2021 	holds = refcount_remove(&db->db_holds, tag);
2022 	ASSERT(holds >= 0);
2023 
2024 	/*
2025 	 * We can't freeze indirects if there is a possibility that they
2026 	 * may be modified in the current syncing context.
2027 	 */
2028 	if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2029 		arc_buf_freeze(db->db_buf);
2030 
2031 	if (holds == db->db_dirtycnt &&
2032 	    db->db_level == 0 && db->db_immediate_evict)
2033 		dbuf_evict_user(db);
2034 
2035 	if (holds == 0) {
2036 		if (db->db_blkid == DMU_BONUS_BLKID) {
2037 			mutex_exit(&db->db_mtx);
2038 
2039 			/*
2040 			 * If the dnode moves here, we cannot cross this barrier
2041 			 * until the move completes.
2042 			 */
2043 			DB_DNODE_ENTER(db);
2044 			(void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count);
2045 			DB_DNODE_EXIT(db);
2046 			/*
2047 			 * The bonus buffer's dnode hold is no longer discounted
2048 			 * in dnode_move(). The dnode cannot move until after
2049 			 * the dnode_rele().
2050 			 */
2051 			dnode_rele(DB_DNODE(db), db);
2052 		} else if (db->db_buf == NULL) {
2053 			/*
2054 			 * This is a special case: we never associated this
2055 			 * dbuf with any data allocated from the ARC.
2056 			 */
2057 			ASSERT(db->db_state == DB_UNCACHED ||
2058 			    db->db_state == DB_NOFILL);
2059 			dbuf_evict(db);
2060 		} else if (arc_released(db->db_buf)) {
2061 			arc_buf_t *buf = db->db_buf;
2062 			/*
2063 			 * This dbuf has anonymous data associated with it.
2064 			 */
2065 			dbuf_set_data(db, NULL);
2066 			VERIFY(arc_buf_remove_ref(buf, db));
2067 			dbuf_evict(db);
2068 		} else {
2069 			VERIFY(!arc_buf_remove_ref(db->db_buf, db));
2070 
2071 			/*
2072 			 * A dbuf will be eligible for eviction if either the
2073 			 * 'primarycache' property is set or a duplicate
2074 			 * copy of this buffer is already cached in the arc.
2075 			 *
2076 			 * In the case of the 'primarycache' a buffer
2077 			 * is considered for eviction if it matches the
2078 			 * criteria set in the property.
2079 			 *
2080 			 * To decide if our buffer is considered a
2081 			 * duplicate, we must call into the arc to determine
2082 			 * if multiple buffers are referencing the same
2083 			 * block on-disk. If so, then we simply evict
2084 			 * ourselves.
2085 			 */
2086 			if (!DBUF_IS_CACHEABLE(db) ||
2087 			    arc_buf_eviction_needed(db->db_buf))
2088 				dbuf_clear(db);
2089 			else
2090 				mutex_exit(&db->db_mtx);
2091 		}
2092 	} else {
2093 		mutex_exit(&db->db_mtx);
2094 	}
2095 }
2096 
2097 #pragma weak dmu_buf_refcount = dbuf_refcount
2098 uint64_t
2099 dbuf_refcount(dmu_buf_impl_t *db)
2100 {
2101 	return (refcount_count(&db->db_holds));
2102 }
2103 
2104 void *
2105 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2106     dmu_buf_evict_func_t *evict_func)
2107 {
2108 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2109 	    user_data_ptr_ptr, evict_func));
2110 }
2111 
2112 void *
2113 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2114     dmu_buf_evict_func_t *evict_func)
2115 {
2116 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2117 
2118 	db->db_immediate_evict = TRUE;
2119 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2120 	    user_data_ptr_ptr, evict_func));
2121 }
2122 
2123 void *
2124 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
2125     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
2126 {
2127 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2128 	ASSERT(db->db_level == 0);
2129 
2130 	ASSERT((user_ptr == NULL) == (evict_func == NULL));
2131 
2132 	mutex_enter(&db->db_mtx);
2133 
2134 	if (db->db_user_ptr == old_user_ptr) {
2135 		db->db_user_ptr = user_ptr;
2136 		db->db_user_data_ptr_ptr = user_data_ptr_ptr;
2137 		db->db_evict_func = evict_func;
2138 
2139 		dbuf_update_data(db);
2140 	} else {
2141 		old_user_ptr = db->db_user_ptr;
2142 	}
2143 
2144 	mutex_exit(&db->db_mtx);
2145 	return (old_user_ptr);
2146 }
2147 
2148 void *
2149 dmu_buf_get_user(dmu_buf_t *db_fake)
2150 {
2151 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2152 	ASSERT(!refcount_is_zero(&db->db_holds));
2153 
2154 	return (db->db_user_ptr);
2155 }
2156 
2157 boolean_t
2158 dmu_buf_freeable(dmu_buf_t *dbuf)
2159 {
2160 	boolean_t res = B_FALSE;
2161 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2162 
2163 	if (db->db_blkptr)
2164 		res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2165 		    db->db_blkptr, db->db_blkptr->blk_birth);
2166 
2167 	return (res);
2168 }
2169 
2170 blkptr_t *
2171 dmu_buf_get_blkptr(dmu_buf_t *db)
2172 {
2173 	dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
2174 	return (dbi->db_blkptr);
2175 }
2176 
2177 static void
2178 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2179 {
2180 	/* ASSERT(dmu_tx_is_syncing(tx) */
2181 	ASSERT(MUTEX_HELD(&db->db_mtx));
2182 
2183 	if (db->db_blkptr != NULL)
2184 		return;
2185 
2186 	if (db->db_blkid == DMU_SPILL_BLKID) {
2187 		db->db_blkptr = &dn->dn_phys->dn_spill;
2188 		BP_ZERO(db->db_blkptr);
2189 		return;
2190 	}
2191 	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2192 		/*
2193 		 * This buffer was allocated at a time when there was
2194 		 * no available blkptrs from the dnode, or it was
2195 		 * inappropriate to hook it in (i.e., nlevels mis-match).
2196 		 */
2197 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2198 		ASSERT(db->db_parent == NULL);
2199 		db->db_parent = dn->dn_dbuf;
2200 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2201 		DBUF_VERIFY(db);
2202 	} else {
2203 		dmu_buf_impl_t *parent = db->db_parent;
2204 		int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2205 
2206 		ASSERT(dn->dn_phys->dn_nlevels > 1);
2207 		if (parent == NULL) {
2208 			mutex_exit(&db->db_mtx);
2209 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
2210 			(void) dbuf_hold_impl(dn, db->db_level+1,
2211 			    db->db_blkid >> epbs, FALSE, db, &parent);
2212 			rw_exit(&dn->dn_struct_rwlock);
2213 			mutex_enter(&db->db_mtx);
2214 			db->db_parent = parent;
2215 		}
2216 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
2217 		    (db->db_blkid & ((1ULL << epbs) - 1));
2218 		DBUF_VERIFY(db);
2219 	}
2220 }
2221 
2222 static void
2223 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2224 {
2225 	dmu_buf_impl_t *db = dr->dr_dbuf;
2226 	dnode_t *dn;
2227 	zio_t *zio;
2228 
2229 	ASSERT(dmu_tx_is_syncing(tx));
2230 
2231 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2232 
2233 	mutex_enter(&db->db_mtx);
2234 
2235 	ASSERT(db->db_level > 0);
2236 	DBUF_VERIFY(db);
2237 
2238 	/* Read the block if it hasn't been read yet. */
2239 	if (db->db_buf == NULL) {
2240 		mutex_exit(&db->db_mtx);
2241 		(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2242 		mutex_enter(&db->db_mtx);
2243 	}
2244 	ASSERT3U(db->db_state, ==, DB_CACHED);
2245 	ASSERT(db->db_buf != NULL);
2246 
2247 	DB_DNODE_ENTER(db);
2248 	dn = DB_DNODE(db);
2249 	/* Indirect block size must match what the dnode thinks it is. */
2250 	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2251 	dbuf_check_blkptr(dn, db);
2252 	DB_DNODE_EXIT(db);
2253 
2254 	/* Provide the pending dirty record to child dbufs */
2255 	db->db_data_pending = dr;
2256 
2257 	mutex_exit(&db->db_mtx);
2258 	dbuf_write(dr, db->db_buf, tx);
2259 
2260 	zio = dr->dr_zio;
2261 	mutex_enter(&dr->dt.di.dr_mtx);
2262 	dbuf_sync_list(&dr->dt.di.dr_children, tx);
2263 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2264 	mutex_exit(&dr->dt.di.dr_mtx);
2265 	zio_nowait(zio);
2266 }
2267 
2268 static void
2269 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2270 {
2271 	arc_buf_t **datap = &dr->dt.dl.dr_data;
2272 	dmu_buf_impl_t *db = dr->dr_dbuf;
2273 	dnode_t *dn;
2274 	objset_t *os;
2275 	uint64_t txg = tx->tx_txg;
2276 
2277 	ASSERT(dmu_tx_is_syncing(tx));
2278 
2279 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2280 
2281 	mutex_enter(&db->db_mtx);
2282 	/*
2283 	 * To be synced, we must be dirtied.  But we
2284 	 * might have been freed after the dirty.
2285 	 */
2286 	if (db->db_state == DB_UNCACHED) {
2287 		/* This buffer has been freed since it was dirtied */
2288 		ASSERT(db->db.db_data == NULL);
2289 	} else if (db->db_state == DB_FILL) {
2290 		/* This buffer was freed and is now being re-filled */
2291 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2292 	} else {
2293 		ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2294 	}
2295 	DBUF_VERIFY(db);
2296 
2297 	DB_DNODE_ENTER(db);
2298 	dn = DB_DNODE(db);
2299 
2300 	if (db->db_blkid == DMU_SPILL_BLKID) {
2301 		mutex_enter(&dn->dn_mtx);
2302 		dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR;
2303 		mutex_exit(&dn->dn_mtx);
2304 	}
2305 
2306 	/*
2307 	 * If this is a bonus buffer, simply copy the bonus data into the
2308 	 * dnode.  It will be written out when the dnode is synced (and it
2309 	 * will be synced, since it must have been dirty for dbuf_sync to
2310 	 * be called).
2311 	 */
2312 	if (db->db_blkid == DMU_BONUS_BLKID) {
2313 		dbuf_dirty_record_t **drp;
2314 
2315 		ASSERT(*datap != NULL);
2316 		ASSERT0(db->db_level);
2317 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
2318 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
2319 		DB_DNODE_EXIT(db);
2320 
2321 		if (*datap != db->db.db_data) {
2322 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
2323 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
2324 		}
2325 		db->db_data_pending = NULL;
2326 		drp = &db->db_last_dirty;
2327 		while (*drp != dr)
2328 			drp = &(*drp)->dr_next;
2329 		ASSERT(dr->dr_next == NULL);
2330 		ASSERT(dr->dr_dbuf == db);
2331 		*drp = dr->dr_next;
2332 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
2333 		ASSERT(db->db_dirtycnt > 0);
2334 		db->db_dirtycnt -= 1;
2335 		dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2336 		return;
2337 	}
2338 
2339 	os = dn->dn_objset;
2340 
2341 	/*
2342 	 * This function may have dropped the db_mtx lock allowing a dmu_sync
2343 	 * operation to sneak in. As a result, we need to ensure that we
2344 	 * don't check the dr_override_state until we have returned from
2345 	 * dbuf_check_blkptr.
2346 	 */
2347 	dbuf_check_blkptr(dn, db);
2348 
2349 	/*
2350 	 * If this buffer is in the middle of an immediate write,
2351 	 * wait for the synchronous IO to complete.
2352 	 */
2353 	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
2354 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
2355 		cv_wait(&db->db_changed, &db->db_mtx);
2356 		ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
2357 	}
2358 
2359 	if (db->db_state != DB_NOFILL &&
2360 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
2361 	    refcount_count(&db->db_holds) > 1 &&
2362 	    dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
2363 	    *datap == db->db_buf) {
2364 		/*
2365 		 * If this buffer is currently "in use" (i.e., there
2366 		 * are active holds and db_data still references it),
2367 		 * then make a copy before we start the write so that
2368 		 * any modifications from the open txg will not leak
2369 		 * into this write.
2370 		 *
2371 		 * NOTE: this copy does not need to be made for
2372 		 * objects only modified in the syncing context (e.g.
2373 		 * DNONE_DNODE blocks).
2374 		 */
2375 		int blksz = arc_buf_size(*datap);
2376 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2377 		*datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2378 		bcopy(db->db.db_data, (*datap)->b_data, blksz);
2379 	}
2380 	db->db_data_pending = dr;
2381 
2382 	mutex_exit(&db->db_mtx);
2383 
2384 	dbuf_write(dr, *datap, tx);
2385 
2386 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2387 	if (dn->dn_object == DMU_META_DNODE_OBJECT) {
2388 		list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2389 		DB_DNODE_EXIT(db);
2390 	} else {
2391 		/*
2392 		 * Although zio_nowait() does not "wait for an IO", it does
2393 		 * initiate the IO. If this is an empty write it seems plausible
2394 		 * that the IO could actually be completed before the nowait
2395 		 * returns. We need to DB_DNODE_EXIT() first in case
2396 		 * zio_nowait() invalidates the dbuf.
2397 		 */
2398 		DB_DNODE_EXIT(db);
2399 		zio_nowait(dr->dr_zio);
2400 	}
2401 }
2402 
2403 void
2404 dbuf_sync_list(list_t *list, dmu_tx_t *tx)
2405 {
2406 	dbuf_dirty_record_t *dr;
2407 
2408 	while (dr = list_head(list)) {
2409 		if (dr->dr_zio != NULL) {
2410 			/*
2411 			 * If we find an already initialized zio then we
2412 			 * are processing the meta-dnode, and we have finished.
2413 			 * The dbufs for all dnodes are put back on the list
2414 			 * during processing, so that we can zio_wait()
2415 			 * these IOs after initiating all child IOs.
2416 			 */
2417 			ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2418 			    DMU_META_DNODE_OBJECT);
2419 			break;
2420 		}
2421 		list_remove(list, dr);
2422 		if (dr->dr_dbuf->db_level > 0)
2423 			dbuf_sync_indirect(dr, tx);
2424 		else
2425 			dbuf_sync_leaf(dr, tx);
2426 	}
2427 }
2428 
2429 /* ARGSUSED */
2430 static void
2431 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2432 {
2433 	dmu_buf_impl_t *db = vdb;
2434 	dnode_t *dn;
2435 	blkptr_t *bp = zio->io_bp;
2436 	blkptr_t *bp_orig = &zio->io_bp_orig;
2437 	spa_t *spa = zio->io_spa;
2438 	int64_t delta;
2439 	uint64_t fill = 0;
2440 	int i;
2441 
2442 	ASSERT(db->db_blkptr == bp);
2443 
2444 	DB_DNODE_ENTER(db);
2445 	dn = DB_DNODE(db);
2446 	delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
2447 	dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
2448 	zio->io_prev_space_delta = delta;
2449 
2450 	if (BP_IS_HOLE(bp)) {
2451 		ASSERT(bp->blk_fill == 0);
2452 		DB_DNODE_EXIT(db);
2453 		return;
2454 	}
2455 
2456 	ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
2457 	    BP_GET_TYPE(bp) == dn->dn_type) ||
2458 	    (db->db_blkid == DMU_SPILL_BLKID &&
2459 	    BP_GET_TYPE(bp) == dn->dn_bonustype));
2460 	ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2461 
2462 	mutex_enter(&db->db_mtx);
2463 
2464 #ifdef ZFS_DEBUG
2465 	if (db->db_blkid == DMU_SPILL_BLKID) {
2466 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2467 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2468 		    db->db_blkptr == &dn->dn_phys->dn_spill);
2469 	}
2470 #endif
2471 
2472 	if (db->db_level == 0) {
2473 		mutex_enter(&dn->dn_mtx);
2474 		if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2475 		    db->db_blkid != DMU_SPILL_BLKID)
2476 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2477 		mutex_exit(&dn->dn_mtx);
2478 
2479 		if (dn->dn_type == DMU_OT_DNODE) {
2480 			dnode_phys_t *dnp = db->db.db_data;
2481 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2482 			    i--, dnp++) {
2483 				if (dnp->dn_type != DMU_OT_NONE)
2484 					fill++;
2485 			}
2486 		} else {
2487 			fill = 1;
2488 		}
2489 	} else {
2490 		blkptr_t *ibp = db->db.db_data;
2491 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2492 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2493 			if (BP_IS_HOLE(ibp))
2494 				continue;
2495 			fill += ibp->blk_fill;
2496 		}
2497 	}
2498 	DB_DNODE_EXIT(db);
2499 
2500 	bp->blk_fill = fill;
2501 
2502 	mutex_exit(&db->db_mtx);
2503 }
2504 
2505 /* ARGSUSED */
2506 static void
2507 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2508 {
2509 	dmu_buf_impl_t *db = vdb;
2510 	blkptr_t *bp = zio->io_bp;
2511 	blkptr_t *bp_orig = &zio->io_bp_orig;
2512 	uint64_t txg = zio->io_txg;
2513 	dbuf_dirty_record_t **drp, *dr;
2514 
2515 	ASSERT0(zio->io_error);
2516 	ASSERT(db->db_blkptr == bp);
2517 
2518 	/*
2519 	 * For nopwrites and rewrites we ensure that the bp matches our
2520 	 * original and bypass all the accounting.
2521 	 */
2522 	if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) {
2523 		ASSERT(BP_EQUAL(bp, bp_orig));
2524 	} else {
2525 		objset_t *os;
2526 		dsl_dataset_t *ds;
2527 		dmu_tx_t *tx;
2528 
2529 		DB_GET_OBJSET(&os, db);
2530 		ds = os->os_dsl_dataset;
2531 		tx = os->os_synctx;
2532 
2533 		(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
2534 		dsl_dataset_block_born(ds, bp, tx);
2535 	}
2536 
2537 	mutex_enter(&db->db_mtx);
2538 
2539 	DBUF_VERIFY(db);
2540 
2541 	drp = &db->db_last_dirty;
2542 	while ((dr = *drp) != db->db_data_pending)
2543 		drp = &dr->dr_next;
2544 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2545 	ASSERT(dr->dr_txg == txg);
2546 	ASSERT(dr->dr_dbuf == db);
2547 	ASSERT(dr->dr_next == NULL);
2548 	*drp = dr->dr_next;
2549 
2550 #ifdef ZFS_DEBUG
2551 	if (db->db_blkid == DMU_SPILL_BLKID) {
2552 		dnode_t *dn;
2553 
2554 		DB_DNODE_ENTER(db);
2555 		dn = DB_DNODE(db);
2556 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2557 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2558 		    db->db_blkptr == &dn->dn_phys->dn_spill);
2559 		DB_DNODE_EXIT(db);
2560 	}
2561 #endif
2562 
2563 	if (db->db_level == 0) {
2564 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2565 		ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2566 		if (db->db_state != DB_NOFILL) {
2567 			if (dr->dt.dl.dr_data != db->db_buf)
2568 				VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
2569 				    db));
2570 			else if (!arc_released(db->db_buf))
2571 				arc_set_callback(db->db_buf, dbuf_do_evict, db);
2572 		}
2573 	} else {
2574 		dnode_t *dn;
2575 
2576 		DB_DNODE_ENTER(db);
2577 		dn = DB_DNODE(db);
2578 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2579 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2580 		if (!BP_IS_HOLE(db->db_blkptr)) {
2581 			int epbs =
2582 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2583 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2584 			    db->db.db_size);
2585 			ASSERT3U(dn->dn_phys->dn_maxblkid
2586 			    >> (db->db_level * epbs), >=, db->db_blkid);
2587 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2588 		}
2589 		DB_DNODE_EXIT(db);
2590 		mutex_destroy(&dr->dt.di.dr_mtx);
2591 		list_destroy(&dr->dt.di.dr_children);
2592 	}
2593 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
2594 
2595 	cv_broadcast(&db->db_changed);
2596 	ASSERT(db->db_dirtycnt > 0);
2597 	db->db_dirtycnt -= 1;
2598 	db->db_data_pending = NULL;
2599 	dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2600 }
2601 
2602 static void
2603 dbuf_write_nofill_ready(zio_t *zio)
2604 {
2605 	dbuf_write_ready(zio, NULL, zio->io_private);
2606 }
2607 
2608 static void
2609 dbuf_write_nofill_done(zio_t *zio)
2610 {
2611 	dbuf_write_done(zio, NULL, zio->io_private);
2612 }
2613 
2614 static void
2615 dbuf_write_override_ready(zio_t *zio)
2616 {
2617 	dbuf_dirty_record_t *dr = zio->io_private;
2618 	dmu_buf_impl_t *db = dr->dr_dbuf;
2619 
2620 	dbuf_write_ready(zio, NULL, db);
2621 }
2622 
2623 static void
2624 dbuf_write_override_done(zio_t *zio)
2625 {
2626 	dbuf_dirty_record_t *dr = zio->io_private;
2627 	dmu_buf_impl_t *db = dr->dr_dbuf;
2628 	blkptr_t *obp = &dr->dt.dl.dr_overridden_by;
2629 
2630 	mutex_enter(&db->db_mtx);
2631 	if (!BP_EQUAL(zio->io_bp, obp)) {
2632 		if (!BP_IS_HOLE(obp))
2633 			dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
2634 		arc_release(dr->dt.dl.dr_data, db);
2635 	}
2636 	mutex_exit(&db->db_mtx);
2637 
2638 	dbuf_write_done(zio, NULL, db);
2639 }
2640 
2641 /* Issue I/O to commit a dirty buffer to disk. */
2642 static void
2643 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
2644 {
2645 	dmu_buf_impl_t *db = dr->dr_dbuf;
2646 	dnode_t *dn;
2647 	objset_t *os;
2648 	dmu_buf_impl_t *parent = db->db_parent;
2649 	uint64_t txg = tx->tx_txg;
2650 	zbookmark_t zb;
2651 	zio_prop_t zp;
2652 	zio_t *zio;
2653 	int wp_flag = 0;
2654 
2655 	DB_DNODE_ENTER(db);
2656 	dn = DB_DNODE(db);
2657 	os = dn->dn_objset;
2658 
2659 	if (db->db_state != DB_NOFILL) {
2660 		if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
2661 			/*
2662 			 * Private object buffers are released here rather
2663 			 * than in dbuf_dirty() since they are only modified
2664 			 * in the syncing context and we don't want the
2665 			 * overhead of making multiple copies of the data.
2666 			 */
2667 			if (BP_IS_HOLE(db->db_blkptr)) {
2668 				arc_buf_thaw(data);
2669 			} else {
2670 				dbuf_release_bp(db);
2671 			}
2672 		}
2673 	}
2674 
2675 	if (parent != dn->dn_dbuf) {
2676 		/* Our parent is an indirect block. */
2677 		/* We have a dirty parent that has been scheduled for write. */
2678 		ASSERT(parent && parent->db_data_pending);
2679 		/* Our parent's buffer is one level closer to the dnode. */
2680 		ASSERT(db->db_level == parent->db_level-1);
2681 		/*
2682 		 * We're about to modify our parent's db_data by modifying
2683 		 * our block pointer, so the parent must be released.
2684 		 */
2685 		ASSERT(arc_released(parent->db_buf));
2686 		zio = parent->db_data_pending->dr_zio;
2687 	} else {
2688 		/* Our parent is the dnode itself. */
2689 		ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
2690 		    db->db_blkid != DMU_SPILL_BLKID) ||
2691 		    (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
2692 		if (db->db_blkid != DMU_SPILL_BLKID)
2693 			ASSERT3P(db->db_blkptr, ==,
2694 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
2695 		zio = dn->dn_zio;
2696 	}
2697 
2698 	ASSERT(db->db_level == 0 || data == db->db_buf);
2699 	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
2700 	ASSERT(zio);
2701 
2702 	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
2703 	    os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
2704 	    db->db.db_object, db->db_level, db->db_blkid);
2705 
2706 	if (db->db_blkid == DMU_SPILL_BLKID)
2707 		wp_flag = WP_SPILL;
2708 	wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
2709 
2710 	dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
2711 	DB_DNODE_EXIT(db);
2712 
2713 	if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
2714 		ASSERT(db->db_state != DB_NOFILL);
2715 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
2716 		    db->db_blkptr, data->b_data, arc_buf_size(data), &zp,
2717 		    dbuf_write_override_ready, dbuf_write_override_done, dr,
2718 		    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2719 		mutex_enter(&db->db_mtx);
2720 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
2721 		zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
2722 		    dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite);
2723 		mutex_exit(&db->db_mtx);
2724 	} else if (db->db_state == DB_NOFILL) {
2725 		ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF);
2726 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
2727 		    db->db_blkptr, NULL, db->db.db_size, &zp,
2728 		    dbuf_write_nofill_ready, dbuf_write_nofill_done, db,
2729 		    ZIO_PRIORITY_ASYNC_WRITE,
2730 		    ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb);
2731 	} else {
2732 		ASSERT(arc_released(data));
2733 		dr->dr_zio = arc_write(zio, os->os_spa, txg,
2734 		    db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db),
2735 		    DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready,
2736 		    dbuf_write_done, db, ZIO_PRIORITY_ASYNC_WRITE,
2737 		    ZIO_FLAG_MUSTSUCCEED, &zb);
2738 	}
2739 }
2740