xref: /illumos-gate/usr/src/uts/common/fs/zfs/dbuf.c (revision fb09f5aad449c97fe309678f3f604982b563a96f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24  * Copyright (c) 2012 by Delphix. All rights reserved.
25  */
26 
27 #include <sys/zfs_context.h>
28 #include <sys/dmu.h>
29 #include <sys/dmu_impl.h>
30 #include <sys/dbuf.h>
31 #include <sys/dmu_objset.h>
32 #include <sys/dsl_dataset.h>
33 #include <sys/dsl_dir.h>
34 #include <sys/dmu_tx.h>
35 #include <sys/spa.h>
36 #include <sys/zio.h>
37 #include <sys/dmu_zfetch.h>
38 #include <sys/sa.h>
39 #include <sys/sa_impl.h>
40 
41 static void dbuf_destroy(dmu_buf_impl_t *db);
42 static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
43 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
44 
45 /*
46  * Global data structures and functions for the dbuf cache.
47  */
48 static kmem_cache_t *dbuf_cache;
49 
50 /* ARGSUSED */
51 static int
52 dbuf_cons(void *vdb, void *unused, int kmflag)
53 {
54 	dmu_buf_impl_t *db = vdb;
55 	bzero(db, sizeof (dmu_buf_impl_t));
56 
57 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
58 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
59 	refcount_create(&db->db_holds);
60 	return (0);
61 }
62 
63 /* ARGSUSED */
64 static void
65 dbuf_dest(void *vdb, void *unused)
66 {
67 	dmu_buf_impl_t *db = vdb;
68 	mutex_destroy(&db->db_mtx);
69 	cv_destroy(&db->db_changed);
70 	refcount_destroy(&db->db_holds);
71 }
72 
73 /*
74  * dbuf hash table routines
75  */
76 static dbuf_hash_table_t dbuf_hash_table;
77 
78 static uint64_t dbuf_hash_count;
79 
80 static uint64_t
81 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
82 {
83 	uintptr_t osv = (uintptr_t)os;
84 	uint64_t crc = -1ULL;
85 
86 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
87 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
88 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
89 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
90 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
91 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
92 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
93 
94 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
95 
96 	return (crc);
97 }
98 
99 #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
100 
101 #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
102 	((dbuf)->db.db_object == (obj) &&		\
103 	(dbuf)->db_objset == (os) &&			\
104 	(dbuf)->db_level == (level) &&			\
105 	(dbuf)->db_blkid == (blkid))
106 
107 dmu_buf_impl_t *
108 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
109 {
110 	dbuf_hash_table_t *h = &dbuf_hash_table;
111 	objset_t *os = dn->dn_objset;
112 	uint64_t obj = dn->dn_object;
113 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
114 	uint64_t idx = hv & h->hash_table_mask;
115 	dmu_buf_impl_t *db;
116 
117 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
118 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
119 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
120 			mutex_enter(&db->db_mtx);
121 			if (db->db_state != DB_EVICTING) {
122 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
123 				return (db);
124 			}
125 			mutex_exit(&db->db_mtx);
126 		}
127 	}
128 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
129 	return (NULL);
130 }
131 
132 /*
133  * Insert an entry into the hash table.  If there is already an element
134  * equal to elem in the hash table, then the already existing element
135  * will be returned and the new element will not be inserted.
136  * Otherwise returns NULL.
137  */
138 static dmu_buf_impl_t *
139 dbuf_hash_insert(dmu_buf_impl_t *db)
140 {
141 	dbuf_hash_table_t *h = &dbuf_hash_table;
142 	objset_t *os = db->db_objset;
143 	uint64_t obj = db->db.db_object;
144 	int level = db->db_level;
145 	uint64_t blkid = db->db_blkid;
146 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
147 	uint64_t idx = hv & h->hash_table_mask;
148 	dmu_buf_impl_t *dbf;
149 
150 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
151 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
152 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
153 			mutex_enter(&dbf->db_mtx);
154 			if (dbf->db_state != DB_EVICTING) {
155 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
156 				return (dbf);
157 			}
158 			mutex_exit(&dbf->db_mtx);
159 		}
160 	}
161 
162 	mutex_enter(&db->db_mtx);
163 	db->db_hash_next = h->hash_table[idx];
164 	h->hash_table[idx] = db;
165 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
166 	atomic_add_64(&dbuf_hash_count, 1);
167 
168 	return (NULL);
169 }
170 
171 /*
172  * Remove an entry from the hash table.  This operation will
173  * fail if there are any existing holds on the db.
174  */
175 static void
176 dbuf_hash_remove(dmu_buf_impl_t *db)
177 {
178 	dbuf_hash_table_t *h = &dbuf_hash_table;
179 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
180 	    db->db_level, db->db_blkid);
181 	uint64_t idx = hv & h->hash_table_mask;
182 	dmu_buf_impl_t *dbf, **dbp;
183 
184 	/*
185 	 * We musn't hold db_mtx to maintin lock ordering:
186 	 * DBUF_HASH_MUTEX > db_mtx.
187 	 */
188 	ASSERT(refcount_is_zero(&db->db_holds));
189 	ASSERT(db->db_state == DB_EVICTING);
190 	ASSERT(!MUTEX_HELD(&db->db_mtx));
191 
192 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
193 	dbp = &h->hash_table[idx];
194 	while ((dbf = *dbp) != db) {
195 		dbp = &dbf->db_hash_next;
196 		ASSERT(dbf != NULL);
197 	}
198 	*dbp = db->db_hash_next;
199 	db->db_hash_next = NULL;
200 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
201 	atomic_add_64(&dbuf_hash_count, -1);
202 }
203 
204 static arc_evict_func_t dbuf_do_evict;
205 
206 static void
207 dbuf_evict_user(dmu_buf_impl_t *db)
208 {
209 	ASSERT(MUTEX_HELD(&db->db_mtx));
210 
211 	if (db->db_level != 0 || db->db_evict_func == NULL)
212 		return;
213 
214 	if (db->db_user_data_ptr_ptr)
215 		*db->db_user_data_ptr_ptr = db->db.db_data;
216 	db->db_evict_func(&db->db, db->db_user_ptr);
217 	db->db_user_ptr = NULL;
218 	db->db_user_data_ptr_ptr = NULL;
219 	db->db_evict_func = NULL;
220 }
221 
222 boolean_t
223 dbuf_is_metadata(dmu_buf_impl_t *db)
224 {
225 	if (db->db_level > 0) {
226 		return (B_TRUE);
227 	} else {
228 		boolean_t is_metadata;
229 
230 		DB_DNODE_ENTER(db);
231 		is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
232 		DB_DNODE_EXIT(db);
233 
234 		return (is_metadata);
235 	}
236 }
237 
238 void
239 dbuf_evict(dmu_buf_impl_t *db)
240 {
241 	ASSERT(MUTEX_HELD(&db->db_mtx));
242 	ASSERT(db->db_buf == NULL);
243 	ASSERT(db->db_data_pending == NULL);
244 
245 	dbuf_clear(db);
246 	dbuf_destroy(db);
247 }
248 
249 void
250 dbuf_init(void)
251 {
252 	uint64_t hsize = 1ULL << 16;
253 	dbuf_hash_table_t *h = &dbuf_hash_table;
254 	int i;
255 
256 	/*
257 	 * The hash table is big enough to fill all of physical memory
258 	 * with an average 4K block size.  The table will take up
259 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
260 	 */
261 	while (hsize * 4096 < physmem * PAGESIZE)
262 		hsize <<= 1;
263 
264 retry:
265 	h->hash_table_mask = hsize - 1;
266 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
267 	if (h->hash_table == NULL) {
268 		/* XXX - we should really return an error instead of assert */
269 		ASSERT(hsize > (1ULL << 10));
270 		hsize >>= 1;
271 		goto retry;
272 	}
273 
274 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
275 	    sizeof (dmu_buf_impl_t),
276 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
277 
278 	for (i = 0; i < DBUF_MUTEXES; i++)
279 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
280 }
281 
282 void
283 dbuf_fini(void)
284 {
285 	dbuf_hash_table_t *h = &dbuf_hash_table;
286 	int i;
287 
288 	for (i = 0; i < DBUF_MUTEXES; i++)
289 		mutex_destroy(&h->hash_mutexes[i]);
290 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
291 	kmem_cache_destroy(dbuf_cache);
292 }
293 
294 /*
295  * Other stuff.
296  */
297 
298 #ifdef ZFS_DEBUG
299 static void
300 dbuf_verify(dmu_buf_impl_t *db)
301 {
302 	dnode_t *dn;
303 	dbuf_dirty_record_t *dr;
304 
305 	ASSERT(MUTEX_HELD(&db->db_mtx));
306 
307 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
308 		return;
309 
310 	ASSERT(db->db_objset != NULL);
311 	DB_DNODE_ENTER(db);
312 	dn = DB_DNODE(db);
313 	if (dn == NULL) {
314 		ASSERT(db->db_parent == NULL);
315 		ASSERT(db->db_blkptr == NULL);
316 	} else {
317 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
318 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
319 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
320 		ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
321 		    db->db_blkid == DMU_SPILL_BLKID ||
322 		    !list_is_empty(&dn->dn_dbufs));
323 	}
324 	if (db->db_blkid == DMU_BONUS_BLKID) {
325 		ASSERT(dn != NULL);
326 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
327 		ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
328 	} else if (db->db_blkid == DMU_SPILL_BLKID) {
329 		ASSERT(dn != NULL);
330 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
331 		ASSERT0(db->db.db_offset);
332 	} else {
333 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
334 	}
335 
336 	for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next)
337 		ASSERT(dr->dr_dbuf == db);
338 
339 	for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next)
340 		ASSERT(dr->dr_dbuf == db);
341 
342 	/*
343 	 * We can't assert that db_size matches dn_datablksz because it
344 	 * can be momentarily different when another thread is doing
345 	 * dnode_set_blksz().
346 	 */
347 	if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
348 		dr = db->db_data_pending;
349 		/*
350 		 * It should only be modified in syncing context, so
351 		 * make sure we only have one copy of the data.
352 		 */
353 		ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
354 	}
355 
356 	/* verify db->db_blkptr */
357 	if (db->db_blkptr) {
358 		if (db->db_parent == dn->dn_dbuf) {
359 			/* db is pointed to by the dnode */
360 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
361 			if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
362 				ASSERT(db->db_parent == NULL);
363 			else
364 				ASSERT(db->db_parent != NULL);
365 			if (db->db_blkid != DMU_SPILL_BLKID)
366 				ASSERT3P(db->db_blkptr, ==,
367 				    &dn->dn_phys->dn_blkptr[db->db_blkid]);
368 		} else {
369 			/* db is pointed to by an indirect block */
370 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
371 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
372 			ASSERT3U(db->db_parent->db.db_object, ==,
373 			    db->db.db_object);
374 			/*
375 			 * dnode_grow_indblksz() can make this fail if we don't
376 			 * have the struct_rwlock.  XXX indblksz no longer
377 			 * grows.  safe to do this now?
378 			 */
379 			if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
380 				ASSERT3P(db->db_blkptr, ==,
381 				    ((blkptr_t *)db->db_parent->db.db_data +
382 				    db->db_blkid % epb));
383 			}
384 		}
385 	}
386 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
387 	    (db->db_buf == NULL || db->db_buf->b_data) &&
388 	    db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
389 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
390 		/*
391 		 * If the blkptr isn't set but they have nonzero data,
392 		 * it had better be dirty, otherwise we'll lose that
393 		 * data when we evict this buffer.
394 		 */
395 		if (db->db_dirtycnt == 0) {
396 			uint64_t *buf = db->db.db_data;
397 			int i;
398 
399 			for (i = 0; i < db->db.db_size >> 3; i++) {
400 				ASSERT(buf[i] == 0);
401 			}
402 		}
403 	}
404 	DB_DNODE_EXIT(db);
405 }
406 #endif
407 
408 static void
409 dbuf_update_data(dmu_buf_impl_t *db)
410 {
411 	ASSERT(MUTEX_HELD(&db->db_mtx));
412 	if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
413 		ASSERT(!refcount_is_zero(&db->db_holds));
414 		*db->db_user_data_ptr_ptr = db->db.db_data;
415 	}
416 }
417 
418 static void
419 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
420 {
421 	ASSERT(MUTEX_HELD(&db->db_mtx));
422 	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
423 	db->db_buf = buf;
424 	if (buf != NULL) {
425 		ASSERT(buf->b_data != NULL);
426 		db->db.db_data = buf->b_data;
427 		if (!arc_released(buf))
428 			arc_set_callback(buf, dbuf_do_evict, db);
429 		dbuf_update_data(db);
430 	} else {
431 		dbuf_evict_user(db);
432 		db->db.db_data = NULL;
433 		if (db->db_state != DB_NOFILL)
434 			db->db_state = DB_UNCACHED;
435 	}
436 }
437 
438 /*
439  * Loan out an arc_buf for read.  Return the loaned arc_buf.
440  */
441 arc_buf_t *
442 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
443 {
444 	arc_buf_t *abuf;
445 
446 	mutex_enter(&db->db_mtx);
447 	if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) {
448 		int blksz = db->db.db_size;
449 		spa_t *spa;
450 
451 		mutex_exit(&db->db_mtx);
452 		DB_GET_SPA(&spa, db);
453 		abuf = arc_loan_buf(spa, blksz);
454 		bcopy(db->db.db_data, abuf->b_data, blksz);
455 	} else {
456 		abuf = db->db_buf;
457 		arc_loan_inuse_buf(abuf, db);
458 		dbuf_set_data(db, NULL);
459 		mutex_exit(&db->db_mtx);
460 	}
461 	return (abuf);
462 }
463 
464 uint64_t
465 dbuf_whichblock(dnode_t *dn, uint64_t offset)
466 {
467 	if (dn->dn_datablkshift) {
468 		return (offset >> dn->dn_datablkshift);
469 	} else {
470 		ASSERT3U(offset, <, dn->dn_datablksz);
471 		return (0);
472 	}
473 }
474 
475 static void
476 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
477 {
478 	dmu_buf_impl_t *db = vdb;
479 
480 	mutex_enter(&db->db_mtx);
481 	ASSERT3U(db->db_state, ==, DB_READ);
482 	/*
483 	 * All reads are synchronous, so we must have a hold on the dbuf
484 	 */
485 	ASSERT(refcount_count(&db->db_holds) > 0);
486 	ASSERT(db->db_buf == NULL);
487 	ASSERT(db->db.db_data == NULL);
488 	if (db->db_level == 0 && db->db_freed_in_flight) {
489 		/* we were freed in flight; disregard any error */
490 		arc_release(buf, db);
491 		bzero(buf->b_data, db->db.db_size);
492 		arc_buf_freeze(buf);
493 		db->db_freed_in_flight = FALSE;
494 		dbuf_set_data(db, buf);
495 		db->db_state = DB_CACHED;
496 	} else if (zio == NULL || zio->io_error == 0) {
497 		dbuf_set_data(db, buf);
498 		db->db_state = DB_CACHED;
499 	} else {
500 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
501 		ASSERT3P(db->db_buf, ==, NULL);
502 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
503 		db->db_state = DB_UNCACHED;
504 	}
505 	cv_broadcast(&db->db_changed);
506 	dbuf_rele_and_unlock(db, NULL);
507 }
508 
509 static void
510 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
511 {
512 	dnode_t *dn;
513 	spa_t *spa;
514 	zbookmark_t zb;
515 	uint32_t aflags = ARC_NOWAIT;
516 	arc_buf_t *pbuf;
517 
518 	DB_DNODE_ENTER(db);
519 	dn = DB_DNODE(db);
520 	ASSERT(!refcount_is_zero(&db->db_holds));
521 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
522 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
523 	ASSERT(MUTEX_HELD(&db->db_mtx));
524 	ASSERT(db->db_state == DB_UNCACHED);
525 	ASSERT(db->db_buf == NULL);
526 
527 	if (db->db_blkid == DMU_BONUS_BLKID) {
528 		int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen);
529 
530 		ASSERT3U(bonuslen, <=, db->db.db_size);
531 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
532 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
533 		if (bonuslen < DN_MAX_BONUSLEN)
534 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
535 		if (bonuslen)
536 			bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen);
537 		DB_DNODE_EXIT(db);
538 		dbuf_update_data(db);
539 		db->db_state = DB_CACHED;
540 		mutex_exit(&db->db_mtx);
541 		return;
542 	}
543 
544 	/*
545 	 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
546 	 * processes the delete record and clears the bp while we are waiting
547 	 * for the dn_mtx (resulting in a "no" from block_freed).
548 	 */
549 	if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
550 	    (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
551 	    BP_IS_HOLE(db->db_blkptr)))) {
552 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
553 
554 		dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa,
555 		    db->db.db_size, db, type));
556 		DB_DNODE_EXIT(db);
557 		bzero(db->db.db_data, db->db.db_size);
558 		db->db_state = DB_CACHED;
559 		*flags |= DB_RF_CACHED;
560 		mutex_exit(&db->db_mtx);
561 		return;
562 	}
563 
564 	spa = dn->dn_objset->os_spa;
565 	DB_DNODE_EXIT(db);
566 
567 	db->db_state = DB_READ;
568 	mutex_exit(&db->db_mtx);
569 
570 	if (DBUF_IS_L2CACHEABLE(db))
571 		aflags |= ARC_L2CACHE;
572 
573 	SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ?
574 	    db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET,
575 	    db->db.db_object, db->db_level, db->db_blkid);
576 
577 	dbuf_add_ref(db, NULL);
578 	/* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */
579 
580 	if (db->db_parent)
581 		pbuf = db->db_parent->db_buf;
582 	else
583 		pbuf = db->db_objset->os_phys_buf;
584 
585 	(void) dsl_read(zio, spa, db->db_blkptr, pbuf,
586 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
587 	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
588 	    &aflags, &zb);
589 	if (aflags & ARC_CACHED)
590 		*flags |= DB_RF_CACHED;
591 }
592 
593 int
594 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
595 {
596 	int err = 0;
597 	int havepzio = (zio != NULL);
598 	int prefetch;
599 	dnode_t *dn;
600 
601 	/*
602 	 * We don't have to hold the mutex to check db_state because it
603 	 * can't be freed while we have a hold on the buffer.
604 	 */
605 	ASSERT(!refcount_is_zero(&db->db_holds));
606 
607 	if (db->db_state == DB_NOFILL)
608 		return (EIO);
609 
610 	DB_DNODE_ENTER(db);
611 	dn = DB_DNODE(db);
612 	if ((flags & DB_RF_HAVESTRUCT) == 0)
613 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
614 
615 	prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
616 	    (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL &&
617 	    DBUF_IS_CACHEABLE(db);
618 
619 	mutex_enter(&db->db_mtx);
620 	if (db->db_state == DB_CACHED) {
621 		mutex_exit(&db->db_mtx);
622 		if (prefetch)
623 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
624 			    db->db.db_size, TRUE);
625 		if ((flags & DB_RF_HAVESTRUCT) == 0)
626 			rw_exit(&dn->dn_struct_rwlock);
627 		DB_DNODE_EXIT(db);
628 	} else if (db->db_state == DB_UNCACHED) {
629 		spa_t *spa = dn->dn_objset->os_spa;
630 
631 		if (zio == NULL)
632 			zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL);
633 		dbuf_read_impl(db, zio, &flags);
634 
635 		/* dbuf_read_impl has dropped db_mtx for us */
636 
637 		if (prefetch)
638 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
639 			    db->db.db_size, flags & DB_RF_CACHED);
640 
641 		if ((flags & DB_RF_HAVESTRUCT) == 0)
642 			rw_exit(&dn->dn_struct_rwlock);
643 		DB_DNODE_EXIT(db);
644 
645 		if (!havepzio)
646 			err = zio_wait(zio);
647 	} else {
648 		mutex_exit(&db->db_mtx);
649 		if (prefetch)
650 			dmu_zfetch(&dn->dn_zfetch, db->db.db_offset,
651 			    db->db.db_size, TRUE);
652 		if ((flags & DB_RF_HAVESTRUCT) == 0)
653 			rw_exit(&dn->dn_struct_rwlock);
654 		DB_DNODE_EXIT(db);
655 
656 		mutex_enter(&db->db_mtx);
657 		if ((flags & DB_RF_NEVERWAIT) == 0) {
658 			while (db->db_state == DB_READ ||
659 			    db->db_state == DB_FILL) {
660 				ASSERT(db->db_state == DB_READ ||
661 				    (flags & DB_RF_HAVESTRUCT) == 0);
662 				cv_wait(&db->db_changed, &db->db_mtx);
663 			}
664 			if (db->db_state == DB_UNCACHED)
665 				err = EIO;
666 		}
667 		mutex_exit(&db->db_mtx);
668 	}
669 
670 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
671 	return (err);
672 }
673 
674 static void
675 dbuf_noread(dmu_buf_impl_t *db)
676 {
677 	ASSERT(!refcount_is_zero(&db->db_holds));
678 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
679 	mutex_enter(&db->db_mtx);
680 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
681 		cv_wait(&db->db_changed, &db->db_mtx);
682 	if (db->db_state == DB_UNCACHED) {
683 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
684 		spa_t *spa;
685 
686 		ASSERT(db->db_buf == NULL);
687 		ASSERT(db->db.db_data == NULL);
688 		DB_GET_SPA(&spa, db);
689 		dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
690 		db->db_state = DB_FILL;
691 	} else if (db->db_state == DB_NOFILL) {
692 		dbuf_set_data(db, NULL);
693 	} else {
694 		ASSERT3U(db->db_state, ==, DB_CACHED);
695 	}
696 	mutex_exit(&db->db_mtx);
697 }
698 
699 /*
700  * This is our just-in-time copy function.  It makes a copy of
701  * buffers, that have been modified in a previous transaction
702  * group, before we modify them in the current active group.
703  *
704  * This function is used in two places: when we are dirtying a
705  * buffer for the first time in a txg, and when we are freeing
706  * a range in a dnode that includes this buffer.
707  *
708  * Note that when we are called from dbuf_free_range() we do
709  * not put a hold on the buffer, we just traverse the active
710  * dbuf list for the dnode.
711  */
712 static void
713 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
714 {
715 	dbuf_dirty_record_t *dr = db->db_last_dirty;
716 
717 	ASSERT(MUTEX_HELD(&db->db_mtx));
718 	ASSERT(db->db.db_data != NULL);
719 	ASSERT(db->db_level == 0);
720 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
721 
722 	if (dr == NULL ||
723 	    (dr->dt.dl.dr_data !=
724 	    ((db->db_blkid  == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
725 		return;
726 
727 	/*
728 	 * If the last dirty record for this dbuf has not yet synced
729 	 * and its referencing the dbuf data, either:
730 	 *	reset the reference to point to a new copy,
731 	 * or (if there a no active holders)
732 	 *	just null out the current db_data pointer.
733 	 */
734 	ASSERT(dr->dr_txg >= txg - 2);
735 	if (db->db_blkid == DMU_BONUS_BLKID) {
736 		/* Note that the data bufs here are zio_bufs */
737 		dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
738 		arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
739 		bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
740 	} else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
741 		int size = db->db.db_size;
742 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
743 		spa_t *spa;
744 
745 		DB_GET_SPA(&spa, db);
746 		dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
747 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
748 	} else {
749 		dbuf_set_data(db, NULL);
750 	}
751 }
752 
753 void
754 dbuf_unoverride(dbuf_dirty_record_t *dr)
755 {
756 	dmu_buf_impl_t *db = dr->dr_dbuf;
757 	blkptr_t *bp = &dr->dt.dl.dr_overridden_by;
758 	uint64_t txg = dr->dr_txg;
759 
760 	ASSERT(MUTEX_HELD(&db->db_mtx));
761 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
762 	ASSERT(db->db_level == 0);
763 
764 	if (db->db_blkid == DMU_BONUS_BLKID ||
765 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
766 		return;
767 
768 	ASSERT(db->db_data_pending != dr);
769 
770 	/* free this block */
771 	if (!BP_IS_HOLE(bp)) {
772 		spa_t *spa;
773 
774 		DB_GET_SPA(&spa, db);
775 		zio_free(spa, txg, bp);
776 	}
777 	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
778 	/*
779 	 * Release the already-written buffer, so we leave it in
780 	 * a consistent dirty state.  Note that all callers are
781 	 * modifying the buffer, so they will immediately do
782 	 * another (redundant) arc_release().  Therefore, leave
783 	 * the buf thawed to save the effort of freezing &
784 	 * immediately re-thawing it.
785 	 */
786 	arc_release(dr->dt.dl.dr_data, db);
787 }
788 
789 /*
790  * Evict (if its unreferenced) or clear (if its referenced) any level-0
791  * data blocks in the free range, so that any future readers will find
792  * empty blocks.  Also, if we happen accross any level-1 dbufs in the
793  * range that have not already been marked dirty, mark them dirty so
794  * they stay in memory.
795  */
796 void
797 dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
798 {
799 	dmu_buf_impl_t *db, *db_next;
800 	uint64_t txg = tx->tx_txg;
801 	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
802 	uint64_t first_l1 = start >> epbs;
803 	uint64_t last_l1 = end >> epbs;
804 
805 	if (end > dn->dn_maxblkid && (end != DMU_SPILL_BLKID)) {
806 		end = dn->dn_maxblkid;
807 		last_l1 = end >> epbs;
808 	}
809 	dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
810 	mutex_enter(&dn->dn_dbufs_mtx);
811 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
812 		db_next = list_next(&dn->dn_dbufs, db);
813 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
814 
815 		if (db->db_level == 1 &&
816 		    db->db_blkid >= first_l1 && db->db_blkid <= last_l1) {
817 			mutex_enter(&db->db_mtx);
818 			if (db->db_last_dirty &&
819 			    db->db_last_dirty->dr_txg < txg) {
820 				dbuf_add_ref(db, FTAG);
821 				mutex_exit(&db->db_mtx);
822 				dbuf_will_dirty(db, tx);
823 				dbuf_rele(db, FTAG);
824 			} else {
825 				mutex_exit(&db->db_mtx);
826 			}
827 		}
828 
829 		if (db->db_level != 0)
830 			continue;
831 		dprintf_dbuf(db, "found buf %s\n", "");
832 		if (db->db_blkid < start || db->db_blkid > end)
833 			continue;
834 
835 		/* found a level 0 buffer in the range */
836 		if (dbuf_undirty(db, tx))
837 			continue;
838 
839 		mutex_enter(&db->db_mtx);
840 		if (db->db_state == DB_UNCACHED ||
841 		    db->db_state == DB_NOFILL ||
842 		    db->db_state == DB_EVICTING) {
843 			ASSERT(db->db.db_data == NULL);
844 			mutex_exit(&db->db_mtx);
845 			continue;
846 		}
847 		if (db->db_state == DB_READ || db->db_state == DB_FILL) {
848 			/* will be handled in dbuf_read_done or dbuf_rele */
849 			db->db_freed_in_flight = TRUE;
850 			mutex_exit(&db->db_mtx);
851 			continue;
852 		}
853 		if (refcount_count(&db->db_holds) == 0) {
854 			ASSERT(db->db_buf);
855 			dbuf_clear(db);
856 			continue;
857 		}
858 		/* The dbuf is referenced */
859 
860 		if (db->db_last_dirty != NULL) {
861 			dbuf_dirty_record_t *dr = db->db_last_dirty;
862 
863 			if (dr->dr_txg == txg) {
864 				/*
865 				 * This buffer is "in-use", re-adjust the file
866 				 * size to reflect that this buffer may
867 				 * contain new data when we sync.
868 				 */
869 				if (db->db_blkid != DMU_SPILL_BLKID &&
870 				    db->db_blkid > dn->dn_maxblkid)
871 					dn->dn_maxblkid = db->db_blkid;
872 				dbuf_unoverride(dr);
873 			} else {
874 				/*
875 				 * This dbuf is not dirty in the open context.
876 				 * Either uncache it (if its not referenced in
877 				 * the open context) or reset its contents to
878 				 * empty.
879 				 */
880 				dbuf_fix_old_data(db, txg);
881 			}
882 		}
883 		/* clear the contents if its cached */
884 		if (db->db_state == DB_CACHED) {
885 			ASSERT(db->db.db_data != NULL);
886 			arc_release(db->db_buf, db);
887 			bzero(db->db.db_data, db->db.db_size);
888 			arc_buf_freeze(db->db_buf);
889 		}
890 
891 		mutex_exit(&db->db_mtx);
892 	}
893 	mutex_exit(&dn->dn_dbufs_mtx);
894 }
895 
896 static int
897 dbuf_block_freeable(dmu_buf_impl_t *db)
898 {
899 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
900 	uint64_t birth_txg = 0;
901 
902 	/*
903 	 * We don't need any locking to protect db_blkptr:
904 	 * If it's syncing, then db_last_dirty will be set
905 	 * so we'll ignore db_blkptr.
906 	 */
907 	ASSERT(MUTEX_HELD(&db->db_mtx));
908 	if (db->db_last_dirty)
909 		birth_txg = db->db_last_dirty->dr_txg;
910 	else if (db->db_blkptr)
911 		birth_txg = db->db_blkptr->blk_birth;
912 
913 	/*
914 	 * If we don't exist or are in a snapshot, we can't be freed.
915 	 * Don't pass the bp to dsl_dataset_block_freeable() since we
916 	 * are holding the db_mtx lock and might deadlock if we are
917 	 * prefetching a dedup-ed block.
918 	 */
919 	if (birth_txg)
920 		return (ds == NULL ||
921 		    dsl_dataset_block_freeable(ds, NULL, birth_txg));
922 	else
923 		return (FALSE);
924 }
925 
926 void
927 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
928 {
929 	arc_buf_t *buf, *obuf;
930 	int osize = db->db.db_size;
931 	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
932 	dnode_t *dn;
933 
934 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
935 
936 	DB_DNODE_ENTER(db);
937 	dn = DB_DNODE(db);
938 
939 	/* XXX does *this* func really need the lock? */
940 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
941 
942 	/*
943 	 * This call to dbuf_will_dirty() with the dn_struct_rwlock held
944 	 * is OK, because there can be no other references to the db
945 	 * when we are changing its size, so no concurrent DB_FILL can
946 	 * be happening.
947 	 */
948 	/*
949 	 * XXX we should be doing a dbuf_read, checking the return
950 	 * value and returning that up to our callers
951 	 */
952 	dbuf_will_dirty(db, tx);
953 
954 	/* create the data buffer for the new block */
955 	buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type);
956 
957 	/* copy old block data to the new block */
958 	obuf = db->db_buf;
959 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
960 	/* zero the remainder */
961 	if (size > osize)
962 		bzero((uint8_t *)buf->b_data + osize, size - osize);
963 
964 	mutex_enter(&db->db_mtx);
965 	dbuf_set_data(db, buf);
966 	VERIFY(arc_buf_remove_ref(obuf, db) == 1);
967 	db->db.db_size = size;
968 
969 	if (db->db_level == 0) {
970 		ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
971 		db->db_last_dirty->dt.dl.dr_data = buf;
972 	}
973 	mutex_exit(&db->db_mtx);
974 
975 	dnode_willuse_space(dn, size-osize, tx);
976 	DB_DNODE_EXIT(db);
977 }
978 
979 void
980 dbuf_release_bp(dmu_buf_impl_t *db)
981 {
982 	objset_t *os;
983 	zbookmark_t zb;
984 
985 	DB_GET_OBJSET(&os, db);
986 	ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
987 	ASSERT(arc_released(os->os_phys_buf) ||
988 	    list_link_active(&os->os_dsl_dataset->ds_synced_link));
989 	ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
990 
991 	zb.zb_objset = os->os_dsl_dataset ?
992 	    os->os_dsl_dataset->ds_object : 0;
993 	zb.zb_object = db->db.db_object;
994 	zb.zb_level = db->db_level;
995 	zb.zb_blkid = db->db_blkid;
996 	(void) arc_release_bp(db->db_buf, db,
997 	    db->db_blkptr, os->os_spa, &zb);
998 }
999 
1000 dbuf_dirty_record_t *
1001 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1002 {
1003 	dnode_t *dn;
1004 	objset_t *os;
1005 	dbuf_dirty_record_t **drp, *dr;
1006 	int drop_struct_lock = FALSE;
1007 	boolean_t do_free_accounting = B_FALSE;
1008 	int txgoff = tx->tx_txg & TXG_MASK;
1009 
1010 	ASSERT(tx->tx_txg != 0);
1011 	ASSERT(!refcount_is_zero(&db->db_holds));
1012 	DMU_TX_DIRTY_BUF(tx, db);
1013 
1014 	DB_DNODE_ENTER(db);
1015 	dn = DB_DNODE(db);
1016 	/*
1017 	 * Shouldn't dirty a regular buffer in syncing context.  Private
1018 	 * objects may be dirtied in syncing context, but only if they
1019 	 * were already pre-dirtied in open context.
1020 	 */
1021 	ASSERT(!dmu_tx_is_syncing(tx) ||
1022 	    BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
1023 	    DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1024 	    dn->dn_objset->os_dsl_dataset == NULL);
1025 	/*
1026 	 * We make this assert for private objects as well, but after we
1027 	 * check if we're already dirty.  They are allowed to re-dirty
1028 	 * in syncing context.
1029 	 */
1030 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1031 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1032 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1033 
1034 	mutex_enter(&db->db_mtx);
1035 	/*
1036 	 * XXX make this true for indirects too?  The problem is that
1037 	 * transactions created with dmu_tx_create_assigned() from
1038 	 * syncing context don't bother holding ahead.
1039 	 */
1040 	ASSERT(db->db_level != 0 ||
1041 	    db->db_state == DB_CACHED || db->db_state == DB_FILL ||
1042 	    db->db_state == DB_NOFILL);
1043 
1044 	mutex_enter(&dn->dn_mtx);
1045 	/*
1046 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
1047 	 * initialize the objset.
1048 	 */
1049 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
1050 	    !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
1051 		dn->dn_dirtyctx =
1052 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
1053 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
1054 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
1055 	}
1056 	mutex_exit(&dn->dn_mtx);
1057 
1058 	if (db->db_blkid == DMU_SPILL_BLKID)
1059 		dn->dn_have_spill = B_TRUE;
1060 
1061 	/*
1062 	 * If this buffer is already dirty, we're done.
1063 	 */
1064 	drp = &db->db_last_dirty;
1065 	ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
1066 	    db->db.db_object == DMU_META_DNODE_OBJECT);
1067 	while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
1068 		drp = &dr->dr_next;
1069 	if (dr && dr->dr_txg == tx->tx_txg) {
1070 		DB_DNODE_EXIT(db);
1071 
1072 		if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
1073 			/*
1074 			 * If this buffer has already been written out,
1075 			 * we now need to reset its state.
1076 			 */
1077 			dbuf_unoverride(dr);
1078 			if (db->db.db_object != DMU_META_DNODE_OBJECT &&
1079 			    db->db_state != DB_NOFILL)
1080 				arc_buf_thaw(db->db_buf);
1081 		}
1082 		mutex_exit(&db->db_mtx);
1083 		return (dr);
1084 	}
1085 
1086 	/*
1087 	 * Only valid if not already dirty.
1088 	 */
1089 	ASSERT(dn->dn_object == 0 ||
1090 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
1091 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
1092 
1093 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
1094 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
1095 	    dn->dn_phys->dn_nlevels > db->db_level ||
1096 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
1097 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
1098 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
1099 
1100 	/*
1101 	 * We should only be dirtying in syncing context if it's the
1102 	 * mos or we're initializing the os or it's a special object.
1103 	 * However, we are allowed to dirty in syncing context provided
1104 	 * we already dirtied it in open context.  Hence we must make
1105 	 * this assertion only if we're not already dirty.
1106 	 */
1107 	os = dn->dn_objset;
1108 	ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) ||
1109 	    os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp));
1110 	ASSERT(db->db.db_size != 0);
1111 
1112 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1113 
1114 	if (db->db_blkid != DMU_BONUS_BLKID) {
1115 		/*
1116 		 * Update the accounting.
1117 		 * Note: we delay "free accounting" until after we drop
1118 		 * the db_mtx.  This keeps us from grabbing other locks
1119 		 * (and possibly deadlocking) in bp_get_dsize() while
1120 		 * also holding the db_mtx.
1121 		 */
1122 		dnode_willuse_space(dn, db->db.db_size, tx);
1123 		do_free_accounting = dbuf_block_freeable(db);
1124 	}
1125 
1126 	/*
1127 	 * If this buffer is dirty in an old transaction group we need
1128 	 * to make a copy of it so that the changes we make in this
1129 	 * transaction group won't leak out when we sync the older txg.
1130 	 */
1131 	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1132 	if (db->db_level == 0) {
1133 		void *data_old = db->db_buf;
1134 
1135 		if (db->db_state != DB_NOFILL) {
1136 			if (db->db_blkid == DMU_BONUS_BLKID) {
1137 				dbuf_fix_old_data(db, tx->tx_txg);
1138 				data_old = db->db.db_data;
1139 			} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1140 				/*
1141 				 * Release the data buffer from the cache so
1142 				 * that we can modify it without impacting
1143 				 * possible other users of this cached data
1144 				 * block.  Note that indirect blocks and
1145 				 * private objects are not released until the
1146 				 * syncing state (since they are only modified
1147 				 * then).
1148 				 */
1149 				arc_release(db->db_buf, db);
1150 				dbuf_fix_old_data(db, tx->tx_txg);
1151 				data_old = db->db_buf;
1152 			}
1153 			ASSERT(data_old != NULL);
1154 		}
1155 		dr->dt.dl.dr_data = data_old;
1156 	} else {
1157 		mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1158 		list_create(&dr->dt.di.dr_children,
1159 		    sizeof (dbuf_dirty_record_t),
1160 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
1161 	}
1162 	dr->dr_dbuf = db;
1163 	dr->dr_txg = tx->tx_txg;
1164 	dr->dr_next = *drp;
1165 	*drp = dr;
1166 
1167 	/*
1168 	 * We could have been freed_in_flight between the dbuf_noread
1169 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1170 	 * happened after the free.
1171 	 */
1172 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1173 	    db->db_blkid != DMU_SPILL_BLKID) {
1174 		mutex_enter(&dn->dn_mtx);
1175 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1176 		mutex_exit(&dn->dn_mtx);
1177 		db->db_freed_in_flight = FALSE;
1178 	}
1179 
1180 	/*
1181 	 * This buffer is now part of this txg
1182 	 */
1183 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1184 	db->db_dirtycnt += 1;
1185 	ASSERT3U(db->db_dirtycnt, <=, 3);
1186 
1187 	mutex_exit(&db->db_mtx);
1188 
1189 	if (db->db_blkid == DMU_BONUS_BLKID ||
1190 	    db->db_blkid == DMU_SPILL_BLKID) {
1191 		mutex_enter(&dn->dn_mtx);
1192 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1193 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1194 		mutex_exit(&dn->dn_mtx);
1195 		dnode_setdirty(dn, tx);
1196 		DB_DNODE_EXIT(db);
1197 		return (dr);
1198 	} else if (do_free_accounting) {
1199 		blkptr_t *bp = db->db_blkptr;
1200 		int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1201 		    bp_get_dsize(os->os_spa, bp) : db->db.db_size;
1202 		/*
1203 		 * This is only a guess -- if the dbuf is dirty
1204 		 * in a previous txg, we don't know how much
1205 		 * space it will use on disk yet.  We should
1206 		 * really have the struct_rwlock to access
1207 		 * db_blkptr, but since this is just a guess,
1208 		 * it's OK if we get an odd answer.
1209 		 */
1210 		ddt_prefetch(os->os_spa, bp);
1211 		dnode_willuse_space(dn, -willfree, tx);
1212 	}
1213 
1214 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1215 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1216 		drop_struct_lock = TRUE;
1217 	}
1218 
1219 	if (db->db_level == 0) {
1220 		dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
1221 		ASSERT(dn->dn_maxblkid >= db->db_blkid);
1222 	}
1223 
1224 	if (db->db_level+1 < dn->dn_nlevels) {
1225 		dmu_buf_impl_t *parent = db->db_parent;
1226 		dbuf_dirty_record_t *di;
1227 		int parent_held = FALSE;
1228 
1229 		if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1230 			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1231 
1232 			parent = dbuf_hold_level(dn, db->db_level+1,
1233 			    db->db_blkid >> epbs, FTAG);
1234 			ASSERT(parent != NULL);
1235 			parent_held = TRUE;
1236 		}
1237 		if (drop_struct_lock)
1238 			rw_exit(&dn->dn_struct_rwlock);
1239 		ASSERT3U(db->db_level+1, ==, parent->db_level);
1240 		di = dbuf_dirty(parent, tx);
1241 		if (parent_held)
1242 			dbuf_rele(parent, FTAG);
1243 
1244 		mutex_enter(&db->db_mtx);
1245 		/*  possible race with dbuf_undirty() */
1246 		if (db->db_last_dirty == dr ||
1247 		    dn->dn_object == DMU_META_DNODE_OBJECT) {
1248 			mutex_enter(&di->dt.di.dr_mtx);
1249 			ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1250 			ASSERT(!list_link_active(&dr->dr_dirty_node));
1251 			list_insert_tail(&di->dt.di.dr_children, dr);
1252 			mutex_exit(&di->dt.di.dr_mtx);
1253 			dr->dr_parent = di;
1254 		}
1255 		mutex_exit(&db->db_mtx);
1256 	} else {
1257 		ASSERT(db->db_level+1 == dn->dn_nlevels);
1258 		ASSERT(db->db_blkid < dn->dn_nblkptr);
1259 		ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
1260 		mutex_enter(&dn->dn_mtx);
1261 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1262 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1263 		mutex_exit(&dn->dn_mtx);
1264 		if (drop_struct_lock)
1265 			rw_exit(&dn->dn_struct_rwlock);
1266 	}
1267 
1268 	dnode_setdirty(dn, tx);
1269 	DB_DNODE_EXIT(db);
1270 	return (dr);
1271 }
1272 
1273 static int
1274 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1275 {
1276 	dnode_t *dn;
1277 	uint64_t txg = tx->tx_txg;
1278 	dbuf_dirty_record_t *dr, **drp;
1279 
1280 	ASSERT(txg != 0);
1281 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1282 
1283 	mutex_enter(&db->db_mtx);
1284 	/*
1285 	 * If this buffer is not dirty, we're done.
1286 	 */
1287 	for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1288 		if (dr->dr_txg <= txg)
1289 			break;
1290 	if (dr == NULL || dr->dr_txg < txg) {
1291 		mutex_exit(&db->db_mtx);
1292 		return (0);
1293 	}
1294 	ASSERT(dr->dr_txg == txg);
1295 	ASSERT(dr->dr_dbuf == db);
1296 
1297 	DB_DNODE_ENTER(db);
1298 	dn = DB_DNODE(db);
1299 
1300 	/*
1301 	 * If this buffer is currently held, we cannot undirty
1302 	 * it, since one of the current holders may be in the
1303 	 * middle of an update.  Note that users of dbuf_undirty()
1304 	 * should not place a hold on the dbuf before the call.
1305 	 * Also note: we can get here with a spill block, so
1306 	 * test for that similar to how dbuf_dirty does.
1307 	 */
1308 	if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1309 		mutex_exit(&db->db_mtx);
1310 		/* Make sure we don't toss this buffer at sync phase */
1311 		if (db->db_blkid != DMU_SPILL_BLKID) {
1312 			mutex_enter(&dn->dn_mtx);
1313 			dnode_clear_range(dn, db->db_blkid, 1, tx);
1314 			mutex_exit(&dn->dn_mtx);
1315 		}
1316 		DB_DNODE_EXIT(db);
1317 		return (0);
1318 	}
1319 
1320 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1321 
1322 	ASSERT(db->db.db_size != 0);
1323 
1324 	/* XXX would be nice to fix up dn_towrite_space[] */
1325 
1326 	*drp = dr->dr_next;
1327 
1328 	/*
1329 	 * Note that there are three places in dbuf_dirty()
1330 	 * where this dirty record may be put on a list.
1331 	 * Make sure to do a list_remove corresponding to
1332 	 * every one of those list_insert calls.
1333 	 */
1334 	if (dr->dr_parent) {
1335 		mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1336 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1337 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1338 	} else if (db->db_blkid == DMU_SPILL_BLKID ||
1339 	    db->db_level+1 == dn->dn_nlevels) {
1340 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1341 		mutex_enter(&dn->dn_mtx);
1342 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1343 		mutex_exit(&dn->dn_mtx);
1344 	}
1345 	DB_DNODE_EXIT(db);
1346 
1347 	if (db->db_level == 0) {
1348 		if (db->db_state != DB_NOFILL) {
1349 			dbuf_unoverride(dr);
1350 
1351 			ASSERT(db->db_buf != NULL);
1352 			ASSERT(dr->dt.dl.dr_data != NULL);
1353 			if (dr->dt.dl.dr_data != db->db_buf)
1354 				VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
1355 				    db) == 1);
1356 		}
1357 	} else {
1358 		ASSERT(db->db_buf != NULL);
1359 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1360 		mutex_destroy(&dr->dt.di.dr_mtx);
1361 		list_destroy(&dr->dt.di.dr_children);
1362 	}
1363 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
1364 
1365 	ASSERT(db->db_dirtycnt > 0);
1366 	db->db_dirtycnt -= 1;
1367 
1368 	if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1369 		arc_buf_t *buf = db->db_buf;
1370 
1371 		ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
1372 		dbuf_set_data(db, NULL);
1373 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
1374 		dbuf_evict(db);
1375 		return (1);
1376 	}
1377 
1378 	mutex_exit(&db->db_mtx);
1379 	return (0);
1380 }
1381 
1382 #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1383 void
1384 dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1385 {
1386 	int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1387 
1388 	ASSERT(tx->tx_txg != 0);
1389 	ASSERT(!refcount_is_zero(&db->db_holds));
1390 
1391 	DB_DNODE_ENTER(db);
1392 	if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
1393 		rf |= DB_RF_HAVESTRUCT;
1394 	DB_DNODE_EXIT(db);
1395 	(void) dbuf_read(db, NULL, rf);
1396 	(void) dbuf_dirty(db, tx);
1397 }
1398 
1399 void
1400 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1401 {
1402 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1403 
1404 	db->db_state = DB_NOFILL;
1405 
1406 	dmu_buf_will_fill(db_fake, tx);
1407 }
1408 
1409 void
1410 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1411 {
1412 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1413 
1414 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1415 	ASSERT(tx->tx_txg != 0);
1416 	ASSERT(db->db_level == 0);
1417 	ASSERT(!refcount_is_zero(&db->db_holds));
1418 
1419 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1420 	    dmu_tx_private_ok(tx));
1421 
1422 	dbuf_noread(db);
1423 	(void) dbuf_dirty(db, tx);
1424 }
1425 
1426 #pragma weak dmu_buf_fill_done = dbuf_fill_done
1427 /* ARGSUSED */
1428 void
1429 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1430 {
1431 	mutex_enter(&db->db_mtx);
1432 	DBUF_VERIFY(db);
1433 
1434 	if (db->db_state == DB_FILL) {
1435 		if (db->db_level == 0 && db->db_freed_in_flight) {
1436 			ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1437 			/* we were freed while filling */
1438 			/* XXX dbuf_undirty? */
1439 			bzero(db->db.db_data, db->db.db_size);
1440 			db->db_freed_in_flight = FALSE;
1441 		}
1442 		db->db_state = DB_CACHED;
1443 		cv_broadcast(&db->db_changed);
1444 	}
1445 	mutex_exit(&db->db_mtx);
1446 }
1447 
1448 /*
1449  * Directly assign a provided arc buf to a given dbuf if it's not referenced
1450  * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf.
1451  */
1452 void
1453 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
1454 {
1455 	ASSERT(!refcount_is_zero(&db->db_holds));
1456 	ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1457 	ASSERT(db->db_level == 0);
1458 	ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA);
1459 	ASSERT(buf != NULL);
1460 	ASSERT(arc_buf_size(buf) == db->db.db_size);
1461 	ASSERT(tx->tx_txg != 0);
1462 
1463 	arc_return_buf(buf, db);
1464 	ASSERT(arc_released(buf));
1465 
1466 	mutex_enter(&db->db_mtx);
1467 
1468 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
1469 		cv_wait(&db->db_changed, &db->db_mtx);
1470 
1471 	ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED);
1472 
1473 	if (db->db_state == DB_CACHED &&
1474 	    refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
1475 		mutex_exit(&db->db_mtx);
1476 		(void) dbuf_dirty(db, tx);
1477 		bcopy(buf->b_data, db->db.db_data, db->db.db_size);
1478 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
1479 		xuio_stat_wbuf_copied();
1480 		return;
1481 	}
1482 
1483 	xuio_stat_wbuf_nocopy();
1484 	if (db->db_state == DB_CACHED) {
1485 		dbuf_dirty_record_t *dr = db->db_last_dirty;
1486 
1487 		ASSERT(db->db_buf != NULL);
1488 		if (dr != NULL && dr->dr_txg == tx->tx_txg) {
1489 			ASSERT(dr->dt.dl.dr_data == db->db_buf);
1490 			if (!arc_released(db->db_buf)) {
1491 				ASSERT(dr->dt.dl.dr_override_state ==
1492 				    DR_OVERRIDDEN);
1493 				arc_release(db->db_buf, db);
1494 			}
1495 			dr->dt.dl.dr_data = buf;
1496 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1);
1497 		} else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
1498 			arc_release(db->db_buf, db);
1499 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 1);
1500 		}
1501 		db->db_buf = NULL;
1502 	}
1503 	ASSERT(db->db_buf == NULL);
1504 	dbuf_set_data(db, buf);
1505 	db->db_state = DB_FILL;
1506 	mutex_exit(&db->db_mtx);
1507 	(void) dbuf_dirty(db, tx);
1508 	dbuf_fill_done(db, tx);
1509 }
1510 
1511 /*
1512  * "Clear" the contents of this dbuf.  This will mark the dbuf
1513  * EVICTING and clear *most* of its references.  Unfortunetely,
1514  * when we are not holding the dn_dbufs_mtx, we can't clear the
1515  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1516  * in this case.  For callers from the DMU we will usually see:
1517  *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1518  * For the arc callback, we will usually see:
1519  *	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1520  * Sometimes, though, we will get a mix of these two:
1521  *	DMU: dbuf_clear()->arc_buf_evict()
1522  *	ARC: dbuf_do_evict()->dbuf_destroy()
1523  */
1524 void
1525 dbuf_clear(dmu_buf_impl_t *db)
1526 {
1527 	dnode_t *dn;
1528 	dmu_buf_impl_t *parent = db->db_parent;
1529 	dmu_buf_impl_t *dndb;
1530 	int dbuf_gone = FALSE;
1531 
1532 	ASSERT(MUTEX_HELD(&db->db_mtx));
1533 	ASSERT(refcount_is_zero(&db->db_holds));
1534 
1535 	dbuf_evict_user(db);
1536 
1537 	if (db->db_state == DB_CACHED) {
1538 		ASSERT(db->db.db_data != NULL);
1539 		if (db->db_blkid == DMU_BONUS_BLKID) {
1540 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
1541 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
1542 		}
1543 		db->db.db_data = NULL;
1544 		db->db_state = DB_UNCACHED;
1545 	}
1546 
1547 	ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1548 	ASSERT(db->db_data_pending == NULL);
1549 
1550 	db->db_state = DB_EVICTING;
1551 	db->db_blkptr = NULL;
1552 
1553 	DB_DNODE_ENTER(db);
1554 	dn = DB_DNODE(db);
1555 	dndb = dn->dn_dbuf;
1556 	if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1557 		list_remove(&dn->dn_dbufs, db);
1558 		(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1559 		membar_producer();
1560 		DB_DNODE_EXIT(db);
1561 		/*
1562 		 * Decrementing the dbuf count means that the hold corresponding
1563 		 * to the removed dbuf is no longer discounted in dnode_move(),
1564 		 * so the dnode cannot be moved until after we release the hold.
1565 		 * The membar_producer() ensures visibility of the decremented
1566 		 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually
1567 		 * release any lock.
1568 		 */
1569 		dnode_rele(dn, db);
1570 		db->db_dnode_handle = NULL;
1571 	} else {
1572 		DB_DNODE_EXIT(db);
1573 	}
1574 
1575 	if (db->db_buf)
1576 		dbuf_gone = arc_buf_evict(db->db_buf);
1577 
1578 	if (!dbuf_gone)
1579 		mutex_exit(&db->db_mtx);
1580 
1581 	/*
1582 	 * If this dbuf is referenced from an indirect dbuf,
1583 	 * decrement the ref count on the indirect dbuf.
1584 	 */
1585 	if (parent && parent != dndb)
1586 		dbuf_rele(parent, db);
1587 }
1588 
1589 static int
1590 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1591     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1592 {
1593 	int nlevels, epbs;
1594 
1595 	*parentp = NULL;
1596 	*bpp = NULL;
1597 
1598 	ASSERT(blkid != DMU_BONUS_BLKID);
1599 
1600 	if (blkid == DMU_SPILL_BLKID) {
1601 		mutex_enter(&dn->dn_mtx);
1602 		if (dn->dn_have_spill &&
1603 		    (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR))
1604 			*bpp = &dn->dn_phys->dn_spill;
1605 		else
1606 			*bpp = NULL;
1607 		dbuf_add_ref(dn->dn_dbuf, NULL);
1608 		*parentp = dn->dn_dbuf;
1609 		mutex_exit(&dn->dn_mtx);
1610 		return (0);
1611 	}
1612 
1613 	if (dn->dn_phys->dn_nlevels == 0)
1614 		nlevels = 1;
1615 	else
1616 		nlevels = dn->dn_phys->dn_nlevels;
1617 
1618 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1619 
1620 	ASSERT3U(level * epbs, <, 64);
1621 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1622 	if (level >= nlevels ||
1623 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1624 		/* the buffer has no parent yet */
1625 		return (ENOENT);
1626 	} else if (level < nlevels-1) {
1627 		/* this block is referenced from an indirect block */
1628 		int err = dbuf_hold_impl(dn, level+1,
1629 		    blkid >> epbs, fail_sparse, NULL, parentp);
1630 		if (err)
1631 			return (err);
1632 		err = dbuf_read(*parentp, NULL,
1633 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1634 		if (err) {
1635 			dbuf_rele(*parentp, NULL);
1636 			*parentp = NULL;
1637 			return (err);
1638 		}
1639 		*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1640 		    (blkid & ((1ULL << epbs) - 1));
1641 		return (0);
1642 	} else {
1643 		/* the block is referenced from the dnode */
1644 		ASSERT3U(level, ==, nlevels-1);
1645 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1646 		    blkid < dn->dn_phys->dn_nblkptr);
1647 		if (dn->dn_dbuf) {
1648 			dbuf_add_ref(dn->dn_dbuf, NULL);
1649 			*parentp = dn->dn_dbuf;
1650 		}
1651 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1652 		return (0);
1653 	}
1654 }
1655 
1656 static dmu_buf_impl_t *
1657 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1658     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1659 {
1660 	objset_t *os = dn->dn_objset;
1661 	dmu_buf_impl_t *db, *odb;
1662 
1663 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1664 	ASSERT(dn->dn_type != DMU_OT_NONE);
1665 
1666 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1667 
1668 	db->db_objset = os;
1669 	db->db.db_object = dn->dn_object;
1670 	db->db_level = level;
1671 	db->db_blkid = blkid;
1672 	db->db_last_dirty = NULL;
1673 	db->db_dirtycnt = 0;
1674 	db->db_dnode_handle = dn->dn_handle;
1675 	db->db_parent = parent;
1676 	db->db_blkptr = blkptr;
1677 
1678 	db->db_user_ptr = NULL;
1679 	db->db_user_data_ptr_ptr = NULL;
1680 	db->db_evict_func = NULL;
1681 	db->db_immediate_evict = 0;
1682 	db->db_freed_in_flight = 0;
1683 
1684 	if (blkid == DMU_BONUS_BLKID) {
1685 		ASSERT3P(parent, ==, dn->dn_dbuf);
1686 		db->db.db_size = DN_MAX_BONUSLEN -
1687 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
1688 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1689 		db->db.db_offset = DMU_BONUS_BLKID;
1690 		db->db_state = DB_UNCACHED;
1691 		/* the bonus dbuf is not placed in the hash table */
1692 		arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1693 		return (db);
1694 	} else if (blkid == DMU_SPILL_BLKID) {
1695 		db->db.db_size = (blkptr != NULL) ?
1696 		    BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE;
1697 		db->db.db_offset = 0;
1698 	} else {
1699 		int blocksize =
1700 		    db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1701 		db->db.db_size = blocksize;
1702 		db->db.db_offset = db->db_blkid * blocksize;
1703 	}
1704 
1705 	/*
1706 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1707 	 * in the hash table *and* added to the dbufs list.
1708 	 * This prevents a possible deadlock with someone
1709 	 * trying to look up this dbuf before its added to the
1710 	 * dn_dbufs list.
1711 	 */
1712 	mutex_enter(&dn->dn_dbufs_mtx);
1713 	db->db_state = DB_EVICTING;
1714 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1715 		/* someone else inserted it first */
1716 		kmem_cache_free(dbuf_cache, db);
1717 		mutex_exit(&dn->dn_dbufs_mtx);
1718 		return (odb);
1719 	}
1720 	list_insert_head(&dn->dn_dbufs, db);
1721 	db->db_state = DB_UNCACHED;
1722 	mutex_exit(&dn->dn_dbufs_mtx);
1723 	arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1724 
1725 	if (parent && parent != dn->dn_dbuf)
1726 		dbuf_add_ref(parent, db);
1727 
1728 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1729 	    refcount_count(&dn->dn_holds) > 0);
1730 	(void) refcount_add(&dn->dn_holds, db);
1731 	(void) atomic_inc_32_nv(&dn->dn_dbufs_count);
1732 
1733 	dprintf_dbuf(db, "db=%p\n", db);
1734 
1735 	return (db);
1736 }
1737 
1738 static int
1739 dbuf_do_evict(void *private)
1740 {
1741 	arc_buf_t *buf = private;
1742 	dmu_buf_impl_t *db = buf->b_private;
1743 
1744 	if (!MUTEX_HELD(&db->db_mtx))
1745 		mutex_enter(&db->db_mtx);
1746 
1747 	ASSERT(refcount_is_zero(&db->db_holds));
1748 
1749 	if (db->db_state != DB_EVICTING) {
1750 		ASSERT(db->db_state == DB_CACHED);
1751 		DBUF_VERIFY(db);
1752 		db->db_buf = NULL;
1753 		dbuf_evict(db);
1754 	} else {
1755 		mutex_exit(&db->db_mtx);
1756 		dbuf_destroy(db);
1757 	}
1758 	return (0);
1759 }
1760 
1761 static void
1762 dbuf_destroy(dmu_buf_impl_t *db)
1763 {
1764 	ASSERT(refcount_is_zero(&db->db_holds));
1765 
1766 	if (db->db_blkid != DMU_BONUS_BLKID) {
1767 		/*
1768 		 * If this dbuf is still on the dn_dbufs list,
1769 		 * remove it from that list.
1770 		 */
1771 		if (db->db_dnode_handle != NULL) {
1772 			dnode_t *dn;
1773 
1774 			DB_DNODE_ENTER(db);
1775 			dn = DB_DNODE(db);
1776 			mutex_enter(&dn->dn_dbufs_mtx);
1777 			list_remove(&dn->dn_dbufs, db);
1778 			(void) atomic_dec_32_nv(&dn->dn_dbufs_count);
1779 			mutex_exit(&dn->dn_dbufs_mtx);
1780 			DB_DNODE_EXIT(db);
1781 			/*
1782 			 * Decrementing the dbuf count means that the hold
1783 			 * corresponding to the removed dbuf is no longer
1784 			 * discounted in dnode_move(), so the dnode cannot be
1785 			 * moved until after we release the hold.
1786 			 */
1787 			dnode_rele(dn, db);
1788 			db->db_dnode_handle = NULL;
1789 		}
1790 		dbuf_hash_remove(db);
1791 	}
1792 	db->db_parent = NULL;
1793 	db->db_buf = NULL;
1794 
1795 	ASSERT(!list_link_active(&db->db_link));
1796 	ASSERT(db->db.db_data == NULL);
1797 	ASSERT(db->db_hash_next == NULL);
1798 	ASSERT(db->db_blkptr == NULL);
1799 	ASSERT(db->db_data_pending == NULL);
1800 
1801 	kmem_cache_free(dbuf_cache, db);
1802 	arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER);
1803 }
1804 
1805 void
1806 dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1807 {
1808 	dmu_buf_impl_t *db = NULL;
1809 	blkptr_t *bp = NULL;
1810 
1811 	ASSERT(blkid != DMU_BONUS_BLKID);
1812 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1813 
1814 	if (dnode_block_freed(dn, blkid))
1815 		return;
1816 
1817 	/* dbuf_find() returns with db_mtx held */
1818 	if (db = dbuf_find(dn, 0, blkid)) {
1819 		/*
1820 		 * This dbuf is already in the cache.  We assume that
1821 		 * it is already CACHED, or else about to be either
1822 		 * read or filled.
1823 		 */
1824 		mutex_exit(&db->db_mtx);
1825 		return;
1826 	}
1827 
1828 	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
1829 		if (bp && !BP_IS_HOLE(bp)) {
1830 			int priority = dn->dn_type == DMU_OT_DDT_ZAP ?
1831 			    ZIO_PRIORITY_DDT_PREFETCH : ZIO_PRIORITY_ASYNC_READ;
1832 			arc_buf_t *pbuf;
1833 			dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
1834 			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
1835 			zbookmark_t zb;
1836 
1837 			SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
1838 			    dn->dn_object, 0, blkid);
1839 
1840 			if (db)
1841 				pbuf = db->db_buf;
1842 			else
1843 				pbuf = dn->dn_objset->os_phys_buf;
1844 
1845 			(void) dsl_read(NULL, dn->dn_objset->os_spa,
1846 			    bp, pbuf, NULL, NULL, priority,
1847 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
1848 			    &aflags, &zb);
1849 		}
1850 		if (db)
1851 			dbuf_rele(db, NULL);
1852 	}
1853 }
1854 
1855 /*
1856  * Returns with db_holds incremented, and db_mtx not held.
1857  * Note: dn_struct_rwlock must be held.
1858  */
1859 int
1860 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1861     void *tag, dmu_buf_impl_t **dbp)
1862 {
1863 	dmu_buf_impl_t *db, *parent = NULL;
1864 
1865 	ASSERT(blkid != DMU_BONUS_BLKID);
1866 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1867 	ASSERT3U(dn->dn_nlevels, >, level);
1868 
1869 	*dbp = NULL;
1870 top:
1871 	/* dbuf_find() returns with db_mtx held */
1872 	db = dbuf_find(dn, level, blkid);
1873 
1874 	if (db == NULL) {
1875 		blkptr_t *bp = NULL;
1876 		int err;
1877 
1878 		ASSERT3P(parent, ==, NULL);
1879 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1880 		if (fail_sparse) {
1881 			if (err == 0 && bp && BP_IS_HOLE(bp))
1882 				err = ENOENT;
1883 			if (err) {
1884 				if (parent)
1885 					dbuf_rele(parent, NULL);
1886 				return (err);
1887 			}
1888 		}
1889 		if (err && err != ENOENT)
1890 			return (err);
1891 		db = dbuf_create(dn, level, blkid, parent, bp);
1892 	}
1893 
1894 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1895 		arc_buf_add_ref(db->db_buf, db);
1896 		if (db->db_buf->b_data == NULL) {
1897 			dbuf_clear(db);
1898 			if (parent) {
1899 				dbuf_rele(parent, NULL);
1900 				parent = NULL;
1901 			}
1902 			goto top;
1903 		}
1904 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1905 	}
1906 
1907 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1908 
1909 	/*
1910 	 * If this buffer is currently syncing out, and we are are
1911 	 * still referencing it from db_data, we need to make a copy
1912 	 * of it in case we decide we want to dirty it again in this txg.
1913 	 */
1914 	if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1915 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
1916 	    db->db_state == DB_CACHED && db->db_data_pending) {
1917 		dbuf_dirty_record_t *dr = db->db_data_pending;
1918 
1919 		if (dr->dt.dl.dr_data == db->db_buf) {
1920 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1921 
1922 			dbuf_set_data(db,
1923 			    arc_buf_alloc(dn->dn_objset->os_spa,
1924 			    db->db.db_size, db, type));
1925 			bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
1926 			    db->db.db_size);
1927 		}
1928 	}
1929 
1930 	(void) refcount_add(&db->db_holds, tag);
1931 	dbuf_update_data(db);
1932 	DBUF_VERIFY(db);
1933 	mutex_exit(&db->db_mtx);
1934 
1935 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
1936 	if (parent)
1937 		dbuf_rele(parent, NULL);
1938 
1939 	ASSERT3P(DB_DNODE(db), ==, dn);
1940 	ASSERT3U(db->db_blkid, ==, blkid);
1941 	ASSERT3U(db->db_level, ==, level);
1942 	*dbp = db;
1943 
1944 	return (0);
1945 }
1946 
1947 dmu_buf_impl_t *
1948 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1949 {
1950 	dmu_buf_impl_t *db;
1951 	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1952 	return (err ? NULL : db);
1953 }
1954 
1955 dmu_buf_impl_t *
1956 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1957 {
1958 	dmu_buf_impl_t *db;
1959 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1960 	return (err ? NULL : db);
1961 }
1962 
1963 void
1964 dbuf_create_bonus(dnode_t *dn)
1965 {
1966 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1967 
1968 	ASSERT(dn->dn_bonus == NULL);
1969 	dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL);
1970 }
1971 
1972 int
1973 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx)
1974 {
1975 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1976 	dnode_t *dn;
1977 
1978 	if (db->db_blkid != DMU_SPILL_BLKID)
1979 		return (ENOTSUP);
1980 	if (blksz == 0)
1981 		blksz = SPA_MINBLOCKSIZE;
1982 	if (blksz > SPA_MAXBLOCKSIZE)
1983 		blksz = SPA_MAXBLOCKSIZE;
1984 	else
1985 		blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE);
1986 
1987 	DB_DNODE_ENTER(db);
1988 	dn = DB_DNODE(db);
1989 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1990 	dbuf_new_size(db, blksz, tx);
1991 	rw_exit(&dn->dn_struct_rwlock);
1992 	DB_DNODE_EXIT(db);
1993 
1994 	return (0);
1995 }
1996 
1997 void
1998 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx)
1999 {
2000 	dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx);
2001 }
2002 
2003 #pragma weak dmu_buf_add_ref = dbuf_add_ref
2004 void
2005 dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
2006 {
2007 	int64_t holds = refcount_add(&db->db_holds, tag);
2008 	ASSERT(holds > 1);
2009 }
2010 
2011 /*
2012  * If you call dbuf_rele() you had better not be referencing the dnode handle
2013  * unless you have some other direct or indirect hold on the dnode. (An indirect
2014  * hold is a hold on one of the dnode's dbufs, including the bonus buffer.)
2015  * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the
2016  * dnode's parent dbuf evicting its dnode handles.
2017  */
2018 #pragma weak dmu_buf_rele = dbuf_rele
2019 void
2020 dbuf_rele(dmu_buf_impl_t *db, void *tag)
2021 {
2022 	mutex_enter(&db->db_mtx);
2023 	dbuf_rele_and_unlock(db, tag);
2024 }
2025 
2026 /*
2027  * dbuf_rele() for an already-locked dbuf.  This is necessary to allow
2028  * db_dirtycnt and db_holds to be updated atomically.
2029  */
2030 void
2031 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
2032 {
2033 	int64_t holds;
2034 
2035 	ASSERT(MUTEX_HELD(&db->db_mtx));
2036 	DBUF_VERIFY(db);
2037 
2038 	/*
2039 	 * Remove the reference to the dbuf before removing its hold on the
2040 	 * dnode so we can guarantee in dnode_move() that a referenced bonus
2041 	 * buffer has a corresponding dnode hold.
2042 	 */
2043 	holds = refcount_remove(&db->db_holds, tag);
2044 	ASSERT(holds >= 0);
2045 
2046 	/*
2047 	 * We can't freeze indirects if there is a possibility that they
2048 	 * may be modified in the current syncing context.
2049 	 */
2050 	if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
2051 		arc_buf_freeze(db->db_buf);
2052 
2053 	if (holds == db->db_dirtycnt &&
2054 	    db->db_level == 0 && db->db_immediate_evict)
2055 		dbuf_evict_user(db);
2056 
2057 	if (holds == 0) {
2058 		if (db->db_blkid == DMU_BONUS_BLKID) {
2059 			mutex_exit(&db->db_mtx);
2060 
2061 			/*
2062 			 * If the dnode moves here, we cannot cross this barrier
2063 			 * until the move completes.
2064 			 */
2065 			DB_DNODE_ENTER(db);
2066 			(void) atomic_dec_32_nv(&DB_DNODE(db)->dn_dbufs_count);
2067 			DB_DNODE_EXIT(db);
2068 			/*
2069 			 * The bonus buffer's dnode hold is no longer discounted
2070 			 * in dnode_move(). The dnode cannot move until after
2071 			 * the dnode_rele().
2072 			 */
2073 			dnode_rele(DB_DNODE(db), db);
2074 		} else if (db->db_buf == NULL) {
2075 			/*
2076 			 * This is a special case: we never associated this
2077 			 * dbuf with any data allocated from the ARC.
2078 			 */
2079 			ASSERT(db->db_state == DB_UNCACHED ||
2080 			    db->db_state == DB_NOFILL);
2081 			dbuf_evict(db);
2082 		} else if (arc_released(db->db_buf)) {
2083 			arc_buf_t *buf = db->db_buf;
2084 			/*
2085 			 * This dbuf has anonymous data associated with it.
2086 			 */
2087 			dbuf_set_data(db, NULL);
2088 			VERIFY(arc_buf_remove_ref(buf, db) == 1);
2089 			dbuf_evict(db);
2090 		} else {
2091 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
2092 			if (!DBUF_IS_CACHEABLE(db))
2093 				dbuf_clear(db);
2094 			else
2095 				mutex_exit(&db->db_mtx);
2096 		}
2097 	} else {
2098 		mutex_exit(&db->db_mtx);
2099 	}
2100 }
2101 
2102 #pragma weak dmu_buf_refcount = dbuf_refcount
2103 uint64_t
2104 dbuf_refcount(dmu_buf_impl_t *db)
2105 {
2106 	return (refcount_count(&db->db_holds));
2107 }
2108 
2109 void *
2110 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2111     dmu_buf_evict_func_t *evict_func)
2112 {
2113 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2114 	    user_data_ptr_ptr, evict_func));
2115 }
2116 
2117 void *
2118 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
2119     dmu_buf_evict_func_t *evict_func)
2120 {
2121 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2122 
2123 	db->db_immediate_evict = TRUE;
2124 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
2125 	    user_data_ptr_ptr, evict_func));
2126 }
2127 
2128 void *
2129 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
2130     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
2131 {
2132 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2133 	ASSERT(db->db_level == 0);
2134 
2135 	ASSERT((user_ptr == NULL) == (evict_func == NULL));
2136 
2137 	mutex_enter(&db->db_mtx);
2138 
2139 	if (db->db_user_ptr == old_user_ptr) {
2140 		db->db_user_ptr = user_ptr;
2141 		db->db_user_data_ptr_ptr = user_data_ptr_ptr;
2142 		db->db_evict_func = evict_func;
2143 
2144 		dbuf_update_data(db);
2145 	} else {
2146 		old_user_ptr = db->db_user_ptr;
2147 	}
2148 
2149 	mutex_exit(&db->db_mtx);
2150 	return (old_user_ptr);
2151 }
2152 
2153 void *
2154 dmu_buf_get_user(dmu_buf_t *db_fake)
2155 {
2156 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2157 	ASSERT(!refcount_is_zero(&db->db_holds));
2158 
2159 	return (db->db_user_ptr);
2160 }
2161 
2162 boolean_t
2163 dmu_buf_freeable(dmu_buf_t *dbuf)
2164 {
2165 	boolean_t res = B_FALSE;
2166 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2167 
2168 	if (db->db_blkptr)
2169 		res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset,
2170 		    db->db_blkptr, db->db_blkptr->blk_birth);
2171 
2172 	return (res);
2173 }
2174 
2175 static void
2176 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
2177 {
2178 	/* ASSERT(dmu_tx_is_syncing(tx) */
2179 	ASSERT(MUTEX_HELD(&db->db_mtx));
2180 
2181 	if (db->db_blkptr != NULL)
2182 		return;
2183 
2184 	if (db->db_blkid == DMU_SPILL_BLKID) {
2185 		db->db_blkptr = &dn->dn_phys->dn_spill;
2186 		BP_ZERO(db->db_blkptr);
2187 		return;
2188 	}
2189 	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
2190 		/*
2191 		 * This buffer was allocated at a time when there was
2192 		 * no available blkptrs from the dnode, or it was
2193 		 * inappropriate to hook it in (i.e., nlevels mis-match).
2194 		 */
2195 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
2196 		ASSERT(db->db_parent == NULL);
2197 		db->db_parent = dn->dn_dbuf;
2198 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
2199 		DBUF_VERIFY(db);
2200 	} else {
2201 		dmu_buf_impl_t *parent = db->db_parent;
2202 		int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2203 
2204 		ASSERT(dn->dn_phys->dn_nlevels > 1);
2205 		if (parent == NULL) {
2206 			mutex_exit(&db->db_mtx);
2207 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
2208 			(void) dbuf_hold_impl(dn, db->db_level+1,
2209 			    db->db_blkid >> epbs, FALSE, db, &parent);
2210 			rw_exit(&dn->dn_struct_rwlock);
2211 			mutex_enter(&db->db_mtx);
2212 			db->db_parent = parent;
2213 		}
2214 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
2215 		    (db->db_blkid & ((1ULL << epbs) - 1));
2216 		DBUF_VERIFY(db);
2217 	}
2218 }
2219 
2220 static void
2221 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2222 {
2223 	dmu_buf_impl_t *db = dr->dr_dbuf;
2224 	dnode_t *dn;
2225 	zio_t *zio;
2226 
2227 	ASSERT(dmu_tx_is_syncing(tx));
2228 
2229 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2230 
2231 	mutex_enter(&db->db_mtx);
2232 
2233 	ASSERT(db->db_level > 0);
2234 	DBUF_VERIFY(db);
2235 
2236 	if (db->db_buf == NULL) {
2237 		mutex_exit(&db->db_mtx);
2238 		(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
2239 		mutex_enter(&db->db_mtx);
2240 	}
2241 	ASSERT3U(db->db_state, ==, DB_CACHED);
2242 	ASSERT(db->db_buf != NULL);
2243 
2244 	DB_DNODE_ENTER(db);
2245 	dn = DB_DNODE(db);
2246 	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2247 	dbuf_check_blkptr(dn, db);
2248 	DB_DNODE_EXIT(db);
2249 
2250 	db->db_data_pending = dr;
2251 
2252 	mutex_exit(&db->db_mtx);
2253 	dbuf_write(dr, db->db_buf, tx);
2254 
2255 	zio = dr->dr_zio;
2256 	mutex_enter(&dr->dt.di.dr_mtx);
2257 	dbuf_sync_list(&dr->dt.di.dr_children, tx);
2258 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2259 	mutex_exit(&dr->dt.di.dr_mtx);
2260 	zio_nowait(zio);
2261 }
2262 
2263 static void
2264 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
2265 {
2266 	arc_buf_t **datap = &dr->dt.dl.dr_data;
2267 	dmu_buf_impl_t *db = dr->dr_dbuf;
2268 	dnode_t *dn;
2269 	objset_t *os;
2270 	uint64_t txg = tx->tx_txg;
2271 
2272 	ASSERT(dmu_tx_is_syncing(tx));
2273 
2274 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
2275 
2276 	mutex_enter(&db->db_mtx);
2277 	/*
2278 	 * To be synced, we must be dirtied.  But we
2279 	 * might have been freed after the dirty.
2280 	 */
2281 	if (db->db_state == DB_UNCACHED) {
2282 		/* This buffer has been freed since it was dirtied */
2283 		ASSERT(db->db.db_data == NULL);
2284 	} else if (db->db_state == DB_FILL) {
2285 		/* This buffer was freed and is now being re-filled */
2286 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
2287 	} else {
2288 		ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
2289 	}
2290 	DBUF_VERIFY(db);
2291 
2292 	DB_DNODE_ENTER(db);
2293 	dn = DB_DNODE(db);
2294 
2295 	if (db->db_blkid == DMU_SPILL_BLKID) {
2296 		mutex_enter(&dn->dn_mtx);
2297 		dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR;
2298 		mutex_exit(&dn->dn_mtx);
2299 	}
2300 
2301 	/*
2302 	 * If this is a bonus buffer, simply copy the bonus data into the
2303 	 * dnode.  It will be written out when the dnode is synced (and it
2304 	 * will be synced, since it must have been dirty for dbuf_sync to
2305 	 * be called).
2306 	 */
2307 	if (db->db_blkid == DMU_BONUS_BLKID) {
2308 		dbuf_dirty_record_t **drp;
2309 
2310 		ASSERT(*datap != NULL);
2311 		ASSERT0(db->db_level);
2312 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
2313 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
2314 		DB_DNODE_EXIT(db);
2315 
2316 		if (*datap != db->db.db_data) {
2317 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
2318 			arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER);
2319 		}
2320 		db->db_data_pending = NULL;
2321 		drp = &db->db_last_dirty;
2322 		while (*drp != dr)
2323 			drp = &(*drp)->dr_next;
2324 		ASSERT(dr->dr_next == NULL);
2325 		ASSERT(dr->dr_dbuf == db);
2326 		*drp = dr->dr_next;
2327 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
2328 		ASSERT(db->db_dirtycnt > 0);
2329 		db->db_dirtycnt -= 1;
2330 		dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2331 		return;
2332 	}
2333 
2334 	os = dn->dn_objset;
2335 
2336 	/*
2337 	 * This function may have dropped the db_mtx lock allowing a dmu_sync
2338 	 * operation to sneak in. As a result, we need to ensure that we
2339 	 * don't check the dr_override_state until we have returned from
2340 	 * dbuf_check_blkptr.
2341 	 */
2342 	dbuf_check_blkptr(dn, db);
2343 
2344 	/*
2345 	 * If this buffer is in the middle of an immediate write,
2346 	 * wait for the synchronous IO to complete.
2347 	 */
2348 	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
2349 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
2350 		cv_wait(&db->db_changed, &db->db_mtx);
2351 		ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
2352 	}
2353 
2354 	if (db->db_state != DB_NOFILL &&
2355 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
2356 	    refcount_count(&db->db_holds) > 1 &&
2357 	    dr->dt.dl.dr_override_state != DR_OVERRIDDEN &&
2358 	    *datap == db->db_buf) {
2359 		/*
2360 		 * If this buffer is currently "in use" (i.e., there
2361 		 * are active holds and db_data still references it),
2362 		 * then make a copy before we start the write so that
2363 		 * any modifications from the open txg will not leak
2364 		 * into this write.
2365 		 *
2366 		 * NOTE: this copy does not need to be made for
2367 		 * objects only modified in the syncing context (e.g.
2368 		 * DNONE_DNODE blocks).
2369 		 */
2370 		int blksz = arc_buf_size(*datap);
2371 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2372 		*datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2373 		bcopy(db->db.db_data, (*datap)->b_data, blksz);
2374 	}
2375 	db->db_data_pending = dr;
2376 
2377 	mutex_exit(&db->db_mtx);
2378 
2379 	dbuf_write(dr, *datap, tx);
2380 
2381 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2382 	if (dn->dn_object == DMU_META_DNODE_OBJECT) {
2383 		list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2384 		DB_DNODE_EXIT(db);
2385 	} else {
2386 		/*
2387 		 * Although zio_nowait() does not "wait for an IO", it does
2388 		 * initiate the IO. If this is an empty write it seems plausible
2389 		 * that the IO could actually be completed before the nowait
2390 		 * returns. We need to DB_DNODE_EXIT() first in case
2391 		 * zio_nowait() invalidates the dbuf.
2392 		 */
2393 		DB_DNODE_EXIT(db);
2394 		zio_nowait(dr->dr_zio);
2395 	}
2396 }
2397 
2398 void
2399 dbuf_sync_list(list_t *list, dmu_tx_t *tx)
2400 {
2401 	dbuf_dirty_record_t *dr;
2402 
2403 	while (dr = list_head(list)) {
2404 		if (dr->dr_zio != NULL) {
2405 			/*
2406 			 * If we find an already initialized zio then we
2407 			 * are processing the meta-dnode, and we have finished.
2408 			 * The dbufs for all dnodes are put back on the list
2409 			 * during processing, so that we can zio_wait()
2410 			 * these IOs after initiating all child IOs.
2411 			 */
2412 			ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2413 			    DMU_META_DNODE_OBJECT);
2414 			break;
2415 		}
2416 		list_remove(list, dr);
2417 		if (dr->dr_dbuf->db_level > 0)
2418 			dbuf_sync_indirect(dr, tx);
2419 		else
2420 			dbuf_sync_leaf(dr, tx);
2421 	}
2422 }
2423 
2424 /* ARGSUSED */
2425 static void
2426 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2427 {
2428 	dmu_buf_impl_t *db = vdb;
2429 	dnode_t *dn;
2430 	blkptr_t *bp = zio->io_bp;
2431 	blkptr_t *bp_orig = &zio->io_bp_orig;
2432 	spa_t *spa = zio->io_spa;
2433 	int64_t delta;
2434 	uint64_t fill = 0;
2435 	int i;
2436 
2437 	ASSERT(db->db_blkptr == bp);
2438 
2439 	DB_DNODE_ENTER(db);
2440 	dn = DB_DNODE(db);
2441 	delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig);
2442 	dnode_diduse_space(dn, delta - zio->io_prev_space_delta);
2443 	zio->io_prev_space_delta = delta;
2444 
2445 	if (BP_IS_HOLE(bp)) {
2446 		ASSERT(bp->blk_fill == 0);
2447 		DB_DNODE_EXIT(db);
2448 		return;
2449 	}
2450 
2451 	ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
2452 	    BP_GET_TYPE(bp) == dn->dn_type) ||
2453 	    (db->db_blkid == DMU_SPILL_BLKID &&
2454 	    BP_GET_TYPE(bp) == dn->dn_bonustype));
2455 	ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2456 
2457 	mutex_enter(&db->db_mtx);
2458 
2459 #ifdef ZFS_DEBUG
2460 	if (db->db_blkid == DMU_SPILL_BLKID) {
2461 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2462 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2463 		    db->db_blkptr == &dn->dn_phys->dn_spill);
2464 	}
2465 #endif
2466 
2467 	if (db->db_level == 0) {
2468 		mutex_enter(&dn->dn_mtx);
2469 		if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
2470 		    db->db_blkid != DMU_SPILL_BLKID)
2471 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2472 		mutex_exit(&dn->dn_mtx);
2473 
2474 		if (dn->dn_type == DMU_OT_DNODE) {
2475 			dnode_phys_t *dnp = db->db.db_data;
2476 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2477 			    i--, dnp++) {
2478 				if (dnp->dn_type != DMU_OT_NONE)
2479 					fill++;
2480 			}
2481 		} else {
2482 			fill = 1;
2483 		}
2484 	} else {
2485 		blkptr_t *ibp = db->db.db_data;
2486 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2487 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2488 			if (BP_IS_HOLE(ibp))
2489 				continue;
2490 			fill += ibp->blk_fill;
2491 		}
2492 	}
2493 	DB_DNODE_EXIT(db);
2494 
2495 	bp->blk_fill = fill;
2496 
2497 	mutex_exit(&db->db_mtx);
2498 }
2499 
2500 /* ARGSUSED */
2501 static void
2502 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2503 {
2504 	dmu_buf_impl_t *db = vdb;
2505 	blkptr_t *bp = zio->io_bp;
2506 	blkptr_t *bp_orig = &zio->io_bp_orig;
2507 	uint64_t txg = zio->io_txg;
2508 	dbuf_dirty_record_t **drp, *dr;
2509 
2510 	ASSERT0(zio->io_error);
2511 	ASSERT(db->db_blkptr == bp);
2512 
2513 	if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
2514 		ASSERT(BP_EQUAL(bp, bp_orig));
2515 	} else {
2516 		objset_t *os;
2517 		dsl_dataset_t *ds;
2518 		dmu_tx_t *tx;
2519 
2520 		DB_GET_OBJSET(&os, db);
2521 		ds = os->os_dsl_dataset;
2522 		tx = os->os_synctx;
2523 
2524 		(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
2525 		dsl_dataset_block_born(ds, bp, tx);
2526 	}
2527 
2528 	mutex_enter(&db->db_mtx);
2529 
2530 	DBUF_VERIFY(db);
2531 
2532 	drp = &db->db_last_dirty;
2533 	while ((dr = *drp) != db->db_data_pending)
2534 		drp = &dr->dr_next;
2535 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2536 	ASSERT(dr->dr_txg == txg);
2537 	ASSERT(dr->dr_dbuf == db);
2538 	ASSERT(dr->dr_next == NULL);
2539 	*drp = dr->dr_next;
2540 
2541 #ifdef ZFS_DEBUG
2542 	if (db->db_blkid == DMU_SPILL_BLKID) {
2543 		dnode_t *dn;
2544 
2545 		DB_DNODE_ENTER(db);
2546 		dn = DB_DNODE(db);
2547 		ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR);
2548 		ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
2549 		    db->db_blkptr == &dn->dn_phys->dn_spill);
2550 		DB_DNODE_EXIT(db);
2551 	}
2552 #endif
2553 
2554 	if (db->db_level == 0) {
2555 		ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2556 		ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2557 		if (db->db_state != DB_NOFILL) {
2558 			if (dr->dt.dl.dr_data != db->db_buf)
2559 				VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data,
2560 				    db) == 1);
2561 			else if (!arc_released(db->db_buf))
2562 				arc_set_callback(db->db_buf, dbuf_do_evict, db);
2563 		}
2564 	} else {
2565 		dnode_t *dn;
2566 
2567 		DB_DNODE_ENTER(db);
2568 		dn = DB_DNODE(db);
2569 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2570 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2571 		if (!BP_IS_HOLE(db->db_blkptr)) {
2572 			int epbs =
2573 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2574 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2575 			    db->db.db_size);
2576 			ASSERT3U(dn->dn_phys->dn_maxblkid
2577 			    >> (db->db_level * epbs), >=, db->db_blkid);
2578 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2579 		}
2580 		DB_DNODE_EXIT(db);
2581 		mutex_destroy(&dr->dt.di.dr_mtx);
2582 		list_destroy(&dr->dt.di.dr_children);
2583 	}
2584 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
2585 
2586 	cv_broadcast(&db->db_changed);
2587 	ASSERT(db->db_dirtycnt > 0);
2588 	db->db_dirtycnt -= 1;
2589 	db->db_data_pending = NULL;
2590 	dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg);
2591 }
2592 
2593 static void
2594 dbuf_write_nofill_ready(zio_t *zio)
2595 {
2596 	dbuf_write_ready(zio, NULL, zio->io_private);
2597 }
2598 
2599 static void
2600 dbuf_write_nofill_done(zio_t *zio)
2601 {
2602 	dbuf_write_done(zio, NULL, zio->io_private);
2603 }
2604 
2605 static void
2606 dbuf_write_override_ready(zio_t *zio)
2607 {
2608 	dbuf_dirty_record_t *dr = zio->io_private;
2609 	dmu_buf_impl_t *db = dr->dr_dbuf;
2610 
2611 	dbuf_write_ready(zio, NULL, db);
2612 }
2613 
2614 static void
2615 dbuf_write_override_done(zio_t *zio)
2616 {
2617 	dbuf_dirty_record_t *dr = zio->io_private;
2618 	dmu_buf_impl_t *db = dr->dr_dbuf;
2619 	blkptr_t *obp = &dr->dt.dl.dr_overridden_by;
2620 
2621 	mutex_enter(&db->db_mtx);
2622 	if (!BP_EQUAL(zio->io_bp, obp)) {
2623 		if (!BP_IS_HOLE(obp))
2624 			dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp);
2625 		arc_release(dr->dt.dl.dr_data, db);
2626 	}
2627 	mutex_exit(&db->db_mtx);
2628 
2629 	dbuf_write_done(zio, NULL, db);
2630 }
2631 
2632 static void
2633 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
2634 {
2635 	dmu_buf_impl_t *db = dr->dr_dbuf;
2636 	dnode_t *dn;
2637 	objset_t *os;
2638 	dmu_buf_impl_t *parent = db->db_parent;
2639 	uint64_t txg = tx->tx_txg;
2640 	zbookmark_t zb;
2641 	zio_prop_t zp;
2642 	zio_t *zio;
2643 	int wp_flag = 0;
2644 
2645 	DB_DNODE_ENTER(db);
2646 	dn = DB_DNODE(db);
2647 	os = dn->dn_objset;
2648 
2649 	if (db->db_state != DB_NOFILL) {
2650 		if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
2651 			/*
2652 			 * Private object buffers are released here rather
2653 			 * than in dbuf_dirty() since they are only modified
2654 			 * in the syncing context and we don't want the
2655 			 * overhead of making multiple copies of the data.
2656 			 */
2657 			if (BP_IS_HOLE(db->db_blkptr)) {
2658 				arc_buf_thaw(data);
2659 			} else {
2660 				dbuf_release_bp(db);
2661 			}
2662 		}
2663 	}
2664 
2665 	if (parent != dn->dn_dbuf) {
2666 		ASSERT(parent && parent->db_data_pending);
2667 		ASSERT(db->db_level == parent->db_level-1);
2668 		ASSERT(arc_released(parent->db_buf));
2669 		zio = parent->db_data_pending->dr_zio;
2670 	} else {
2671 		ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
2672 		    db->db_blkid != DMU_SPILL_BLKID) ||
2673 		    (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
2674 		if (db->db_blkid != DMU_SPILL_BLKID)
2675 			ASSERT3P(db->db_blkptr, ==,
2676 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
2677 		zio = dn->dn_zio;
2678 	}
2679 
2680 	ASSERT(db->db_level == 0 || data == db->db_buf);
2681 	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
2682 	ASSERT(zio);
2683 
2684 	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
2685 	    os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
2686 	    db->db.db_object, db->db_level, db->db_blkid);
2687 
2688 	if (db->db_blkid == DMU_SPILL_BLKID)
2689 		wp_flag = WP_SPILL;
2690 	wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0;
2691 
2692 	dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
2693 	DB_DNODE_EXIT(db);
2694 
2695 	if (db->db_level == 0 && dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
2696 		ASSERT(db->db_state != DB_NOFILL);
2697 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
2698 		    db->db_blkptr, data->b_data, arc_buf_size(data), &zp,
2699 		    dbuf_write_override_ready, dbuf_write_override_done, dr,
2700 		    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2701 		mutex_enter(&db->db_mtx);
2702 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
2703 		zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by,
2704 		    dr->dt.dl.dr_copies);
2705 		mutex_exit(&db->db_mtx);
2706 	} else if (db->db_state == DB_NOFILL) {
2707 		ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF);
2708 		dr->dr_zio = zio_write(zio, os->os_spa, txg,
2709 		    db->db_blkptr, NULL, db->db.db_size, &zp,
2710 		    dbuf_write_nofill_ready, dbuf_write_nofill_done, db,
2711 		    ZIO_PRIORITY_ASYNC_WRITE,
2712 		    ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb);
2713 	} else {
2714 		ASSERT(arc_released(data));
2715 		dr->dr_zio = arc_write(zio, os->os_spa, txg,
2716 		    db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), &zp,
2717 		    dbuf_write_ready, dbuf_write_done, db,
2718 		    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2719 	}
2720 }
2721