xref: /illumos-gate/usr/src/uts/common/fs/zfs/dbuf.c (revision 88b7b0f29b20b808b9e06071885b1d6a3ddb6328)
1fa9e4066Sahrens /*
2fa9e4066Sahrens  * CDDL HEADER START
3fa9e4066Sahrens  *
4fa9e4066Sahrens  * The contents of this file are subject to the terms of the
5f65e61c0Sahrens  * Common Development and Distribution License (the "License").
6f65e61c0Sahrens  * You may not use this file except in compliance with the License.
7fa9e4066Sahrens  *
8fa9e4066Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens  * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens  * See the License for the specific language governing permissions
11fa9e4066Sahrens  * and limitations under the License.
12fa9e4066Sahrens  *
13fa9e4066Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens  *
19fa9e4066Sahrens  * CDDL HEADER END
20fa9e4066Sahrens  */
21fa9e4066Sahrens /*
221ab7f2deSmaybee  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23fa9e4066Sahrens  * Use is subject to license terms.
24fa9e4066Sahrens  */
25fa9e4066Sahrens 
26fa9e4066Sahrens #include <sys/zfs_context.h>
27fa9e4066Sahrens #include <sys/dmu.h>
28fa9e4066Sahrens #include <sys/dmu_impl.h>
29fa9e4066Sahrens #include <sys/dbuf.h>
30fa9e4066Sahrens #include <sys/dmu_objset.h>
31fa9e4066Sahrens #include <sys/dsl_dataset.h>
32fa9e4066Sahrens #include <sys/dsl_dir.h>
33fa9e4066Sahrens #include <sys/dmu_tx.h>
34fa9e4066Sahrens #include <sys/spa.h>
35fa9e4066Sahrens #include <sys/zio.h>
36fa9e4066Sahrens #include <sys/dmu_zfetch.h>
37fa9e4066Sahrens 
38fa9e4066Sahrens static void dbuf_destroy(dmu_buf_impl_t *db);
39fa9e4066Sahrens static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
40088f3894Sahrens static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
41c717a561Smaybee static arc_done_func_t dbuf_write_ready;
42fa9e4066Sahrens static arc_done_func_t dbuf_write_done;
43fa9e4066Sahrens 
44fa9e4066Sahrens /*
45fa9e4066Sahrens  * Global data structures and functions for the dbuf cache.
46fa9e4066Sahrens  */
47fa9e4066Sahrens static kmem_cache_t *dbuf_cache;
48fa9e4066Sahrens 
49fa9e4066Sahrens /* ARGSUSED */
50fa9e4066Sahrens static int
51fa9e4066Sahrens dbuf_cons(void *vdb, void *unused, int kmflag)
52fa9e4066Sahrens {
53fa9e4066Sahrens 	dmu_buf_impl_t *db = vdb;
54fa9e4066Sahrens 	bzero(db, sizeof (dmu_buf_impl_t));
55fa9e4066Sahrens 
56fa9e4066Sahrens 	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);
57fa9e4066Sahrens 	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
58fa9e4066Sahrens 	refcount_create(&db->db_holds);
59fa9e4066Sahrens 	return (0);
60fa9e4066Sahrens }
61fa9e4066Sahrens 
62fa9e4066Sahrens /* ARGSUSED */
63fa9e4066Sahrens static void
64fa9e4066Sahrens dbuf_dest(void *vdb, void *unused)
65fa9e4066Sahrens {
66fa9e4066Sahrens 	dmu_buf_impl_t *db = vdb;
67fa9e4066Sahrens 	mutex_destroy(&db->db_mtx);
68fa9e4066Sahrens 	cv_destroy(&db->db_changed);
69fa9e4066Sahrens 	refcount_destroy(&db->db_holds);
70fa9e4066Sahrens }
71fa9e4066Sahrens 
72fa9e4066Sahrens /*
73fa9e4066Sahrens  * dbuf hash table routines
74fa9e4066Sahrens  */
75fa9e4066Sahrens static dbuf_hash_table_t dbuf_hash_table;
76fa9e4066Sahrens 
77fa9e4066Sahrens static uint64_t dbuf_hash_count;
78fa9e4066Sahrens 
79fa9e4066Sahrens static uint64_t
80fa9e4066Sahrens dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)
81fa9e4066Sahrens {
82fa9e4066Sahrens 	uintptr_t osv = (uintptr_t)os;
83fa9e4066Sahrens 	uint64_t crc = -1ULL;
84fa9e4066Sahrens 
85fa9e4066Sahrens 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
86fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];
87fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];
88fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
89fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
90fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];
91fa9e4066Sahrens 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];
92fa9e4066Sahrens 
93fa9e4066Sahrens 	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);
94fa9e4066Sahrens 
95fa9e4066Sahrens 	return (crc);
96fa9e4066Sahrens }
97fa9e4066Sahrens 
98fa9e4066Sahrens #define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);
99fa9e4066Sahrens 
100fa9e4066Sahrens #define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\
101fa9e4066Sahrens 	((dbuf)->db.db_object == (obj) &&		\
102fa9e4066Sahrens 	(dbuf)->db_objset == (os) &&			\
103fa9e4066Sahrens 	(dbuf)->db_level == (level) &&			\
104fa9e4066Sahrens 	(dbuf)->db_blkid == (blkid))
105fa9e4066Sahrens 
106fa9e4066Sahrens dmu_buf_impl_t *
107fa9e4066Sahrens dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)
108fa9e4066Sahrens {
109fa9e4066Sahrens 	dbuf_hash_table_t *h = &dbuf_hash_table;
110fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
111fa9e4066Sahrens 	uint64_t obj = dn->dn_object;
112fa9e4066Sahrens 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
113fa9e4066Sahrens 	uint64_t idx = hv & h->hash_table_mask;
114fa9e4066Sahrens 	dmu_buf_impl_t *db;
115fa9e4066Sahrens 
116fa9e4066Sahrens 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
117fa9e4066Sahrens 	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
118fa9e4066Sahrens 		if (DBUF_EQUAL(db, os, obj, level, blkid)) {
119fa9e4066Sahrens 			mutex_enter(&db->db_mtx);
120ea8dc4b6Seschrock 			if (db->db_state != DB_EVICTING) {
121fa9e4066Sahrens 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
122fa9e4066Sahrens 				return (db);
123fa9e4066Sahrens 			}
124fa9e4066Sahrens 			mutex_exit(&db->db_mtx);
125fa9e4066Sahrens 		}
126fa9e4066Sahrens 	}
127fa9e4066Sahrens 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
128fa9e4066Sahrens 	return (NULL);
129fa9e4066Sahrens }
130fa9e4066Sahrens 
131fa9e4066Sahrens /*
132fa9e4066Sahrens  * Insert an entry into the hash table.  If there is already an element
133fa9e4066Sahrens  * equal to elem in the hash table, then the already existing element
134fa9e4066Sahrens  * will be returned and the new element will not be inserted.
135fa9e4066Sahrens  * Otherwise returns NULL.
136fa9e4066Sahrens  */
137fa9e4066Sahrens static dmu_buf_impl_t *
138fa9e4066Sahrens dbuf_hash_insert(dmu_buf_impl_t *db)
139fa9e4066Sahrens {
140fa9e4066Sahrens 	dbuf_hash_table_t *h = &dbuf_hash_table;
141fa9e4066Sahrens 	objset_impl_t *os = db->db_objset;
142fa9e4066Sahrens 	uint64_t obj = db->db.db_object;
143fa9e4066Sahrens 	int level = db->db_level;
144fa9e4066Sahrens 	uint64_t blkid = db->db_blkid;
145fa9e4066Sahrens 	uint64_t hv = DBUF_HASH(os, obj, level, blkid);
146fa9e4066Sahrens 	uint64_t idx = hv & h->hash_table_mask;
147fa9e4066Sahrens 	dmu_buf_impl_t *dbf;
148fa9e4066Sahrens 
149fa9e4066Sahrens 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
150fa9e4066Sahrens 	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {
151fa9e4066Sahrens 		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {
152fa9e4066Sahrens 			mutex_enter(&dbf->db_mtx);
153ea8dc4b6Seschrock 			if (dbf->db_state != DB_EVICTING) {
154fa9e4066Sahrens 				mutex_exit(DBUF_HASH_MUTEX(h, idx));
155fa9e4066Sahrens 				return (dbf);
156fa9e4066Sahrens 			}
157fa9e4066Sahrens 			mutex_exit(&dbf->db_mtx);
158fa9e4066Sahrens 		}
159fa9e4066Sahrens 	}
160fa9e4066Sahrens 
161fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
162fa9e4066Sahrens 	db->db_hash_next = h->hash_table[idx];
163fa9e4066Sahrens 	h->hash_table[idx] = db;
164fa9e4066Sahrens 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
165fa9e4066Sahrens 	atomic_add_64(&dbuf_hash_count, 1);
166fa9e4066Sahrens 
167fa9e4066Sahrens 	return (NULL);
168fa9e4066Sahrens }
169fa9e4066Sahrens 
170fa9e4066Sahrens /*
171fa9e4066Sahrens  * Remove an entry from the hash table.  This operation will
172fa9e4066Sahrens  * fail if there are any existing holds on the db.
173fa9e4066Sahrens  */
174fa9e4066Sahrens static void
175fa9e4066Sahrens dbuf_hash_remove(dmu_buf_impl_t *db)
176fa9e4066Sahrens {
177fa9e4066Sahrens 	dbuf_hash_table_t *h = &dbuf_hash_table;
178fa9e4066Sahrens 	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,
179fa9e4066Sahrens 	    db->db_level, db->db_blkid);
180fa9e4066Sahrens 	uint64_t idx = hv & h->hash_table_mask;
181fa9e4066Sahrens 	dmu_buf_impl_t *dbf, **dbp;
182fa9e4066Sahrens 
183fa9e4066Sahrens 	/*
184fa9e4066Sahrens 	 * We musn't hold db_mtx to maintin lock ordering:
185fa9e4066Sahrens 	 * DBUF_HASH_MUTEX > db_mtx.
186fa9e4066Sahrens 	 */
187fa9e4066Sahrens 	ASSERT(refcount_is_zero(&db->db_holds));
188ea8dc4b6Seschrock 	ASSERT(db->db_state == DB_EVICTING);
189fa9e4066Sahrens 	ASSERT(!MUTEX_HELD(&db->db_mtx));
190fa9e4066Sahrens 
191fa9e4066Sahrens 	mutex_enter(DBUF_HASH_MUTEX(h, idx));
192fa9e4066Sahrens 	dbp = &h->hash_table[idx];
193fa9e4066Sahrens 	while ((dbf = *dbp) != db) {
194fa9e4066Sahrens 		dbp = &dbf->db_hash_next;
195fa9e4066Sahrens 		ASSERT(dbf != NULL);
196fa9e4066Sahrens 	}
197fa9e4066Sahrens 	*dbp = db->db_hash_next;
198fa9e4066Sahrens 	db->db_hash_next = NULL;
199fa9e4066Sahrens 	mutex_exit(DBUF_HASH_MUTEX(h, idx));
200fa9e4066Sahrens 	atomic_add_64(&dbuf_hash_count, -1);
201fa9e4066Sahrens }
202fa9e4066Sahrens 
203ea8dc4b6Seschrock static arc_evict_func_t dbuf_do_evict;
204fa9e4066Sahrens 
205fa9e4066Sahrens static void
206fa9e4066Sahrens dbuf_evict_user(dmu_buf_impl_t *db)
207fa9e4066Sahrens {
208fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
209fa9e4066Sahrens 
210c717a561Smaybee 	if (db->db_level != 0 || db->db_evict_func == NULL)
211fa9e4066Sahrens 		return;
212fa9e4066Sahrens 
213c717a561Smaybee 	if (db->db_user_data_ptr_ptr)
214c717a561Smaybee 		*db->db_user_data_ptr_ptr = db->db.db_data;
215c717a561Smaybee 	db->db_evict_func(&db->db, db->db_user_ptr);
216c717a561Smaybee 	db->db_user_ptr = NULL;
217c717a561Smaybee 	db->db_user_data_ptr_ptr = NULL;
218c717a561Smaybee 	db->db_evict_func = NULL;
219fa9e4066Sahrens }
220fa9e4066Sahrens 
221ea8dc4b6Seschrock void
222ea8dc4b6Seschrock dbuf_evict(dmu_buf_impl_t *db)
223ea8dc4b6Seschrock {
224ea8dc4b6Seschrock 	ASSERT(MUTEX_HELD(&db->db_mtx));
225ea8dc4b6Seschrock 	ASSERT(db->db_buf == NULL);
226c717a561Smaybee 	ASSERT(db->db_data_pending == NULL);
227ea8dc4b6Seschrock 
228ea8dc4b6Seschrock 	dbuf_clear(db);
229ea8dc4b6Seschrock 	dbuf_destroy(db);
230ea8dc4b6Seschrock }
231ea8dc4b6Seschrock 
232fa9e4066Sahrens void
233fa9e4066Sahrens dbuf_init(void)
234fa9e4066Sahrens {
235ea8dc4b6Seschrock 	uint64_t hsize = 1ULL << 16;
236fa9e4066Sahrens 	dbuf_hash_table_t *h = &dbuf_hash_table;
237fa9e4066Sahrens 	int i;
238fa9e4066Sahrens 
239fa9e4066Sahrens 	/*
240fa9e4066Sahrens 	 * The hash table is big enough to fill all of physical memory
241ea8dc4b6Seschrock 	 * with an average 4K block size.  The table will take up
242ea8dc4b6Seschrock 	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).
243fa9e4066Sahrens 	 */
244ea8dc4b6Seschrock 	while (hsize * 4096 < physmem * PAGESIZE)
245fa9e4066Sahrens 		hsize <<= 1;
246fa9e4066Sahrens 
247ea8dc4b6Seschrock retry:
248fa9e4066Sahrens 	h->hash_table_mask = hsize - 1;
249ea8dc4b6Seschrock 	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);
250ea8dc4b6Seschrock 	if (h->hash_table == NULL) {
251ea8dc4b6Seschrock 		/* XXX - we should really return an error instead of assert */
252ea8dc4b6Seschrock 		ASSERT(hsize > (1ULL << 10));
253ea8dc4b6Seschrock 		hsize >>= 1;
254ea8dc4b6Seschrock 		goto retry;
255ea8dc4b6Seschrock 	}
256fa9e4066Sahrens 
257fa9e4066Sahrens 	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",
258fa9e4066Sahrens 	    sizeof (dmu_buf_impl_t),
259fa9e4066Sahrens 	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);
260fa9e4066Sahrens 
261fa9e4066Sahrens 	for (i = 0; i < DBUF_MUTEXES; i++)
262fa9e4066Sahrens 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
263fa9e4066Sahrens }
264fa9e4066Sahrens 
265fa9e4066Sahrens void
266fa9e4066Sahrens dbuf_fini(void)
267fa9e4066Sahrens {
268fa9e4066Sahrens 	dbuf_hash_table_t *h = &dbuf_hash_table;
269fa9e4066Sahrens 	int i;
270fa9e4066Sahrens 
271fa9e4066Sahrens 	for (i = 0; i < DBUF_MUTEXES; i++)
272fa9e4066Sahrens 		mutex_destroy(&h->hash_mutexes[i]);
273fa9e4066Sahrens 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
274fa9e4066Sahrens 	kmem_cache_destroy(dbuf_cache);
275fa9e4066Sahrens }
276fa9e4066Sahrens 
277fa9e4066Sahrens /*
278fa9e4066Sahrens  * Other stuff.
279fa9e4066Sahrens  */
280fa9e4066Sahrens 
2819c9dc39aSek #ifdef ZFS_DEBUG
282fa9e4066Sahrens static void
283fa9e4066Sahrens dbuf_verify(dmu_buf_impl_t *db)
284fa9e4066Sahrens {
285fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
286fa9e4066Sahrens 
287fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
288fa9e4066Sahrens 
289fa9e4066Sahrens 	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))
290fa9e4066Sahrens 		return;
291fa9e4066Sahrens 
292fa9e4066Sahrens 	ASSERT(db->db_objset != NULL);
293fa9e4066Sahrens 	if (dn == NULL) {
294fa9e4066Sahrens 		ASSERT(db->db_parent == NULL);
295fa9e4066Sahrens 		ASSERT(db->db_blkptr == NULL);
296fa9e4066Sahrens 	} else {
297fa9e4066Sahrens 		ASSERT3U(db->db.db_object, ==, dn->dn_object);
298fa9e4066Sahrens 		ASSERT3P(db->db_objset, ==, dn->dn_objset);
299fa9e4066Sahrens 		ASSERT3U(db->db_level, <, dn->dn_nlevels);
300ea8dc4b6Seschrock 		ASSERT(db->db_blkid == DB_BONUS_BLKID ||
301ea8dc4b6Seschrock 		    list_head(&dn->dn_dbufs));
302fa9e4066Sahrens 	}
303fa9e4066Sahrens 	if (db->db_blkid == DB_BONUS_BLKID) {
304fa9e4066Sahrens 		ASSERT(dn != NULL);
3051934e92fSmaybee 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
306fa9e4066Sahrens 		ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID);
307fa9e4066Sahrens 	} else {
308fa9e4066Sahrens 		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
309fa9e4066Sahrens 	}
310fa9e4066Sahrens 
311*88b7b0f2SMatthew Ahrens 	/*
312*88b7b0f2SMatthew Ahrens 	 * We can't assert that db_size matches dn_datablksz because it
313*88b7b0f2SMatthew Ahrens 	 * can be momentarily different when another thread is doing
314*88b7b0f2SMatthew Ahrens 	 * dnode_set_blksz().
315*88b7b0f2SMatthew Ahrens 	 */
316*88b7b0f2SMatthew Ahrens 	if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
317*88b7b0f2SMatthew Ahrens 		dbuf_dirty_record_t *dr = db->db_data_pending;
318*88b7b0f2SMatthew Ahrens 		/*
319*88b7b0f2SMatthew Ahrens 		 * It should only be modified in syncing context, so
320*88b7b0f2SMatthew Ahrens 		 * make sure we only have one copy of the data.
321*88b7b0f2SMatthew Ahrens 		 */
322*88b7b0f2SMatthew Ahrens 		ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
323fa9e4066Sahrens 	}
324fa9e4066Sahrens 
325fa9e4066Sahrens 	/* verify db->db_blkptr */
326fa9e4066Sahrens 	if (db->db_blkptr) {
327fa9e4066Sahrens 		if (db->db_parent == dn->dn_dbuf) {
328fa9e4066Sahrens 			/* db is pointed to by the dnode */
329fa9e4066Sahrens 			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
330ea8dc4b6Seschrock 			if (db->db.db_object == DMU_META_DNODE_OBJECT)
331fa9e4066Sahrens 				ASSERT(db->db_parent == NULL);
332fa9e4066Sahrens 			else
333fa9e4066Sahrens 				ASSERT(db->db_parent != NULL);
334fa9e4066Sahrens 			ASSERT3P(db->db_blkptr, ==,
335fa9e4066Sahrens 			    &dn->dn_phys->dn_blkptr[db->db_blkid]);
336fa9e4066Sahrens 		} else {
337fa9e4066Sahrens 			/* db is pointed to by an indirect block */
338fa9e4066Sahrens 			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;
339fa9e4066Sahrens 			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
340fa9e4066Sahrens 			ASSERT3U(db->db_parent->db.db_object, ==,
341fa9e4066Sahrens 			    db->db.db_object);
342fa9e4066Sahrens 			/*
343fa9e4066Sahrens 			 * dnode_grow_indblksz() can make this fail if we don't
344fa9e4066Sahrens 			 * have the struct_rwlock.  XXX indblksz no longer
345fa9e4066Sahrens 			 * grows.  safe to do this now?
346fa9e4066Sahrens 			 */
347fa9e4066Sahrens 			if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) {
348fa9e4066Sahrens 				ASSERT3P(db->db_blkptr, ==,
349fa9e4066Sahrens 				    ((blkptr_t *)db->db_parent->db.db_data +
350fa9e4066Sahrens 				    db->db_blkid % epb));
351fa9e4066Sahrens 			}
352fa9e4066Sahrens 		}
353fa9e4066Sahrens 	}
354fa9e4066Sahrens 	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
355fa9e4066Sahrens 	    db->db.db_data && db->db_blkid != DB_BONUS_BLKID &&
356fa9e4066Sahrens 	    db->db_state != DB_FILL && !dn->dn_free_txg) {
357fa9e4066Sahrens 		/*
358fa9e4066Sahrens 		 * If the blkptr isn't set but they have nonzero data,
359fa9e4066Sahrens 		 * it had better be dirty, otherwise we'll lose that
360fa9e4066Sahrens 		 * data when we evict this buffer.
361fa9e4066Sahrens 		 */
362fa9e4066Sahrens 		if (db->db_dirtycnt == 0) {
363fa9e4066Sahrens 			uint64_t *buf = db->db.db_data;
364fa9e4066Sahrens 			int i;
365fa9e4066Sahrens 
366fa9e4066Sahrens 			for (i = 0; i < db->db.db_size >> 3; i++) {
367fa9e4066Sahrens 				ASSERT(buf[i] == 0);
368fa9e4066Sahrens 			}
369fa9e4066Sahrens 		}
370fa9e4066Sahrens 	}
371fa9e4066Sahrens }
3729c9dc39aSek #endif
373fa9e4066Sahrens 
374fa9e4066Sahrens static void
375fa9e4066Sahrens dbuf_update_data(dmu_buf_impl_t *db)
376fa9e4066Sahrens {
377fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
378c717a561Smaybee 	if (db->db_level == 0 && db->db_user_data_ptr_ptr) {
379fa9e4066Sahrens 		ASSERT(!refcount_is_zero(&db->db_holds));
380c717a561Smaybee 		*db->db_user_data_ptr_ptr = db->db.db_data;
381fa9e4066Sahrens 	}
382fa9e4066Sahrens }
383fa9e4066Sahrens 
384fa9e4066Sahrens static void
385fa9e4066Sahrens dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
386fa9e4066Sahrens {
387fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
388ea8dc4b6Seschrock 	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));
389fa9e4066Sahrens 	db->db_buf = buf;
390ea8dc4b6Seschrock 	if (buf != NULL) {
391ea8dc4b6Seschrock 		ASSERT(buf->b_data != NULL);
392ea8dc4b6Seschrock 		db->db.db_data = buf->b_data;
393ea8dc4b6Seschrock 		if (!arc_released(buf))
394ea8dc4b6Seschrock 			arc_set_callback(buf, dbuf_do_evict, db);
395ea8dc4b6Seschrock 		dbuf_update_data(db);
396ea8dc4b6Seschrock 	} else {
397ea8dc4b6Seschrock 		dbuf_evict_user(db);
398ea8dc4b6Seschrock 		db->db.db_data = NULL;
399ea8dc4b6Seschrock 		db->db_state = DB_UNCACHED;
400ea8dc4b6Seschrock 	}
401fa9e4066Sahrens }
402fa9e4066Sahrens 
403fa9e4066Sahrens uint64_t
404fa9e4066Sahrens dbuf_whichblock(dnode_t *dn, uint64_t offset)
405fa9e4066Sahrens {
406fa9e4066Sahrens 	if (dn->dn_datablkshift) {
407fa9e4066Sahrens 		return (offset >> dn->dn_datablkshift);
408fa9e4066Sahrens 	} else {
409fa9e4066Sahrens 		ASSERT3U(offset, <, dn->dn_datablksz);
410fa9e4066Sahrens 		return (0);
411fa9e4066Sahrens 	}
412fa9e4066Sahrens }
413fa9e4066Sahrens 
414fa9e4066Sahrens static void
415fa9e4066Sahrens dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)
416fa9e4066Sahrens {
417fa9e4066Sahrens 	dmu_buf_impl_t *db = vdb;
418fa9e4066Sahrens 
419fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
420fa9e4066Sahrens 	ASSERT3U(db->db_state, ==, DB_READ);
421fa9e4066Sahrens 	/*
422fa9e4066Sahrens 	 * All reads are synchronous, so we must have a hold on the dbuf
423fa9e4066Sahrens 	 */
424fa9e4066Sahrens 	ASSERT(refcount_count(&db->db_holds) > 0);
425ea8dc4b6Seschrock 	ASSERT(db->db_buf == NULL);
426fa9e4066Sahrens 	ASSERT(db->db.db_data == NULL);
427c717a561Smaybee 	if (db->db_level == 0 && db->db_freed_in_flight) {
428fa9e4066Sahrens 		/* we were freed in flight; disregard any error */
429fa9e4066Sahrens 		arc_release(buf, db);
430fa9e4066Sahrens 		bzero(buf->b_data, db->db.db_size);
4316b4acc8bSahrens 		arc_buf_freeze(buf);
432c717a561Smaybee 		db->db_freed_in_flight = FALSE;
433fa9e4066Sahrens 		dbuf_set_data(db, buf);
434fa9e4066Sahrens 		db->db_state = DB_CACHED;
435fa9e4066Sahrens 	} else if (zio == NULL || zio->io_error == 0) {
436fa9e4066Sahrens 		dbuf_set_data(db, buf);
437fa9e4066Sahrens 		db->db_state = DB_CACHED;
438fa9e4066Sahrens 	} else {
439fa9e4066Sahrens 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
440fa9e4066Sahrens 		ASSERT3P(db->db_buf, ==, NULL);
441ea8dc4b6Seschrock 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
442ea8dc4b6Seschrock 		db->db_state = DB_UNCACHED;
443fa9e4066Sahrens 	}
444fa9e4066Sahrens 	cv_broadcast(&db->db_changed);
445fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
446ea8dc4b6Seschrock 	dbuf_rele(db, NULL);
447fa9e4066Sahrens }
448fa9e4066Sahrens 
449ea8dc4b6Seschrock static void
45013506d1eSmaybee dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)
451fa9e4066Sahrens {
452088f3894Sahrens 	dnode_t *dn = db->db_dnode;
453ea8dc4b6Seschrock 	zbookmark_t zb;
45413506d1eSmaybee 	uint32_t aflags = ARC_NOWAIT;
455088f3894Sahrens 	arc_buf_t *pbuf;
456fa9e4066Sahrens 
457fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
458fa9e4066Sahrens 	/* We need the struct_rwlock to prevent db_blkptr from changing. */
459088f3894Sahrens 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
460ea8dc4b6Seschrock 	ASSERT(MUTEX_HELD(&db->db_mtx));
461ea8dc4b6Seschrock 	ASSERT(db->db_state == DB_UNCACHED);
462ea8dc4b6Seschrock 	ASSERT(db->db_buf == NULL);
463fa9e4066Sahrens 
464fa9e4066Sahrens 	if (db->db_blkid == DB_BONUS_BLKID) {
465088f3894Sahrens 		int bonuslen = dn->dn_bonuslen;
4661934e92fSmaybee 
4671934e92fSmaybee 		ASSERT3U(bonuslen, <=, db->db.db_size);
468ea8dc4b6Seschrock 		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);
46925c2057bSmaybee 		arc_space_consume(DN_MAX_BONUSLEN);
4701934e92fSmaybee 		if (bonuslen < DN_MAX_BONUSLEN)
471ea8dc4b6Seschrock 			bzero(db->db.db_data, DN_MAX_BONUSLEN);
472088f3894Sahrens 		bcopy(DN_BONUS(dn->dn_phys), db->db.db_data,
4731934e92fSmaybee 		    bonuslen);
474ea8dc4b6Seschrock 		dbuf_update_data(db);
475fa9e4066Sahrens 		db->db_state = DB_CACHED;
476fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
477fa9e4066Sahrens 		return;
478fa9e4066Sahrens 	}
479fa9e4066Sahrens 
4801c8564a7SMark Maybee 	/*
4811c8564a7SMark Maybee 	 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync()
4821c8564a7SMark Maybee 	 * processes the delete record and clears the bp while we are waiting
4831c8564a7SMark Maybee 	 * for the dn_mtx (resulting in a "no" from block_freed).
4841c8564a7SMark Maybee 	 */
485088f3894Sahrens 	if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) ||
4861c8564a7SMark Maybee 	    (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) ||
4871c8564a7SMark Maybee 	    BP_IS_HOLE(db->db_blkptr)))) {
488ad23a2dbSjohansen 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
489ad23a2dbSjohansen 
490088f3894Sahrens 		dbuf_set_data(db, arc_buf_alloc(dn->dn_objset->os_spa,
491ad23a2dbSjohansen 		    db->db.db_size, db, type));
492fa9e4066Sahrens 		bzero(db->db.db_data, db->db.db_size);
493fa9e4066Sahrens 		db->db_state = DB_CACHED;
49413506d1eSmaybee 		*flags |= DB_RF_CACHED;
495fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
496fa9e4066Sahrens 		return;
497fa9e4066Sahrens 	}
498fa9e4066Sahrens 
499fa9e4066Sahrens 	db->db_state = DB_READ;
500fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
501fa9e4066Sahrens 
5023baa08fcSek 	if (DBUF_IS_L2CACHEABLE(db))
5033baa08fcSek 		aflags |= ARC_L2CACHE;
5043baa08fcSek 
505ea8dc4b6Seschrock 	zb.zb_objset = db->db_objset->os_dsl_dataset ?
506ea8dc4b6Seschrock 	    db->db_objset->os_dsl_dataset->ds_object : 0;
507ea8dc4b6Seschrock 	zb.zb_object = db->db.db_object;
508ea8dc4b6Seschrock 	zb.zb_level = db->db_level;
509ea8dc4b6Seschrock 	zb.zb_blkid = db->db_blkid;
510ea8dc4b6Seschrock 
511ea8dc4b6Seschrock 	dbuf_add_ref(db, NULL);
512fa9e4066Sahrens 	/* ZIO_FLAG_CANFAIL callers have to check the parent zio's error */
513088f3894Sahrens 
514088f3894Sahrens 	if (db->db_parent)
515088f3894Sahrens 		pbuf = db->db_parent->db_buf;
516088f3894Sahrens 	else
517088f3894Sahrens 		pbuf = db->db_objset->os_phys_buf;
518088f3894Sahrens 
519088f3894Sahrens 	(void) arc_read(zio, dn->dn_objset->os_spa, db->db_blkptr, pbuf,
520fa9e4066Sahrens 	    dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ,
52113506d1eSmaybee 	    (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED,
52213506d1eSmaybee 	    &aflags, &zb);
52313506d1eSmaybee 	if (aflags & ARC_CACHED)
52413506d1eSmaybee 		*flags |= DB_RF_CACHED;
525fa9e4066Sahrens }
526fa9e4066Sahrens 
527ea8dc4b6Seschrock int
528ea8dc4b6Seschrock dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags)
529fa9e4066Sahrens {
530ea8dc4b6Seschrock 	int err = 0;
531ea8dc4b6Seschrock 	int havepzio = (zio != NULL);
53213506d1eSmaybee 	int prefetch;
533fa9e4066Sahrens 
534fa9e4066Sahrens 	/*
535fa9e4066Sahrens 	 * We don't have to hold the mutex to check db_state because it
536fa9e4066Sahrens 	 * can't be freed while we have a hold on the buffer.
537fa9e4066Sahrens 	 */
538fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
539fa9e4066Sahrens 
540ea8dc4b6Seschrock 	if ((flags & DB_RF_HAVESTRUCT) == 0)
541ea8dc4b6Seschrock 		rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER);
542ea8dc4b6Seschrock 
54313506d1eSmaybee 	prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
5443baa08fcSek 	    (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL &&
5453baa08fcSek 	    DBUF_IS_CACHEABLE(db);
54613506d1eSmaybee 
547ea8dc4b6Seschrock 	mutex_enter(&db->db_mtx);
548ea8dc4b6Seschrock 	if (db->db_state == DB_CACHED) {
549ea8dc4b6Seschrock 		mutex_exit(&db->db_mtx);
55013506d1eSmaybee 		if (prefetch)
55113506d1eSmaybee 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
55213506d1eSmaybee 			    db->db.db_size, TRUE);
553fa9e4066Sahrens 		if ((flags & DB_RF_HAVESTRUCT) == 0)
554ea8dc4b6Seschrock 			rw_exit(&db->db_dnode->dn_struct_rwlock);
555ea8dc4b6Seschrock 	} else if (db->db_state == DB_UNCACHED) {
556ea8dc4b6Seschrock 		if (zio == NULL) {
557ea8dc4b6Seschrock 			zio = zio_root(db->db_dnode->dn_objset->os_spa,
558ea8dc4b6Seschrock 			    NULL, NULL, ZIO_FLAG_CANFAIL);
559ea8dc4b6Seschrock 		}
56013506d1eSmaybee 		dbuf_read_impl(db, zio, &flags);
56113506d1eSmaybee 
562ea8dc4b6Seschrock 		/* dbuf_read_impl has dropped db_mtx for us */
563ea8dc4b6Seschrock 
56413506d1eSmaybee 		if (prefetch)
565ea8dc4b6Seschrock 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
56613506d1eSmaybee 			    db->db.db_size, flags & DB_RF_CACHED);
567ea8dc4b6Seschrock 
568fa9e4066Sahrens 		if ((flags & DB_RF_HAVESTRUCT) == 0)
569fa9e4066Sahrens 			rw_exit(&db->db_dnode->dn_struct_rwlock);
570fa9e4066Sahrens 
571ea8dc4b6Seschrock 		if (!havepzio)
572ea8dc4b6Seschrock 			err = zio_wait(zio);
573ea8dc4b6Seschrock 	} else {
57413506d1eSmaybee 		mutex_exit(&db->db_mtx);
57513506d1eSmaybee 		if (prefetch)
57613506d1eSmaybee 			dmu_zfetch(&db->db_dnode->dn_zfetch, db->db.db_offset,
57713506d1eSmaybee 			    db->db.db_size, TRUE);
578ea8dc4b6Seschrock 		if ((flags & DB_RF_HAVESTRUCT) == 0)
579ea8dc4b6Seschrock 			rw_exit(&db->db_dnode->dn_struct_rwlock);
58013506d1eSmaybee 
58113506d1eSmaybee 		mutex_enter(&db->db_mtx);
582ea8dc4b6Seschrock 		if ((flags & DB_RF_NEVERWAIT) == 0) {
583ea8dc4b6Seschrock 			while (db->db_state == DB_READ ||
584ea8dc4b6Seschrock 			    db->db_state == DB_FILL) {
585ea8dc4b6Seschrock 				ASSERT(db->db_state == DB_READ ||
586ea8dc4b6Seschrock 				    (flags & DB_RF_HAVESTRUCT) == 0);
587ea8dc4b6Seschrock 				cv_wait(&db->db_changed, &db->db_mtx);
588ea8dc4b6Seschrock 			}
589ea8dc4b6Seschrock 			if (db->db_state == DB_UNCACHED)
590ea8dc4b6Seschrock 				err = EIO;
591ea8dc4b6Seschrock 		}
592ea8dc4b6Seschrock 		mutex_exit(&db->db_mtx);
593fa9e4066Sahrens 	}
594fa9e4066Sahrens 
595ea8dc4b6Seschrock 	ASSERT(err || havepzio || db->db_state == DB_CACHED);
596ea8dc4b6Seschrock 	return (err);
597fa9e4066Sahrens }
598fa9e4066Sahrens 
599fa9e4066Sahrens static void
600fa9e4066Sahrens dbuf_noread(dmu_buf_impl_t *db)
601fa9e4066Sahrens {
602fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
603ea8dc4b6Seschrock 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
604fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
605fa9e4066Sahrens 	while (db->db_state == DB_READ || db->db_state == DB_FILL)
606fa9e4066Sahrens 		cv_wait(&db->db_changed, &db->db_mtx);
607fa9e4066Sahrens 	if (db->db_state == DB_UNCACHED) {
608ad23a2dbSjohansen 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
609ad23a2dbSjohansen 
610ea8dc4b6Seschrock 		ASSERT(db->db_buf == NULL);
611fa9e4066Sahrens 		ASSERT(db->db.db_data == NULL);
612fa9e4066Sahrens 		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
613ad23a2dbSjohansen 		    db->db.db_size, db, type));
614fa9e4066Sahrens 		db->db_state = DB_FILL;
615fa9e4066Sahrens 	} else {
616fa9e4066Sahrens 		ASSERT3U(db->db_state, ==, DB_CACHED);
617fa9e4066Sahrens 	}
618fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
619fa9e4066Sahrens }
620fa9e4066Sahrens 
621fa9e4066Sahrens /*
622fa9e4066Sahrens  * This is our just-in-time copy function.  It makes a copy of
623fa9e4066Sahrens  * buffers, that have been modified in a previous transaction
624fa9e4066Sahrens  * group, before we modify them in the current active group.
625fa9e4066Sahrens  *
626fa9e4066Sahrens  * This function is used in two places: when we are dirtying a
627fa9e4066Sahrens  * buffer for the first time in a txg, and when we are freeing
628fa9e4066Sahrens  * a range in a dnode that includes this buffer.
629fa9e4066Sahrens  *
630fa9e4066Sahrens  * Note that when we are called from dbuf_free_range() we do
631fa9e4066Sahrens  * not put a hold on the buffer, we just traverse the active
632fa9e4066Sahrens  * dbuf list for the dnode.
633fa9e4066Sahrens  */
634fa9e4066Sahrens static void
635fa9e4066Sahrens dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
636fa9e4066Sahrens {
637c717a561Smaybee 	dbuf_dirty_record_t *dr = db->db_last_dirty;
638fa9e4066Sahrens 
639fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
640fa9e4066Sahrens 	ASSERT(db->db.db_data != NULL);
641c717a561Smaybee 	ASSERT(db->db_level == 0);
642c717a561Smaybee 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
643fa9e4066Sahrens 
6444d31c452Smaybee 	if (dr == NULL ||
6454d31c452Smaybee 	    (dr->dt.dl.dr_data !=
6464d31c452Smaybee 	    ((db->db_blkid  == DB_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
647fa9e4066Sahrens 		return;
648fa9e4066Sahrens 
649fa9e4066Sahrens 	/*
650c717a561Smaybee 	 * If the last dirty record for this dbuf has not yet synced
651c717a561Smaybee 	 * and its referencing the dbuf data, either:
652c717a561Smaybee 	 * 	reset the reference to point to a new copy,
653c717a561Smaybee 	 * or (if there a no active holders)
654c717a561Smaybee 	 *	just null out the current db_data pointer.
655fa9e4066Sahrens 	 */
656c717a561Smaybee 	ASSERT(dr->dr_txg >= txg - 2);
657c717a561Smaybee 	if (db->db_blkid == DB_BONUS_BLKID) {
658c717a561Smaybee 		/* Note that the data bufs here are zio_bufs */
659c717a561Smaybee 		dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN);
6600e8c6158Smaybee 		arc_space_consume(DN_MAX_BONUSLEN);
661c717a561Smaybee 		bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN);
662c717a561Smaybee 	} else if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
663c717a561Smaybee 		int size = db->db.db_size;
664c717a561Smaybee 		arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
665c717a561Smaybee 		dr->dt.dl.dr_data = arc_buf_alloc(
666c717a561Smaybee 		    db->db_dnode->dn_objset->os_spa, size, db, type);
667c717a561Smaybee 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
668c717a561Smaybee 	} else {
669c717a561Smaybee 		dbuf_set_data(db, NULL);
670fa9e4066Sahrens 	}
671fa9e4066Sahrens }
672fa9e4066Sahrens 
673c717a561Smaybee void
674c717a561Smaybee dbuf_unoverride(dbuf_dirty_record_t *dr)
675ea8dc4b6Seschrock {
676c717a561Smaybee 	dmu_buf_impl_t *db = dr->dr_dbuf;
677c717a561Smaybee 	uint64_t txg = dr->dr_txg;
678ea8dc4b6Seschrock 
679ea8dc4b6Seschrock 	ASSERT(MUTEX_HELD(&db->db_mtx));
680c717a561Smaybee 	ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC);
681c717a561Smaybee 	ASSERT(db->db_level == 0);
682ea8dc4b6Seschrock 
683c717a561Smaybee 	if (db->db_blkid == DB_BONUS_BLKID ||
684c717a561Smaybee 	    dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN)
685c717a561Smaybee 		return;
686ea8dc4b6Seschrock 
687c717a561Smaybee 	/* free this block */
688c717a561Smaybee 	if (!BP_IS_HOLE(&dr->dt.dl.dr_overridden_by)) {
689c717a561Smaybee 		/* XXX can get silent EIO here */
690088f3894Sahrens 		(void) dsl_free(NULL,
691088f3894Sahrens 		    spa_get_dsl(db->db_dnode->dn_objset->os_spa),
692c717a561Smaybee 		    txg, &dr->dt.dl.dr_overridden_by, NULL, NULL, ARC_WAIT);
693fa9e4066Sahrens 	}
694c717a561Smaybee 	dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
695c717a561Smaybee 	/*
696c717a561Smaybee 	 * Release the already-written buffer, so we leave it in
697c717a561Smaybee 	 * a consistent dirty state.  Note that all callers are
698c717a561Smaybee 	 * modifying the buffer, so they will immediately do
699c717a561Smaybee 	 * another (redundant) arc_release().  Therefore, leave
700c717a561Smaybee 	 * the buf thawed to save the effort of freezing &
701c717a561Smaybee 	 * immediately re-thawing it.
702c717a561Smaybee 	 */
703c717a561Smaybee 	arc_release(dr->dt.dl.dr_data, db);
704fa9e4066Sahrens }
705fa9e4066Sahrens 
706cdb0ab79Smaybee /*
707cdb0ab79Smaybee  * Evict (if its unreferenced) or clear (if its referenced) any level-0
708cdb0ab79Smaybee  * data blocks in the free range, so that any future readers will find
709cdb0ab79Smaybee  * empty blocks.  Also, if we happen accross any level-1 dbufs in the
710cdb0ab79Smaybee  * range that have not already been marked dirty, mark them dirty so
711cdb0ab79Smaybee  * they stay in memory.
712cdb0ab79Smaybee  */
713fa9e4066Sahrens void
714cdb0ab79Smaybee dbuf_free_range(dnode_t *dn, uint64_t start, uint64_t end, dmu_tx_t *tx)
715fa9e4066Sahrens {
716fa9e4066Sahrens 	dmu_buf_impl_t *db, *db_next;
717fa9e4066Sahrens 	uint64_t txg = tx->tx_txg;
718cdb0ab79Smaybee 	int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
719cdb0ab79Smaybee 	uint64_t first_l1 = start >> epbs;
720cdb0ab79Smaybee 	uint64_t last_l1 = end >> epbs;
721fa9e4066Sahrens 
722cdb0ab79Smaybee 	if (end > dn->dn_maxblkid) {
723cdb0ab79Smaybee 		end = dn->dn_maxblkid;
724cdb0ab79Smaybee 		last_l1 = end >> epbs;
725cdb0ab79Smaybee 	}
726cdb0ab79Smaybee 	dprintf_dnode(dn, "start=%llu end=%llu\n", start, end);
727fa9e4066Sahrens 	mutex_enter(&dn->dn_dbufs_mtx);
728fa9e4066Sahrens 	for (db = list_head(&dn->dn_dbufs); db; db = db_next) {
729fa9e4066Sahrens 		db_next = list_next(&dn->dn_dbufs, db);
730ea8dc4b6Seschrock 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
731cdb0ab79Smaybee 
732cdb0ab79Smaybee 		if (db->db_level == 1 &&
733cdb0ab79Smaybee 		    db->db_blkid >= first_l1 && db->db_blkid <= last_l1) {
734cdb0ab79Smaybee 			mutex_enter(&db->db_mtx);
735cdb0ab79Smaybee 			if (db->db_last_dirty &&
736cdb0ab79Smaybee 			    db->db_last_dirty->dr_txg < txg) {
737cdb0ab79Smaybee 				dbuf_add_ref(db, FTAG);
738cdb0ab79Smaybee 				mutex_exit(&db->db_mtx);
739cdb0ab79Smaybee 				dbuf_will_dirty(db, tx);
740cdb0ab79Smaybee 				dbuf_rele(db, FTAG);
741cdb0ab79Smaybee 			} else {
742cdb0ab79Smaybee 				mutex_exit(&db->db_mtx);
743cdb0ab79Smaybee 			}
744cdb0ab79Smaybee 		}
745cdb0ab79Smaybee 
746ea8dc4b6Seschrock 		if (db->db_level != 0)
747fa9e4066Sahrens 			continue;
748fa9e4066Sahrens 		dprintf_dbuf(db, "found buf %s\n", "");
749cdb0ab79Smaybee 		if (db->db_blkid < start || db->db_blkid > end)
750fa9e4066Sahrens 			continue;
751fa9e4066Sahrens 
752fa9e4066Sahrens 		/* found a level 0 buffer in the range */
753fa9e4066Sahrens 		if (dbuf_undirty(db, tx))
754fa9e4066Sahrens 			continue;
755fa9e4066Sahrens 
756fa9e4066Sahrens 		mutex_enter(&db->db_mtx);
757ea8dc4b6Seschrock 		if (db->db_state == DB_UNCACHED ||
758ea8dc4b6Seschrock 		    db->db_state == DB_EVICTING) {
759fa9e4066Sahrens 			ASSERT(db->db.db_data == NULL);
760fa9e4066Sahrens 			mutex_exit(&db->db_mtx);
761fa9e4066Sahrens 			continue;
762fa9e4066Sahrens 		}
763c543ec06Sahrens 		if (db->db_state == DB_READ || db->db_state == DB_FILL) {
764c543ec06Sahrens 			/* will be handled in dbuf_read_done or dbuf_rele */
765c717a561Smaybee 			db->db_freed_in_flight = TRUE;
766fa9e4066Sahrens 			mutex_exit(&db->db_mtx);
767fa9e4066Sahrens 			continue;
768fa9e4066Sahrens 		}
769ea8dc4b6Seschrock 		if (refcount_count(&db->db_holds) == 0) {
770ea8dc4b6Seschrock 			ASSERT(db->db_buf);
771ea8dc4b6Seschrock 			dbuf_clear(db);
772ea8dc4b6Seschrock 			continue;
773ea8dc4b6Seschrock 		}
774c717a561Smaybee 		/* The dbuf is referenced */
775fa9e4066Sahrens 
776c717a561Smaybee 		if (db->db_last_dirty != NULL) {
777c717a561Smaybee 			dbuf_dirty_record_t *dr = db->db_last_dirty;
778c717a561Smaybee 
779c717a561Smaybee 			if (dr->dr_txg == txg) {
78044eda4d7Smaybee 				/*
781c717a561Smaybee 				 * This buffer is "in-use", re-adjust the file
782c717a561Smaybee 				 * size to reflect that this buffer may
783c717a561Smaybee 				 * contain new data when we sync.
78444eda4d7Smaybee 				 */
785c717a561Smaybee 				if (db->db_blkid > dn->dn_maxblkid)
786c717a561Smaybee 					dn->dn_maxblkid = db->db_blkid;
787c717a561Smaybee 				dbuf_unoverride(dr);
788c717a561Smaybee 			} else {
789c717a561Smaybee 				/*
790c717a561Smaybee 				 * This dbuf is not dirty in the open context.
791c717a561Smaybee 				 * Either uncache it (if its not referenced in
792c717a561Smaybee 				 * the open context) or reset its contents to
793c717a561Smaybee 				 * empty.
794c717a561Smaybee 				 */
795c717a561Smaybee 				dbuf_fix_old_data(db, txg);
79644eda4d7Smaybee 			}
797ea8dc4b6Seschrock 		}
798c717a561Smaybee 		/* clear the contents if its cached */
799ea8dc4b6Seschrock 		if (db->db_state == DB_CACHED) {
800ea8dc4b6Seschrock 			ASSERT(db->db.db_data != NULL);
801fa9e4066Sahrens 			arc_release(db->db_buf, db);
802fa9e4066Sahrens 			bzero(db->db.db_data, db->db.db_size);
8036b4acc8bSahrens 			arc_buf_freeze(db->db_buf);
804fa9e4066Sahrens 		}
805ea8dc4b6Seschrock 
806fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
807fa9e4066Sahrens 	}
808fa9e4066Sahrens 	mutex_exit(&dn->dn_dbufs_mtx);
809fa9e4066Sahrens }
810fa9e4066Sahrens 
811fa9e4066Sahrens static int
8121934e92fSmaybee dbuf_block_freeable(dmu_buf_impl_t *db)
813fa9e4066Sahrens {
814fa9e4066Sahrens 	dsl_dataset_t *ds = db->db_objset->os_dsl_dataset;
815fa9e4066Sahrens 	uint64_t birth_txg = 0;
816fa9e4066Sahrens 
817fa9e4066Sahrens 	/*
818fa9e4066Sahrens 	 * We don't need any locking to protect db_blkptr:
819c717a561Smaybee 	 * If it's syncing, then db_last_dirty will be set
820c717a561Smaybee 	 * so we'll ignore db_blkptr.
821fa9e4066Sahrens 	 */
822c717a561Smaybee 	ASSERT(MUTEX_HELD(&db->db_mtx));
823c717a561Smaybee 	if (db->db_last_dirty)
824c717a561Smaybee 		birth_txg = db->db_last_dirty->dr_txg;
825fa9e4066Sahrens 	else if (db->db_blkptr)
826fa9e4066Sahrens 		birth_txg = db->db_blkptr->blk_birth;
827fa9e4066Sahrens 
8281934e92fSmaybee 	/* If we don't exist or are in a snapshot, we can't be freed */
829fa9e4066Sahrens 	if (birth_txg)
8301934e92fSmaybee 		return (ds == NULL ||
8311934e92fSmaybee 		    dsl_dataset_block_freeable(ds, birth_txg));
832fa9e4066Sahrens 	else
8331934e92fSmaybee 		return (FALSE);
834fa9e4066Sahrens }
835fa9e4066Sahrens 
836fa9e4066Sahrens void
837fa9e4066Sahrens dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
838fa9e4066Sahrens {
839fa9e4066Sahrens 	arc_buf_t *buf, *obuf;
840fa9e4066Sahrens 	int osize = db->db.db_size;
841ad23a2dbSjohansen 	arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
842fa9e4066Sahrens 
843ea8dc4b6Seschrock 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
844ea8dc4b6Seschrock 
845fa9e4066Sahrens 	/* XXX does *this* func really need the lock? */
846fa9e4066Sahrens 	ASSERT(RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock));
847fa9e4066Sahrens 
848fa9e4066Sahrens 	/*
849fa9e4066Sahrens 	 * This call to dbuf_will_dirty() with the dn_struct_rwlock held
850fa9e4066Sahrens 	 * is OK, because there can be no other references to the db
851fa9e4066Sahrens 	 * when we are changing its size, so no concurrent DB_FILL can
852fa9e4066Sahrens 	 * be happening.
853fa9e4066Sahrens 	 */
854ea8dc4b6Seschrock 	/*
855ea8dc4b6Seschrock 	 * XXX we should be doing a dbuf_read, checking the return
856ea8dc4b6Seschrock 	 * value and returning that up to our callers
857ea8dc4b6Seschrock 	 */
858fa9e4066Sahrens 	dbuf_will_dirty(db, tx);
859fa9e4066Sahrens 
860fa9e4066Sahrens 	/* create the data buffer for the new block */
861ad23a2dbSjohansen 	buf = arc_buf_alloc(db->db_dnode->dn_objset->os_spa, size, db, type);
862fa9e4066Sahrens 
863fa9e4066Sahrens 	/* copy old block data to the new block */
864fa9e4066Sahrens 	obuf = db->db_buf;
865f65e61c0Sahrens 	bcopy(obuf->b_data, buf->b_data, MIN(osize, size));
866fa9e4066Sahrens 	/* zero the remainder */
867f65e61c0Sahrens 	if (size > osize)
868f65e61c0Sahrens 		bzero((uint8_t *)buf->b_data + osize, size - osize);
869fa9e4066Sahrens 
870fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
871fa9e4066Sahrens 	dbuf_set_data(db, buf);
872ea8dc4b6Seschrock 	VERIFY(arc_buf_remove_ref(obuf, db) == 1);
873fa9e4066Sahrens 	db->db.db_size = size;
874fa9e4066Sahrens 
875c717a561Smaybee 	if (db->db_level == 0) {
876c717a561Smaybee 		ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg);
877c717a561Smaybee 		db->db_last_dirty->dt.dl.dr_data = buf;
878c717a561Smaybee 	}
879fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
880fa9e4066Sahrens 
881fa9e4066Sahrens 	dnode_willuse_space(db->db_dnode, size-osize, tx);
882fa9e4066Sahrens }
883fa9e4066Sahrens 
884c717a561Smaybee dbuf_dirty_record_t *
885fa9e4066Sahrens dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
886fa9e4066Sahrens {
887fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
888fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
889c717a561Smaybee 	dbuf_dirty_record_t **drp, *dr;
890fa9e4066Sahrens 	int drop_struct_lock = FALSE;
891d3469faaSMark Maybee 	boolean_t do_free_accounting = B_FALSE;
892fa9e4066Sahrens 	int txgoff = tx->tx_txg & TXG_MASK;
893fa9e4066Sahrens 
894fa9e4066Sahrens 	ASSERT(tx->tx_txg != 0);
895fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
8969c9dc39aSek 	DMU_TX_DIRTY_BUF(tx, db);
897fa9e4066Sahrens 
898fa9e4066Sahrens 	/*
899fa9e4066Sahrens 	 * Shouldn't dirty a regular buffer in syncing context.  Private
900fa9e4066Sahrens 	 * objects may be dirtied in syncing context, but only if they
901fa9e4066Sahrens 	 * were already pre-dirtied in open context.
902fa9e4066Sahrens 	 * XXX We may want to prohibit dirtying in syncing context even
903fa9e4066Sahrens 	 * if they did pre-dirty.
904fa9e4066Sahrens 	 */
905c717a561Smaybee 	ASSERT(!dmu_tx_is_syncing(tx) ||
906c717a561Smaybee 	    BP_IS_HOLE(dn->dn_objset->os_rootbp) ||
907c717a561Smaybee 	    dn->dn_object == DMU_META_DNODE_OBJECT ||
908c717a561Smaybee 	    dn->dn_objset->os_dsl_dataset == NULL ||
909c717a561Smaybee 	    dsl_dir_is_private(dn->dn_objset->os_dsl_dataset->ds_dir));
910fa9e4066Sahrens 
911fa9e4066Sahrens 	/*
912fa9e4066Sahrens 	 * We make this assert for private objects as well, but after we
913fa9e4066Sahrens 	 * check if we're already dirty.  They are allowed to re-dirty
914fa9e4066Sahrens 	 * in syncing context.
915fa9e4066Sahrens 	 */
916ea8dc4b6Seschrock 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
917c717a561Smaybee 	    dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
918fa9e4066Sahrens 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
919fa9e4066Sahrens 
920fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
921fa9e4066Sahrens 	/*
922c717a561Smaybee 	 * XXX make this true for indirects too?  The problem is that
923c717a561Smaybee 	 * transactions created with dmu_tx_create_assigned() from
924c717a561Smaybee 	 * syncing context don't bother holding ahead.
925fa9e4066Sahrens 	 */
926c717a561Smaybee 	ASSERT(db->db_level != 0 ||
927c717a561Smaybee 	    db->db_state == DB_CACHED || db->db_state == DB_FILL);
928fa9e4066Sahrens 
929fa9e4066Sahrens 	mutex_enter(&dn->dn_mtx);
930fa9e4066Sahrens 	/*
931fa9e4066Sahrens 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
932fa9e4066Sahrens 	 * initialize the objset.
933fa9e4066Sahrens 	 */
934fa9e4066Sahrens 	if (dn->dn_dirtyctx == DN_UNDIRTIED &&
935c717a561Smaybee 	    !BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
936fa9e4066Sahrens 		dn->dn_dirtyctx =
937fa9e4066Sahrens 		    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN);
938fa9e4066Sahrens 		ASSERT(dn->dn_dirtyctx_firstset == NULL);
939fa9e4066Sahrens 		dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP);
940fa9e4066Sahrens 	}
941fa9e4066Sahrens 	mutex_exit(&dn->dn_mtx);
942fa9e4066Sahrens 
943fa9e4066Sahrens 	/*
944fa9e4066Sahrens 	 * If this buffer is already dirty, we're done.
945fa9e4066Sahrens 	 */
946c717a561Smaybee 	drp = &db->db_last_dirty;
947c717a561Smaybee 	ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg ||
948c717a561Smaybee 	    db->db.db_object == DMU_META_DNODE_OBJECT);
9497e2186e3Sbonwick 	while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg)
9507e2186e3Sbonwick 		drp = &dr->dr_next;
9517e2186e3Sbonwick 	if (dr && dr->dr_txg == tx->tx_txg) {
952c717a561Smaybee 		if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) {
953c717a561Smaybee 			/*
954c717a561Smaybee 			 * If this buffer has already been written out,
955c717a561Smaybee 			 * we now need to reset its state.
956c717a561Smaybee 			 */
9577e2186e3Sbonwick 			dbuf_unoverride(dr);
958c717a561Smaybee 			if (db->db.db_object != DMU_META_DNODE_OBJECT)
959c717a561Smaybee 				arc_buf_thaw(db->db_buf);
960c717a561Smaybee 		}
961fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
9627e2186e3Sbonwick 		return (dr);
963fa9e4066Sahrens 	}
964fa9e4066Sahrens 
965fa9e4066Sahrens 	/*
966fa9e4066Sahrens 	 * Only valid if not already dirty.
967fa9e4066Sahrens 	 */
968fa9e4066Sahrens 	ASSERT(dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx ==
969fa9e4066Sahrens 	    (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN));
970fa9e4066Sahrens 
971fa9e4066Sahrens 	ASSERT3U(dn->dn_nlevels, >, db->db_level);
972fa9e4066Sahrens 	ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
973fa9e4066Sahrens 	    dn->dn_phys->dn_nlevels > db->db_level ||
974fa9e4066Sahrens 	    dn->dn_next_nlevels[txgoff] > db->db_level ||
975fa9e4066Sahrens 	    dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
976fa9e4066Sahrens 	    dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
977fa9e4066Sahrens 
978fa9e4066Sahrens 	/*
979fa9e4066Sahrens 	 * We should only be dirtying in syncing context if it's the
980fa9e4066Sahrens 	 * mos, a spa os, or we're initializing the os.  However, we are
981fa9e4066Sahrens 	 * allowed to dirty in syncing context provided we already
982fa9e4066Sahrens 	 * dirtied it in open context.  Hence we must make this
983fa9e4066Sahrens 	 * assertion only if we're not already dirty.
984fa9e4066Sahrens 	 */
985fa9e4066Sahrens 	ASSERT(!dmu_tx_is_syncing(tx) ||
986fa9e4066Sahrens 	    os->os_dsl_dataset == NULL ||
987fa9e4066Sahrens 	    !dsl_dir_is_private(os->os_dsl_dataset->ds_dir) ||
988c717a561Smaybee 	    !BP_IS_HOLE(os->os_rootbp));
989fa9e4066Sahrens 	ASSERT(db->db.db_size != 0);
990fa9e4066Sahrens 
991fa9e4066Sahrens 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
992fa9e4066Sahrens 
9931934e92fSmaybee 	if (db->db_blkid != DB_BONUS_BLKID) {
9941934e92fSmaybee 		/*
9951934e92fSmaybee 		 * Update the accounting.
996d3469faaSMark Maybee 		 * Note: we delay "free accounting" until after we drop
997d3469faaSMark Maybee 		 * the db_mtx.  This keeps us from grabbing other locks
998d3469faaSMark Maybee 		 * (and possibly deadlocking) in bp_get_dasize() while
999d3469faaSMark Maybee 		 * also holding the db_mtx.
10001934e92fSmaybee 		 */
10011934e92fSmaybee 		dnode_willuse_space(dn, db->db.db_size, tx);
1002d3469faaSMark Maybee 		do_free_accounting = dbuf_block_freeable(db);
10031934e92fSmaybee 	}
10041934e92fSmaybee 
1005ea8dc4b6Seschrock 	/*
1006ea8dc4b6Seschrock 	 * If this buffer is dirty in an old transaction group we need
1007ea8dc4b6Seschrock 	 * to make a copy of it so that the changes we make in this
1008ea8dc4b6Seschrock 	 * transaction group won't leak out when we sync the older txg.
1009ea8dc4b6Seschrock 	 */
1010c717a561Smaybee 	dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP);
1011c717a561Smaybee 	if (db->db_level == 0) {
1012c717a561Smaybee 		void *data_old = db->db_buf;
1013c717a561Smaybee 
1014c717a561Smaybee 		if (db->db_blkid == DB_BONUS_BLKID) {
1015c717a561Smaybee 			dbuf_fix_old_data(db, tx->tx_txg);
1016c717a561Smaybee 			data_old = db->db.db_data;
1017c717a561Smaybee 		} else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
1018c717a561Smaybee 			/*
1019c717a561Smaybee 			 * Release the data buffer from the cache so that we
1020c717a561Smaybee 			 * can modify it without impacting possible other users
1021c717a561Smaybee 			 * of this cached data block.  Note that indirect
1022c717a561Smaybee 			 * blocks and private objects are not released until the
1023c717a561Smaybee 			 * syncing state (since they are only modified then).
1024c717a561Smaybee 			 */
1025fa9e4066Sahrens 			arc_release(db->db_buf, db);
1026fa9e4066Sahrens 			dbuf_fix_old_data(db, tx->tx_txg);
1027c717a561Smaybee 			data_old = db->db_buf;
1028fa9e4066Sahrens 		}
1029c717a561Smaybee 		ASSERT(data_old != NULL);
1030c717a561Smaybee 		dr->dt.dl.dr_data = data_old;
1031c717a561Smaybee 	} else {
1032c717a561Smaybee 		mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL);
1033c717a561Smaybee 		list_create(&dr->dt.di.dr_children,
1034c717a561Smaybee 		    sizeof (dbuf_dirty_record_t),
1035c717a561Smaybee 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
1036fa9e4066Sahrens 	}
1037c717a561Smaybee 	dr->dr_dbuf = db;
1038c717a561Smaybee 	dr->dr_txg = tx->tx_txg;
1039c717a561Smaybee 	dr->dr_next = *drp;
1040c717a561Smaybee 	*drp = dr;
1041fa9e4066Sahrens 
1042fa9e4066Sahrens 	/*
1043fa9e4066Sahrens 	 * We could have been freed_in_flight between the dbuf_noread
1044fa9e4066Sahrens 	 * and dbuf_dirty.  We win, as though the dbuf_noread() had
1045fa9e4066Sahrens 	 * happened after the free.
1046fa9e4066Sahrens 	 */
1047fa9e4066Sahrens 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID) {
1048c717a561Smaybee 		mutex_enter(&dn->dn_mtx);
1049fa9e4066Sahrens 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1050c717a561Smaybee 		mutex_exit(&dn->dn_mtx);
1051c717a561Smaybee 		db->db_freed_in_flight = FALSE;
1052fa9e4066Sahrens 	}
1053fa9e4066Sahrens 
1054fa9e4066Sahrens 	/*
1055fa9e4066Sahrens 	 * This buffer is now part of this txg
1056fa9e4066Sahrens 	 */
1057fa9e4066Sahrens 	dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
1058fa9e4066Sahrens 	db->db_dirtycnt += 1;
1059fa9e4066Sahrens 	ASSERT3U(db->db_dirtycnt, <=, 3);
1060fa9e4066Sahrens 
1061fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1062fa9e4066Sahrens 
1063fa9e4066Sahrens 	if (db->db_blkid == DB_BONUS_BLKID) {
1064c717a561Smaybee 		mutex_enter(&dn->dn_mtx);
1065c717a561Smaybee 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1066c717a561Smaybee 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1067c717a561Smaybee 		mutex_exit(&dn->dn_mtx);
1068fa9e4066Sahrens 		dnode_setdirty(dn, tx);
1069c717a561Smaybee 		return (dr);
1070d3469faaSMark Maybee 	} else if (do_free_accounting) {
1071d3469faaSMark Maybee 		blkptr_t *bp = db->db_blkptr;
1072d3469faaSMark Maybee 		int64_t willfree = (bp && !BP_IS_HOLE(bp)) ?
1073d3469faaSMark Maybee 		    bp_get_dasize(os->os_spa, bp) : db->db.db_size;
1074d3469faaSMark Maybee 		/*
1075d3469faaSMark Maybee 		 * This is only a guess -- if the dbuf is dirty
1076d3469faaSMark Maybee 		 * in a previous txg, we don't know how much
1077d3469faaSMark Maybee 		 * space it will use on disk yet.  We should
1078d3469faaSMark Maybee 		 * really have the struct_rwlock to access
1079d3469faaSMark Maybee 		 * db_blkptr, but since this is just a guess,
1080d3469faaSMark Maybee 		 * it's OK if we get an odd answer.
1081d3469faaSMark Maybee 		 */
1082d3469faaSMark Maybee 		dnode_willuse_space(dn, -willfree, tx);
1083fa9e4066Sahrens 	}
1084fa9e4066Sahrens 
1085fa9e4066Sahrens 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
1086fa9e4066Sahrens 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
1087fa9e4066Sahrens 		drop_struct_lock = TRUE;
1088fa9e4066Sahrens 	}
1089fa9e4066Sahrens 
10908346f03fSJonathan W Adams 	if (db->db_level == 0) {
10918346f03fSJonathan W Adams 		dnode_new_blkid(dn, db->db_blkid, tx, drop_struct_lock);
10928346f03fSJonathan W Adams 		ASSERT(dn->dn_maxblkid >= db->db_blkid);
10938346f03fSJonathan W Adams 	}
10948346f03fSJonathan W Adams 
109544eda4d7Smaybee 	if (db->db_level+1 < dn->dn_nlevels) {
1096c717a561Smaybee 		dmu_buf_impl_t *parent = db->db_parent;
1097c717a561Smaybee 		dbuf_dirty_record_t *di;
1098c717a561Smaybee 		int parent_held = FALSE;
1099c717a561Smaybee 
1100c717a561Smaybee 		if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
1101c717a561Smaybee 			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1102c717a561Smaybee 
1103c717a561Smaybee 			parent = dbuf_hold_level(dn, db->db_level+1,
1104c717a561Smaybee 			    db->db_blkid >> epbs, FTAG);
1105c717a561Smaybee 			parent_held = TRUE;
1106c717a561Smaybee 		}
1107fa9e4066Sahrens 		if (drop_struct_lock)
1108fa9e4066Sahrens 			rw_exit(&dn->dn_struct_rwlock);
1109c717a561Smaybee 		ASSERT3U(db->db_level+1, ==, parent->db_level);
1110c717a561Smaybee 		di = dbuf_dirty(parent, tx);
1111c717a561Smaybee 		if (parent_held)
1112c717a561Smaybee 			dbuf_rele(parent, FTAG);
1113c717a561Smaybee 
1114c717a561Smaybee 		mutex_enter(&db->db_mtx);
1115c717a561Smaybee 		/*  possible race with dbuf_undirty() */
1116c717a561Smaybee 		if (db->db_last_dirty == dr ||
1117c717a561Smaybee 		    dn->dn_object == DMU_META_DNODE_OBJECT) {
1118c717a561Smaybee 			mutex_enter(&di->dt.di.dr_mtx);
1119c717a561Smaybee 			ASSERT3U(di->dr_txg, ==, tx->tx_txg);
1120c717a561Smaybee 			ASSERT(!list_link_active(&dr->dr_dirty_node));
1121c717a561Smaybee 			list_insert_tail(&di->dt.di.dr_children, dr);
1122c717a561Smaybee 			mutex_exit(&di->dt.di.dr_mtx);
1123c717a561Smaybee 			dr->dr_parent = di;
1124c717a561Smaybee 		}
1125c717a561Smaybee 		mutex_exit(&db->db_mtx);
1126fa9e4066Sahrens 	} else {
1127c717a561Smaybee 		ASSERT(db->db_level+1 == dn->dn_nlevels);
1128c717a561Smaybee 		ASSERT(db->db_blkid < dn->dn_nblkptr);
1129c717a561Smaybee 		ASSERT(db->db_parent == NULL ||
1130c717a561Smaybee 		    db->db_parent == db->db_dnode->dn_dbuf);
1131c717a561Smaybee 		mutex_enter(&dn->dn_mtx);
1132c717a561Smaybee 		ASSERT(!list_link_active(&dr->dr_dirty_node));
1133c717a561Smaybee 		list_insert_tail(&dn->dn_dirty_records[txgoff], dr);
1134c717a561Smaybee 		mutex_exit(&dn->dn_mtx);
1135fa9e4066Sahrens 		if (drop_struct_lock)
1136fa9e4066Sahrens 			rw_exit(&dn->dn_struct_rwlock);
1137fa9e4066Sahrens 	}
1138fa9e4066Sahrens 
1139fa9e4066Sahrens 	dnode_setdirty(dn, tx);
1140c717a561Smaybee 	return (dr);
1141fa9e4066Sahrens }
1142fa9e4066Sahrens 
1143fa9e4066Sahrens static int
1144fa9e4066Sahrens dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1145fa9e4066Sahrens {
1146fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
1147c717a561Smaybee 	uint64_t txg = tx->tx_txg;
114817f17c2dSbonwick 	dbuf_dirty_record_t *dr, **drp;
1149fa9e4066Sahrens 
1150c717a561Smaybee 	ASSERT(txg != 0);
1151ea8dc4b6Seschrock 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1152fa9e4066Sahrens 
1153fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
1154fa9e4066Sahrens 
1155fa9e4066Sahrens 	/*
1156fa9e4066Sahrens 	 * If this buffer is not dirty, we're done.
1157fa9e4066Sahrens 	 */
115817f17c2dSbonwick 	for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next)
1159c717a561Smaybee 		if (dr->dr_txg <= txg)
1160c717a561Smaybee 			break;
1161c717a561Smaybee 	if (dr == NULL || dr->dr_txg < txg) {
1162fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
1163fa9e4066Sahrens 		return (0);
1164fa9e4066Sahrens 	}
1165c717a561Smaybee 	ASSERT(dr->dr_txg == txg);
1166fa9e4066Sahrens 
1167fa9e4066Sahrens 	/*
1168fa9e4066Sahrens 	 * If this buffer is currently held, we cannot undirty
1169fa9e4066Sahrens 	 * it, since one of the current holders may be in the
1170fa9e4066Sahrens 	 * middle of an update.  Note that users of dbuf_undirty()
1171fa9e4066Sahrens 	 * should not place a hold on the dbuf before the call.
1172fa9e4066Sahrens 	 */
1173fa9e4066Sahrens 	if (refcount_count(&db->db_holds) > db->db_dirtycnt) {
1174fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
117544eda4d7Smaybee 		/* Make sure we don't toss this buffer at sync phase */
1176fa9e4066Sahrens 		mutex_enter(&dn->dn_mtx);
1177fa9e4066Sahrens 		dnode_clear_range(dn, db->db_blkid, 1, tx);
1178fa9e4066Sahrens 		mutex_exit(&dn->dn_mtx);
1179fa9e4066Sahrens 		return (0);
1180fa9e4066Sahrens 	}
1181fa9e4066Sahrens 
1182fa9e4066Sahrens 	dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
1183fa9e4066Sahrens 
1184fa9e4066Sahrens 	ASSERT(db->db.db_size != 0);
1185fa9e4066Sahrens 
1186fa9e4066Sahrens 	/* XXX would be nice to fix up dn_towrite_space[] */
1187fa9e4066Sahrens 
118817f17c2dSbonwick 	*drp = dr->dr_next;
1189c717a561Smaybee 
1190c717a561Smaybee 	if (dr->dr_parent) {
1191c717a561Smaybee 		mutex_enter(&dr->dr_parent->dt.di.dr_mtx);
1192c717a561Smaybee 		list_remove(&dr->dr_parent->dt.di.dr_children, dr);
1193c717a561Smaybee 		mutex_exit(&dr->dr_parent->dt.di.dr_mtx);
1194c717a561Smaybee 	} else if (db->db_level+1 == dn->dn_nlevels) {
1195cdb0ab79Smaybee 		ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
1196c717a561Smaybee 		mutex_enter(&dn->dn_mtx);
1197c717a561Smaybee 		list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr);
1198c717a561Smaybee 		mutex_exit(&dn->dn_mtx);
1199c717a561Smaybee 	}
1200c717a561Smaybee 
1201c717a561Smaybee 	if (db->db_level == 0) {
1202c717a561Smaybee 		dbuf_unoverride(dr);
1203c717a561Smaybee 
1204c717a561Smaybee 		ASSERT(db->db_buf != NULL);
1205c717a561Smaybee 		ASSERT(dr->dt.dl.dr_data != NULL);
1206c717a561Smaybee 		if (dr->dt.dl.dr_data != db->db_buf)
1207c717a561Smaybee 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1);
1208c717a561Smaybee 	} else {
1209c717a561Smaybee 		ASSERT(db->db_buf != NULL);
1210c717a561Smaybee 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1211c25056deSgw 		mutex_destroy(&dr->dt.di.dr_mtx);
1212c25056deSgw 		list_destroy(&dr->dt.di.dr_children);
1213c717a561Smaybee 	}
1214c717a561Smaybee 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
1215fa9e4066Sahrens 
1216fa9e4066Sahrens 	ASSERT(db->db_dirtycnt > 0);
1217fa9e4066Sahrens 	db->db_dirtycnt -= 1;
1218fa9e4066Sahrens 
1219c717a561Smaybee 	if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
1220ea8dc4b6Seschrock 		arc_buf_t *buf = db->db_buf;
1221fa9e4066Sahrens 
1222ea8dc4b6Seschrock 		ASSERT(arc_released(buf));
1223ea8dc4b6Seschrock 		dbuf_set_data(db, NULL);
1224ea8dc4b6Seschrock 		VERIFY(arc_buf_remove_ref(buf, db) == 1);
1225fa9e4066Sahrens 		dbuf_evict(db);
1226fa9e4066Sahrens 		return (1);
1227fa9e4066Sahrens 	}
1228fa9e4066Sahrens 
1229fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1230fa9e4066Sahrens 	return (0);
1231fa9e4066Sahrens }
1232fa9e4066Sahrens 
1233fa9e4066Sahrens #pragma weak dmu_buf_will_dirty = dbuf_will_dirty
1234fa9e4066Sahrens void
1235fa9e4066Sahrens dbuf_will_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
1236fa9e4066Sahrens {
12371ab7f2deSmaybee 	int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH;
1238fa9e4066Sahrens 
1239fa9e4066Sahrens 	ASSERT(tx->tx_txg != 0);
1240fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
1241fa9e4066Sahrens 
1242fa9e4066Sahrens 	if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock))
1243fa9e4066Sahrens 		rf |= DB_RF_HAVESTRUCT;
1244ea8dc4b6Seschrock 	(void) dbuf_read(db, NULL, rf);
1245c717a561Smaybee 	(void) dbuf_dirty(db, tx);
1246fa9e4066Sahrens }
1247fa9e4066Sahrens 
1248fa9e4066Sahrens void
1249ea8dc4b6Seschrock dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx)
1250fa9e4066Sahrens {
1251ea8dc4b6Seschrock 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1252ea8dc4b6Seschrock 
1253ea8dc4b6Seschrock 	ASSERT(db->db_blkid != DB_BONUS_BLKID);
1254fa9e4066Sahrens 	ASSERT(tx->tx_txg != 0);
1255fa9e4066Sahrens 	ASSERT(db->db_level == 0);
1256fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
1257fa9e4066Sahrens 
1258ea8dc4b6Seschrock 	ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
1259fa9e4066Sahrens 	    dmu_tx_private_ok(tx));
1260fa9e4066Sahrens 
1261fa9e4066Sahrens 	dbuf_noread(db);
1262c717a561Smaybee 	(void) dbuf_dirty(db, tx);
1263fa9e4066Sahrens }
1264fa9e4066Sahrens 
1265fa9e4066Sahrens #pragma weak dmu_buf_fill_done = dbuf_fill_done
1266fa9e4066Sahrens /* ARGSUSED */
1267fa9e4066Sahrens void
1268fa9e4066Sahrens dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx)
1269fa9e4066Sahrens {
1270fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
12719c9dc39aSek 	DBUF_VERIFY(db);
1272fa9e4066Sahrens 
1273fa9e4066Sahrens 	if (db->db_state == DB_FILL) {
1274c717a561Smaybee 		if (db->db_level == 0 && db->db_freed_in_flight) {
1275ea8dc4b6Seschrock 			ASSERT(db->db_blkid != DB_BONUS_BLKID);
1276fa9e4066Sahrens 			/* we were freed while filling */
1277fa9e4066Sahrens 			/* XXX dbuf_undirty? */
1278fa9e4066Sahrens 			bzero(db->db.db_data, db->db.db_size);
1279c717a561Smaybee 			db->db_freed_in_flight = FALSE;
1280fa9e4066Sahrens 		}
1281fa9e4066Sahrens 		db->db_state = DB_CACHED;
1282fa9e4066Sahrens 		cv_broadcast(&db->db_changed);
1283fa9e4066Sahrens 	}
1284fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1285fa9e4066Sahrens }
1286fa9e4066Sahrens 
1287ea8dc4b6Seschrock /*
1288ea8dc4b6Seschrock  * "Clear" the contents of this dbuf.  This will mark the dbuf
1289ea8dc4b6Seschrock  * EVICTING and clear *most* of its references.  Unfortunetely,
1290ea8dc4b6Seschrock  * when we are not holding the dn_dbufs_mtx, we can't clear the
1291ea8dc4b6Seschrock  * entry in the dn_dbufs list.  We have to wait until dbuf_destroy()
1292ea8dc4b6Seschrock  * in this case.  For callers from the DMU we will usually see:
1293ea8dc4b6Seschrock  *	dbuf_clear()->arc_buf_evict()->dbuf_do_evict()->dbuf_destroy()
1294ea8dc4b6Seschrock  * For the arc callback, we will usually see:
1295ea8dc4b6Seschrock  * 	dbuf_do_evict()->dbuf_clear();dbuf_destroy()
1296ea8dc4b6Seschrock  * Sometimes, though, we will get a mix of these two:
1297ea8dc4b6Seschrock  *	DMU: dbuf_clear()->arc_buf_evict()
1298ea8dc4b6Seschrock  *	ARC: dbuf_do_evict()->dbuf_destroy()
1299ea8dc4b6Seschrock  */
1300ea8dc4b6Seschrock void
1301fa9e4066Sahrens dbuf_clear(dmu_buf_impl_t *db)
1302fa9e4066Sahrens {
1303fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
1304ea8dc4b6Seschrock 	dmu_buf_impl_t *parent = db->db_parent;
1305c543ec06Sahrens 	dmu_buf_impl_t *dndb = dn->dn_dbuf;
1306ea8dc4b6Seschrock 	int dbuf_gone = FALSE;
1307fa9e4066Sahrens 
1308fa9e4066Sahrens 	ASSERT(MUTEX_HELD(&db->db_mtx));
1309fa9e4066Sahrens 	ASSERT(refcount_is_zero(&db->db_holds));
1310fa9e4066Sahrens 
1311ea8dc4b6Seschrock 	dbuf_evict_user(db);
1312ea8dc4b6Seschrock 
1313fa9e4066Sahrens 	if (db->db_state == DB_CACHED) {
1314ea8dc4b6Seschrock 		ASSERT(db->db.db_data != NULL);
13150e8c6158Smaybee 		if (db->db_blkid == DB_BONUS_BLKID) {
1316ea8dc4b6Seschrock 			zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN);
13170e8c6158Smaybee 			arc_space_return(DN_MAX_BONUSLEN);
13180e8c6158Smaybee 		}
1319fa9e4066Sahrens 		db->db.db_data = NULL;
1320fa9e4066Sahrens 		db->db_state = DB_UNCACHED;
1321fa9e4066Sahrens 	}
1322fa9e4066Sahrens 
1323fa9e4066Sahrens 	ASSERT3U(db->db_state, ==, DB_UNCACHED);
1324fa9e4066Sahrens 	ASSERT(db->db_data_pending == NULL);
1325fa9e4066Sahrens 
1326ea8dc4b6Seschrock 	db->db_state = DB_EVICTING;
1327ea8dc4b6Seschrock 	db->db_blkptr = NULL;
1328ea8dc4b6Seschrock 
1329ea8dc4b6Seschrock 	if (db->db_blkid != DB_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) {
1330ea8dc4b6Seschrock 		list_remove(&dn->dn_dbufs, db);
1331ea8dc4b6Seschrock 		dnode_rele(dn, db);
13321934e92fSmaybee 		db->db_dnode = NULL;
1333ea8dc4b6Seschrock 	}
1334ea8dc4b6Seschrock 
1335ea8dc4b6Seschrock 	if (db->db_buf)
1336ea8dc4b6Seschrock 		dbuf_gone = arc_buf_evict(db->db_buf);
1337ea8dc4b6Seschrock 
1338ea8dc4b6Seschrock 	if (!dbuf_gone)
1339ea8dc4b6Seschrock 		mutex_exit(&db->db_mtx);
1340fa9e4066Sahrens 
1341fa9e4066Sahrens 	/*
1342fa9e4066Sahrens 	 * If this dbuf is referened from an indirect dbuf,
1343fa9e4066Sahrens 	 * decrement the ref count on the indirect dbuf.
1344fa9e4066Sahrens 	 */
1345c543ec06Sahrens 	if (parent && parent != dndb)
1346ea8dc4b6Seschrock 		dbuf_rele(parent, db);
1347fa9e4066Sahrens }
1348fa9e4066Sahrens 
1349fa9e4066Sahrens static int
1350fa9e4066Sahrens dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse,
1351fa9e4066Sahrens     dmu_buf_impl_t **parentp, blkptr_t **bpp)
1352fa9e4066Sahrens {
1353fa9e4066Sahrens 	int nlevels, epbs;
1354fa9e4066Sahrens 
13550b69c2f0Sahrens 	*parentp = NULL;
13560b69c2f0Sahrens 	*bpp = NULL;
13570b69c2f0Sahrens 
1358ea8dc4b6Seschrock 	ASSERT(blkid != DB_BONUS_BLKID);
1359ea8dc4b6Seschrock 
1360fa9e4066Sahrens 	if (dn->dn_phys->dn_nlevels == 0)
1361fa9e4066Sahrens 		nlevels = 1;
1362fa9e4066Sahrens 	else
1363fa9e4066Sahrens 		nlevels = dn->dn_phys->dn_nlevels;
1364fa9e4066Sahrens 
1365fa9e4066Sahrens 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
1366fa9e4066Sahrens 
1367fa9e4066Sahrens 	ASSERT3U(level * epbs, <, 64);
1368fa9e4066Sahrens 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1369ea8dc4b6Seschrock 	if (level >= nlevels ||
1370fa9e4066Sahrens 	    (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) {
1371fa9e4066Sahrens 		/* the buffer has no parent yet */
1372fa9e4066Sahrens 		return (ENOENT);
1373fa9e4066Sahrens 	} else if (level < nlevels-1) {
1374fa9e4066Sahrens 		/* this block is referenced from an indirect block */
1375fa9e4066Sahrens 		int err = dbuf_hold_impl(dn, level+1,
1376fa9e4066Sahrens 		    blkid >> epbs, fail_sparse, NULL, parentp);
1377fa9e4066Sahrens 		if (err)
1378fa9e4066Sahrens 			return (err);
1379ea8dc4b6Seschrock 		err = dbuf_read(*parentp, NULL,
1380ea8dc4b6Seschrock 		    (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL));
1381c543ec06Sahrens 		if (err) {
1382c543ec06Sahrens 			dbuf_rele(*parentp, NULL);
1383c543ec06Sahrens 			*parentp = NULL;
1384c543ec06Sahrens 			return (err);
1385ea8dc4b6Seschrock 		}
1386c543ec06Sahrens 		*bpp = ((blkptr_t *)(*parentp)->db.db_data) +
1387c543ec06Sahrens 		    (blkid & ((1ULL << epbs) - 1));
1388c543ec06Sahrens 		return (0);
1389fa9e4066Sahrens 	} else {
1390fa9e4066Sahrens 		/* the block is referenced from the dnode */
1391fa9e4066Sahrens 		ASSERT3U(level, ==, nlevels-1);
1392fa9e4066Sahrens 		ASSERT(dn->dn_phys->dn_nblkptr == 0 ||
1393fa9e4066Sahrens 		    blkid < dn->dn_phys->dn_nblkptr);
1394c543ec06Sahrens 		if (dn->dn_dbuf) {
1395c543ec06Sahrens 			dbuf_add_ref(dn->dn_dbuf, NULL);
1396c543ec06Sahrens 			*parentp = dn->dn_dbuf;
1397c543ec06Sahrens 		}
1398fa9e4066Sahrens 		*bpp = &dn->dn_phys->dn_blkptr[blkid];
1399fa9e4066Sahrens 		return (0);
1400fa9e4066Sahrens 	}
1401fa9e4066Sahrens }
1402fa9e4066Sahrens 
1403fa9e4066Sahrens static dmu_buf_impl_t *
1404fa9e4066Sahrens dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
1405fa9e4066Sahrens     dmu_buf_impl_t *parent, blkptr_t *blkptr)
1406fa9e4066Sahrens {
1407fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
1408fa9e4066Sahrens 	dmu_buf_impl_t *db, *odb;
1409fa9e4066Sahrens 
1410fa9e4066Sahrens 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1411fa9e4066Sahrens 	ASSERT(dn->dn_type != DMU_OT_NONE);
1412fa9e4066Sahrens 
1413fa9e4066Sahrens 	db = kmem_cache_alloc(dbuf_cache, KM_SLEEP);
1414fa9e4066Sahrens 
1415fa9e4066Sahrens 	db->db_objset = os;
1416fa9e4066Sahrens 	db->db.db_object = dn->dn_object;
1417fa9e4066Sahrens 	db->db_level = level;
1418fa9e4066Sahrens 	db->db_blkid = blkid;
1419c717a561Smaybee 	db->db_last_dirty = NULL;
1420ea8dc4b6Seschrock 	db->db_dirtycnt = 0;
1421ea8dc4b6Seschrock 	db->db_dnode = dn;
1422ea8dc4b6Seschrock 	db->db_parent = parent;
1423ea8dc4b6Seschrock 	db->db_blkptr = blkptr;
1424fa9e4066Sahrens 
1425c717a561Smaybee 	db->db_user_ptr = NULL;
1426c717a561Smaybee 	db->db_user_data_ptr_ptr = NULL;
1427c717a561Smaybee 	db->db_evict_func = NULL;
1428c717a561Smaybee 	db->db_immediate_evict = 0;
1429c717a561Smaybee 	db->db_freed_in_flight = 0;
1430ea8dc4b6Seschrock 
1431ea8dc4b6Seschrock 	if (blkid == DB_BONUS_BLKID) {
1432ea8dc4b6Seschrock 		ASSERT3P(parent, ==, dn->dn_dbuf);
14331934e92fSmaybee 		db->db.db_size = DN_MAX_BONUSLEN -
14341934e92fSmaybee 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
14351934e92fSmaybee 		ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1436fa9e4066Sahrens 		db->db.db_offset = DB_BONUS_BLKID;
1437ea8dc4b6Seschrock 		db->db_state = DB_UNCACHED;
1438ea8dc4b6Seschrock 		/* the bonus dbuf is not placed in the hash table */
14390e8c6158Smaybee 		arc_space_consume(sizeof (dmu_buf_impl_t));
1440ea8dc4b6Seschrock 		return (db);
1441fa9e4066Sahrens 	} else {
1442fa9e4066Sahrens 		int blocksize =
1443fa9e4066Sahrens 		    db->db_level ? 1<<dn->dn_indblkshift :  dn->dn_datablksz;
1444fa9e4066Sahrens 		db->db.db_size = blocksize;
1445fa9e4066Sahrens 		db->db.db_offset = db->db_blkid * blocksize;
1446fa9e4066Sahrens 	}
1447fa9e4066Sahrens 
1448fa9e4066Sahrens 	/*
1449fa9e4066Sahrens 	 * Hold the dn_dbufs_mtx while we get the new dbuf
1450fa9e4066Sahrens 	 * in the hash table *and* added to the dbufs list.
1451fa9e4066Sahrens 	 * This prevents a possible deadlock with someone
1452fa9e4066Sahrens 	 * trying to look up this dbuf before its added to the
1453fa9e4066Sahrens 	 * dn_dbufs list.
1454fa9e4066Sahrens 	 */
1455fa9e4066Sahrens 	mutex_enter(&dn->dn_dbufs_mtx);
1456ea8dc4b6Seschrock 	db->db_state = DB_EVICTING;
1457fa9e4066Sahrens 	if ((odb = dbuf_hash_insert(db)) != NULL) {
1458fa9e4066Sahrens 		/* someone else inserted it first */
1459fa9e4066Sahrens 		kmem_cache_free(dbuf_cache, db);
1460fa9e4066Sahrens 		mutex_exit(&dn->dn_dbufs_mtx);
1461fa9e4066Sahrens 		return (odb);
1462fa9e4066Sahrens 	}
1463fa9e4066Sahrens 	list_insert_head(&dn->dn_dbufs, db);
1464ea8dc4b6Seschrock 	db->db_state = DB_UNCACHED;
1465fa9e4066Sahrens 	mutex_exit(&dn->dn_dbufs_mtx);
14660e8c6158Smaybee 	arc_space_consume(sizeof (dmu_buf_impl_t));
1467fa9e4066Sahrens 
1468fa9e4066Sahrens 	if (parent && parent != dn->dn_dbuf)
1469fa9e4066Sahrens 		dbuf_add_ref(parent, db);
1470fa9e4066Sahrens 
1471ea8dc4b6Seschrock 	ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT ||
1472ea8dc4b6Seschrock 	    refcount_count(&dn->dn_holds) > 0);
1473fa9e4066Sahrens 	(void) refcount_add(&dn->dn_holds, db);
1474fa9e4066Sahrens 
1475fa9e4066Sahrens 	dprintf_dbuf(db, "db=%p\n", db);
1476fa9e4066Sahrens 
1477fa9e4066Sahrens 	return (db);
1478fa9e4066Sahrens }
1479fa9e4066Sahrens 
1480fa9e4066Sahrens static int
1481ea8dc4b6Seschrock dbuf_do_evict(void *private)
1482fa9e4066Sahrens {
1483ea8dc4b6Seschrock 	arc_buf_t *buf = private;
1484ea8dc4b6Seschrock 	dmu_buf_impl_t *db = buf->b_private;
1485fa9e4066Sahrens 
1486ea8dc4b6Seschrock 	if (!MUTEX_HELD(&db->db_mtx))
1487ea8dc4b6Seschrock 		mutex_enter(&db->db_mtx);
1488fa9e4066Sahrens 
1489ea8dc4b6Seschrock 	ASSERT(refcount_is_zero(&db->db_holds));
1490fa9e4066Sahrens 
1491ea8dc4b6Seschrock 	if (db->db_state != DB_EVICTING) {
1492ea8dc4b6Seschrock 		ASSERT(db->db_state == DB_CACHED);
1493ea8dc4b6Seschrock 		DBUF_VERIFY(db);
1494ea8dc4b6Seschrock 		db->db_buf = NULL;
1495ea8dc4b6Seschrock 		dbuf_evict(db);
1496ea8dc4b6Seschrock 	} else {
1497ea8dc4b6Seschrock 		mutex_exit(&db->db_mtx);
1498ea8dc4b6Seschrock 		dbuf_destroy(db);
1499fa9e4066Sahrens 	}
1500ea8dc4b6Seschrock 	return (0);
1501fa9e4066Sahrens }
1502fa9e4066Sahrens 
1503fa9e4066Sahrens static void
1504fa9e4066Sahrens dbuf_destroy(dmu_buf_impl_t *db)
1505fa9e4066Sahrens {
1506fa9e4066Sahrens 	ASSERT(refcount_is_zero(&db->db_holds));
1507fa9e4066Sahrens 
1508ea8dc4b6Seschrock 	if (db->db_blkid != DB_BONUS_BLKID) {
1509ea8dc4b6Seschrock 		/*
1510ea8dc4b6Seschrock 		 * If this dbuf is still on the dn_dbufs list,
1511ea8dc4b6Seschrock 		 * remove it from that list.
1512ea8dc4b6Seschrock 		 */
15131934e92fSmaybee 		if (db->db_dnode) {
15141934e92fSmaybee 			dnode_t *dn = db->db_dnode;
15151934e92fSmaybee 
15161934e92fSmaybee 			mutex_enter(&dn->dn_dbufs_mtx);
1517ea8dc4b6Seschrock 			list_remove(&dn->dn_dbufs, db);
1518c543ec06Sahrens 			mutex_exit(&dn->dn_dbufs_mtx);
1519ea8dc4b6Seschrock 
1520ea8dc4b6Seschrock 			dnode_rele(dn, db);
15211934e92fSmaybee 			db->db_dnode = NULL;
1522ea8dc4b6Seschrock 		}
1523ea8dc4b6Seschrock 		dbuf_hash_remove(db);
1524ea8dc4b6Seschrock 	}
1525ea8dc4b6Seschrock 	db->db_parent = NULL;
1526ea8dc4b6Seschrock 	db->db_buf = NULL;
1527ea8dc4b6Seschrock 
1528f82bfe17Sgw 	ASSERT(!list_link_active(&db->db_link));
1529fa9e4066Sahrens 	ASSERT(db->db.db_data == NULL);
1530fa9e4066Sahrens 	ASSERT(db->db_hash_next == NULL);
1531fa9e4066Sahrens 	ASSERT(db->db_blkptr == NULL);
1532fa9e4066Sahrens 	ASSERT(db->db_data_pending == NULL);
1533fa9e4066Sahrens 
1534fa9e4066Sahrens 	kmem_cache_free(dbuf_cache, db);
15350e8c6158Smaybee 	arc_space_return(sizeof (dmu_buf_impl_t));
1536fa9e4066Sahrens }
1537fa9e4066Sahrens 
1538fa9e4066Sahrens void
1539fa9e4066Sahrens dbuf_prefetch(dnode_t *dn, uint64_t blkid)
1540fa9e4066Sahrens {
154113506d1eSmaybee 	dmu_buf_impl_t *db = NULL;
1542fa9e4066Sahrens 	blkptr_t *bp = NULL;
1543fa9e4066Sahrens 
1544fa9e4066Sahrens 	ASSERT(blkid != DB_BONUS_BLKID);
1545fa9e4066Sahrens 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1546fa9e4066Sahrens 
1547fa9e4066Sahrens 	if (dnode_block_freed(dn, blkid))
1548fa9e4066Sahrens 		return;
1549fa9e4066Sahrens 
1550fa9e4066Sahrens 	/* dbuf_find() returns with db_mtx held */
1551fa9e4066Sahrens 	if (db = dbuf_find(dn, 0, blkid)) {
155213506d1eSmaybee 		if (refcount_count(&db->db_holds) > 0) {
155313506d1eSmaybee 			/*
155413506d1eSmaybee 			 * This dbuf is active.  We assume that it is
155513506d1eSmaybee 			 * already CACHED, or else about to be either
155613506d1eSmaybee 			 * read or filled.
155713506d1eSmaybee 			 */
155813506d1eSmaybee 			mutex_exit(&db->db_mtx);
155913506d1eSmaybee 			return;
156013506d1eSmaybee 		}
1561fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
15620b69c2f0Sahrens 		db = NULL;
1563fa9e4066Sahrens 	}
1564fa9e4066Sahrens 
156513506d1eSmaybee 	if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) {
1566fa9e4066Sahrens 		if (bp && !BP_IS_HOLE(bp)) {
1567088f3894Sahrens 			arc_buf_t *pbuf;
156813506d1eSmaybee 			uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH;
1569ea8dc4b6Seschrock 			zbookmark_t zb;
1570ea8dc4b6Seschrock 			zb.zb_objset = dn->dn_objset->os_dsl_dataset ?
1571ea8dc4b6Seschrock 			    dn->dn_objset->os_dsl_dataset->ds_object : 0;
1572ea8dc4b6Seschrock 			zb.zb_object = dn->dn_object;
1573ea8dc4b6Seschrock 			zb.zb_level = 0;
1574ea8dc4b6Seschrock 			zb.zb_blkid = blkid;
1575ea8dc4b6Seschrock 
1576088f3894Sahrens 			if (db)
1577088f3894Sahrens 				pbuf = db->db_buf;
1578088f3894Sahrens 			else
1579088f3894Sahrens 				pbuf = dn->dn_objset->os_phys_buf;
1580088f3894Sahrens 
1581088f3894Sahrens 			(void) arc_read(NULL, dn->dn_objset->os_spa,
1582088f3894Sahrens 			    bp, pbuf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
1583fa9e4066Sahrens 			    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE,
158413506d1eSmaybee 			    &aflags, &zb);
1585fa9e4066Sahrens 		}
158613506d1eSmaybee 		if (db)
158713506d1eSmaybee 			dbuf_rele(db, NULL);
1588fa9e4066Sahrens 	}
1589fa9e4066Sahrens }
1590fa9e4066Sahrens 
1591fa9e4066Sahrens /*
1592fa9e4066Sahrens  * Returns with db_holds incremented, and db_mtx not held.
1593fa9e4066Sahrens  * Note: dn_struct_rwlock must be held.
1594fa9e4066Sahrens  */
1595fa9e4066Sahrens int
1596fa9e4066Sahrens dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse,
1597fa9e4066Sahrens     void *tag, dmu_buf_impl_t **dbp)
1598fa9e4066Sahrens {
1599fa9e4066Sahrens 	dmu_buf_impl_t *db, *parent = NULL;
1600fa9e4066Sahrens 
1601ea8dc4b6Seschrock 	ASSERT(blkid != DB_BONUS_BLKID);
1602fa9e4066Sahrens 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
1603fa9e4066Sahrens 	ASSERT3U(dn->dn_nlevels, >, level);
1604fa9e4066Sahrens 
1605fa9e4066Sahrens 	*dbp = NULL;
1606ea8dc4b6Seschrock top:
1607fa9e4066Sahrens 	/* dbuf_find() returns with db_mtx held */
1608fa9e4066Sahrens 	db = dbuf_find(dn, level, blkid);
1609fa9e4066Sahrens 
1610fa9e4066Sahrens 	if (db == NULL) {
1611fa9e4066Sahrens 		blkptr_t *bp = NULL;
1612fa9e4066Sahrens 		int err;
1613fa9e4066Sahrens 
1614c543ec06Sahrens 		ASSERT3P(parent, ==, NULL);
1615fa9e4066Sahrens 		err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp);
1616fa9e4066Sahrens 		if (fail_sparse) {
1617fa9e4066Sahrens 			if (err == 0 && bp && BP_IS_HOLE(bp))
1618fa9e4066Sahrens 				err = ENOENT;
1619fa9e4066Sahrens 			if (err) {
1620c543ec06Sahrens 				if (parent)
1621ea8dc4b6Seschrock 					dbuf_rele(parent, NULL);
1622fa9e4066Sahrens 				return (err);
1623fa9e4066Sahrens 			}
1624fa9e4066Sahrens 		}
1625ea8dc4b6Seschrock 		if (err && err != ENOENT)
1626ea8dc4b6Seschrock 			return (err);
1627fa9e4066Sahrens 		db = dbuf_create(dn, level, blkid, parent, bp);
1628fa9e4066Sahrens 	}
1629fa9e4066Sahrens 
1630ea8dc4b6Seschrock 	if (db->db_buf && refcount_is_zero(&db->db_holds)) {
1631ea8dc4b6Seschrock 		arc_buf_add_ref(db->db_buf, db);
1632ea8dc4b6Seschrock 		if (db->db_buf->b_data == NULL) {
1633ea8dc4b6Seschrock 			dbuf_clear(db);
1634c543ec06Sahrens 			if (parent) {
1635c543ec06Sahrens 				dbuf_rele(parent, NULL);
1636c543ec06Sahrens 				parent = NULL;
1637c543ec06Sahrens 			}
1638ea8dc4b6Seschrock 			goto top;
1639ea8dc4b6Seschrock 		}
1640ea8dc4b6Seschrock 		ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
1641ea8dc4b6Seschrock 	}
1642ea8dc4b6Seschrock 
1643ea8dc4b6Seschrock 	ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
1644ea8dc4b6Seschrock 
1645fa9e4066Sahrens 	/*
1646c717a561Smaybee 	 * If this buffer is currently syncing out, and we are are
1647c717a561Smaybee 	 * still referencing it from db_data, we need to make a copy
1648c717a561Smaybee 	 * of it in case we decide we want to dirty it again in this txg.
1649fa9e4066Sahrens 	 */
1650c717a561Smaybee 	if (db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
1651ea8dc4b6Seschrock 	    dn->dn_object != DMU_META_DNODE_OBJECT &&
1652c717a561Smaybee 	    db->db_state == DB_CACHED && db->db_data_pending) {
1653c717a561Smaybee 		dbuf_dirty_record_t *dr = db->db_data_pending;
1654fa9e4066Sahrens 
1655c717a561Smaybee 		if (dr->dt.dl.dr_data == db->db_buf) {
1656c717a561Smaybee 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1657c717a561Smaybee 
1658c717a561Smaybee 			dbuf_set_data(db,
1659c717a561Smaybee 			    arc_buf_alloc(db->db_dnode->dn_objset->os_spa,
1660c717a561Smaybee 			    db->db.db_size, db, type));
1661c717a561Smaybee 			bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data,
1662c717a561Smaybee 			    db->db.db_size);
1663c717a561Smaybee 		}
1664fa9e4066Sahrens 	}
1665fa9e4066Sahrens 
1666ea8dc4b6Seschrock 	(void) refcount_add(&db->db_holds, tag);
1667fa9e4066Sahrens 	dbuf_update_data(db);
16689c9dc39aSek 	DBUF_VERIFY(db);
1669fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1670fa9e4066Sahrens 
1671fa9e4066Sahrens 	/* NOTE: we can't rele the parent until after we drop the db_mtx */
1672c543ec06Sahrens 	if (parent)
1673ea8dc4b6Seschrock 		dbuf_rele(parent, NULL);
1674fa9e4066Sahrens 
1675fa9e4066Sahrens 	ASSERT3P(db->db_dnode, ==, dn);
1676fa9e4066Sahrens 	ASSERT3U(db->db_blkid, ==, blkid);
1677fa9e4066Sahrens 	ASSERT3U(db->db_level, ==, level);
1678fa9e4066Sahrens 	*dbp = db;
1679fa9e4066Sahrens 
1680fa9e4066Sahrens 	return (0);
1681fa9e4066Sahrens }
1682fa9e4066Sahrens 
1683fa9e4066Sahrens dmu_buf_impl_t *
1684ea8dc4b6Seschrock dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag)
1685fa9e4066Sahrens {
1686fa9e4066Sahrens 	dmu_buf_impl_t *db;
1687ea8dc4b6Seschrock 	int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db);
1688ea8dc4b6Seschrock 	return (err ? NULL : db);
1689fa9e4066Sahrens }
1690fa9e4066Sahrens 
1691fa9e4066Sahrens dmu_buf_impl_t *
1692fa9e4066Sahrens dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag)
1693fa9e4066Sahrens {
1694fa9e4066Sahrens 	dmu_buf_impl_t *db;
1695ea8dc4b6Seschrock 	int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db);
1696ea8dc4b6Seschrock 	return (err ? NULL : db);
1697fa9e4066Sahrens }
1698fa9e4066Sahrens 
16991934e92fSmaybee void
1700ea8dc4b6Seschrock dbuf_create_bonus(dnode_t *dn)
1701fa9e4066Sahrens {
1702ea8dc4b6Seschrock 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
1703ea8dc4b6Seschrock 
1704ea8dc4b6Seschrock 	ASSERT(dn->dn_bonus == NULL);
17051934e92fSmaybee 	dn->dn_bonus = dbuf_create(dn, 0, DB_BONUS_BLKID, dn->dn_dbuf, NULL);
1706fa9e4066Sahrens }
1707fa9e4066Sahrens 
1708ea8dc4b6Seschrock #pragma weak dmu_buf_add_ref = dbuf_add_ref
1709fa9e4066Sahrens void
1710fa9e4066Sahrens dbuf_add_ref(dmu_buf_impl_t *db, void *tag)
1711fa9e4066Sahrens {
1712ea8dc4b6Seschrock 	int64_t holds = refcount_add(&db->db_holds, tag);
1713ea8dc4b6Seschrock 	ASSERT(holds > 1);
1714fa9e4066Sahrens }
1715fa9e4066Sahrens 
1716ea8dc4b6Seschrock #pragma weak dmu_buf_rele = dbuf_rele
1717fa9e4066Sahrens void
1718ea8dc4b6Seschrock dbuf_rele(dmu_buf_impl_t *db, void *tag)
1719fa9e4066Sahrens {
1720fa9e4066Sahrens 	int64_t holds;
1721fa9e4066Sahrens 
1722fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
17239c9dc39aSek 	DBUF_VERIFY(db);
1724fa9e4066Sahrens 
1725fa9e4066Sahrens 	holds = refcount_remove(&db->db_holds, tag);
1726ea8dc4b6Seschrock 	ASSERT(holds >= 0);
1727ea8dc4b6Seschrock 
1728c717a561Smaybee 	/*
1729c717a561Smaybee 	 * We can't freeze indirects if there is a possibility that they
1730c717a561Smaybee 	 * may be modified in the current syncing context.
1731c717a561Smaybee 	 */
1732c717a561Smaybee 	if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0))
17336b4acc8bSahrens 		arc_buf_freeze(db->db_buf);
17346b4acc8bSahrens 
1735ea8dc4b6Seschrock 	if (holds == db->db_dirtycnt &&
1736c717a561Smaybee 	    db->db_level == 0 && db->db_immediate_evict)
1737ea8dc4b6Seschrock 		dbuf_evict_user(db);
1738fa9e4066Sahrens 
1739fa9e4066Sahrens 	if (holds == 0) {
1740ea8dc4b6Seschrock 		if (db->db_blkid == DB_BONUS_BLKID) {
1741ea8dc4b6Seschrock 			mutex_exit(&db->db_mtx);
1742ea8dc4b6Seschrock 			dnode_rele(db->db_dnode, db);
1743ea8dc4b6Seschrock 		} else if (db->db_buf == NULL) {
1744ea8dc4b6Seschrock 			/*
1745ea8dc4b6Seschrock 			 * This is a special case: we never associated this
1746ea8dc4b6Seschrock 			 * dbuf with any data allocated from the ARC.
1747ea8dc4b6Seschrock 			 */
1748ea8dc4b6Seschrock 			ASSERT3U(db->db_state, ==, DB_UNCACHED);
1749ea8dc4b6Seschrock 			dbuf_evict(db);
17506b4acc8bSahrens 		} else if (arc_released(db->db_buf)) {
1751ea8dc4b6Seschrock 			arc_buf_t *buf = db->db_buf;
1752ea8dc4b6Seschrock 			/*
1753ea8dc4b6Seschrock 			 * This dbuf has anonymous data associated with it.
1754ea8dc4b6Seschrock 			 */
1755ea8dc4b6Seschrock 			dbuf_set_data(db, NULL);
1756ea8dc4b6Seschrock 			VERIFY(arc_buf_remove_ref(buf, db) == 1);
1757ea8dc4b6Seschrock 			dbuf_evict(db);
1758ea8dc4b6Seschrock 		} else {
1759ea8dc4b6Seschrock 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
17603baa08fcSek 			if (!DBUF_IS_CACHEABLE(db))
17613baa08fcSek 				dbuf_clear(db);
17623baa08fcSek 			else
17633baa08fcSek 				mutex_exit(&db->db_mtx);
1764ea8dc4b6Seschrock 		}
1765fa9e4066Sahrens 	} else {
1766fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
1767fa9e4066Sahrens 	}
1768fa9e4066Sahrens }
1769fa9e4066Sahrens 
1770fa9e4066Sahrens #pragma weak dmu_buf_refcount = dbuf_refcount
1771fa9e4066Sahrens uint64_t
1772fa9e4066Sahrens dbuf_refcount(dmu_buf_impl_t *db)
1773fa9e4066Sahrens {
1774fa9e4066Sahrens 	return (refcount_count(&db->db_holds));
1775fa9e4066Sahrens }
1776fa9e4066Sahrens 
1777fa9e4066Sahrens void *
1778fa9e4066Sahrens dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1779fa9e4066Sahrens     dmu_buf_evict_func_t *evict_func)
1780fa9e4066Sahrens {
1781fa9e4066Sahrens 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1782fa9e4066Sahrens 	    user_data_ptr_ptr, evict_func));
1783fa9e4066Sahrens }
1784fa9e4066Sahrens 
1785fa9e4066Sahrens void *
1786fa9e4066Sahrens dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr,
1787fa9e4066Sahrens     dmu_buf_evict_func_t *evict_func)
1788fa9e4066Sahrens {
1789fa9e4066Sahrens 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1790fa9e4066Sahrens 
1791c717a561Smaybee 	db->db_immediate_evict = TRUE;
1792fa9e4066Sahrens 	return (dmu_buf_update_user(db_fake, NULL, user_ptr,
1793fa9e4066Sahrens 	    user_data_ptr_ptr, evict_func));
1794fa9e4066Sahrens }
1795fa9e4066Sahrens 
1796fa9e4066Sahrens void *
1797fa9e4066Sahrens dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
1798fa9e4066Sahrens     void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func)
1799fa9e4066Sahrens {
1800fa9e4066Sahrens 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1801fa9e4066Sahrens 	ASSERT(db->db_level == 0);
1802fa9e4066Sahrens 
1803fa9e4066Sahrens 	ASSERT((user_ptr == NULL) == (evict_func == NULL));
1804fa9e4066Sahrens 
1805fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
1806fa9e4066Sahrens 
1807c717a561Smaybee 	if (db->db_user_ptr == old_user_ptr) {
1808c717a561Smaybee 		db->db_user_ptr = user_ptr;
1809c717a561Smaybee 		db->db_user_data_ptr_ptr = user_data_ptr_ptr;
1810c717a561Smaybee 		db->db_evict_func = evict_func;
1811fa9e4066Sahrens 
1812fa9e4066Sahrens 		dbuf_update_data(db);
1813fa9e4066Sahrens 	} else {
1814c717a561Smaybee 		old_user_ptr = db->db_user_ptr;
1815fa9e4066Sahrens 	}
1816fa9e4066Sahrens 
1817fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1818fa9e4066Sahrens 	return (old_user_ptr);
1819fa9e4066Sahrens }
1820fa9e4066Sahrens 
1821fa9e4066Sahrens void *
1822fa9e4066Sahrens dmu_buf_get_user(dmu_buf_t *db_fake)
1823fa9e4066Sahrens {
1824fa9e4066Sahrens 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
1825fa9e4066Sahrens 	ASSERT(!refcount_is_zero(&db->db_holds));
1826fa9e4066Sahrens 
1827c717a561Smaybee 	return (db->db_user_ptr);
1828fa9e4066Sahrens }
1829fa9e4066Sahrens 
1830c717a561Smaybee static void
1831c717a561Smaybee dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
1832fa9e4066Sahrens {
1833c717a561Smaybee 	/* ASSERT(dmu_tx_is_syncing(tx) */
1834c717a561Smaybee 	ASSERT(MUTEX_HELD(&db->db_mtx));
1835c717a561Smaybee 
1836c717a561Smaybee 	if (db->db_blkptr != NULL)
1837c717a561Smaybee 		return;
1838c717a561Smaybee 
1839c717a561Smaybee 	if (db->db_level == dn->dn_phys->dn_nlevels-1) {
1840c717a561Smaybee 		/*
1841c717a561Smaybee 		 * This buffer was allocated at a time when there was
1842c717a561Smaybee 		 * no available blkptrs from the dnode, or it was
1843c717a561Smaybee 		 * inappropriate to hook it in (i.e., nlevels mis-match).
1844c717a561Smaybee 		 */
1845c717a561Smaybee 		ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
1846c717a561Smaybee 		ASSERT(db->db_parent == NULL);
1847c717a561Smaybee 		db->db_parent = dn->dn_dbuf;
1848c717a561Smaybee 		db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
1849c717a561Smaybee 		DBUF_VERIFY(db);
1850c717a561Smaybee 	} else {
1851c717a561Smaybee 		dmu_buf_impl_t *parent = db->db_parent;
1852c717a561Smaybee 		int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
1853c717a561Smaybee 
1854c717a561Smaybee 		ASSERT(dn->dn_phys->dn_nlevels > 1);
1855c717a561Smaybee 		if (parent == NULL) {
1856c717a561Smaybee 			mutex_exit(&db->db_mtx);
1857c717a561Smaybee 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
1858c717a561Smaybee 			(void) dbuf_hold_impl(dn, db->db_level+1,
1859c717a561Smaybee 			    db->db_blkid >> epbs, FALSE, db, &parent);
1860c717a561Smaybee 			rw_exit(&dn->dn_struct_rwlock);
1861c717a561Smaybee 			mutex_enter(&db->db_mtx);
1862c717a561Smaybee 			db->db_parent = parent;
1863c717a561Smaybee 		}
1864c717a561Smaybee 		db->db_blkptr = (blkptr_t *)parent->db.db_data +
1865c717a561Smaybee 		    (db->db_blkid & ((1ULL << epbs) - 1));
1866c717a561Smaybee 		DBUF_VERIFY(db);
1867c717a561Smaybee 	}
1868c717a561Smaybee }
1869c717a561Smaybee 
1870c717a561Smaybee static void
1871c717a561Smaybee dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
1872c717a561Smaybee {
1873c717a561Smaybee 	dmu_buf_impl_t *db = dr->dr_dbuf;
1874c717a561Smaybee 	dnode_t *dn = db->db_dnode;
1875c717a561Smaybee 	zio_t *zio;
1876c717a561Smaybee 
1877c717a561Smaybee 	ASSERT(dmu_tx_is_syncing(tx));
1878c717a561Smaybee 
1879c717a561Smaybee 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
1880c717a561Smaybee 
1881c717a561Smaybee 	mutex_enter(&db->db_mtx);
1882c717a561Smaybee 
1883c717a561Smaybee 	ASSERT(db->db_level > 0);
1884c717a561Smaybee 	DBUF_VERIFY(db);
1885c717a561Smaybee 
1886c717a561Smaybee 	if (db->db_buf == NULL) {
1887c717a561Smaybee 		mutex_exit(&db->db_mtx);
1888c717a561Smaybee 		(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
1889c717a561Smaybee 		mutex_enter(&db->db_mtx);
1890c717a561Smaybee 	}
1891c717a561Smaybee 	ASSERT3U(db->db_state, ==, DB_CACHED);
1892c717a561Smaybee 	ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
1893c717a561Smaybee 	ASSERT(db->db_buf != NULL);
1894c717a561Smaybee 
1895c717a561Smaybee 	dbuf_check_blkptr(dn, db);
1896c717a561Smaybee 
1897c717a561Smaybee 	db->db_data_pending = dr;
1898c717a561Smaybee 
1899af2c4821Smaybee 	mutex_exit(&db->db_mtx);
1900088f3894Sahrens 	dbuf_write(dr, db->db_buf, tx);
1901c717a561Smaybee 
1902c717a561Smaybee 	zio = dr->dr_zio;
1903c717a561Smaybee 	mutex_enter(&dr->dt.di.dr_mtx);
1904c717a561Smaybee 	dbuf_sync_list(&dr->dt.di.dr_children, tx);
1905c717a561Smaybee 	ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
1906c717a561Smaybee 	mutex_exit(&dr->dt.di.dr_mtx);
1907c717a561Smaybee 	zio_nowait(zio);
1908c717a561Smaybee }
1909c717a561Smaybee 
1910c717a561Smaybee static void
1911c717a561Smaybee dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx)
1912c717a561Smaybee {
1913c717a561Smaybee 	arc_buf_t **datap = &dr->dt.dl.dr_data;
1914c717a561Smaybee 	dmu_buf_impl_t *db = dr->dr_dbuf;
1915fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
1916fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
1917c717a561Smaybee 	uint64_t txg = tx->tx_txg;
1918fa9e4066Sahrens 	int blksz;
1919fa9e4066Sahrens 
1920fa9e4066Sahrens 	ASSERT(dmu_tx_is_syncing(tx));
1921fa9e4066Sahrens 
1922fa9e4066Sahrens 	dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
1923fa9e4066Sahrens 
1924fa9e4066Sahrens 	mutex_enter(&db->db_mtx);
1925fa9e4066Sahrens 	/*
1926fa9e4066Sahrens 	 * To be synced, we must be dirtied.  But we
1927fa9e4066Sahrens 	 * might have been freed after the dirty.
1928fa9e4066Sahrens 	 */
1929fa9e4066Sahrens 	if (db->db_state == DB_UNCACHED) {
1930fa9e4066Sahrens 		/* This buffer has been freed since it was dirtied */
1931fa9e4066Sahrens 		ASSERT(db->db.db_data == NULL);
1932fa9e4066Sahrens 	} else if (db->db_state == DB_FILL) {
1933fa9e4066Sahrens 		/* This buffer was freed and is now being re-filled */
1934c717a561Smaybee 		ASSERT(db->db.db_data != dr->dt.dl.dr_data);
1935fa9e4066Sahrens 	} else {
1936fa9e4066Sahrens 		ASSERT3U(db->db_state, ==, DB_CACHED);
1937fa9e4066Sahrens 	}
19389c9dc39aSek 	DBUF_VERIFY(db);
1939fa9e4066Sahrens 
1940fa9e4066Sahrens 	/*
1941c717a561Smaybee 	 * If this is a bonus buffer, simply copy the bonus data into the
1942c717a561Smaybee 	 * dnode.  It will be written out when the dnode is synced (and it
1943c717a561Smaybee 	 * will be synced, since it must have been dirty for dbuf_sync to
1944c717a561Smaybee 	 * be called).
1945fa9e4066Sahrens 	 */
1946ea8dc4b6Seschrock 	if (db->db_blkid == DB_BONUS_BLKID) {
1947c717a561Smaybee 		dbuf_dirty_record_t **drp;
19481934e92fSmaybee 
1949ea8dc4b6Seschrock 		ASSERT(*datap != NULL);
1950ea8dc4b6Seschrock 		ASSERT3U(db->db_level, ==, 0);
1951ea8dc4b6Seschrock 		ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN);
1952ea8dc4b6Seschrock 		bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen);
19530e8c6158Smaybee 		if (*datap != db->db.db_data) {
1954ea8dc4b6Seschrock 			zio_buf_free(*datap, DN_MAX_BONUSLEN);
19550e8c6158Smaybee 			arc_space_return(DN_MAX_BONUSLEN);
19560e8c6158Smaybee 		}
1957ea8dc4b6Seschrock 		db->db_data_pending = NULL;
1958c717a561Smaybee 		drp = &db->db_last_dirty;
1959c717a561Smaybee 		while (*drp != dr)
1960c717a561Smaybee 			drp = &(*drp)->dr_next;
196117f17c2dSbonwick 		ASSERT(dr->dr_next == NULL);
196217f17c2dSbonwick 		*drp = dr->dr_next;
1963c717a561Smaybee 		kmem_free(dr, sizeof (dbuf_dirty_record_t));
1964ea8dc4b6Seschrock 		ASSERT(db->db_dirtycnt > 0);
1965ea8dc4b6Seschrock 		db->db_dirtycnt -= 1;
1966ea8dc4b6Seschrock 		mutex_exit(&db->db_mtx);
1967ea8dc4b6Seschrock 		dbuf_rele(db, (void *)(uintptr_t)txg);
1968ea8dc4b6Seschrock 		return;
1969ea8dc4b6Seschrock 	}
1970ea8dc4b6Seschrock 
1971f82bfe17Sgw 	/*
1972f82bfe17Sgw 	 * This function may have dropped the db_mtx lock allowing a dmu_sync
1973f82bfe17Sgw 	 * operation to sneak in. As a result, we need to ensure that we
1974f82bfe17Sgw 	 * don't check the dr_override_state until we have returned from
1975f82bfe17Sgw 	 * dbuf_check_blkptr.
1976f82bfe17Sgw 	 */
1977f82bfe17Sgw 	dbuf_check_blkptr(dn, db);
1978f82bfe17Sgw 
1979c717a561Smaybee 	/*
1980c717a561Smaybee 	 * If this buffer is in the middle of an immdiate write,
1981c717a561Smaybee 	 * wait for the synchronous IO to complete.
1982c717a561Smaybee 	 */
1983c717a561Smaybee 	while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) {
1984c717a561Smaybee 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
1985c717a561Smaybee 		cv_wait(&db->db_changed, &db->db_mtx);
1986c717a561Smaybee 		ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN);
1987c717a561Smaybee 	}
1988c5c6ffa0Smaybee 
1989c717a561Smaybee 	/*
1990c717a561Smaybee 	 * If this dbuf has already been written out via an immediate write,
1991c717a561Smaybee 	 * just complete the write by copying over the new block pointer and
1992c717a561Smaybee 	 * updating the accounting via the write-completion functions.
1993c717a561Smaybee 	 */
1994c717a561Smaybee 	if (dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1995c717a561Smaybee 		zio_t zio_fake;
1996c717a561Smaybee 
1997c717a561Smaybee 		zio_fake.io_private = &db;
1998c717a561Smaybee 		zio_fake.io_error = 0;
1999c717a561Smaybee 		zio_fake.io_bp = db->db_blkptr;
2000c717a561Smaybee 		zio_fake.io_bp_orig = *db->db_blkptr;
2001c717a561Smaybee 		zio_fake.io_txg = txg;
2002e14bb325SJeff Bonwick 		zio_fake.io_flags = 0;
2003c717a561Smaybee 
2004c717a561Smaybee 		*db->db_blkptr = dr->dt.dl.dr_overridden_by;
2005c717a561Smaybee 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
2006c717a561Smaybee 		db->db_data_pending = dr;
2007c717a561Smaybee 		dr->dr_zio = &zio_fake;
2008c717a561Smaybee 		mutex_exit(&db->db_mtx);
2009c717a561Smaybee 
2010e14bb325SJeff Bonwick 		ASSERT(!DVA_EQUAL(BP_IDENTITY(zio_fake.io_bp),
2011f4a72450SJeff Bonwick 		    BP_IDENTITY(&zio_fake.io_bp_orig)) ||
2012f4a72450SJeff Bonwick 		    BP_IS_HOLE(zio_fake.io_bp));
2013e14bb325SJeff Bonwick 
2014c717a561Smaybee 		if (BP_IS_OLDER(&zio_fake.io_bp_orig, txg))
2015cdb0ab79Smaybee 			(void) dsl_dataset_block_kill(os->os_dsl_dataset,
2016c717a561Smaybee 			    &zio_fake.io_bp_orig, dn->dn_zio, tx);
2017c717a561Smaybee 
2018c717a561Smaybee 		dbuf_write_ready(&zio_fake, db->db_buf, db);
2019c717a561Smaybee 		dbuf_write_done(&zio_fake, db->db_buf, db);
2020c717a561Smaybee 
2021c717a561Smaybee 		return;
2022c717a561Smaybee 	}
2023c717a561Smaybee 
2024c717a561Smaybee 	blksz = arc_buf_size(*datap);
2025c717a561Smaybee 
2026c717a561Smaybee 	if (dn->dn_object != DMU_META_DNODE_OBJECT) {
2027fa9e4066Sahrens 		/*
2028fa9e4066Sahrens 		 * If this buffer is currently "in use" (i.e., there are
2029fa9e4066Sahrens 		 * active holds and db_data still references it), then make
2030fa9e4066Sahrens 		 * a copy before we start the write so that any modifications
2031fa9e4066Sahrens 		 * from the open txg will not leak into this write.
2032fa9e4066Sahrens 		 *
2033fa9e4066Sahrens 		 * NOTE: this copy does not need to be made for objects only
2034c717a561Smaybee 		 * modified in the syncing context (e.g. DNONE_DNODE blocks).
2035fa9e4066Sahrens 		 */
2036c717a561Smaybee 		if (refcount_count(&db->db_holds) > 1 && *datap == db->db_buf) {
2037c717a561Smaybee 			arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2038c717a561Smaybee 			*datap = arc_buf_alloc(os->os_spa, blksz, db, type);
2039c717a561Smaybee 			bcopy(db->db.db_data, (*datap)->b_data, blksz);
2040fa9e4066Sahrens 		}
2041fa9e4066Sahrens 	}
2042fa9e4066Sahrens 
2043c717a561Smaybee 	ASSERT(*datap != NULL);
2044c717a561Smaybee 	db->db_data_pending = dr;
2045fa9e4066Sahrens 
2046c717a561Smaybee 	mutex_exit(&db->db_mtx);
2047fa9e4066Sahrens 
2048088f3894Sahrens 	dbuf_write(dr, *datap, tx);
2049fa9e4066Sahrens 
2050c717a561Smaybee 	ASSERT(!list_link_active(&dr->dr_dirty_node));
2051c717a561Smaybee 	if (dn->dn_object == DMU_META_DNODE_OBJECT)
2052c717a561Smaybee 		list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr);
2053c717a561Smaybee 	else
2054c717a561Smaybee 		zio_nowait(dr->dr_zio);
2055c717a561Smaybee }
205623b11526Smaybee 
2057c717a561Smaybee void
2058c717a561Smaybee dbuf_sync_list(list_t *list, dmu_tx_t *tx)
2059c717a561Smaybee {
2060c717a561Smaybee 	dbuf_dirty_record_t *dr;
2061c717a561Smaybee 
2062c717a561Smaybee 	while (dr = list_head(list)) {
2063c717a561Smaybee 		if (dr->dr_zio != NULL) {
2064c717a561Smaybee 			/*
2065c717a561Smaybee 			 * If we find an already initialized zio then we
2066c717a561Smaybee 			 * are processing the meta-dnode, and we have finished.
2067c717a561Smaybee 			 * The dbufs for all dnodes are put back on the list
2068c717a561Smaybee 			 * during processing, so that we can zio_wait()
2069c717a561Smaybee 			 * these IOs after initiating all child IOs.
2070c717a561Smaybee 			 */
2071c717a561Smaybee 			ASSERT3U(dr->dr_dbuf->db.db_object, ==,
2072c717a561Smaybee 			    DMU_META_DNODE_OBJECT);
2073c717a561Smaybee 			break;
207423b11526Smaybee 		}
2075c717a561Smaybee 		list_remove(list, dr);
2076c717a561Smaybee 		if (dr->dr_dbuf->db_level > 0)
2077c717a561Smaybee 			dbuf_sync_indirect(dr, tx);
2078c717a561Smaybee 		else
2079c717a561Smaybee 			dbuf_sync_leaf(dr, tx);
208023b11526Smaybee 	}
2081c717a561Smaybee }
208223b11526Smaybee 
2083c717a561Smaybee static void
2084088f3894Sahrens dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx)
2085c717a561Smaybee {
2086c717a561Smaybee 	dmu_buf_impl_t *db = dr->dr_dbuf;
2087c717a561Smaybee 	dnode_t *dn = db->db_dnode;
2088c717a561Smaybee 	objset_impl_t *os = dn->dn_objset;
2089c717a561Smaybee 	dmu_buf_impl_t *parent = db->db_parent;
2090c717a561Smaybee 	uint64_t txg = tx->tx_txg;
2091c717a561Smaybee 	zbookmark_t zb;
2092088f3894Sahrens 	writeprops_t wp = { 0 };
2093c717a561Smaybee 	zio_t *zio;
2094fa9e4066Sahrens 
2095088f3894Sahrens 	if (!BP_IS_HOLE(db->db_blkptr) &&
2096088f3894Sahrens 	    (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE)) {
2097088f3894Sahrens 		/*
2098088f3894Sahrens 		 * Private object buffers are released here rather
2099088f3894Sahrens 		 * than in dbuf_dirty() since they are only modified
2100088f3894Sahrens 		 * in the syncing context and we don't want the
2101088f3894Sahrens 		 * overhead of making multiple copies of the data.
2102088f3894Sahrens 		 */
2103088f3894Sahrens 		arc_release(data, db);
2104088f3894Sahrens 	} else {
2105088f3894Sahrens 		ASSERT(arc_released(data));
2106088f3894Sahrens 		/* XXX why do we need to thaw here? */
2107088f3894Sahrens 		arc_buf_thaw(data);
2108088f3894Sahrens 	}
2109088f3894Sahrens 
2110c717a561Smaybee 	if (parent != dn->dn_dbuf) {
2111c717a561Smaybee 		ASSERT(parent && parent->db_data_pending);
2112fa9e4066Sahrens 		ASSERT(db->db_level == parent->db_level-1);
2113c717a561Smaybee 		ASSERT(arc_released(parent->db_buf));
2114c717a561Smaybee 		zio = parent->db_data_pending->dr_zio;
2115ea8dc4b6Seschrock 	} else {
2116fa9e4066Sahrens 		ASSERT(db->db_level == dn->dn_phys->dn_nlevels-1);
2117fa9e4066Sahrens 		ASSERT3P(db->db_blkptr, ==,
2118fa9e4066Sahrens 		    &dn->dn_phys->dn_blkptr[db->db_blkid]);
2119c717a561Smaybee 		zio = dn->dn_zio;
2120ea8dc4b6Seschrock 	}
2121fa9e4066Sahrens 
2122c717a561Smaybee 	ASSERT(db->db_level == 0 || data == db->db_buf);
2123c717a561Smaybee 	ASSERT3U(db->db_blkptr->blk_birth, <=, txg);
2124c717a561Smaybee 	ASSERT(zio);
2125fa9e4066Sahrens 
2126ea8dc4b6Seschrock 	zb.zb_objset = os->os_dsl_dataset ? os->os_dsl_dataset->ds_object : 0;
2127ea8dc4b6Seschrock 	zb.zb_object = db->db.db_object;
2128ea8dc4b6Seschrock 	zb.zb_level = db->db_level;
2129ea8dc4b6Seschrock 	zb.zb_blkid = db->db_blkid;
213044cd46caSbillm 
2131088f3894Sahrens 	wp.wp_type = dn->dn_type;
2132088f3894Sahrens 	wp.wp_level = db->db_level;
2133088f3894Sahrens 	wp.wp_copies = os->os_copies;
2134088f3894Sahrens 	wp.wp_dncompress = dn->dn_compress;
2135088f3894Sahrens 	wp.wp_oscompress = os->os_compress;
2136088f3894Sahrens 	wp.wp_dnchecksum = dn->dn_checksum;
2137088f3894Sahrens 	wp.wp_oschecksum = os->os_checksum;
2138088f3894Sahrens 
2139c717a561Smaybee 	if (BP_IS_OLDER(db->db_blkptr, txg))
2140cdb0ab79Smaybee 		(void) dsl_dataset_block_kill(
2141c717a561Smaybee 		    os->os_dsl_dataset, db->db_blkptr, zio, tx);
2142fa9e4066Sahrens 
2143088f3894Sahrens 	dr->dr_zio = arc_write(zio, os->os_spa, &wp,
21443baa08fcSek 	    DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr,
21453baa08fcSek 	    data, dbuf_write_ready, dbuf_write_done, db,
2146e14bb325SJeff Bonwick 	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
2147fa9e4066Sahrens }
2148fa9e4066Sahrens 
2149fa9e4066Sahrens /* ARGSUSED */
2150fa9e4066Sahrens static void
2151c717a561Smaybee dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb)
2152fa9e4066Sahrens {
2153fa9e4066Sahrens 	dmu_buf_impl_t *db = vdb;
2154fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
2155fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
2156e14bb325SJeff Bonwick 	blkptr_t *bp = zio->io_bp;
2157c717a561Smaybee 	blkptr_t *bp_orig = &zio->io_bp_orig;
2158fa9e4066Sahrens 	uint64_t fill = 0;
2159c717a561Smaybee 	int old_size, new_size, i;
2160fa9e4066Sahrens 
2161e14bb325SJeff Bonwick 	ASSERT(db->db_blkptr == bp);
2162e14bb325SJeff Bonwick 
2163c717a561Smaybee 	dprintf_dbuf_bp(db, bp_orig, "bp_orig: %s", "");
2164fa9e4066Sahrens 
2165c717a561Smaybee 	old_size = bp_get_dasize(os->os_spa, bp_orig);
2166e14bb325SJeff Bonwick 	new_size = bp_get_dasize(os->os_spa, bp);
2167fa9e4066Sahrens 
2168e14bb325SJeff Bonwick 	dnode_diduse_space(dn, new_size - old_size);
2169fa9e4066Sahrens 
2170e14bb325SJeff Bonwick 	if (BP_IS_HOLE(bp)) {
2171c717a561Smaybee 		dsl_dataset_t *ds = os->os_dsl_dataset;
2172c717a561Smaybee 		dmu_tx_t *tx = os->os_synctx;
2173fa9e4066Sahrens 
2174c717a561Smaybee 		if (bp_orig->blk_birth == tx->tx_txg)
2175e14bb325SJeff Bonwick 			(void) dsl_dataset_block_kill(ds, bp_orig, zio, tx);
2176e14bb325SJeff Bonwick 		ASSERT3U(bp->blk_fill, ==, 0);
2177c717a561Smaybee 		return;
2178c717a561Smaybee 	}
2179c5c6ffa0Smaybee 
2180e14bb325SJeff Bonwick 	ASSERT(BP_GET_TYPE(bp) == dn->dn_type);
2181e14bb325SJeff Bonwick 	ASSERT(BP_GET_LEVEL(bp) == db->db_level);
2182e14bb325SJeff Bonwick 
2183c717a561Smaybee 	mutex_enter(&db->db_mtx);
2184fa9e4066Sahrens 
2185fa9e4066Sahrens 	if (db->db_level == 0) {
2186fa9e4066Sahrens 		mutex_enter(&dn->dn_mtx);
2187c717a561Smaybee 		if (db->db_blkid > dn->dn_phys->dn_maxblkid)
2188fa9e4066Sahrens 			dn->dn_phys->dn_maxblkid = db->db_blkid;
2189fa9e4066Sahrens 		mutex_exit(&dn->dn_mtx);
2190fa9e4066Sahrens 
2191fa9e4066Sahrens 		if (dn->dn_type == DMU_OT_DNODE) {
2192fa9e4066Sahrens 			dnode_phys_t *dnp = db->db.db_data;
2193fa9e4066Sahrens 			for (i = db->db.db_size >> DNODE_SHIFT; i > 0;
2194fa9e4066Sahrens 			    i--, dnp++) {
2195fa9e4066Sahrens 				if (dnp->dn_type != DMU_OT_NONE)
2196fa9e4066Sahrens 					fill++;
2197fa9e4066Sahrens 			}
2198fa9e4066Sahrens 		} else {
2199c717a561Smaybee 			fill = 1;
2200fa9e4066Sahrens 		}
2201fa9e4066Sahrens 	} else {
2202e14bb325SJeff Bonwick 		blkptr_t *ibp = db->db.db_data;
2203fa9e4066Sahrens 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2204e14bb325SJeff Bonwick 		for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
2205e14bb325SJeff Bonwick 			if (BP_IS_HOLE(ibp))
2206fa9e4066Sahrens 				continue;
2207e14bb325SJeff Bonwick 			ASSERT3U(BP_GET_LSIZE(ibp), ==,
2208fa9e4066Sahrens 			    db->db_level == 1 ? dn->dn_datablksz :
2209fa9e4066Sahrens 			    (1<<dn->dn_phys->dn_indblkshift));
2210e14bb325SJeff Bonwick 			fill += ibp->blk_fill;
2211fa9e4066Sahrens 		}
2212fa9e4066Sahrens 	}
2213fa9e4066Sahrens 
2214e14bb325SJeff Bonwick 	bp->blk_fill = fill;
2215c717a561Smaybee 
2216c717a561Smaybee 	mutex_exit(&db->db_mtx);
2217c717a561Smaybee 
2218e14bb325SJeff Bonwick 	if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
2219e14bb325SJeff Bonwick 		ASSERT(DVA_EQUAL(BP_IDENTITY(bp), BP_IDENTITY(bp_orig)));
2220e14bb325SJeff Bonwick 	} else {
2221c717a561Smaybee 		dsl_dataset_t *ds = os->os_dsl_dataset;
2222c717a561Smaybee 		dmu_tx_t *tx = os->os_synctx;
2223c717a561Smaybee 
2224c717a561Smaybee 		if (bp_orig->blk_birth == tx->tx_txg)
2225e14bb325SJeff Bonwick 			(void) dsl_dataset_block_kill(ds, bp_orig, zio, tx);
2226e14bb325SJeff Bonwick 		dsl_dataset_block_born(ds, bp, tx);
2227fa9e4066Sahrens 	}
2228c717a561Smaybee }
2229fa9e4066Sahrens 
2230c717a561Smaybee /* ARGSUSED */
2231c717a561Smaybee static void
2232c717a561Smaybee dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb)
2233c717a561Smaybee {
2234c717a561Smaybee 	dmu_buf_impl_t *db = vdb;
2235c717a561Smaybee 	uint64_t txg = zio->io_txg;
2236c717a561Smaybee 	dbuf_dirty_record_t **drp, *dr;
2237c717a561Smaybee 
2238c717a561Smaybee 	ASSERT3U(zio->io_error, ==, 0);
2239c717a561Smaybee 
2240c717a561Smaybee 	mutex_enter(&db->db_mtx);
2241c717a561Smaybee 
2242c717a561Smaybee 	drp = &db->db_last_dirty;
224317f17c2dSbonwick 	while ((dr = *drp) != db->db_data_pending)
224417f17c2dSbonwick 		drp = &dr->dr_next;
224517f17c2dSbonwick 	ASSERT(!list_link_active(&dr->dr_dirty_node));
224617f17c2dSbonwick 	ASSERT(dr->dr_txg == txg);
224717f17c2dSbonwick 	ASSERT(dr->dr_next == NULL);
224817f17c2dSbonwick 	*drp = dr->dr_next;
2249c717a561Smaybee 
2250c717a561Smaybee 	if (db->db_level == 0) {
2251c717a561Smaybee 		ASSERT(db->db_blkid != DB_BONUS_BLKID);
2252c717a561Smaybee 		ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2253c717a561Smaybee 
2254c717a561Smaybee 		if (dr->dt.dl.dr_data != db->db_buf)
2255c717a561Smaybee 			VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db) == 1);
2256c717a561Smaybee 		else if (!BP_IS_HOLE(db->db_blkptr))
2257c717a561Smaybee 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2258c717a561Smaybee 		else
2259c717a561Smaybee 			ASSERT(arc_released(db->db_buf));
2260c717a561Smaybee 	} else {
2261c717a561Smaybee 		dnode_t *dn = db->db_dnode;
2262c717a561Smaybee 
2263c717a561Smaybee 		ASSERT(list_head(&dr->dt.di.dr_children) == NULL);
2264c717a561Smaybee 		ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
2265c717a561Smaybee 		if (!BP_IS_HOLE(db->db_blkptr)) {
2266c717a561Smaybee 			int epbs =
2267c717a561Smaybee 			    dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2268c717a561Smaybee 			ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
2269c717a561Smaybee 			    db->db.db_size);
2270c717a561Smaybee 			ASSERT3U(dn->dn_phys->dn_maxblkid
2271c717a561Smaybee 			    >> (db->db_level * epbs), >=, db->db_blkid);
2272c717a561Smaybee 			arc_set_callback(db->db_buf, dbuf_do_evict, db);
2273c717a561Smaybee 		}
2274c25056deSgw 		mutex_destroy(&dr->dt.di.dr_mtx);
2275c25056deSgw 		list_destroy(&dr->dt.di.dr_children);
2276c717a561Smaybee 	}
2277c717a561Smaybee 	kmem_free(dr, sizeof (dbuf_dirty_record_t));
2278fa9e4066Sahrens 
2279fa9e4066Sahrens 	cv_broadcast(&db->db_changed);
2280fa9e4066Sahrens 	ASSERT(db->db_dirtycnt > 0);
2281fa9e4066Sahrens 	db->db_dirtycnt -= 1;
2282c717a561Smaybee 	db->db_data_pending = NULL;
2283fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
2284fa9e4066Sahrens 
2285c717a561Smaybee 	dprintf_dbuf_bp(db, zio->io_bp, "bp: %s", "");
2286fa9e4066Sahrens 
2287ea8dc4b6Seschrock 	dbuf_rele(db, (void *)(uintptr_t)txg);
2288fa9e4066Sahrens }
2289