xref: /illumos-gate/usr/src/uts/common/fs/zfs/dnode_sync.c (revision 55434c770c89aa1b84474f2559a106803511aba0)
1fa9e4066Sahrens /*
2fa9e4066Sahrens  * CDDL HEADER START
3fa9e4066Sahrens  *
4fa9e4066Sahrens  * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock  * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock  * You may not use this file except in compliance with the License.
7fa9e4066Sahrens  *
8fa9e4066Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens  * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens  * See the License for the specific language governing permissions
11fa9e4066Sahrens  * and limitations under the License.
12fa9e4066Sahrens  *
13fa9e4066Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens  *
19fa9e4066Sahrens  * CDDL HEADER END
20fa9e4066Sahrens  */
21fa9e4066Sahrens /*
22*55434c77Sek  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23fa9e4066Sahrens  * Use is subject to license terms.
24fa9e4066Sahrens  */
25fa9e4066Sahrens 
26fa9e4066Sahrens #pragma ident	"%Z%%M%	%I%	%E% SMI"
27fa9e4066Sahrens 
28fa9e4066Sahrens #include <sys/zfs_context.h>
29fa9e4066Sahrens #include <sys/dbuf.h>
30fa9e4066Sahrens #include <sys/dnode.h>
31fa9e4066Sahrens #include <sys/dmu.h>
32fa9e4066Sahrens #include <sys/dmu_tx.h>
33fa9e4066Sahrens #include <sys/dmu_objset.h>
34fa9e4066Sahrens #include <sys/dsl_dataset.h>
35fa9e4066Sahrens #include <sys/spa.h>
36fa9e4066Sahrens #include <sys/zio.h>
37fa9e4066Sahrens 
38fa9e4066Sahrens static void
39fa9e4066Sahrens dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)
40fa9e4066Sahrens {
41fa9e4066Sahrens 	dmu_buf_impl_t *db;
42fa9e4066Sahrens 	int i;
43fa9e4066Sahrens 	uint64_t txg = tx->tx_txg;
44fa9e4066Sahrens 
45fa9e4066Sahrens 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
46fa9e4066Sahrens 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
47fa9e4066Sahrens 	/* this dnode can't be paged out because it's dirty */
48fa9e4066Sahrens 
49fa9e4066Sahrens 	db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);
50ea8dc4b6Seschrock 	ASSERT(db != NULL);
51fa9e4066Sahrens 	for (i = 0; i < dn->dn_phys->dn_nblkptr; i++)
52fa9e4066Sahrens 		if (!BP_IS_HOLE(&dn->dn_phys->dn_blkptr[i]))
53fa9e4066Sahrens 			break;
54fa9e4066Sahrens 	if (i != dn->dn_phys->dn_nblkptr) {
55fa9e4066Sahrens 		ASSERT(list_link_active(&db->db_dirty_node[txg&TXG_MASK]));
56fa9e4066Sahrens 
57ea8dc4b6Seschrock 		(void) dbuf_read(db, NULL,
58ea8dc4b6Seschrock 		    DB_RF_HAVESTRUCT | DB_RF_MUST_SUCCEED);
59fa9e4066Sahrens 		arc_release(db->db_buf, db);
60fa9e4066Sahrens 		/* copy dnode's block pointers to new indirect block */
61fa9e4066Sahrens 		ASSERT3U(sizeof (blkptr_t) * dn->dn_phys->dn_nblkptr, <=,
62fa9e4066Sahrens 		    db->db.db_size);
63fa9e4066Sahrens 		bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,
64fa9e4066Sahrens 		    sizeof (blkptr_t) * dn->dn_phys->dn_nblkptr);
656b4acc8bSahrens 		arc_buf_freeze(db->db_buf);
66fa9e4066Sahrens 	}
67fa9e4066Sahrens 
68fa9e4066Sahrens 	dn->dn_phys->dn_nlevels += 1;
69fa9e4066Sahrens 	dprintf("os=%p obj=%llu, increase to %d\n",
70fa9e4066Sahrens 		dn->dn_objset, dn->dn_object,
71fa9e4066Sahrens 		dn->dn_phys->dn_nlevels);
72fa9e4066Sahrens 
73fa9e4066Sahrens 	/* set dbuf's parent pointers to new indirect buf */
74fa9e4066Sahrens 	for (i = 0; i < dn->dn_phys->dn_nblkptr; i++) {
75fa9e4066Sahrens 		dmu_buf_impl_t *child =
76fa9e4066Sahrens 		    dbuf_find(dn, dn->dn_phys->dn_nlevels-2, i);
77fa9e4066Sahrens 		if (child == NULL)
78fa9e4066Sahrens 			continue;
79fa9e4066Sahrens 		if (child->db_dnode == NULL) {
80fa9e4066Sahrens 			mutex_exit(&child->db_mtx);
81fa9e4066Sahrens 			continue;
82fa9e4066Sahrens 		}
83fa9e4066Sahrens 
84fa9e4066Sahrens 		if (child->db_parent == NULL ||
85fa9e4066Sahrens 		    child->db_parent == dn->dn_dbuf) {
86fa9e4066Sahrens 			dprintf_dbuf_bp(child, child->db_blkptr,
87fa9e4066Sahrens 			    "changing db_blkptr to new indirect %s", "");
88fa9e4066Sahrens 			child->db_parent = db;
89fa9e4066Sahrens 			dbuf_add_ref(db, child);
90fa9e4066Sahrens 			if (db->db.db_data) {
91fa9e4066Sahrens 				child->db_blkptr =
92fa9e4066Sahrens 				    (blkptr_t *)db->db.db_data + i;
93fa9e4066Sahrens 			} else {
94fa9e4066Sahrens 				child->db_blkptr = NULL;
95fa9e4066Sahrens 			}
96fa9e4066Sahrens 			dprintf_dbuf_bp(child, child->db_blkptr,
97fa9e4066Sahrens 			    "changed db_blkptr to new indirect %s", "");
98fa9e4066Sahrens 		}
99fa9e4066Sahrens 		ASSERT3P(child->db_parent, ==, db);
100fa9e4066Sahrens 
101fa9e4066Sahrens 		mutex_exit(&child->db_mtx);
102fa9e4066Sahrens 	}
103fa9e4066Sahrens 
104fa9e4066Sahrens 	bzero(dn->dn_phys->dn_blkptr,
105fa9e4066Sahrens 		sizeof (blkptr_t) * dn->dn_phys->dn_nblkptr);
106fa9e4066Sahrens 
107ea8dc4b6Seschrock 	dbuf_rele(db, FTAG);
108fa9e4066Sahrens }
109fa9e4066Sahrens 
110fa9e4066Sahrens static void
111fa9e4066Sahrens free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)
112fa9e4066Sahrens {
113fa9e4066Sahrens 	objset_impl_t *os = dn->dn_objset;
114fa9e4066Sahrens 	uint64_t bytesfreed = 0;
115fa9e4066Sahrens 	int i;
116fa9e4066Sahrens 
117fa9e4066Sahrens 	dprintf("os=%p obj=%llx num=%d\n", os, dn->dn_object, num);
118fa9e4066Sahrens 
119fa9e4066Sahrens 	for (i = 0; i < num; i++, bp++) {
120fa9e4066Sahrens 		if (BP_IS_HOLE(bp))
121fa9e4066Sahrens 			continue;
122fa9e4066Sahrens 
12399653d4eSeschrock 		bytesfreed += bp_get_dasize(os->os_spa, bp);
12499653d4eSeschrock 		ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));
125fa9e4066Sahrens 		dsl_dataset_block_kill(os->os_dsl_dataset, bp, tx);
126fa9e4066Sahrens 	}
127fa9e4066Sahrens 	dnode_diduse_space(dn, -bytesfreed);
128fa9e4066Sahrens }
129fa9e4066Sahrens 
1309c9dc39aSek #ifdef ZFS_DEBUG
131fa9e4066Sahrens static void
132fa9e4066Sahrens free_verify(dmu_buf_impl_t *db, uint64_t start, uint64_t end, dmu_tx_t *tx)
133fa9e4066Sahrens {
134fa9e4066Sahrens 	int off, num;
135fa9e4066Sahrens 	int i, err, epbs;
136fa9e4066Sahrens 	uint64_t txg = tx->tx_txg;
137fa9e4066Sahrens 
138fa9e4066Sahrens 	epbs = db->db_dnode->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
139fa9e4066Sahrens 	off = start - (db->db_blkid * 1<<epbs);
140fa9e4066Sahrens 	num = end - start + 1;
141fa9e4066Sahrens 
142fa9e4066Sahrens 	ASSERT3U(off, >=, 0);
143fa9e4066Sahrens 	ASSERT3U(num, >=, 0);
144fa9e4066Sahrens 	ASSERT3U(db->db_level, >, 0);
145fa9e4066Sahrens 	ASSERT3U(db->db.db_size, ==, 1<<db->db_dnode->dn_phys->dn_indblkshift);
146fa9e4066Sahrens 	ASSERT3U(off+num, <=, db->db.db_size >> SPA_BLKPTRSHIFT);
147fa9e4066Sahrens 	ASSERT(db->db_blkptr != NULL);
148fa9e4066Sahrens 
149fa9e4066Sahrens 	for (i = off; i < off+num; i++) {
150fa9e4066Sahrens 		uint64_t *buf;
151fa9e4066Sahrens 		int j;
152fa9e4066Sahrens 		dmu_buf_impl_t *child;
153fa9e4066Sahrens 
154fa9e4066Sahrens 		ASSERT(db->db_level == 1);
155fa9e4066Sahrens 
156fa9e4066Sahrens 		rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER);
157fa9e4066Sahrens 		err = dbuf_hold_impl(db->db_dnode, db->db_level-1,
158fa9e4066Sahrens 			(db->db_blkid << epbs) + i, TRUE, FTAG, &child);
159fa9e4066Sahrens 		rw_exit(&db->db_dnode->dn_struct_rwlock);
160fa9e4066Sahrens 		if (err == ENOENT)
161fa9e4066Sahrens 			continue;
162fa9e4066Sahrens 		ASSERT(err == 0);
163fa9e4066Sahrens 		ASSERT(child->db_level == 0);
164fa9e4066Sahrens 		ASSERT(!list_link_active(&child->db_dirty_node[txg&TXG_MASK]));
165fa9e4066Sahrens 
166fa9e4066Sahrens 		/* db_data_old better be zeroed */
167fa9e4066Sahrens 		if (child->db_d.db_data_old[txg & TXG_MASK]) {
1686b4acc8bSahrens 			buf = child->db_d.db_data_old[txg & TXG_MASK]->b_data;
169fa9e4066Sahrens 			for (j = 0; j < child->db.db_size >> 3; j++) {
170fa9e4066Sahrens 				if (buf[j] != 0) {
171fa9e4066Sahrens 					panic("freed data not zero: "
172fa9e4066Sahrens 					    "child=%p i=%d off=%d num=%d\n",
173fa9e4066Sahrens 					    child, i, off, num);
174fa9e4066Sahrens 				}
175fa9e4066Sahrens 			}
176fa9e4066Sahrens 		}
177fa9e4066Sahrens 
178fa9e4066Sahrens 		/*
179fa9e4066Sahrens 		 * db_data better be zeroed unless it's dirty in a
180fa9e4066Sahrens 		 * future txg.
181fa9e4066Sahrens 		 */
182fa9e4066Sahrens 		mutex_enter(&child->db_mtx);
183fa9e4066Sahrens 		buf = child->db.db_data;
184fa9e4066Sahrens 		if (buf != NULL && child->db_state != DB_FILL &&
185fa9e4066Sahrens 		    !list_link_active(&child->db_dirty_node
186fa9e4066Sahrens 			[(txg+1) & TXG_MASK]) &&
187fa9e4066Sahrens 		    !list_link_active(&child->db_dirty_node
188fa9e4066Sahrens 			[(txg+2) & TXG_MASK])) {
189fa9e4066Sahrens 			for (j = 0; j < child->db.db_size >> 3; j++) {
190fa9e4066Sahrens 				if (buf[j] != 0) {
191fa9e4066Sahrens 					panic("freed data not zero: "
192fa9e4066Sahrens 					    "child=%p i=%d off=%d num=%d\n",
193fa9e4066Sahrens 					    child, i, off, num);
194fa9e4066Sahrens 				}
195fa9e4066Sahrens 			}
196fa9e4066Sahrens 		}
197fa9e4066Sahrens 		mutex_exit(&child->db_mtx);
198fa9e4066Sahrens 
199ea8dc4b6Seschrock 		dbuf_rele(child, FTAG);
200fa9e4066Sahrens 	}
201fa9e4066Sahrens }
2029c9dc39aSek #endif
203fa9e4066Sahrens 
204fa9e4066Sahrens static int
205fa9e4066Sahrens free_children(dmu_buf_impl_t *db, uint64_t blkid, uint64_t nblks, int trunc,
206fa9e4066Sahrens     dmu_tx_t *tx)
207fa9e4066Sahrens {
208fa9e4066Sahrens 	dnode_t *dn = db->db_dnode;
209fa9e4066Sahrens 	blkptr_t *bp;
210fa9e4066Sahrens 	dmu_buf_impl_t *subdb;
211fa9e4066Sahrens 	uint64_t start, end, dbstart, dbend, i;
212fa9e4066Sahrens 	int epbs, shift, err;
21323b11526Smaybee 	int txgoff = tx->tx_txg & TXG_MASK;
214fa9e4066Sahrens 	int all = TRUE;
215fa9e4066Sahrens 
216ea8dc4b6Seschrock 	(void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
217fa9e4066Sahrens 	arc_release(db->db_buf, db);
218fa9e4066Sahrens 	bp = (blkptr_t *)db->db.db_data;
219fa9e4066Sahrens 
220fa9e4066Sahrens 	epbs = db->db_dnode->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
221fa9e4066Sahrens 	shift = (db->db_level - 1) * epbs;
222fa9e4066Sahrens 	dbstart = db->db_blkid << epbs;
223fa9e4066Sahrens 	start = blkid >> shift;
224fa9e4066Sahrens 	if (dbstart < start) {
225fa9e4066Sahrens 		bp += start - dbstart;
226fa9e4066Sahrens 		all = FALSE;
227fa9e4066Sahrens 	} else {
228fa9e4066Sahrens 		start = dbstart;
229fa9e4066Sahrens 	}
230fa9e4066Sahrens 	dbend = ((db->db_blkid + 1) << epbs) - 1;
231fa9e4066Sahrens 	end = (blkid + nblks - 1) >> shift;
232fa9e4066Sahrens 	if (dbend <= end)
233fa9e4066Sahrens 		end = dbend;
234fa9e4066Sahrens 	else if (all)
235fa9e4066Sahrens 		all = trunc;
236fa9e4066Sahrens 	ASSERT3U(start, <=, end);
237fa9e4066Sahrens 
238fa9e4066Sahrens 	if (db->db_level == 1) {
2399c9dc39aSek 		FREE_VERIFY(db, start, end, tx);
240fa9e4066Sahrens 		free_blocks(dn, bp, end-start+1, tx);
2416b4acc8bSahrens 		arc_buf_freeze(db->db_buf);
24223b11526Smaybee 		ASSERT(all || list_link_active(&db->db_dirty_node[txgoff]));
243fa9e4066Sahrens 		return (all);
244fa9e4066Sahrens 	}
245fa9e4066Sahrens 
246fa9e4066Sahrens 	for (i = start; i <= end; i++, bp++) {
247fa9e4066Sahrens 		if (BP_IS_HOLE(bp))
248fa9e4066Sahrens 			continue;
249fa9e4066Sahrens 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
250fa9e4066Sahrens 		err = dbuf_hold_impl(dn, db->db_level-1, i, TRUE, FTAG, &subdb);
251fa9e4066Sahrens 		ASSERT3U(err, ==, 0);
252fa9e4066Sahrens 		rw_exit(&dn->dn_struct_rwlock);
253fa9e4066Sahrens 
254fa9e4066Sahrens 		if (free_children(subdb, blkid, nblks, trunc, tx)) {
255fa9e4066Sahrens 			ASSERT3P(subdb->db_blkptr, ==, bp);
256fa9e4066Sahrens 			free_blocks(dn, bp, 1, tx);
25723b11526Smaybee 		} else {
25823b11526Smaybee 			all = FALSE;
259fa9e4066Sahrens 		}
260ea8dc4b6Seschrock 		dbuf_rele(subdb, FTAG);
261fa9e4066Sahrens 	}
2626b4acc8bSahrens 	arc_buf_freeze(db->db_buf);
263fa9e4066Sahrens #ifdef ZFS_DEBUG
264fa9e4066Sahrens 	bp -= (end-start)+1;
265fa9e4066Sahrens 	for (i = start; i <= end; i++, bp++) {
266fa9e4066Sahrens 		if (i == start && blkid != 0)
267fa9e4066Sahrens 			continue;
268fa9e4066Sahrens 		else if (i == end && !trunc)
269fa9e4066Sahrens 			continue;
270fa9e4066Sahrens 		ASSERT3U(bp->blk_birth, ==, 0);
271fa9e4066Sahrens 	}
272fa9e4066Sahrens #endif
27323b11526Smaybee 	ASSERT(all || list_link_active(&db->db_dirty_node[txgoff]));
274fa9e4066Sahrens 	return (all);
275fa9e4066Sahrens }
276fa9e4066Sahrens 
277fa9e4066Sahrens /*
278fa9e4066Sahrens  * free_range: Traverse the indicated range of the provided file
279fa9e4066Sahrens  * and "free" all the blocks contained there.
280fa9e4066Sahrens  */
281fa9e4066Sahrens static void
282fa9e4066Sahrens dnode_sync_free_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx)
283fa9e4066Sahrens {
284fa9e4066Sahrens 	blkptr_t *bp = dn->dn_phys->dn_blkptr;
285fa9e4066Sahrens 	dmu_buf_impl_t *db;
286fa9e4066Sahrens 	int trunc, start, end, shift, i, err;
287fa9e4066Sahrens 	int dnlevel = dn->dn_phys->dn_nlevels;
288fa9e4066Sahrens 
289fa9e4066Sahrens 	if (blkid > dn->dn_phys->dn_maxblkid)
290fa9e4066Sahrens 		return;
291fa9e4066Sahrens 
292fa9e4066Sahrens 	ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);
293fa9e4066Sahrens 	trunc = blkid + nblks > dn->dn_phys->dn_maxblkid;
294fa9e4066Sahrens 	if (trunc)
295fa9e4066Sahrens 		nblks = dn->dn_phys->dn_maxblkid - blkid + 1;
296fa9e4066Sahrens 
297fa9e4066Sahrens 	/* There are no indirect blocks in the object */
298fa9e4066Sahrens 	if (dnlevel == 1) {
299fa9e4066Sahrens 		if (blkid >= dn->dn_phys->dn_nblkptr) {
300fa9e4066Sahrens 			/* this range was never made persistent */
301fa9e4066Sahrens 			return;
302fa9e4066Sahrens 		}
303fa9e4066Sahrens 		ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);
304fa9e4066Sahrens 		free_blocks(dn, bp + blkid, nblks, tx);
305fa9e4066Sahrens 		if (trunc) {
306fa9e4066Sahrens 			uint64_t off = (dn->dn_phys->dn_maxblkid + 1) *
307fa9e4066Sahrens 			    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);
308fa9e4066Sahrens 			dn->dn_phys->dn_maxblkid = (blkid ? blkid - 1 : 0);
309fa9e4066Sahrens 			ASSERT(off < dn->dn_phys->dn_maxblkid ||
310fa9e4066Sahrens 			    dn->dn_phys->dn_maxblkid == 0 ||
3116754306eSahrens 			    dnode_next_offset(dn, FALSE, &off,
3126754306eSahrens 			    1, 1, 0) != 0);
313fa9e4066Sahrens 		}
314fa9e4066Sahrens 		return;
315fa9e4066Sahrens 	}
316fa9e4066Sahrens 
317fa9e4066Sahrens 	shift = (dnlevel - 1) * (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);
318fa9e4066Sahrens 	start = blkid >> shift;
319fa9e4066Sahrens 	ASSERT(start < dn->dn_phys->dn_nblkptr);
320fa9e4066Sahrens 	end = (blkid + nblks - 1) >> shift;
321fa9e4066Sahrens 	bp += start;
322fa9e4066Sahrens 	for (i = start; i <= end; i++, bp++) {
323fa9e4066Sahrens 		if (BP_IS_HOLE(bp))
324fa9e4066Sahrens 			continue;
325fa9e4066Sahrens 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
326fa9e4066Sahrens 		err = dbuf_hold_impl(dn, dnlevel-1, i, TRUE, FTAG, &db);
327fa9e4066Sahrens 		ASSERT3U(err, ==, 0);
328fa9e4066Sahrens 		rw_exit(&dn->dn_struct_rwlock);
329fa9e4066Sahrens 
330fa9e4066Sahrens 		if (free_children(db, blkid, nblks, trunc, tx)) {
331fa9e4066Sahrens 			ASSERT3P(db->db_blkptr, ==, bp);
332fa9e4066Sahrens 			free_blocks(dn, bp, 1, tx);
333fa9e4066Sahrens 		}
334ea8dc4b6Seschrock 		dbuf_rele(db, FTAG);
335fa9e4066Sahrens 	}
336fa9e4066Sahrens 	if (trunc) {
337fa9e4066Sahrens 		uint64_t off = (dn->dn_phys->dn_maxblkid + 1) *
338fa9e4066Sahrens 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);
339fa9e4066Sahrens 		dn->dn_phys->dn_maxblkid = (blkid ? blkid - 1 : 0);
340fa9e4066Sahrens 		ASSERT(off < dn->dn_phys->dn_maxblkid ||
341fa9e4066Sahrens 		    dn->dn_phys->dn_maxblkid == 0 ||
3426754306eSahrens 		    dnode_next_offset(dn, FALSE, &off, 1, 1, 0) != 0);
343fa9e4066Sahrens 	}
344fa9e4066Sahrens }
345fa9e4066Sahrens 
346ea8dc4b6Seschrock /*
347ea8dc4b6Seschrock  * Try to kick all the dnodes dbufs out of the cache...
348ea8dc4b6Seschrock  */
349436b2950Sperrin int
350436b2950Sperrin dnode_evict_dbufs(dnode_t *dn, int try)
351ea8dc4b6Seschrock {
352c543ec06Sahrens 	int progress;
353c543ec06Sahrens 	int pass = 0;
354c543ec06Sahrens 
355c543ec06Sahrens 	do {
356d90a49d6Smaybee 		dmu_buf_impl_t *db, marker;
357c543ec06Sahrens 		int evicting = FALSE;
358c543ec06Sahrens 
359c543ec06Sahrens 		progress = FALSE;
360c543ec06Sahrens 		mutex_enter(&dn->dn_dbufs_mtx);
361d90a49d6Smaybee 		list_insert_tail(&dn->dn_dbufs, &marker);
362d90a49d6Smaybee 		db = list_head(&dn->dn_dbufs);
363d90a49d6Smaybee 		for (; db != &marker; db = list_head(&dn->dn_dbufs)) {
364d90a49d6Smaybee 			list_remove(&dn->dn_dbufs, db);
365d90a49d6Smaybee 			list_insert_tail(&dn->dn_dbufs, db);
366ea8dc4b6Seschrock 
367ea8dc4b6Seschrock 			mutex_enter(&db->db_mtx);
368c543ec06Sahrens 			if (db->db_state == DB_EVICTING) {
369c543ec06Sahrens 				progress = TRUE;
370c543ec06Sahrens 				evicting = TRUE;
371c543ec06Sahrens 				mutex_exit(&db->db_mtx);
372c543ec06Sahrens 			} else if (refcount_is_zero(&db->db_holds)) {
373c543ec06Sahrens 				progress = TRUE;
374c543ec06Sahrens 				ASSERT(!arc_released(db->db_buf));
375c543ec06Sahrens 				dbuf_clear(db); /* exits db_mtx for us */
376c543ec06Sahrens 			} else {
377c543ec06Sahrens 				mutex_exit(&db->db_mtx);
378c543ec06Sahrens 			}
379c543ec06Sahrens 
380ea8dc4b6Seschrock 		}
381d90a49d6Smaybee 		list_remove(&dn->dn_dbufs, &marker);
382c543ec06Sahrens 		/*
383c543ec06Sahrens 		 * NB: we need to drop dn_dbufs_mtx between passes so
384c543ec06Sahrens 		 * that any DB_EVICTING dbufs can make progress.
385c543ec06Sahrens 		 * Ideally, we would have some cv we could wait on, but
386c543ec06Sahrens 		 * since we don't, just wait a bit to give the other
387c543ec06Sahrens 		 * thread a chance to run.
388c543ec06Sahrens 		 */
389c543ec06Sahrens 		mutex_exit(&dn->dn_dbufs_mtx);
390c543ec06Sahrens 		if (evicting)
391c543ec06Sahrens 			delay(1);
392c543ec06Sahrens 		pass++;
393c543ec06Sahrens 		ASSERT(pass < 100); /* sanity check */
394c543ec06Sahrens 	} while (progress);
395c543ec06Sahrens 
396c543ec06Sahrens 	/*
397436b2950Sperrin 	 * This function works fine even if it can't evict everything.
398436b2950Sperrin 	 * If were only asked to try to evict everything then
399436b2950Sperrin 	 * return an error if we can't. Otherwise panic as the caller
400436b2950Sperrin 	 * expects total eviction.
401c543ec06Sahrens 	 */
402c543ec06Sahrens 	if (list_head(&dn->dn_dbufs) != NULL) {
403436b2950Sperrin 		if (try) {
404436b2950Sperrin 			return (1);
405436b2950Sperrin 		} else {
406436b2950Sperrin 			panic("dangling dbufs (dn=%p, dbuf=%p)\n",
407436b2950Sperrin 			    dn, list_head(&dn->dn_dbufs));
408436b2950Sperrin 		}
409ea8dc4b6Seschrock 	}
410c543ec06Sahrens 
411ea8dc4b6Seschrock 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
412ea8dc4b6Seschrock 	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
413ea8dc4b6Seschrock 		mutex_enter(&dn->dn_bonus->db_mtx);
414ea8dc4b6Seschrock 		dbuf_evict(dn->dn_bonus);
415ea8dc4b6Seschrock 		dn->dn_bonus = NULL;
416ea8dc4b6Seschrock 	}
417ea8dc4b6Seschrock 	rw_exit(&dn->dn_struct_rwlock);
418436b2950Sperrin 	return (0);
419ea8dc4b6Seschrock }
420ea8dc4b6Seschrock 
421fa9e4066Sahrens static int
422fa9e4066Sahrens dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
423fa9e4066Sahrens {
424fa9e4066Sahrens 	dmu_buf_impl_t *db;
425fa9e4066Sahrens 	int txgoff = tx->tx_txg & TXG_MASK;
426fa9e4066Sahrens 
427fa9e4066Sahrens 	ASSERT(dmu_tx_is_syncing(tx));
428fa9e4066Sahrens 
429fa9e4066Sahrens 	/* Undirty all buffers */
430fa9e4066Sahrens 	while (db = list_head(&dn->dn_dirty_dbufs[txgoff])) {
431fa9e4066Sahrens 		mutex_enter(&db->db_mtx);
432fa9e4066Sahrens 		/* XXX - use dbuf_undirty()? */
433fa9e4066Sahrens 		list_remove(&dn->dn_dirty_dbufs[txgoff], db);
434fa9e4066Sahrens 		if (db->db_level == 0) {
435ea8dc4b6Seschrock 			ASSERT(db->db_blkid == DB_BONUS_BLKID ||
436ea8dc4b6Seschrock 			    db->db_d.db_data_old[txgoff] == db->db_buf);
437fa9e4066Sahrens 			if (db->db_d.db_overridden_by[txgoff])
438fa9e4066Sahrens 				dbuf_unoverride(db, tx->tx_txg);
439fa9e4066Sahrens 			db->db_d.db_data_old[txgoff] = NULL;
440fa9e4066Sahrens 		}
441fa9e4066Sahrens 		db->db_dirtycnt -= 1;
442fa9e4066Sahrens 		mutex_exit(&db->db_mtx);
443ea8dc4b6Seschrock 		dbuf_rele(db, (void *)(uintptr_t)tx->tx_txg);
444fa9e4066Sahrens 	}
445fa9e4066Sahrens 
446436b2950Sperrin 	(void) dnode_evict_dbufs(dn, 0);
447ea8dc4b6Seschrock 	ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL);
448ea8dc4b6Seschrock 
449ea8dc4b6Seschrock 	/*
450ea8dc4b6Seschrock 	 * XXX - It would be nice to assert this, but we may still
451ea8dc4b6Seschrock 	 * have residual holds from async evictions from the arc...
452ea8dc4b6Seschrock 	 *
453*55434c77Sek 	 * zfs_obj_to_path() also depends on this being
454*55434c77Sek 	 * commented out.
455*55434c77Sek 	 *
456ea8dc4b6Seschrock 	 * ASSERT3U(refcount_count(&dn->dn_holds), ==, 1);
457ea8dc4b6Seschrock 	 */
458fa9e4066Sahrens 
459fa9e4066Sahrens 	/* Undirty next bits */
460fa9e4066Sahrens 	dn->dn_next_nlevels[txgoff] = 0;
461fa9e4066Sahrens 	dn->dn_next_indblkshift[txgoff] = 0;
462c543ec06Sahrens 	dn->dn_next_blksz[txgoff] = 0;
463fa9e4066Sahrens 
464fa9e4066Sahrens 	/* free up all the blocks in the file. */
465fa9e4066Sahrens 	dnode_sync_free_range(dn, 0, dn->dn_phys->dn_maxblkid+1, tx);
46699653d4eSeschrock 	ASSERT3U(DN_USED_BYTES(dn->dn_phys), ==, 0);
467fa9e4066Sahrens 
468fa9e4066Sahrens 	/* ASSERT(blkptrs are zero); */
469fa9e4066Sahrens 	ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
470fa9e4066Sahrens 	ASSERT(dn->dn_type != DMU_OT_NONE);
471fa9e4066Sahrens 
472fa9e4066Sahrens 	ASSERT(dn->dn_free_txg > 0);
473fa9e4066Sahrens 	if (dn->dn_allocated_txg != dn->dn_free_txg)
474fa9e4066Sahrens 		dbuf_will_dirty(dn->dn_dbuf, tx);
475fa9e4066Sahrens 	bzero(dn->dn_phys, sizeof (dnode_phys_t));
476fa9e4066Sahrens 
477fa9e4066Sahrens 	mutex_enter(&dn->dn_mtx);
478fa9e4066Sahrens 	dn->dn_type = DMU_OT_NONE;
479fa9e4066Sahrens 	dn->dn_maxblkid = 0;
480fa9e4066Sahrens 	dn->dn_allocated_txg = 0;
481fa9e4066Sahrens 	mutex_exit(&dn->dn_mtx);
482fa9e4066Sahrens 
483ea8dc4b6Seschrock 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
484fa9e4066Sahrens 
485fa9e4066Sahrens 	dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);
486fa9e4066Sahrens 	/*
487fa9e4066Sahrens 	 * Now that we've released our hold, the dnode may
488fa9e4066Sahrens 	 * be evicted, so we musn't access it.
489fa9e4066Sahrens 	 */
490fa9e4066Sahrens 	return (1);
491fa9e4066Sahrens }
492fa9e4066Sahrens 
493fa9e4066Sahrens /*
494fa9e4066Sahrens  * Write out the dnode's dirty buffers at the specified level.
495fa9e4066Sahrens  * This may create more dirty buffers at the next level up.
496fa9e4066Sahrens  *
497fa9e4066Sahrens  * NOTE: The dnode is kept in memory by being dirty.  Once the
498fa9e4066Sahrens  * dirty bit is cleared, it may be evicted.  Beware of this!
499fa9e4066Sahrens  */
500fa9e4066Sahrens int
501fa9e4066Sahrens dnode_sync(dnode_t *dn, int level, zio_t *zio, dmu_tx_t *tx)
502fa9e4066Sahrens {
503fa9e4066Sahrens 	free_range_t *rp;
504fa9e4066Sahrens 	int txgoff = tx->tx_txg & TXG_MASK;
505fa9e4066Sahrens 	dnode_phys_t *dnp = dn->dn_phys;
506fa9e4066Sahrens 
507fa9e4066Sahrens 	ASSERT(dmu_tx_is_syncing(tx));
508fa9e4066Sahrens 	ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);
5099c9dc39aSek 	DNODE_VERIFY(dn);
510c543ec06Sahrens 
511fa9e4066Sahrens 	/*
512fa9e4066Sahrens 	 * Make sure the dbuf for the dn_phys is released before we modify it.
513fa9e4066Sahrens 	 */
514fa9e4066Sahrens 	if (dn->dn_dbuf)
515fa9e4066Sahrens 		arc_release(dn->dn_dbuf->db_buf, dn->dn_dbuf);
516fa9e4066Sahrens 
517fa9e4066Sahrens 	mutex_enter(&dn->dn_mtx);
518fa9e4066Sahrens 	if (dn->dn_allocated_txg == tx->tx_txg) {
519fa9e4066Sahrens 		/* The dnode is newly allocated or reallocated */
520fa9e4066Sahrens 		if (dnp->dn_type == DMU_OT_NONE) {
521fa9e4066Sahrens 			/* this is a first alloc, not a realloc */
522fa9e4066Sahrens 			/* XXX shouldn't the phys already be zeroed? */
523fa9e4066Sahrens 			bzero(dnp, DNODE_CORE_SIZE);
524fa9e4066Sahrens 			dnp->dn_nlevels = 1;
525fa9e4066Sahrens 		}
526fa9e4066Sahrens 
527fa9e4066Sahrens 		if (dn->dn_nblkptr > dnp->dn_nblkptr) {
528fa9e4066Sahrens 			/* zero the new blkptrs we are gaining */
529fa9e4066Sahrens 			bzero(dnp->dn_blkptr + dnp->dn_nblkptr,
530fa9e4066Sahrens 			    sizeof (blkptr_t) *
531fa9e4066Sahrens 			    (dn->dn_nblkptr - dnp->dn_nblkptr));
532fa9e4066Sahrens 		}
533fa9e4066Sahrens 		dnp->dn_type = dn->dn_type;
534fa9e4066Sahrens 		dnp->dn_bonustype = dn->dn_bonustype;
535fa9e4066Sahrens 		dnp->dn_bonuslen = dn->dn_bonuslen;
536fa9e4066Sahrens 		dnp->dn_nblkptr = dn->dn_nblkptr;
537fa9e4066Sahrens 	}
538fa9e4066Sahrens 
539f676ed34Sahrens 	ASSERT(level != 0 || dnp->dn_nlevels > 1 ||
540f676ed34Sahrens 	    BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
541f676ed34Sahrens 	    BP_GET_LSIZE(&dnp->dn_blkptr[0]) ==
542f676ed34Sahrens 	    dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
543f676ed34Sahrens 
544c543ec06Sahrens 	if (dn->dn_next_blksz[txgoff]) {
545c543ec06Sahrens 		ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],
546fa9e4066Sahrens 		    SPA_MINBLOCKSIZE) == 0);
547f676ed34Sahrens 		ASSERT(BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
548347a31bcSahrens 		    list_head(&dn->dn_dirty_dbufs[txgoff]) != NULL ||
549347a31bcSahrens 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==
550347a31bcSahrens 		    dnp->dn_datablkszsec);
551fa9e4066Sahrens 		dnp->dn_datablkszsec =
552c543ec06Sahrens 		    dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;
553c543ec06Sahrens 		dn->dn_next_blksz[txgoff] = 0;
554fa9e4066Sahrens 	}
555fa9e4066Sahrens 
556fa9e4066Sahrens 	if (dn->dn_next_indblkshift[txgoff]) {
557fa9e4066Sahrens 		ASSERT(dnp->dn_nlevels == 1);
558fa9e4066Sahrens 		dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];
559fa9e4066Sahrens 		dn->dn_next_indblkshift[txgoff] = 0;
560fa9e4066Sahrens 	}
561fa9e4066Sahrens 
562fa9e4066Sahrens 	/*
563fa9e4066Sahrens 	 * Just take the live (open-context) values for checksum and compress.
564fa9e4066Sahrens 	 * Strictly speaking it's a future leak, but nothing bad happens if we
565fa9e4066Sahrens 	 * start using the new checksum or compress algorithm a little early.
566fa9e4066Sahrens 	 */
567fa9e4066Sahrens 	dnp->dn_checksum = dn->dn_checksum;
568fa9e4066Sahrens 	dnp->dn_compress = dn->dn_compress;
569fa9e4066Sahrens 
570fa9e4066Sahrens 	mutex_exit(&dn->dn_mtx);
571fa9e4066Sahrens 
572fa9e4066Sahrens 	/* process all the "freed" ranges in the file */
573fa9e4066Sahrens 	if (dn->dn_free_txg == 0 || dn->dn_free_txg > tx->tx_txg) {
57423b11526Smaybee 		for (rp = avl_last(&dn->dn_ranges[txgoff]); rp != NULL;
57523b11526Smaybee 		    rp = AVL_PREV(&dn->dn_ranges[txgoff], rp))
576fa9e4066Sahrens 			dnode_sync_free_range(dn,
577fa9e4066Sahrens 			    rp->fr_blkid, rp->fr_nblks, tx);
578fa9e4066Sahrens 	}
579fa9e4066Sahrens 	mutex_enter(&dn->dn_mtx);
580fa9e4066Sahrens 	for (rp = avl_first(&dn->dn_ranges[txgoff]); rp; ) {
581fa9e4066Sahrens 		free_range_t *last = rp;
582fa9e4066Sahrens 		rp = AVL_NEXT(&dn->dn_ranges[txgoff], rp);
583fa9e4066Sahrens 		avl_remove(&dn->dn_ranges[txgoff], last);
584fa9e4066Sahrens 		kmem_free(last, sizeof (free_range_t));
585fa9e4066Sahrens 	}
586fa9e4066Sahrens 	mutex_exit(&dn->dn_mtx);
587fa9e4066Sahrens 
588fa9e4066Sahrens 	if (dn->dn_free_txg > 0 && dn->dn_free_txg <= tx->tx_txg) {
589fa9e4066Sahrens 		ASSERT3U(level, ==, 0);
590fa9e4066Sahrens 		return (dnode_sync_free(dn, tx));
591fa9e4066Sahrens 	}
592fa9e4066Sahrens 
593fa9e4066Sahrens 	if (dn->dn_next_nlevels[txgoff]) {
594fa9e4066Sahrens 		int new_lvl = dn->dn_next_nlevels[txgoff];
595fa9e4066Sahrens 
596fa9e4066Sahrens 		rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
597fa9e4066Sahrens 		while (new_lvl > dnp->dn_nlevels)
598fa9e4066Sahrens 			dnode_increase_indirection(dn, tx);
599fa9e4066Sahrens 		rw_exit(&dn->dn_struct_rwlock);
600fa9e4066Sahrens 		dn->dn_next_nlevels[txgoff] = 0;
601fa9e4066Sahrens 	}
602fa9e4066Sahrens 
603fa9e4066Sahrens 	if (level == dnp->dn_nlevels) {
604fa9e4066Sahrens 		uint64_t off = (dn->dn_phys->dn_maxblkid + 1) *
605fa9e4066Sahrens 		    (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);
606fa9e4066Sahrens 
607fa9e4066Sahrens 		/* we've already synced out all data and indirect blocks */
608fa9e4066Sahrens 		/* there are no more dirty dbufs under this dnode */
609fa9e4066Sahrens 		ASSERT3P(list_head(&dn->dn_dirty_dbufs[txgoff]), ==, NULL);
610fa9e4066Sahrens 		ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= tx->tx_txg);
611fa9e4066Sahrens 
612fa9e4066Sahrens 		/* NB: the "off < maxblkid" is to catch overflow */
613fa9e4066Sahrens 		/*
614fa9e4066Sahrens 		 * NB: if blocksize is changing, we could get confused,
615fa9e4066Sahrens 		 * so only bother if there are multiple blocks and thus
616fa9e4066Sahrens 		 * it can't be changing.
617fa9e4066Sahrens 		 */
6186754306eSahrens 		ASSERT(off < dn->dn_phys->dn_maxblkid ||
619fa9e4066Sahrens 		    dn->dn_phys->dn_maxblkid == 0 ||
6206754306eSahrens 		    dnode_next_offset(dn, FALSE, &off, 1, 1, 0) != 0);
621fa9e4066Sahrens 
622f676ed34Sahrens 		ASSERT(dnp->dn_nlevels > 1 ||
623f676ed34Sahrens 		    BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
624f676ed34Sahrens 		    BP_GET_LSIZE(&dnp->dn_blkptr[0]) ==
625f676ed34Sahrens 		    dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
626f676ed34Sahrens 
627ea8dc4b6Seschrock 		if (dn->dn_object != DMU_META_DNODE_OBJECT) {
628fa9e4066Sahrens 			dbuf_will_dirty(dn->dn_dbuf, tx);
629fa9e4066Sahrens 			dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);
630fa9e4066Sahrens 		}
631fa9e4066Sahrens 
632fa9e4066Sahrens 		/*
633fa9e4066Sahrens 		 * Now that we've dropped the reference, the dnode may
634fa9e4066Sahrens 		 * be evicted, so we musn't access it.
635fa9e4066Sahrens 		 */
636fa9e4066Sahrens 		return (1);
637fa9e4066Sahrens 	} else {
638fa9e4066Sahrens 		dmu_buf_impl_t *db, *db_next;
639fa9e4066Sahrens 		list_t *list = &dn->dn_dirty_dbufs[txgoff];
640fa9e4066Sahrens 		/*
641fa9e4066Sahrens 		 * Iterate over the list, removing and sync'ing dbufs
642fa9e4066Sahrens 		 * which are on the level we want, and leaving others.
643fa9e4066Sahrens 		 */
644fa9e4066Sahrens 		for (db = list_head(list); db; db = db_next) {
645fa9e4066Sahrens 			db_next = list_next(list, db);
646fa9e4066Sahrens 			if (db->db_level == level) {
647fa9e4066Sahrens 				list_remove(list, db);
648fa9e4066Sahrens 				dbuf_sync(db, zio, tx);
649fa9e4066Sahrens 			}
650fa9e4066Sahrens 		}
651fa9e4066Sahrens 		return (0);
652fa9e4066Sahrens 	}
653fa9e4066Sahrens }
654