1fa9e4066Sahrens /*
2fa9e4066Sahrens * CDDL HEADER START
3fa9e4066Sahrens *
4fa9e4066Sahrens * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock * You may not use this file except in compliance with the License.
7fa9e4066Sahrens *
8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens * See the License for the specific language governing permissions
11fa9e4066Sahrens * and limitations under the License.
12fa9e4066Sahrens *
13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens *
19fa9e4066Sahrens * CDDL HEADER END
20fa9e4066Sahrens */
21ad135b5dSChristopher Siden
22fa9e4066Sahrens /*
2306e0070dSMark Shellenbaum * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24c2919acbSMatthew Ahrens * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
25bc9014e6SJustin Gibbs * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
267611ab7bSPatrick Mooney * Copyright 2020 Oxide Computer Company
27fa9e4066Sahrens */
28fa9e4066Sahrens
29fa9e4066Sahrens #include <sys/zfs_context.h>
30fa9e4066Sahrens #include <sys/dbuf.h>
31fa9e4066Sahrens #include <sys/dnode.h>
32fa9e4066Sahrens #include <sys/dmu.h>
33fa9e4066Sahrens #include <sys/dmu_tx.h>
34fa9e4066Sahrens #include <sys/dmu_objset.h>
35eb633035STom Caputi #include <sys/dmu_recv.h>
36fa9e4066Sahrens #include <sys/dsl_dataset.h>
37fa9e4066Sahrens #include <sys/spa.h>
38bf16b11eSMatthew Ahrens #include <sys/range_tree.h>
3943466aaeSMax Grossman #include <sys/zfeature.h>
40fa9e4066Sahrens
41fa9e4066Sahrens static void
dnode_increase_indirection(dnode_t * dn,dmu_tx_t * tx)42fa9e4066Sahrens dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx)
43fa9e4066Sahrens {
44fa9e4066Sahrens dmu_buf_impl_t *db;
45c717a561Smaybee int txgoff = tx->tx_txg & TXG_MASK;
46c717a561Smaybee int nblkptr = dn->dn_phys->dn_nblkptr;
47c717a561Smaybee int old_toplvl = dn->dn_phys->dn_nlevels - 1;
48c717a561Smaybee int new_level = dn->dn_next_nlevels[txgoff];
49fa9e4066Sahrens int i;
50fa9e4066Sahrens
51c717a561Smaybee rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
52c717a561Smaybee
53c717a561Smaybee /* this dnode can't be paged out because it's dirty */
54fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
55c717a561Smaybee ASSERT(new_level > 1 && dn->dn_phys->dn_nlevels > 0);
56fa9e4066Sahrens
57fa9e4066Sahrens db = dbuf_hold_level(dn, dn->dn_phys->dn_nlevels, 0, FTAG);
58ea8dc4b6Seschrock ASSERT(db != NULL);
59c717a561Smaybee
60c717a561Smaybee dn->dn_phys->dn_nlevels = new_level;
61f82bfe17Sgw dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset,
62f82bfe17Sgw dn->dn_object, dn->dn_phys->dn_nlevels);
63c717a561Smaybee
64*9704bf7fSPaul Dagnelie /*
65*9704bf7fSPaul Dagnelie * Lock ordering requires that we hold the children's db_mutexes (by
66*9704bf7fSPaul Dagnelie * calling dbuf_find()) before holding the parent's db_rwlock. The lock
67*9704bf7fSPaul Dagnelie * order is imposed by dbuf_read's steps of "grab the lock to protect
68*9704bf7fSPaul Dagnelie * db_parent, get db_parent, hold db_parent's db_rwlock".
69*9704bf7fSPaul Dagnelie */
70*9704bf7fSPaul Dagnelie dmu_buf_impl_t *children[DN_MAX_NBLKPTR];
71*9704bf7fSPaul Dagnelie ASSERT3U(nblkptr, <=, DN_MAX_NBLKPTR);
72*9704bf7fSPaul Dagnelie for (i = 0; i < nblkptr; i++) {
73*9704bf7fSPaul Dagnelie children[i] =
74*9704bf7fSPaul Dagnelie dbuf_find(dn->dn_objset, dn->dn_object, old_toplvl, i);
75*9704bf7fSPaul Dagnelie }
76*9704bf7fSPaul Dagnelie
778df0bcf0SPaul Dagnelie /* transfer dnode's block pointers to new indirect block */
788df0bcf0SPaul Dagnelie (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT);
79*9704bf7fSPaul Dagnelie if (dn->dn_dbuf != NULL)
80*9704bf7fSPaul Dagnelie rw_enter(&dn->dn_dbuf->db_rwlock, RW_WRITER);
81*9704bf7fSPaul Dagnelie rw_enter(&db->db_rwlock, RW_WRITER);
828df0bcf0SPaul Dagnelie ASSERT(db->db.db_data);
838df0bcf0SPaul Dagnelie ASSERT(arc_released(db->db_buf));
848df0bcf0SPaul Dagnelie ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size);
858df0bcf0SPaul Dagnelie bcopy(dn->dn_phys->dn_blkptr, db->db.db_data,
868df0bcf0SPaul Dagnelie sizeof (blkptr_t) * nblkptr);
878df0bcf0SPaul Dagnelie arc_buf_freeze(db->db_buf);
88fa9e4066Sahrens
89fa9e4066Sahrens /* set dbuf's parent pointers to new indirect buf */
90c717a561Smaybee for (i = 0; i < nblkptr; i++) {
91*9704bf7fSPaul Dagnelie dmu_buf_impl_t *child = children[i];
92c717a561Smaybee
93fa9e4066Sahrens if (child == NULL)
94fa9e4066Sahrens continue;
95744947dcSTom Erickson #ifdef DEBUG
96744947dcSTom Erickson DB_DNODE_ENTER(child);
97744947dcSTom Erickson ASSERT3P(DB_DNODE(child), ==, dn);
98744947dcSTom Erickson DB_DNODE_EXIT(child);
99744947dcSTom Erickson #endif /* DEBUG */
100c717a561Smaybee if (child->db_parent && child->db_parent != dn->dn_dbuf) {
101c717a561Smaybee ASSERT(child->db_parent->db_level == db->db_level);
102c717a561Smaybee ASSERT(child->db_blkptr !=
103c717a561Smaybee &dn->dn_phys->dn_blkptr[child->db_blkid]);
104fa9e4066Sahrens mutex_exit(&child->db_mtx);
105fa9e4066Sahrens continue;
106fa9e4066Sahrens }
107c717a561Smaybee ASSERT(child->db_parent == NULL ||
108c717a561Smaybee child->db_parent == dn->dn_dbuf);
109c717a561Smaybee
110c717a561Smaybee child->db_parent = db;
111c717a561Smaybee dbuf_add_ref(db, child);
112c717a561Smaybee if (db->db.db_data)
113c717a561Smaybee child->db_blkptr = (blkptr_t *)db->db.db_data + i;
114c717a561Smaybee else
115c717a561Smaybee child->db_blkptr = NULL;
116c717a561Smaybee dprintf_dbuf_bp(child, child->db_blkptr,
117c717a561Smaybee "changed db_blkptr to new indirect %s", "");
118fa9e4066Sahrens
119fa9e4066Sahrens mutex_exit(&child->db_mtx);
120fa9e4066Sahrens }
121fa9e4066Sahrens
122c717a561Smaybee bzero(dn->dn_phys->dn_blkptr, sizeof (blkptr_t) * nblkptr);
123fa9e4066Sahrens
124*9704bf7fSPaul Dagnelie rw_exit(&db->db_rwlock);
125*9704bf7fSPaul Dagnelie if (dn->dn_dbuf != NULL)
126*9704bf7fSPaul Dagnelie rw_exit(&dn->dn_dbuf->db_rwlock);
127*9704bf7fSPaul Dagnelie
128ea8dc4b6Seschrock dbuf_rele(db, FTAG);
129c717a561Smaybee
130c717a561Smaybee rw_exit(&dn->dn_struct_rwlock);
131fa9e4066Sahrens }
132fa9e4066Sahrens
13343466aaeSMax Grossman static void
free_blocks(dnode_t * dn,blkptr_t * bp,int num,dmu_tx_t * tx)134fa9e4066Sahrens free_blocks(dnode_t *dn, blkptr_t *bp, int num, dmu_tx_t *tx)
135fa9e4066Sahrens {
136cdb0ab79Smaybee dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
137fa9e4066Sahrens uint64_t bytesfreed = 0;
138fa9e4066Sahrens
139cdb0ab79Smaybee dprintf("ds=%p obj=%llx num=%d\n", ds, dn->dn_object, num);
140fa9e4066Sahrens
14143466aaeSMax Grossman for (int i = 0; i < num; i++, bp++) {
142fa9e4066Sahrens if (BP_IS_HOLE(bp))
143fa9e4066Sahrens continue;
144fa9e4066Sahrens
145b24ab676SJeff Bonwick bytesfreed += dsl_dataset_block_kill(ds, bp, tx, B_FALSE);
14699653d4eSeschrock ASSERT3U(bytesfreed, <=, DN_USED_BYTES(dn->dn_phys));
14743466aaeSMax Grossman
14843466aaeSMax Grossman /*
14943466aaeSMax Grossman * Save some useful information on the holes being
15043466aaeSMax Grossman * punched, including logical size, type, and indirection
15143466aaeSMax Grossman * level. Retaining birth time enables detection of when
15243466aaeSMax Grossman * holes are punched for reducing the number of free
15343466aaeSMax Grossman * records transmitted during a zfs send.
15443466aaeSMax Grossman */
15543466aaeSMax Grossman
15643466aaeSMax Grossman uint64_t lsize = BP_GET_LSIZE(bp);
15743466aaeSMax Grossman dmu_object_type_t type = BP_GET_TYPE(bp);
15843466aaeSMax Grossman uint64_t lvl = BP_GET_LEVEL(bp);
15943466aaeSMax Grossman
160c717a561Smaybee bzero(bp, sizeof (blkptr_t));
16143466aaeSMax Grossman
16243466aaeSMax Grossman if (spa_feature_is_active(dn->dn_objset->os_spa,
16343466aaeSMax Grossman SPA_FEATURE_HOLE_BIRTH)) {
16443466aaeSMax Grossman BP_SET_LSIZE(bp, lsize);
16543466aaeSMax Grossman BP_SET_TYPE(bp, type);
16643466aaeSMax Grossman BP_SET_LEVEL(bp, lvl);
16743466aaeSMax Grossman BP_SET_BIRTH(bp, dmu_tx_get_txg(tx), 0);
16843466aaeSMax Grossman }
169fa9e4066Sahrens }
170fa9e4066Sahrens dnode_diduse_space(dn, -bytesfreed);
171fa9e4066Sahrens }
172fa9e4066Sahrens
1739c9dc39aSek #ifdef ZFS_DEBUG
174fa9e4066Sahrens static void
free_verify(dmu_buf_impl_t * db,uint64_t start,uint64_t end,dmu_tx_t * tx)175fa9e4066Sahrens free_verify(dmu_buf_impl_t *db, uint64_t start, uint64_t end, dmu_tx_t *tx)
176fa9e4066Sahrens {
177fa9e4066Sahrens int off, num;
178fa9e4066Sahrens int i, err, epbs;
179fa9e4066Sahrens uint64_t txg = tx->tx_txg;
180744947dcSTom Erickson dnode_t *dn;
181fa9e4066Sahrens
182744947dcSTom Erickson DB_DNODE_ENTER(db);
183744947dcSTom Erickson dn = DB_DNODE(db);
184744947dcSTom Erickson epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
185fa9e4066Sahrens off = start - (db->db_blkid * 1<<epbs);
186fa9e4066Sahrens num = end - start + 1;
187fa9e4066Sahrens
188fa9e4066Sahrens ASSERT3U(off, >=, 0);
189fa9e4066Sahrens ASSERT3U(num, >=, 0);
190fa9e4066Sahrens ASSERT3U(db->db_level, >, 0);
191744947dcSTom Erickson ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
192fa9e4066Sahrens ASSERT3U(off+num, <=, db->db.db_size >> SPA_BLKPTRSHIFT);
193fa9e4066Sahrens ASSERT(db->db_blkptr != NULL);
194fa9e4066Sahrens
195fa9e4066Sahrens for (i = off; i < off+num; i++) {
196fa9e4066Sahrens uint64_t *buf;
197fa9e4066Sahrens dmu_buf_impl_t *child;
198c717a561Smaybee dbuf_dirty_record_t *dr;
199c717a561Smaybee int j;
200fa9e4066Sahrens
201fa9e4066Sahrens ASSERT(db->db_level == 1);
202fa9e4066Sahrens
203744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER);
204*9704bf7fSPaul Dagnelie err = dbuf_hold_impl(dn, db->db_level - 1,
205a2cdcdd2SPaul Dagnelie (db->db_blkid << epbs) + i, TRUE, FALSE, FTAG, &child);
206744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock);
207fa9e4066Sahrens if (err == ENOENT)
208fa9e4066Sahrens continue;
209fa9e4066Sahrens ASSERT(err == 0);
210fa9e4066Sahrens ASSERT(child->db_level == 0);
211c717a561Smaybee dr = child->db_last_dirty;
212c717a561Smaybee while (dr && dr->dr_txg > txg)
213c717a561Smaybee dr = dr->dr_next;
214c717a561Smaybee ASSERT(dr == NULL || dr->dr_txg == txg);
215c717a561Smaybee
216c717a561Smaybee /* data_old better be zeroed */
217c717a561Smaybee if (dr) {
218c717a561Smaybee buf = dr->dt.dl.dr_data->b_data;
219fa9e4066Sahrens for (j = 0; j < child->db.db_size >> 3; j++) {
220fa9e4066Sahrens if (buf[j] != 0) {
221fa9e4066Sahrens panic("freed data not zero: "
222fa9e4066Sahrens "child=%p i=%d off=%d num=%d\n",
223903a11ebSrh (void *)child, i, off, num);
224fa9e4066Sahrens }
225fa9e4066Sahrens }
226fa9e4066Sahrens }
227fa9e4066Sahrens
228fa9e4066Sahrens /*
229fa9e4066Sahrens * db_data better be zeroed unless it's dirty in a
230fa9e4066Sahrens * future txg.
231fa9e4066Sahrens */
232fa9e4066Sahrens mutex_enter(&child->db_mtx);
233fa9e4066Sahrens buf = child->db.db_data;
234fa9e4066Sahrens if (buf != NULL && child->db_state != DB_FILL &&
235c717a561Smaybee child->db_last_dirty == NULL) {
236fa9e4066Sahrens for (j = 0; j < child->db.db_size >> 3; j++) {
237fa9e4066Sahrens if (buf[j] != 0) {
238fa9e4066Sahrens panic("freed data not zero: "
239fa9e4066Sahrens "child=%p i=%d off=%d num=%d\n",
240903a11ebSrh (void *)child, i, off, num);
241fa9e4066Sahrens }
242fa9e4066Sahrens }
243fa9e4066Sahrens }
244fa9e4066Sahrens mutex_exit(&child->db_mtx);
245fa9e4066Sahrens
246ea8dc4b6Seschrock dbuf_rele(child, FTAG);
247fa9e4066Sahrens }
248744947dcSTom Erickson DB_DNODE_EXIT(db);
249fa9e4066Sahrens }
2509c9dc39aSek #endif
251fa9e4066Sahrens
252738e2a3cSPaul Dagnelie /*
253738e2a3cSPaul Dagnelie * We don't usually free the indirect blocks here. If in one txg we have a
254738e2a3cSPaul Dagnelie * free_range and a write to the same indirect block, it's important that we
255738e2a3cSPaul Dagnelie * preserve the hole's birth times. Therefore, we don't free any any indirect
256738e2a3cSPaul Dagnelie * blocks in free_children(). If an indirect block happens to turn into all
257738e2a3cSPaul Dagnelie * holes, it will be freed by dbuf_write_children_ready, which happens at a
258738e2a3cSPaul Dagnelie * point in the syncing process where we know for certain the contents of the
259738e2a3cSPaul Dagnelie * indirect block.
260738e2a3cSPaul Dagnelie *
261738e2a3cSPaul Dagnelie * However, if we're freeing a dnode, its space accounting must go to zero
262738e2a3cSPaul Dagnelie * before we actually try to free the dnode, or we will trip an assertion. In
263738e2a3cSPaul Dagnelie * addition, we know the case described above cannot occur, because the dnode is
264738e2a3cSPaul Dagnelie * being freed. Therefore, we free the indirect blocks immediately in that
265738e2a3cSPaul Dagnelie * case.
266738e2a3cSPaul Dagnelie */
26743466aaeSMax Grossman static void
free_children(dmu_buf_impl_t * db,uint64_t blkid,uint64_t nblks,boolean_t free_indirects,dmu_tx_t * tx)26843466aaeSMax Grossman free_children(dmu_buf_impl_t *db, uint64_t blkid, uint64_t nblks,
269738e2a3cSPaul Dagnelie boolean_t free_indirects, dmu_tx_t *tx)
270fa9e4066Sahrens {
271744947dcSTom Erickson dnode_t *dn;
272fa9e4066Sahrens blkptr_t *bp;
273fa9e4066Sahrens dmu_buf_impl_t *subdb;
2741a01181fSGeorge Wilson uint64_t start, end, dbstart, dbend;
2751a01181fSGeorge Wilson unsigned int epbs, shift, i;
276cdb0ab79Smaybee
277cdb0ab79Smaybee /*
278cdb0ab79Smaybee * There is a small possibility that this block will not be cached:
279cdb0ab79Smaybee * 1 - if level > 1 and there are no children with level <= 1
28043466aaeSMax Grossman * 2 - if this block was evicted since we read it from
28143466aaeSMax Grossman * dmu_tx_hold_free().
282cdb0ab79Smaybee */
283cdb0ab79Smaybee if (db->db_state != DB_CACHED)
284cdb0ab79Smaybee (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
285fa9e4066Sahrens
28699a19144SMatthew Ahrens /*
28799a19144SMatthew Ahrens * If we modify this indirect block, and we are not freeing the
28899a19144SMatthew Ahrens * dnode (!free_indirects), then this indirect block needs to get
28999a19144SMatthew Ahrens * written to disk by dbuf_write(). If it is dirty, we know it will
29099a19144SMatthew Ahrens * be written (otherwise, we would have incorrect on-disk state
29199a19144SMatthew Ahrens * because the space would be freed but still referenced by the BP
29299a19144SMatthew Ahrens * in this indirect block). Therefore we VERIFY that it is
29399a19144SMatthew Ahrens * dirty.
29499a19144SMatthew Ahrens *
29599a19144SMatthew Ahrens * Our VERIFY covers some cases that do not actually have to be
29699a19144SMatthew Ahrens * dirty, but the open-context code happens to dirty. E.g. if the
29799a19144SMatthew Ahrens * blocks we are freeing are all holes, because in that case, we
29899a19144SMatthew Ahrens * are only freeing part of this indirect block, so it is an
29999a19144SMatthew Ahrens * ancestor of the first or last block to be freed. The first and
30099a19144SMatthew Ahrens * last L1 indirect blocks are always dirtied by dnode_free_range().
30199a19144SMatthew Ahrens */
302*9704bf7fSPaul Dagnelie db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
30399a19144SMatthew Ahrens VERIFY(BP_GET_FILL(db->db_blkptr) == 0 || db->db_dirtycnt > 0);
304*9704bf7fSPaul Dagnelie dmu_buf_unlock_parent(db, dblt, FTAG);
30599a19144SMatthew Ahrens
3063f9d6ad7SLin Ling dbuf_release_bp(db);
30743466aaeSMax Grossman bp = db->db.db_data;
308fa9e4066Sahrens
309744947dcSTom Erickson DB_DNODE_ENTER(db);
310744947dcSTom Erickson dn = DB_DNODE(db);
311744947dcSTom Erickson epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
3121a01181fSGeorge Wilson ASSERT3U(epbs, <, 31);
313fa9e4066Sahrens shift = (db->db_level - 1) * epbs;
314fa9e4066Sahrens dbstart = db->db_blkid << epbs;
315fa9e4066Sahrens start = blkid >> shift;
316fa9e4066Sahrens if (dbstart < start) {
317fa9e4066Sahrens bp += start - dbstart;
318fa9e4066Sahrens } else {
319fa9e4066Sahrens start = dbstart;
320fa9e4066Sahrens }
321fa9e4066Sahrens dbend = ((db->db_blkid + 1) << epbs) - 1;
322fa9e4066Sahrens end = (blkid + nblks - 1) >> shift;
323fa9e4066Sahrens if (dbend <= end)
324fa9e4066Sahrens end = dbend;
32543466aaeSMax Grossman
326fa9e4066Sahrens ASSERT3U(start, <=, end);
327fa9e4066Sahrens
328fa9e4066Sahrens if (db->db_level == 1) {
3299c9dc39aSek FREE_VERIFY(db, start, end, tx);
330*9704bf7fSPaul Dagnelie rw_enter(&db->db_rwlock, RW_WRITER);
331*9704bf7fSPaul Dagnelie free_blocks(dn, bp, end - start + 1, tx);
332*9704bf7fSPaul Dagnelie rw_exit(&db->db_rwlock);
33343466aaeSMax Grossman } else {
3341a01181fSGeorge Wilson for (uint64_t id = start; id <= end; id++, bp++) {
33543466aaeSMax Grossman if (BP_IS_HOLE(bp))
33643466aaeSMax Grossman continue;
33743466aaeSMax Grossman rw_enter(&dn->dn_struct_rwlock, RW_READER);
33843466aaeSMax Grossman VERIFY0(dbuf_hold_impl(dn, db->db_level - 1,
3391a01181fSGeorge Wilson id, TRUE, FALSE, FTAG, &subdb));
34043466aaeSMax Grossman rw_exit(&dn->dn_struct_rwlock);
34143466aaeSMax Grossman ASSERT3P(bp, ==, subdb->db_blkptr);
34243466aaeSMax Grossman
343738e2a3cSPaul Dagnelie free_children(subdb, blkid, nblks, free_indirects, tx);
34443466aaeSMax Grossman dbuf_rele(subdb, FTAG);
34543466aaeSMax Grossman }
346fa9e4066Sahrens }
347fa9e4066Sahrens
348738e2a3cSPaul Dagnelie if (free_indirects) {
349*9704bf7fSPaul Dagnelie rw_enter(&db->db_rwlock, RW_WRITER);
350738e2a3cSPaul Dagnelie for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++)
351738e2a3cSPaul Dagnelie ASSERT(BP_IS_HOLE(bp));
35243466aaeSMax Grossman bzero(db->db.db_data, db->db.db_size);
35343466aaeSMax Grossman free_blocks(dn, db->db_blkptr, 1, tx);
354*9704bf7fSPaul Dagnelie rw_exit(&db->db_rwlock);
355fa9e4066Sahrens }
35643466aaeSMax Grossman
357744947dcSTom Erickson DB_DNODE_EXIT(db);
3586b4acc8bSahrens arc_buf_freeze(db->db_buf);
359fa9e4066Sahrens }
360fa9e4066Sahrens
361fa9e4066Sahrens /*
362f7170741SWill Andrews * Traverse the indicated range of the provided file
363fa9e4066Sahrens * and "free" all the blocks contained there.
364fa9e4066Sahrens */
365fa9e4066Sahrens static void
dnode_sync_free_range_impl(dnode_t * dn,uint64_t blkid,uint64_t nblks,boolean_t free_indirects,dmu_tx_t * tx)366bf16b11eSMatthew Ahrens dnode_sync_free_range_impl(dnode_t *dn, uint64_t blkid, uint64_t nblks,
367738e2a3cSPaul Dagnelie boolean_t free_indirects, dmu_tx_t *tx)
368fa9e4066Sahrens {
369fa9e4066Sahrens blkptr_t *bp = dn->dn_phys->dn_blkptr;
370fa9e4066Sahrens int dnlevel = dn->dn_phys->dn_nlevels;
37143466aaeSMax Grossman boolean_t trunc = B_FALSE;
372fa9e4066Sahrens
373fa9e4066Sahrens if (blkid > dn->dn_phys->dn_maxblkid)
374fa9e4066Sahrens return;
375fa9e4066Sahrens
376fa9e4066Sahrens ASSERT(dn->dn_phys->dn_maxblkid < UINT64_MAX);
37743466aaeSMax Grossman if (blkid + nblks > dn->dn_phys->dn_maxblkid) {
378fa9e4066Sahrens nblks = dn->dn_phys->dn_maxblkid - blkid + 1;
37943466aaeSMax Grossman trunc = B_TRUE;
38043466aaeSMax Grossman }
381fa9e4066Sahrens
382fa9e4066Sahrens /* There are no indirect blocks in the object */
383fa9e4066Sahrens if (dnlevel == 1) {
384fa9e4066Sahrens if (blkid >= dn->dn_phys->dn_nblkptr) {
385fa9e4066Sahrens /* this range was never made persistent */
386fa9e4066Sahrens return;
387fa9e4066Sahrens }
388fa9e4066Sahrens ASSERT3U(blkid + nblks, <=, dn->dn_phys->dn_nblkptr);
38943466aaeSMax Grossman free_blocks(dn, bp + blkid, nblks, tx);
39043466aaeSMax Grossman } else {
39143466aaeSMax Grossman int shift = (dnlevel - 1) *
39243466aaeSMax Grossman (dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT);
39343466aaeSMax Grossman int start = blkid >> shift;
39443466aaeSMax Grossman int end = (blkid + nblks - 1) >> shift;
39543466aaeSMax Grossman dmu_buf_impl_t *db;
39643466aaeSMax Grossman
39743466aaeSMax Grossman ASSERT(start < dn->dn_phys->dn_nblkptr);
39843466aaeSMax Grossman bp += start;
39943466aaeSMax Grossman for (int i = start; i <= end; i++, bp++) {
40043466aaeSMax Grossman if (BP_IS_HOLE(bp))
40143466aaeSMax Grossman continue;
40243466aaeSMax Grossman rw_enter(&dn->dn_struct_rwlock, RW_READER);
40343466aaeSMax Grossman VERIFY0(dbuf_hold_impl(dn, dnlevel - 1, i,
404a2cdcdd2SPaul Dagnelie TRUE, FALSE, FTAG, &db));
40543466aaeSMax Grossman rw_exit(&dn->dn_struct_rwlock);
406738e2a3cSPaul Dagnelie free_children(db, blkid, nblks, free_indirects, tx);
40743466aaeSMax Grossman dbuf_rele(db, FTAG);
408fa9e4066Sahrens }
409fa9e4066Sahrens }
41043466aaeSMax Grossman
411eb633035STom Caputi /*
412eb633035STom Caputi * Do not truncate the maxblkid if we are performing a raw
413eb633035STom Caputi * receive. The raw receive sets the maxblkid manually and
414eb633035STom Caputi * must not be overridden. Usually, the last DRR_FREE record
415eb633035STom Caputi * will be at the maxblkid, because the source system sets
416eb633035STom Caputi * the maxblkid when truncating. However, if the last block
417eb633035STom Caputi * was freed by overwriting with zeros and being compressed
418eb633035STom Caputi * away to a hole, the source system will generate a DRR_FREE
419eb633035STom Caputi * record while leaving the maxblkid after the end of that
420eb633035STom Caputi * record. In this case we need to leave the maxblkid as
421eb633035STom Caputi * indicated in the DRR_OBJECT record, so that it matches the
422eb633035STom Caputi * source system, ensuring that the cryptographic hashes will
423eb633035STom Caputi * match.
424eb633035STom Caputi */
425eb633035STom Caputi if (trunc && !dn->dn_objset->os_raw_receive) {
42643466aaeSMax Grossman dn->dn_phys->dn_maxblkid = blkid == 0 ? 0 : blkid - 1;
42743466aaeSMax Grossman
428fa9e4066Sahrens uint64_t off = (dn->dn_phys->dn_maxblkid + 1) *
429fa9e4066Sahrens (dn->dn_phys->dn_datablkszsec << SPA_MINBLOCKSHIFT);
430fa9e4066Sahrens ASSERT(off < dn->dn_phys->dn_maxblkid ||
431fa9e4066Sahrens dn->dn_phys->dn_maxblkid == 0 ||
432cdb0ab79Smaybee dnode_next_offset(dn, 0, &off, 1, 1, 0) != 0);
433fa9e4066Sahrens }
434fa9e4066Sahrens }
435fa9e4066Sahrens
436bf16b11eSMatthew Ahrens typedef struct dnode_sync_free_range_arg {
437bf16b11eSMatthew Ahrens dnode_t *dsfra_dnode;
438bf16b11eSMatthew Ahrens dmu_tx_t *dsfra_tx;
439738e2a3cSPaul Dagnelie boolean_t dsfra_free_indirects;
440bf16b11eSMatthew Ahrens } dnode_sync_free_range_arg_t;
441bf16b11eSMatthew Ahrens
442bf16b11eSMatthew Ahrens static void
dnode_sync_free_range(void * arg,uint64_t blkid,uint64_t nblks)443bf16b11eSMatthew Ahrens dnode_sync_free_range(void *arg, uint64_t blkid, uint64_t nblks)
444bf16b11eSMatthew Ahrens {
445bf16b11eSMatthew Ahrens dnode_sync_free_range_arg_t *dsfra = arg;
446bf16b11eSMatthew Ahrens dnode_t *dn = dsfra->dsfra_dnode;
447bf16b11eSMatthew Ahrens
448bf16b11eSMatthew Ahrens mutex_exit(&dn->dn_mtx);
449738e2a3cSPaul Dagnelie dnode_sync_free_range_impl(dn, blkid, nblks,
450738e2a3cSPaul Dagnelie dsfra->dsfra_free_indirects, dsfra->dsfra_tx);
451bf16b11eSMatthew Ahrens mutex_enter(&dn->dn_mtx);
452bf16b11eSMatthew Ahrens }
453bf16b11eSMatthew Ahrens
454ea8dc4b6Seschrock /*
455f7170741SWill Andrews * Try to kick all the dnode's dbufs out of the cache...
456ea8dc4b6Seschrock */
4571934e92fSmaybee void
dnode_evict_dbufs(dnode_t * dn)4581934e92fSmaybee dnode_evict_dbufs(dnode_t *dn)
459ea8dc4b6Seschrock {
460bc9014e6SJustin Gibbs dmu_buf_impl_t db_marker;
461bc9014e6SJustin Gibbs dmu_buf_impl_t *db, *db_next;
462c543ec06Sahrens
463bc9014e6SJustin Gibbs mutex_enter(&dn->dn_dbufs_mtx);
464bc9014e6SJustin Gibbs for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
465c543ec06Sahrens
466744947dcSTom Erickson #ifdef DEBUG
467bc9014e6SJustin Gibbs DB_DNODE_ENTER(db);
468bc9014e6SJustin Gibbs ASSERT3P(DB_DNODE(db), ==, dn);
469bc9014e6SJustin Gibbs DB_DNODE_EXIT(db);
470744947dcSTom Erickson #endif /* DEBUG */
471ea8dc4b6Seschrock
472bc9014e6SJustin Gibbs mutex_enter(&db->db_mtx);
473bc9014e6SJustin Gibbs if (db->db_state != DB_EVICTING &&
474e914ace2STim Schumacher zfs_refcount_is_zero(&db->db_holds)) {
475bc9014e6SJustin Gibbs db_marker.db_level = db->db_level;
476bc9014e6SJustin Gibbs db_marker.db_blkid = db->db_blkid;
477bc9014e6SJustin Gibbs db_marker.db_state = DB_SEARCH;
478bc9014e6SJustin Gibbs avl_insert_here(&dn->dn_dbufs, &db_marker, db,
479bc9014e6SJustin Gibbs AVL_BEFORE);
480bc9014e6SJustin Gibbs
481c2919acbSMatthew Ahrens /*
482c2919acbSMatthew Ahrens * We need to use the "marker" dbuf rather than
483c2919acbSMatthew Ahrens * simply getting the next dbuf, because
484c2919acbSMatthew Ahrens * dbuf_destroy() may actually remove multiple dbufs.
485c2919acbSMatthew Ahrens * It can call itself recursively on the parent dbuf,
486c2919acbSMatthew Ahrens * which may also be removed from dn_dbufs. The code
487c2919acbSMatthew Ahrens * flow would look like:
488c2919acbSMatthew Ahrens *
489c2919acbSMatthew Ahrens * dbuf_destroy():
490c2919acbSMatthew Ahrens * dnode_rele_and_unlock(parent_dbuf, evicting=TRUE):
491c2919acbSMatthew Ahrens * if (!cacheable || pending_evict)
492c2919acbSMatthew Ahrens * dbuf_destroy()
493c2919acbSMatthew Ahrens */
494dcbf3bd6SGeorge Wilson dbuf_destroy(db);
495bc9014e6SJustin Gibbs
496bc9014e6SJustin Gibbs db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);
497bc9014e6SJustin Gibbs avl_remove(&dn->dn_dbufs, &db_marker);
498bc9014e6SJustin Gibbs } else {
499d2058105SJustin T. Gibbs db->db_pending_evict = TRUE;
500bc9014e6SJustin Gibbs mutex_exit(&db->db_mtx);
501bc9014e6SJustin Gibbs db_next = AVL_NEXT(&dn->dn_dbufs, db);
502ea8dc4b6Seschrock }
503bc9014e6SJustin Gibbs }
504bc9014e6SJustin Gibbs mutex_exit(&dn->dn_dbufs_mtx);
505c543ec06Sahrens
506cd485b49SJustin T. Gibbs dnode_evict_bonus(dn);
507cd485b49SJustin T. Gibbs }
508cd485b49SJustin T. Gibbs
509cd485b49SJustin T. Gibbs void
dnode_evict_bonus(dnode_t * dn)510cd485b49SJustin T. Gibbs dnode_evict_bonus(dnode_t *dn)
511cd485b49SJustin T. Gibbs {
512ea8dc4b6Seschrock rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
513d2058105SJustin T. Gibbs if (dn->dn_bonus != NULL) {
514e914ace2STim Schumacher if (zfs_refcount_is_zero(&dn->dn_bonus->db_holds)) {
515d2058105SJustin T. Gibbs mutex_enter(&dn->dn_bonus->db_mtx);
516dcbf3bd6SGeorge Wilson dbuf_destroy(dn->dn_bonus);
517d2058105SJustin T. Gibbs dn->dn_bonus = NULL;
518d2058105SJustin T. Gibbs } else {
519d2058105SJustin T. Gibbs dn->dn_bonus->db_pending_evict = TRUE;
520d2058105SJustin T. Gibbs }
521ea8dc4b6Seschrock }
522ea8dc4b6Seschrock rw_exit(&dn->dn_struct_rwlock);
523ea8dc4b6Seschrock }
524ea8dc4b6Seschrock
525c717a561Smaybee static void
dnode_undirty_dbufs(list_t * list)526c717a561Smaybee dnode_undirty_dbufs(list_t *list)
527fa9e4066Sahrens {
528c717a561Smaybee dbuf_dirty_record_t *dr;
529fa9e4066Sahrens
530c717a561Smaybee while (dr = list_head(list)) {
531c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf;
532c717a561Smaybee uint64_t txg = dr->dr_txg;
533fa9e4066Sahrens
534b24ab676SJeff Bonwick if (db->db_level != 0)
535b24ab676SJeff Bonwick dnode_undirty_dbufs(&dr->dt.di.dr_children);
536b24ab676SJeff Bonwick
537fa9e4066Sahrens mutex_enter(&db->db_mtx);
538fa9e4066Sahrens /* XXX - use dbuf_undirty()? */
539c717a561Smaybee list_remove(list, dr);
540c717a561Smaybee ASSERT(db->db_last_dirty == dr);
541c717a561Smaybee db->db_last_dirty = NULL;
542c717a561Smaybee db->db_dirtycnt -= 1;
543fa9e4066Sahrens if (db->db_level == 0) {
5440a586ceaSMark Shellenbaum ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
545c717a561Smaybee dr->dt.dl.dr_data == db->db_buf);
546c717a561Smaybee dbuf_unoverride(dr);
547d2b3cbbdSJorgen Lundman } else {
548d2b3cbbdSJorgen Lundman mutex_destroy(&dr->dt.di.dr_mtx);
549d2b3cbbdSJorgen Lundman list_destroy(&dr->dt.di.dr_children);
550fa9e4066Sahrens }
551c717a561Smaybee kmem_free(dr, sizeof (dbuf_dirty_record_t));
552c2919acbSMatthew Ahrens dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg, B_FALSE);
553fa9e4066Sahrens }
554c717a561Smaybee }
555fa9e4066Sahrens
556c717a561Smaybee static void
dnode_sync_free(dnode_t * dn,dmu_tx_t * tx)557c717a561Smaybee dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
558c717a561Smaybee {
559c717a561Smaybee int txgoff = tx->tx_txg & TXG_MASK;
560c717a561Smaybee
561c717a561Smaybee ASSERT(dmu_tx_is_syncing(tx));
562c717a561Smaybee
563cdb0ab79Smaybee /*
564cdb0ab79Smaybee * Our contents should have been freed in dnode_sync() by the
565cdb0ab79Smaybee * free range record inserted by the caller of dnode_free().
566cdb0ab79Smaybee */
567fb09f5aaSMadhav Suresh ASSERT0(DN_USED_BYTES(dn->dn_phys));
568cdb0ab79Smaybee ASSERT(BP_IS_HOLE(dn->dn_phys->dn_blkptr));
569cdb0ab79Smaybee
570c717a561Smaybee dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
5711934e92fSmaybee dnode_evict_dbufs(dn);
572ea8dc4b6Seschrock
573ea8dc4b6Seschrock /*
574ea8dc4b6Seschrock * XXX - It would be nice to assert this, but we may still
575ea8dc4b6Seschrock * have residual holds from async evictions from the arc...
576ea8dc4b6Seschrock *
57755434c77Sek * zfs_obj_to_path() also depends on this being
57855434c77Sek * commented out.
57955434c77Sek *
580e914ace2STim Schumacher * ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 1);
581ea8dc4b6Seschrock */
582fa9e4066Sahrens
583fa9e4066Sahrens /* Undirty next bits */
584fa9e4066Sahrens dn->dn_next_nlevels[txgoff] = 0;
585fa9e4066Sahrens dn->dn_next_indblkshift[txgoff] = 0;
586c543ec06Sahrens dn->dn_next_blksz[txgoff] = 0;
587eb633035STom Caputi dn->dn_next_maxblkid[txgoff] = 0;
588fa9e4066Sahrens
589fa9e4066Sahrens /* ASSERT(blkptrs are zero); */
590fa9e4066Sahrens ASSERT(dn->dn_phys->dn_type != DMU_OT_NONE);
591fa9e4066Sahrens ASSERT(dn->dn_type != DMU_OT_NONE);
592fa9e4066Sahrens
593fa9e4066Sahrens ASSERT(dn->dn_free_txg > 0);
594fa9e4066Sahrens if (dn->dn_allocated_txg != dn->dn_free_txg)
59543466aaeSMax Grossman dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);
59654811da5SToomas Soome bzero(dn->dn_phys, sizeof (dnode_phys_t) * dn->dn_num_slots);
59754811da5SToomas Soome dnode_free_interior_slots(dn);
598fa9e4066Sahrens
599fa9e4066Sahrens mutex_enter(&dn->dn_mtx);
600fa9e4066Sahrens dn->dn_type = DMU_OT_NONE;
601fa9e4066Sahrens dn->dn_maxblkid = 0;
602fa9e4066Sahrens dn->dn_allocated_txg = 0;
603758f6e0bSgw dn->dn_free_txg = 0;
6040a586ceaSMark Shellenbaum dn->dn_have_spill = B_FALSE;
60554811da5SToomas Soome dn->dn_num_slots = 1;
606fa9e4066Sahrens mutex_exit(&dn->dn_mtx);
607fa9e4066Sahrens
608ea8dc4b6Seschrock ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
609fa9e4066Sahrens
610fa9e4066Sahrens dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);
611fa9e4066Sahrens /*
612fa9e4066Sahrens * Now that we've released our hold, the dnode may
613eb633035STom Caputi * be evicted, so we mustn't access it.
614fa9e4066Sahrens */
615fa9e4066Sahrens }
616fa9e4066Sahrens
617fa9e4066Sahrens /*
618c717a561Smaybee * Write out the dnode's dirty buffers.
619fa9e4066Sahrens */
620c717a561Smaybee void
dnode_sync(dnode_t * dn,dmu_tx_t * tx)621c717a561Smaybee dnode_sync(dnode_t *dn, dmu_tx_t *tx)
622fa9e4066Sahrens {
623eb633035STom Caputi objset_t *os = dn->dn_objset;
624fa9e4066Sahrens dnode_phys_t *dnp = dn->dn_phys;
625c717a561Smaybee int txgoff = tx->tx_txg & TXG_MASK;
626c717a561Smaybee list_t *list = &dn->dn_dirty_records[txgoff];
62714843421SMatthew Ahrens static const dnode_phys_t zerodn = { 0 };
6280a586ceaSMark Shellenbaum boolean_t kill_spill = B_FALSE;
629fa9e4066Sahrens
630fa9e4066Sahrens ASSERT(dmu_tx_is_syncing(tx));
631fa9e4066Sahrens ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg);
63214843421SMatthew Ahrens ASSERT(dnp->dn_type != DMU_OT_NONE ||
63354811da5SToomas Soome bcmp(dnp, &zerodn, DNODE_MIN_SIZE) == 0);
6349c9dc39aSek DNODE_VERIFY(dn);
635c543ec06Sahrens
636c717a561Smaybee ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf));
637fa9e4066Sahrens
638eb633035STom Caputi /*
639eb633035STom Caputi * Do user accounting if it is enabled and this is not
640eb633035STom Caputi * an encrypted receive.
641eb633035STom Caputi */
642eb633035STom Caputi if (dmu_objset_userused_enabled(os) &&
643eb633035STom Caputi !DMU_OBJECT_IS_SPECIAL(dn->dn_object) &&
644eb633035STom Caputi (!os->os_encrypted || !dmu_objset_is_receiving(os))) {
6450a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx);
6460a586ceaSMark Shellenbaum dn->dn_oldused = DN_USED_BYTES(dn->dn_phys);
6470a586ceaSMark Shellenbaum dn->dn_oldflags = dn->dn_phys->dn_flags;
64814843421SMatthew Ahrens dn->dn_phys->dn_flags |= DNODE_FLAG_USERUSED_ACCOUNTED;
649f67950b2SNasf-Fan if (dmu_objset_userobjused_enabled(dn->dn_objset))
650f67950b2SNasf-Fan dn->dn_phys->dn_flags |=
651f67950b2SNasf-Fan DNODE_FLAG_USEROBJUSED_ACCOUNTED;
6520a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx);
65306e0070dSMark Shellenbaum dmu_objset_userquota_get_ids(dn, B_FALSE, tx);
65414843421SMatthew Ahrens } else {
655eb633035STom Caputi /* Once we account for it, we should always account for it */
65614843421SMatthew Ahrens ASSERT(!(dn->dn_phys->dn_flags &
65714843421SMatthew Ahrens DNODE_FLAG_USERUSED_ACCOUNTED));
658f67950b2SNasf-Fan ASSERT(!(dn->dn_phys->dn_flags &
659f67950b2SNasf-Fan DNODE_FLAG_USEROBJUSED_ACCOUNTED));
66014843421SMatthew Ahrens }
66114843421SMatthew Ahrens
662fa9e4066Sahrens mutex_enter(&dn->dn_mtx);
663fa9e4066Sahrens if (dn->dn_allocated_txg == tx->tx_txg) {
664fa9e4066Sahrens /* The dnode is newly allocated or reallocated */
665fa9e4066Sahrens if (dnp->dn_type == DMU_OT_NONE) {
666fa9e4066Sahrens /* this is a first alloc, not a realloc */
667fa9e4066Sahrens dnp->dn_nlevels = 1;
668da03de99SMark Maybee dnp->dn_nblkptr = dn->dn_nblkptr;
669fa9e4066Sahrens }
670fa9e4066Sahrens
671fa9e4066Sahrens dnp->dn_type = dn->dn_type;
672fa9e4066Sahrens dnp->dn_bonustype = dn->dn_bonustype;
673fa9e4066Sahrens dnp->dn_bonuslen = dn->dn_bonuslen;
674fa9e4066Sahrens }
67554811da5SToomas Soome
67654811da5SToomas Soome dnp->dn_extra_slots = dn->dn_num_slots - 1;
67754811da5SToomas Soome
678c717a561Smaybee ASSERT(dnp->dn_nlevels > 1 ||
679f676ed34Sahrens BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
6805d7b4d43SMatthew Ahrens BP_IS_EMBEDDED(&dnp->dn_blkptr[0]) ||
681f676ed34Sahrens BP_GET_LSIZE(&dnp->dn_blkptr[0]) ==
682f676ed34Sahrens dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
6835d7b4d43SMatthew Ahrens ASSERT(dnp->dn_nlevels < 2 ||
6845d7b4d43SMatthew Ahrens BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
6855d7b4d43SMatthew Ahrens BP_GET_LSIZE(&dnp->dn_blkptr[0]) == 1 << dnp->dn_indblkshift);
686f676ed34Sahrens
6872acef22dSMatthew Ahrens if (dn->dn_next_type[txgoff] != 0) {
6882acef22dSMatthew Ahrens dnp->dn_type = dn->dn_type;
6892acef22dSMatthew Ahrens dn->dn_next_type[txgoff] = 0;
6902acef22dSMatthew Ahrens }
6912acef22dSMatthew Ahrens
6922acef22dSMatthew Ahrens if (dn->dn_next_blksz[txgoff] != 0) {
693c543ec06Sahrens ASSERT(P2PHASE(dn->dn_next_blksz[txgoff],
694fa9e4066Sahrens SPA_MINBLOCKSIZE) == 0);
695f676ed34Sahrens ASSERT(BP_IS_HOLE(&dnp->dn_blkptr[0]) ||
696cdb0ab79Smaybee dn->dn_maxblkid == 0 || list_head(list) != NULL ||
697347a31bcSahrens dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT ==
698bf16b11eSMatthew Ahrens dnp->dn_datablkszsec ||
69986714001SSerapheim Dimitropoulos !range_tree_is_empty(dn->dn_free_ranges[txgoff]));
700fa9e4066Sahrens dnp->dn_datablkszsec =
701c543ec06Sahrens dn->dn_next_blksz[txgoff] >> SPA_MINBLOCKSHIFT;
702c543ec06Sahrens dn->dn_next_blksz[txgoff] = 0;
703fa9e4066Sahrens }
704fa9e4066Sahrens
7052acef22dSMatthew Ahrens if (dn->dn_next_bonuslen[txgoff] != 0) {
7061934e92fSmaybee if (dn->dn_next_bonuslen[txgoff] == DN_ZERO_BONUSLEN)
7071934e92fSmaybee dnp->dn_bonuslen = 0;
7081934e92fSmaybee else
7091934e92fSmaybee dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff];
71054811da5SToomas Soome ASSERT(dnp->dn_bonuslen <=
71154811da5SToomas Soome DN_SLOTS_TO_BONUSLEN(dnp->dn_extra_slots + 1));
7121934e92fSmaybee dn->dn_next_bonuslen[txgoff] = 0;
7131934e92fSmaybee }
7141934e92fSmaybee
7152acef22dSMatthew Ahrens if (dn->dn_next_bonustype[txgoff] != 0) {
716ad135b5dSChristopher Siden ASSERT(DMU_OT_IS_VALID(dn->dn_next_bonustype[txgoff]));
7170a586ceaSMark Shellenbaum dnp->dn_bonustype = dn->dn_next_bonustype[txgoff];
7180a586ceaSMark Shellenbaum dn->dn_next_bonustype[txgoff] = 0;
7190a586ceaSMark Shellenbaum }
7200a586ceaSMark Shellenbaum
72143466aaeSMax Grossman boolean_t freeing_dnode = dn->dn_free_txg > 0 &&
72243466aaeSMax Grossman dn->dn_free_txg <= tx->tx_txg;
72343466aaeSMax Grossman
7240a586ceaSMark Shellenbaum /*
725e6518318SMatthew Ahrens * Remove the spill block if we have been explicitly asked to
726e6518318SMatthew Ahrens * remove it, or if the object is being removed.
7270a586ceaSMark Shellenbaum */
728e6518318SMatthew Ahrens if (dn->dn_rm_spillblk[txgoff] || freeing_dnode) {
729e6518318SMatthew Ahrens if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
7300a586ceaSMark Shellenbaum kill_spill = B_TRUE;
7310a586ceaSMark Shellenbaum dn->dn_rm_spillblk[txgoff] = 0;
7320a586ceaSMark Shellenbaum }
7330a586ceaSMark Shellenbaum
7342acef22dSMatthew Ahrens if (dn->dn_next_indblkshift[txgoff] != 0) {
735fa9e4066Sahrens ASSERT(dnp->dn_nlevels == 1);
736fa9e4066Sahrens dnp->dn_indblkshift = dn->dn_next_indblkshift[txgoff];
737fa9e4066Sahrens dn->dn_next_indblkshift[txgoff] = 0;
738fa9e4066Sahrens }
739fa9e4066Sahrens
740fa9e4066Sahrens /*
741fa9e4066Sahrens * Just take the live (open-context) values for checksum and compress.
742fa9e4066Sahrens * Strictly speaking it's a future leak, but nothing bad happens if we
743fa9e4066Sahrens * start using the new checksum or compress algorithm a little early.
744fa9e4066Sahrens */
745fa9e4066Sahrens dnp->dn_checksum = dn->dn_checksum;
746fa9e4066Sahrens dnp->dn_compress = dn->dn_compress;
747fa9e4066Sahrens
748fa9e4066Sahrens mutex_exit(&dn->dn_mtx);
749fa9e4066Sahrens
7500a586ceaSMark Shellenbaum if (kill_spill) {
75154811da5SToomas Soome free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx);
7520a586ceaSMark Shellenbaum mutex_enter(&dn->dn_mtx);
7530a586ceaSMark Shellenbaum dnp->dn_flags &= ~DNODE_FLAG_SPILL_BLKPTR;
7540a586ceaSMark Shellenbaum mutex_exit(&dn->dn_mtx);
7550a586ceaSMark Shellenbaum }
7560a586ceaSMark Shellenbaum
757fa9e4066Sahrens /* process all the "freed" ranges in the file */
758bf16b11eSMatthew Ahrens if (dn->dn_free_ranges[txgoff] != NULL) {
759bf16b11eSMatthew Ahrens dnode_sync_free_range_arg_t dsfra;
760bf16b11eSMatthew Ahrens dsfra.dsfra_dnode = dn;
761bf16b11eSMatthew Ahrens dsfra.dsfra_tx = tx;
762738e2a3cSPaul Dagnelie dsfra.dsfra_free_indirects = freeing_dnode;
7637611ab7bSPatrick Mooney mutex_enter(&dn->dn_mtx);
764738e2a3cSPaul Dagnelie if (freeing_dnode) {
765738e2a3cSPaul Dagnelie ASSERT(range_tree_contains(dn->dn_free_ranges[txgoff],
766738e2a3cSPaul Dagnelie 0, dn->dn_maxblkid + 1));
767738e2a3cSPaul Dagnelie }
7687611ab7bSPatrick Mooney /*
7697611ab7bSPatrick Mooney * Because dnode_sync_free_range() must drop dn_mtx during its
7707611ab7bSPatrick Mooney * processing, using it as a callback to range_tree_vacate() is
7717611ab7bSPatrick Mooney * not safe. No other operations (besides destroy) are allowed
7727611ab7bSPatrick Mooney * once range_tree_vacate() has begun, and dropping dn_mtx
7737611ab7bSPatrick Mooney * would leave a window open for another thread to observe that
7747611ab7bSPatrick Mooney * invalid (and unsafe) state.
7757611ab7bSPatrick Mooney */
7767611ab7bSPatrick Mooney range_tree_walk(dn->dn_free_ranges[txgoff],
777bf16b11eSMatthew Ahrens dnode_sync_free_range, &dsfra);
7787611ab7bSPatrick Mooney range_tree_vacate(dn->dn_free_ranges[txgoff], NULL, NULL);
779bf16b11eSMatthew Ahrens range_tree_destroy(dn->dn_free_ranges[txgoff]);
780bf16b11eSMatthew Ahrens dn->dn_free_ranges[txgoff] = NULL;
781cdb0ab79Smaybee mutex_exit(&dn->dn_mtx);
782fa9e4066Sahrens }
7831934e92fSmaybee
78443466aaeSMax Grossman if (freeing_dnode) {
785af346df5SNed Bass dn->dn_objset->os_freed_dnodes++;
786c717a561Smaybee dnode_sync_free(dn, tx);
787c717a561Smaybee return;
788fa9e4066Sahrens }
789fa9e4066Sahrens
79054811da5SToomas Soome if (dn->dn_num_slots > DNODE_MIN_SLOTS) {
79154811da5SToomas Soome dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
79254811da5SToomas Soome mutex_enter(&ds->ds_lock);
79354811da5SToomas Soome ds->ds_feature_activation_needed[SPA_FEATURE_LARGE_DNODE] =
79454811da5SToomas Soome B_TRUE;
79554811da5SToomas Soome mutex_exit(&ds->ds_lock);
79654811da5SToomas Soome }
79754811da5SToomas Soome
798e503a685SGeorge Wilson if (dn->dn_next_nlevels[txgoff]) {
799e503a685SGeorge Wilson dnode_increase_indirection(dn, tx);
800e503a685SGeorge Wilson dn->dn_next_nlevels[txgoff] = 0;
801e503a685SGeorge Wilson }
802e503a685SGeorge Wilson
803eb633035STom Caputi /*
804eb633035STom Caputi * This must be done after dnode_sync_free_range()
805eb633035STom Caputi * and dnode_increase_indirection(). See dnode_new_blkid()
806eb633035STom Caputi * for an explanation of the high bit being set.
807eb633035STom Caputi */
808eb633035STom Caputi if (dn->dn_next_maxblkid[txgoff]) {
809eb633035STom Caputi mutex_enter(&dn->dn_mtx);
810eb633035STom Caputi dnp->dn_maxblkid =
811eb633035STom Caputi dn->dn_next_maxblkid[txgoff] & ~DMU_NEXT_MAXBLKID_SET;
812eb633035STom Caputi dn->dn_next_maxblkid[txgoff] = 0;
813eb633035STom Caputi mutex_exit(&dn->dn_mtx);
814eb633035STom Caputi }
815eb633035STom Caputi
816da03de99SMark Maybee if (dn->dn_next_nblkptr[txgoff]) {
817da03de99SMark Maybee /* this should only happen on a realloc */
818da03de99SMark Maybee ASSERT(dn->dn_allocated_txg == tx->tx_txg);
819da03de99SMark Maybee if (dn->dn_next_nblkptr[txgoff] > dnp->dn_nblkptr) {
820da03de99SMark Maybee /* zero the new blkptrs we are gaining */
821da03de99SMark Maybee bzero(dnp->dn_blkptr + dnp->dn_nblkptr,
822da03de99SMark Maybee sizeof (blkptr_t) *
823da03de99SMark Maybee (dn->dn_next_nblkptr[txgoff] - dnp->dn_nblkptr));
824da03de99SMark Maybee #ifdef ZFS_DEBUG
825da03de99SMark Maybee } else {
826da03de99SMark Maybee int i;
827da03de99SMark Maybee ASSERT(dn->dn_next_nblkptr[txgoff] < dnp->dn_nblkptr);
828da03de99SMark Maybee /* the blkptrs we are losing better be unallocated */
829da03de99SMark Maybee for (i = dn->dn_next_nblkptr[txgoff];
830da03de99SMark Maybee i < dnp->dn_nblkptr; i++)
831da03de99SMark Maybee ASSERT(BP_IS_HOLE(&dnp->dn_blkptr[i]));
832da03de99SMark Maybee #endif
833da03de99SMark Maybee }
834da03de99SMark Maybee mutex_enter(&dn->dn_mtx);
835da03de99SMark Maybee dnp->dn_nblkptr = dn->dn_next_nblkptr[txgoff];
836da03de99SMark Maybee dn->dn_next_nblkptr[txgoff] = 0;
837da03de99SMark Maybee mutex_exit(&dn->dn_mtx);
838da03de99SMark Maybee }
839da03de99SMark Maybee
84046e1baa6SMatthew Ahrens dbuf_sync_list(list, dn->dn_phys->dn_nlevels - 1, tx);
841fa9e4066Sahrens
84214843421SMatthew Ahrens if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
843c717a561Smaybee ASSERT3P(list_head(list), ==, NULL);
844c717a561Smaybee dnode_rele(dn, (void *)(uintptr_t)tx->tx_txg);
845fa9e4066Sahrens }
846c717a561Smaybee
847c717a561Smaybee /*
848c717a561Smaybee * Although we have dropped our reference to the dnode, it
849c717a561Smaybee * can't be evicted until its written, and we haven't yet
850c717a561Smaybee * initiated the IO for the dnode's dbuf.
851c717a561Smaybee */
852fa9e4066Sahrens }
853