2fa9e406ahrens * CDDL HEADER START
3fa9e406ahrens *
4fa9e406ahrens * The contents of this file are subject to the terms of the
5f65e61cahrens * Common Development and Distribution License (the "License").
6f65e61cahrens * You may not use this file except in compliance with the License.
7fa9e406ahrens *
8fa9e406ahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e406ahrens * or http://www.opensolaris.org/os/licensing.
10fa9e406ahrens * See the License for the specific language governing permissions
11fa9e406ahrens * and limitations under the License.
12fa9e406ahrens *
13fa9e406ahrens * When distributing Covered Code, include this CDDL HEADER in each
14fa9e406ahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e406ahrens * If applicable, add the following below this CDDL HEADER, with the
16fa9e406ahrens * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e406ahrens * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e406ahrens *
19fa9e406ahrens * CDDL HEADER END
20fa9e406ahrens */
2201025c8John Harres * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
239dccfd2Albert Lee * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24b7b2590Matthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
25c3d26abMatthew Ahrens * Copyright (c) 2014 Integros [integros.com]
269dccfd2Albert Lee */
28fa9e406ahrens#include <sys/dmu.h>
29fa9e406ahrens#include <sys/dmu_impl.h>
30fa9e406ahrens#include <sys/dbuf.h>
31fa9e406ahrens#include <sys/dmu_tx.h>
32fa9e406ahrens#include <sys/dmu_objset.h>
3361e255cMatthew Ahrens#include <sys/dsl_dataset.h>
3461e255cMatthew Ahrens#include <sys/dsl_dir.h>
35fa9e406ahrens#include <sys/dsl_pool.h>
3661e255cMatthew Ahrens#include <sys/zap_impl.h>
37fa9e406ahrens#include <sys/spa.h>
380a586ceMark Shellenbaum#include <sys/sa.h>
390a586ceMark Shellenbaum#include <sys/sa_impl.h>
40fa9e406ahrens#include <sys/zfs_context.h>
410a586ceMark Shellenbaum#include <sys/varargs.h>
43ea8dc4beschrocktypedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn,
44ea8dc4beschrock    uint64_t arg1, uint64_t arg2);
47fa9e406ahrensdmu_tx_t *
481d452cfahrensdmu_tx_create_dd(dsl_dir_t *dd)
50fa9e406ahrens	dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_SLEEP);
51fa9e406ahrens	tx->tx_dir = dd;
524445fffMatthew Ahrens	if (dd != NULL)
53fa9e406ahrens		tx->tx_pool = dd->dd_pool;
54fa9e406ahrens	list_create(&tx->tx_holds, sizeof (dmu_tx_hold_t),
558a2f1b9ahrens	    offsetof(dmu_tx_hold_t, txh_node));
56d20e665Ricardo M. Correia	list_create(&tx->tx_callbacks, sizeof (dmu_tx_callback_t),
57d20e665Ricardo M. Correia	    offsetof(dmu_tx_callback_t, dcb_node));
5869962b5Matthew Ahrens	tx->tx_start = gethrtime();
59fa9e406ahrens	return (tx);
62fa9e406ahrensdmu_tx_t *
63fa9e406ahrensdmu_tx_create(objset_t *os)
65503ad85Matthew Ahrens	dmu_tx_t *tx = dmu_tx_create_dd(os->os_dsl_dataset->ds_dir);
66fa9e406ahrens	tx->tx_objset = os;
67fa9e406ahrens	return (tx);
70fa9e406ahrensdmu_tx_t *
71fa9e406ahrensdmu_tx_create_assigned(struct dsl_pool *dp, uint64_t txg)
731d452cfahrens	dmu_tx_t *tx = dmu_tx_create_dd(NULL);
75b7b2590Matthew Ahrens	txg_verify(dp->dp_spa, txg);
76fa9e406ahrens	tx->tx_pool = dp;
77fa9e406ahrens	tx->tx_txg = txg;
78fa9e406ahrens	tx->tx_anyobj = TRUE;
80fa9e406ahrens	return (tx);
84fa9e406ahrensdmu_tx_is_syncing(dmu_tx_t *tx)
86fa9e406ahrens	return (tx->tx_anyobj);
90fa9e406ahrensdmu_tx_private_ok(dmu_tx_t *tx)
92ea8dc4beschrock	return (tx->tx_anyobj);
958a2f1b9ahrensstatic dmu_tx_hold_t *
96b0c42cdbzzzdmu_tx_hold_dnode_impl(dmu_tx_t *tx, dnode_t *dn, enum dmu_tx_hold_type type,
97b0c42cdbzzz    uint64_t arg1, uint64_t arg2)
998a2f1b9ahrens	dmu_tx_hold_t *txh;
101b0c42cdbzzz	if (dn != NULL) {
102e914aceTim Schumacher		(void) zfs_refcount_add(&dn->dn_holds, tx);
103b0c42cdbzzz		if (tx->tx_txg != 0) {
104fa9e406ahrens			mutex_enter(&dn->dn_mtx);
105fa9e406ahrens			/*
106fa9e406ahrens			 * dn->dn_assigned_txg == tx->tx_txg doesn't pose a
107fa9e406ahrens			 * problem, but there's no way for it to happen (for
108fa9e406ahrens			 * now, at least).
109fa9e406ahrens			 */
110fa9e406ahrens			ASSERT(dn->dn_assigned_txg == 0);
111fa9e406ahrens			dn->dn_assigned_txg = tx->tx_txg;
112e914aceTim Schumacher			(void) zfs_refcount_add(&dn->dn_tx_holds, tx);
113fa9e406ahrens			mutex_exit(&dn->dn_mtx);
114fa9e406ahrens		}
115fa9e406ahrens	}
1178a2f1b9ahrens	txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_SLEEP);
1188a2f1b9ahrens	txh->txh_tx = tx;
1198a2f1b9ahrens	txh->txh_dnode = dn;
120e914aceTim Schumacher	zfs_refcount_create(&txh->txh_space_towrite);
121e914aceTim Schumacher	zfs_refcount_create(&txh->txh_memory_tohold);
1228a2f1b9ahrens	txh->txh_type = type;
1238a2f1b9ahrens	txh->txh_arg1 = arg1;
1248a2f1b9ahrens	txh->txh_arg2 = arg2;
1258a2f1b9ahrens	list_insert_tail(&tx->tx_holds, txh);
1278a2f1b9ahrens	return (txh);
130b0c42cdbzzzstatic dmu_tx_hold_t *
131b0c42cdbzzzdmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object,
132b0c42cdbzzz    enum dmu_tx_hold_type type, uint64_t arg1, uint64_t arg2)
134b0c42cdbzzz	dnode_t *dn = NULL;
135b0c42cdbzzz	dmu_tx_hold_t *txh;
136b0c42cdbzzz	int err;
138b0c42cdbzzz	if (object != DMU_NEW_OBJECT) {
139b0c42cdbzzz		err = dnode_hold(os, object, FTAG, &dn);
140b0c42cdbzzz		if (err != 0) {
141b0c42cdbzzz			tx->tx_err = err;
142b0c42cdbzzz			return (NULL);
143b0c42cdbzzz		}
144b0c42cdbzzz	}
145b0c42cdbzzz	txh = dmu_tx_hold_dnode_impl(tx, dn, type, arg1, arg2);
146b0c42cdbzzz	if (dn != NULL)
147b0c42cdbzzz		dnode_rele(dn, FTAG);
148b0c42cdbzzz	return (txh);
152b0c42cdbzzzdmu_tx_add_new_object(dmu_tx_t *tx, dnode_t *dn)
154fa9e406ahrens	/*
155fa9e406ahrens	 * If we're syncing, they can manipulate any object anyhow, and
156fa9e406ahrens	 * the hold on the dnode_t can cause problems.
157fa9e406ahrens	 */
158b0c42cdbzzz	if (!dmu_tx_is_syncing(tx))
159b0c42cdbzzz		(void) dmu_tx_hold_dnode_impl(tx, dn, THT_NEWOBJECT, 0, 0);
16261e255cMatthew Ahrens/*
16361e255cMatthew Ahrens * This function reads specified data from disk.  The specified data will
16461e255cMatthew Ahrens * be needed to perform the transaction -- i.e, it will be read after
16561e255cMatthew Ahrens * we do dmu_tx_assign().  There are two reasons that we read the data now
16661e255cMatthew Ahrens * (before dmu_tx_assign()):
16761e255cMatthew Ahrens *
16861e255cMatthew Ahrens * 1. Reading it now has potentially better performance.  The transaction
16961e255cMatthew Ahrens * has not yet been assigned, so the TXG is not held open, and also the
17061e255cMatthew Ahrens * caller typically has less locks held when calling dmu_tx_hold_*() than
17161e255cMatthew Ahrens * after the transaction has been assigned.  This reduces the lock (and txg)
17261e255cMatthew Ahrens * hold times, thus reducing lock contention.
17361e255cMatthew Ahrens *
17461e255cMatthew Ahrens * 2. It is easier for callers (primarily the ZPL) to handle i/o errors
17561e255cMatthew Ahrens * that are detected before they start making changes to the DMU state
17661e255cMatthew Ahrens * (i.e. now).  Once the transaction has been assigned, and some DMU
17761e255cMatthew Ahrens * state has been changed, it can be difficult to recover from an i/o
17861e255cMatthew Ahrens * error (e.g. to undo the changes already made in memory at the DMU
17961e255cMatthew Ahrens * layer).  Typically code to do so does not exist in the caller -- it
18061e255cMatthew Ahrens * assumes that the data has already been cached and thus i/o errors are
18161e255cMatthew Ahrens * not possible.
18261e255cMatthew Ahrens *
18361e255cMatthew Ahrens * It has been observed that the i/o initiated here can be a performance
18461e255cMatthew Ahrens * problem, and it appears to be optional, because we don't look at the
18561e255cMatthew Ahrens * data which is read.  However, removing this read would only serve to
18661e255cMatthew Ahrens * move the work elsewhere (after the dmu_tx_assign()), where it may
18761e255cMatthew Ahrens * have a greater impact on performance (in addition to the impact on
18861e255cMatthew Ahrens * fault tolerance noted above).
18961e255cMatthew Ahrens */
190ea8dc4beschrockstatic int
191ea8dc4beschrockdmu_tx_check_ioerr(zio_t *zio, dnode_t *dn, int level, uint64_t blkid)
193ea8dc4beschrock	int err;
194ea8dc4beschrock	dmu_buf_impl_t *db;
196ea8dc4beschrock	rw_enter(&dn->dn_struct_rwlock, RW_READER);
197ea8dc4beschrock	db = dbuf_hold_level(dn, level, blkid, FTAG);
198ea8dc4beschrock	rw_exit(&dn->dn_struct_rwlock);
199ea8dc4beschrock	if (db == NULL)
200be6fd75Matthew Ahrens		return (SET_ERROR(EIO));
2011ab7f2dmaybee	err = dbuf_read(db, zio, DB_RF_CANFAIL | DB_RF_NOPREFETCH);
202ea8dc4beschrock	dbuf_rele(db, FTAG);
203ea8dc4beschrock	return (err);
206fa9e406ahrens/* ARGSUSED */
207fa9e406ahrensstatic void
2088a2f1b9ahrensdmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
2108a2f1b9ahrens	dnode_t *dn = txh->txh_dnode;
2118a2f1b9ahrens	int err = 0;
213fa9e406ahrens	if (len == 0)
214fa9e406ahrens		return;
216e914aceTim Schumacher	(void) zfs_refcount_add_many(&txh->txh_space_towrite, len, FTAG);
2174a7f2a7Mark Maybee
218e914aceTim Schumacher	if (zfs_refcount_count(&txh->txh_space_towrite) > 2 * DMU_MAX_ACCESS)
21961e255cMatthew Ahrens		err = SET_ERROR(EFBIG);
22161e255cMatthew Ahrens	if (dn == NULL)
22261e255cMatthew Ahrens		return;
22461e255cMatthew Ahrens	/*
22561e255cMatthew Ahrens	 * For i/o error checking, read the blocks that will be needed
22661e255cMatthew Ahrens	 * to perform the write: the first and last level-0 blocks (if
22761e255cMatthew Ahrens	 * they are not aligned, i.e. if they are partial-block writes),
22861e255cMatthew Ahrens	 * and all the level-1 blocks.
22961e255cMatthew Ahrens	 */
23061e255cMatthew Ahrens	if (dn->dn_maxblkid == 0) {
23161e255cMatthew Ahrens		if (off < dn->dn_datablksz &&
23261e255cMatthew Ahrens		    (off > 0 || len < dn->dn_datablksz)) {
23361e255cMatthew Ahrens			err = dmu_tx_check_ioerr(NULL, dn, 0, 0);
23461e255cMatthew Ahrens			if (err != 0) {
23561e255cMatthew Ahrens				txh->txh_tx->tx_err = err;
236ea8dc4beschrock			}
2374a7f2a7Mark Maybee		}
23861e255cMatthew Ahrens	} else {
23961e255cMatthew Ahrens		zio_t *zio = zio_root(dn->dn_objset->os_spa,
24061e255cMatthew Ahrens		    NULL, NULL, ZIO_FLAG_CANFAIL);
2414a7f2a7Mark Maybee
24261e255cMatthew Ahrens		/* first level-0 block */
24361e255cMatthew Ahrens		uint64_t start = off >> dn->dn_datablkshift;
24461e255cMatthew Ahrens		if (P2PHASE(off, dn->dn_datablksz) || len < dn->dn_datablksz) {
24561e255cMatthew Ahrens			err = dmu_tx_check_ioerr(zio, dn, 0, start);
24661e255cMatthew Ahrens			if (err != 0) {
24761e255cMatthew Ahrens				txh->txh_tx->tx_err = err;
24861e255cMatthew Ahrens			}
249b24ab67Jeff Bonwick		}
2504a7f2a7Mark Maybee
25161e255cMatthew Ahrens		/* last level-0 block */
25261e255cMatthew Ahrens		uint64_t end = (off + len - 1) >> dn->dn_datablkshift;
25361e255cMatthew Ahrens		if (end != start && end <= dn->dn_maxblkid &&
25461e255cMatthew Ahrens		    P2PHASE(off + len, dn->dn_datablksz)) {
25561e255cMatthew Ahrens			err = dmu_tx_check_ioerr(zio, dn, 0, end);
25661e255cMatthew Ahrens			if (err != 0) {
25701025c8John Harres				txh->txh_tx->tx_err = err;
25801025c8John Harres			}
25961e255cMatthew Ahrens		}
26001025c8John Harres
26161e255cMatthew Ahrens		/* level-1 blocks */
26261e255cMatthew Ahrens		if (dn->dn_nlevels > 1) {
26361e255cMatthew Ahrens			int shft = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
26461e255cMatthew Ahrens			for (uint64_t i = (start >> shft) + 1;
26561e255cMatthew Ahrens			    i < end >> shft; i++) {
26661e255cMatthew Ahrens				err = dmu_tx_check_ioerr(zio, dn, 1, i);
26761e255cMatthew Ahrens				if (err != 0) {
26861e255cMatthew Ahrens					txh->txh_tx->tx_err = err;
2690c779adMatthew Ahrens				}
2704a7f2a7Mark Maybee			}
2714a7f2a7Mark Maybee		}
27361e255cMatthew Ahrens		err = zio_wait(zio);
27461e255cMatthew Ahrens		if (err != 0) {
27561e255cMatthew Ahrens			txh->txh_tx->tx_err = err;
2764a7f2a7Mark Maybee		}
277fa9e406ahrens	}
280fa9e406ahrensstatic void
2818a2f1b9ahrensdmu_tx_count_dnode(dmu_tx_hold_t *txh)
283e914aceTim Schumacher	(void) zfs_refcount_add_many(&txh->txh_space_towrite, DNODE_MIN_SIZE,
284e914aceTim Schumacher	    FTAG);
288fa9e406ahrensdmu_tx_hold_write(dmu_tx_t *tx, uint64_t object, uint64_t off, int len)
2908a2f1b9ahrens	dmu_tx_hold_t *txh;
29261e255cMatthew Ahrens	ASSERT0(tx->tx_txg);
29361e255cMatthew Ahrens	ASSERT3U(len, <=, DMU_MAX_ACCESS);
294dd6ef53maybee	ASSERT(len == 0 || UINT64_MAX - off >= len - 1);
2968a2f1b9ahrens	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
2978a2f1b9ahrens	    object, THT_WRITE, off, len);
298b0c42cdbzzz	if (txh != NULL) {
299b0c42cdbzzz		dmu_tx_count_write(txh, off, len);
300b0c42cdbzzz		dmu_tx_count_dnode(txh);
301b0c42cdbzzz	}
3055cabbc6Prashanth Sreenivasadmu_tx_hold_remap_l1indirect(dmu_tx_t *tx, uint64_t object)
3065cabbc6Prashanth Sreenivasa{
3075cabbc6Prashanth Sreenivasa	dmu_tx_hold_t *txh;
3085cabbc6Prashanth Sreenivasa
3095cabbc6Prashanth Sreenivasa	ASSERT(tx->tx_txg == 0);
3105cabbc6Prashanth Sreenivasa	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
3115cabbc6Prashanth Sreenivasa	    object, THT_WRITE, 0, 0);
3125cabbc6Prashanth Sreenivasa	if (txh == NULL)
3135cabbc6Prashanth Sreenivasa		return;
3145cabbc6Prashanth Sreenivasa
3155cabbc6Prashanth Sreenivasa	dnode_t *dn = txh->txh_dnode;
316e914aceTim Schumacher	(void) zfs_refcount_add_many(&txh->txh_space_towrite,
3175cabbc6Prashanth Sreenivasa	    1ULL << dn->dn_indblkshift, FTAG);
3185cabbc6Prashanth Sreenivasa	dmu_tx_count_dnode(txh);
3195cabbc6Prashanth Sreenivasa}
3205cabbc6Prashanth Sreenivasa
3215cabbc6Prashanth Sreenivasavoid
322b0c42cdbzzzdmu_tx_hold_write_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, int len)
324b0c42cdbzzz	dmu_tx_hold_t *txh;
326b0c42cdbzzz	ASSERT0(tx->tx_txg);
327b0c42cdbzzz	ASSERT3U(len, <=, DMU_MAX_ACCESS);
328b0c42cdbzzz	ASSERT(len == 0 || UINT64_MAX - off >= len - 1);
330b0c42cdbzzz	txh = dmu_tx_hold_dnode_impl(tx, dn, THT_WRITE, off, len);
331b0c42cdbzzz	if (txh != NULL) {
332b0c42cdbzzz		dmu_tx_count_write(txh, off, len);
333b0c42cdbzzz		dmu_tx_count_dnode(txh);
334b0c42cdbzzz	}
3374bb7380Matthew Ahrens/*
3384bb7380Matthew Ahrens * This function marks the transaction as being a "net free".  The end
3394bb7380Matthew Ahrens * result is that refquotas will be disabled for this transaction, and
3404bb7380Matthew Ahrens * this transaction will be able to use half of the pool space overhead
3414bb7380Matthew Ahrens * (see dsl_pool_adjustedsize()).  Therefore this function should only
3424bb7380Matthew Ahrens * be called for transactions that we expect will not cause a net increase
3434bb7380Matthew Ahrens * in the amount of space used (but it's OK if that is occasionally not true).
3444bb7380Matthew Ahrens */
3454bb7380Matthew Ahrensvoid
3464bb7380Matthew Ahrensdmu_tx_mark_netfree(dmu_tx_t *tx)
3474bb7380Matthew Ahrens{
34861e255cMatthew Ahrens	tx->tx_netfree = B_TRUE;
3494bb7380Matthew Ahrens}
3504bb7380Matthew Ahrens
351b0c42cdbzzzstatic void
352b0c42cdbzzzdmu_tx_hold_free_impl(dmu_tx_hold_t *txh, uint64_t off, uint64_t len)
354b0c42cdbzzz	dmu_tx_t *tx;
355b0c42cdbzzz	dnode_t *dn;
3562f3d878Matthew Ahrens	int err;
358b0c42cdbzzz	tx = txh->txh_tx;
3598a2f1b9ahrens	ASSERT(tx->tx_txg == 0);
361b0c42cdbzzz	dn = txh->txh_dnode;
36269962b5Matthew Ahrens	dmu_tx_count_dnode(txh);
36461e255cMatthew Ahrens	if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz)
365fa9e406ahrens		return;
366fa9e406ahrens	if (len == DMU_OBJECT_END)
36761e255cMatthew Ahrens		len = (dn->dn_maxblkid + 1) * dn->dn_datablksz - off;
369ea8dc4beschrock	/*
3702f3d878Matthew Ahrens	 * For i/o error checking, we read the first and last level-0
3712f3d878Matthew Ahrens	 * blocks if they are not aligned, and all the level-1 blocks.
3722f3d878Matthew Ahrens	 *
3732f3d878Matthew Ahrens	 * Note:  dbuf_free_range() assumes that we have not instantiated
3742f3d878Matthew Ahrens	 * any level-0 dbufs that will be completely freed.  Therefore we must
3752f3d878Matthew Ahrens	 * exercise care to not read or count the first and last blocks
3762f3d878Matthew Ahrens	 * if they are blocksize-aligned.
3772f3d878Matthew Ahrens	 */
3782f3d878Matthew Ahrens	if (dn->dn_datablkshift == 0) {
379713d6c2Matthew Ahrens		if (off != 0 || len < dn->dn_datablksz)
3805253393Matthew Ahrens			dmu_tx_count_write(txh, 0, dn->dn_datablksz);
3812f3d878Matthew Ahrens	} else {
3822f3d878Matthew Ahrens		/* first block will be modified if it is not aligned */
3832f3d878Matthew Ahrens		if (!IS_P2ALIGNED(off, 1 << dn->dn_datablkshift))
3842f3d878Matthew Ahrens			dmu_tx_count_write(txh, off, 1);
3852f3d878Matthew Ahrens		/* last block will be modified if it is not aligned */
3862f3d878Matthew Ahrens		if (!IS_P2ALIGNED(off + len, 1 << dn->dn_datablkshift))
38761e255cMatthew Ahrens			dmu_tx_count_write(txh, off + len, 1);
3882f3d878Matthew Ahrens	}
3892f3d878Matthew Ahrens
3902f3d878Matthew Ahrens	/*
3912f3d878Matthew Ahrens	 * Check level-1 blocks.
392ea8dc4beschrock	 */
39398572acahrens	if (dn->dn_nlevels > 1) {
3942f3d878Matthew Ahrens		int shift = dn->dn_datablkshift + dn->dn_indblkshift -
39598572acahrens		    SPA_BLKPTRSHIFT;
3962f3d878Matthew Ahrens		uint64_t start = off >> shift;
3972f3d878Matthew Ahrens		uint64_t end = (off + len) >> shift;
3982f3d878Matthew Ahrens
3992f3d878Matthew Ahrens		ASSERT(dn->dn_indblkshift != 0);
401bb411a0Matthew Ahrens		/*
402bb411a0Matthew Ahrens		 * dnode_reallocate() can result in an object with indirect
403bb411a0Matthew Ahrens		 * blocks having an odd data block size.  In this case,
404bb411a0Matthew Ahrens		 * just check the single block.
405bb411a0Matthew Ahrens		 */
406bb411a0Matthew Ahrens		if (dn->dn_datablkshift == 0)
407bb411a0Matthew Ahrens			start = end = 0;
408bb411a0Matthew Ahrens
40961e255cMatthew Ahrens		zio_t *zio = zio_root(tx->tx_pool->dp_spa,
41098572acahrens		    NULL, NULL, ZIO_FLAG_CANFAIL);
4112f3d878Matthew Ahrens		for (uint64_t i = start; i <= end; i++) {
41298572acahrens			uint64_t ibyte = i << shift;
413cdb0ab7maybee			err = dnode_next_offset(dn, 0, &ibyte, 2, 1, 0);
41498572acahrens			i = ibyte >> shift;
41546e1baaMatthew Ahrens			if (err == ESRCH || i > end)
41698572acahrens				break;
41761e255cMatthew Ahrens			if (err != 0) {
41898572acahrens				tx->tx_err = err;
41961e255cMatthew Ahrens				(void) zio_wait(zio);
42098572acahrens				return;
42198572acahrens			}
423e914aceTim Schumacher			(void) zfs_refcount_add_many(&txh->txh_memory_tohold,
42461e255cMatthew Ahrens			    1 << dn->dn_indblkshift, FTAG);
42561e255cMatthew Ahrens
42698572acahrens			err = dmu_tx_check_ioerr(zio, dn, 1, i);
42761e255cMatthew Ahrens			if (err != 0) {
42898572acahrens				tx->tx_err = err;
42961e255cMatthew Ahrens				(void) zio_wait(zio);
43098572acahrens				return;
43198572acahrens			}
43298572acahrens		}
43398572acahrens		err = zio_wait(zio);
43461e255cMatthew Ahrens		if (err != 0) {
435ea8dc4beschrock			tx->tx_err = err;
436ea8dc4beschrock			return;
437ea8dc4beschrock		}
438ea8dc4beschrock	}
442b0c42cdbzzzdmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len)
444b0c42cdbzzz	dmu_tx_hold_t *txh;
446b0c42cdbzzz	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
447b0c42cdbzzz	    object, THT_FREE, off, len);
448b0c42cdbzzz	if (txh != NULL)
449b0c42cdbzzz		(void) dmu_tx_hold_free_impl(txh, off, len);
453b0c42cdbzzzdmu_tx_hold_free_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, uint64_t len)
455b0c42cdbzzz	dmu_tx_hold_t *txh;
457b0c42cdbzzz	txh = dmu_tx_hold_dnode_impl(tx, dn, THT_FREE, off, len);
458b0c42cdbzzz	if (txh != NULL)
459b0c42cdbzzz		(void) dmu_tx_hold_free_impl(txh, off, len);
462b0c42cdbzzzstatic void
463411be58Matthew Ahrensdmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, const char *name)
465b0c42cdbzzz	dmu_tx_t *tx = txh->txh_tx;
466b0c42cdbzzz	dnode_t *dn;
4670c779adMatthew Ahrens	int err;
4698a2f1b9ahrens	ASSERT(tx->tx_txg == 0);
471b0c42cdbzzz	dn = txh->txh_dnode;
4738a2f1b9ahrens	dmu_tx_count_dnode(txh);
47561e255cMatthew Ahrens	/*
47661e255cMatthew Ahrens	 * Modifying a almost-full microzap is around the worst case (128KB)
47761e255cMatthew Ahrens	 *
47861e255cMatthew Ahrens	 * If it is a fat zap, the worst case would be 7*16KB=112KB:
47961e255cMatthew Ahrens	 * - 3 blocks overwritten: target leaf, ptrtbl block, header block
48061e255cMatthew Ahrens	 * - 4 new blocks written if adding:
48161e255cMatthew Ahrens	 *    - 2 blocks for possibly split leaves,
48261e255cMatthew Ahrens	 *    - 2 grown ptrtbl blocks
48361e255cMatthew Ahrens	 */
484e914aceTim Schumacher	(void) zfs_refcount_add_many(&txh->txh_space_towrite,
48561e255cMatthew Ahrens	    MZAP_MAX_BLKSZ, FTAG);
48661e255cMatthew Ahrens
48761e255cMatthew Ahrens	if (dn == NULL)
488fa9e406ahrens		return;
490ad135b5Christopher Siden	ASSERT3P(DMU_OT_BYTESWAP(dn->dn_type), ==, DMU_BSWAP_ZAP);
49261e255cMatthew Ahrens	if (dn->dn_maxblkid == 0 || name == NULL) {
493fa9e406ahrens		/*
49461e255cMatthew Ahrens		 * This is a microzap (only one block), or we don't know
49561e255cMatthew Ahrens		 * the name.  Check the first block for i/o errors.
496fa9e406ahrens		 */
497ea8dc4beschrock		err = dmu_tx_check_ioerr(NULL, dn, 0, 0);
49861e255cMatthew Ahrens		if (err != 0) {
499ea8dc4beschrock			tx->tx_err = err;
5000c779adMatthew Ahrens		}
50161e255cMatthew Ahrens	} else {
502ea8dc4beschrock		/*
50361e255cMatthew Ahrens		 * Access the name so that we'll check for i/o errors to
50461e255cMatthew Ahrens		 * the leaf blocks, etc.  We ignore ENOENT, as this name
50561e255cMatthew Ahrens		 * may not yet exist.
506ea8dc4beschrock		 */
50779d7283Matthew Ahrens		err = zap_lookup_by_dnode(dn, name, 8, 0, NULL);
50861e255cMatthew Ahrens		if (err == EIO || err == ECKSUM || err == ENXIO) {
509ea8dc4beschrock			tx->tx_err = err;
5100c779adMatthew Ahrens		}
5110c779adMatthew Ahrens	}
515b0c42cdbzzzdmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name)
517b0c42cdbzzz	dmu_tx_hold_t *txh;
519b0c42cdbzzz	ASSERT0(tx->tx_txg);
521b0c42cdbzzz	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
522b0c42cdbzzz	    object, THT_ZAP, add, (uintptr_t)name);
523b0c42cdbzzz	if (txh != NULL)
524411be58Matthew Ahrens		dmu_tx_hold_zap_impl(txh, name);
528b0c42cdbzzzdmu_tx_hold_zap_by_dnode(dmu_tx_t *tx, dnode_t *dn, int add, const char *name)
530b0c42cdbzzz	dmu_tx_hold_t *txh;
532b0c42cdbzzz	ASSERT0(tx->tx_txg);
533b0c42cdbzzz	ASSERT(dn != NULL);
535b0c42cdbzzz	txh = dmu_tx_hold_dnode_impl(tx, dn, THT_ZAP, add, (uintptr_t)name);
536b0c42cdbzzz	if (txh != NULL)
537411be58Matthew Ahrens		dmu_tx_hold_zap_impl(txh, name);
541fa9e406ahrensdmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object)
5438a2f1b9ahrens	dmu_tx_hold_t *txh;
5458a2f1b9ahrens	ASSERT(tx->tx_txg == 0);
5478a2f1b9ahrens	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
5488a2f1b9ahrens	    object, THT_BONUS, 0, 0);
5498a2f1b9ahrens	if (txh)
5508a2f1b9ahrens		dmu_tx_count_dnode(txh);
554b0c42cdbzzzdmu_tx_hold_bonus_by_dnode(dmu_tx_t *tx, dnode_t *dn)
556b0c42cdbzzz	dmu_tx_hold_t *txh;
558b0c42cdbzzz	ASSERT0(tx->tx_txg);
560b0c42cdbzzz	txh = dmu_tx_hold_dnode_impl(tx, dn, THT_BONUS, 0, 0);
561b0c42cdbzzz	if (txh)
562b0c42cdbzzz		dmu_tx_count_dnode(txh);
566fa9e406ahrensdmu_tx_hold_space(dmu_tx_t *tx, uint64_t space)
5688a2f1b9ahrens	dmu_tx_hold_t *txh;
569fa9e406ahrens	ASSERT(tx->tx_txg == 0);
5718a2f1b9ahrens	txh = dmu_tx_hold_object_impl(tx, tx->tx_objset,
5728a2f1b9ahrens	    DMU_NEW_OBJECT, THT_SPACE, space, 0);
574e914aceTim Schumacher	(void) zfs_refcount_add_many(&txh->txh_space_towrite, space, FTAG);
5779c9dc39ek#ifdef ZFS_DEBUG
579fa9e406ahrensdmu_tx_dirty_buf(dmu_tx_t *tx, dmu_buf_impl_t *db)
58161e255cMatthew Ahrens	boolean_t match_object = B_FALSE;
58261e255cMatthew Ahrens	boolean_t match_offset = B_FALSE;
584744947dTom Erickson	DB_DNODE_ENTER(db);
58561e255cMatthew Ahrens	dnode_t *dn = DB_DNODE(db);
586fa9e406ahrens	ASSERT(tx->tx_txg != 0);
587503ad85Matthew Ahrens	ASSERT(tx->tx_objset == NULL || dn->dn_objset == tx->tx_objset);
588fa9e406ahrens	ASSERT3U(dn->dn_object, ==, db->db.db_object);
590744947dTom Erickson	if (tx->tx_anyobj) {
591744947dTom Erickson		DB_DNODE_EXIT(db);
592fa9e406ahrens		return;
593744947dTom Erickson	}
595fa9e406ahrens	/* XXX No checking on the meta dnode for now */
596744947dTom Erickson	if (db->db.db_object == DMU_META_DNODE_OBJECT) {
597744947dTom Erickson		DB_DNODE_EXIT(db);
598fa9e406ahrens		return;
599744947dTom Erickson	}
60161e255cMatthew Ahrens	for (dmu_tx_hold_t *txh = list_head(&tx->tx_holds); txh != NULL;
6028a2f1b9ahrens	    txh = list_next(&tx->tx_holds, txh)) {
603fa9e406ahrens		ASSERT(dn == NULL || dn->dn_assigned_txg == tx->tx_txg);
6048a2f1b9ahrens		if (txh->txh_dnode == dn && txh->txh_type != THT_NEWOBJECT)
605fa9e406ahrens			match_object = TRUE;
6068a2f1b9ahrens		if (txh->txh_dnode == NULL || txh->txh_dnode == dn) {
607fa9e406ahrens			int datablkshift = dn->dn_datablkshift ?
608fa9e406ahrens			    dn->dn_datablkshift : SPA_MAXBLOCKSHIFT;
609fa9e406ahrens			int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
610fa9e406ahrens			int shift = datablkshift + epbs * db->db_level;
611fa9e406ahrens			uint64_t beginblk = shift >= 64 ? 0 :
6128a2f1b9ahrens			    (txh->txh_arg1 >> shift);
613fa9e406ahrens			uint64_t endblk = shift >= 64 ? 0 :
6148a2f1b9ahrens			    ((txh->txh_arg1 + txh->txh_arg2 - 1) >> shift);
615fa9e406ahrens			uint64_t blkid = db->db_blkid;
6178a2f1b9ahrens			/* XXX txh_arg2 better not be zero... */
6198a2f1b9ahrens			dprintf("found txh type %x beginblk=%llx endblk=%llx\n",
6208a2f1b9ahrens			    txh->txh_type, beginblk, endblk);
6228a2f1b9ahrens			switch (txh->txh_type) {
623fa9e406ahrens			case THT_WRITE:
624fa9e406ahrens				if (blkid >= beginblk && blkid <= endblk)
625fa9e406ahrens					match_offset = TRUE;
626fa9e406ahrens				/*
627fa9e406ahrens				 * We will let this hold work for the bonus
6280a586ceMark Shellenbaum				 * or spill buffer so that we don't need to
6290a586ceMark Shellenbaum				 * hold it when creating a new object.
630fa9e406ahrens				 */
6310a586ceMark Shellenbaum				if (blkid == DMU_BONUS_BLKID ||
6320a586ceMark Shellenbaum				    blkid == DMU_SPILL_BLKID)
633fa9e406ahrens					match_offset = TRUE;
634fa9e406ahrens				/*
635fa9e406ahrens				 * They might have to increase nlevels,
636fa9e406ahrens				 * thus dirtying the new TLIBs.  Or the
637fa9e406ahrens				 * might have to change the block size,
638fa9e406ahrens				 * thus dirying the new lvl=0 blk=0.
639fa9e406ahrens				 */
640fa9e406ahrens				if (blkid == 0)
641fa9e406ahrens					match_offset = TRUE;
642fa9e406ahrens				break;
643fa9e406ahrens			case THT_FREE:
644cdb0ab7maybee				/*
645cdb0ab7maybee				 * We will dirty all the level 1 blocks in
646cdb0ab7maybee				 * the free range and perhaps the first and
647cdb0ab7maybee				 * last level 0 block.
648cdb0ab7maybee				 */
649cdb0ab7maybee				if (blkid >= beginblk && (blkid <= endblk ||
650cdb0ab7maybee				    txh->txh_arg2 == DMU_OBJECT_END))
651fa9e406ahrens					match_offset = TRUE;
652fa9e406ahrens				break;
6530a586ceMark Shellenbaum			case THT_SPILL:
6540a586ceMark Shellenbaum				if (blkid == DMU_SPILL_BLKID)
6550a586ceMark Shellenbaum					match_offset = TRUE;
6560a586ceMark Shellenbaum				break;
657fa9e406ahrens			case THT_BONUS:
6580a586ceMark Shellenbaum				if (blkid == DMU_BONUS_BLKID)
659fa9e406ahrens					match_offset = TRUE;
660fa9e406ahrens				break;
661fa9e406ahrens			case THT_ZAP:
662fa9e406ahrens				match_offset = TRUE;
663fa9e406ahrens				break;
664fa9e406ahrens			case THT_NEWOBJECT:
665fa9e406ahrens				match_object = TRUE;
666fa9e406ahrens				break;
667fa9e406ahrens			default:
6688a2f1b9ahrens				ASSERT(!"bad txh_type");
669fa9e406ahrens			}
670fa9e406ahrens		}
671744947dTom Erickson		if (match_object && match_offset) {
672744947dTom Erickson			DB_DNODE_EXIT(db);
673fa9e406ahrens			return;
674744947dTom Erickson		}
675fa9e406ahrens	}
676744947dTom Erickson	DB_DNODE_EXIT(db);
677fa9e406ahrens	panic("dirtying dbuf obj=%llx lvl=%u blkid=%llx but not tx_held\n",
678fa9e406ahrens	    (u_longlong_t)db->db.db_object, db->db_level,
679fa9e406ahrens	    (u_longlong_t)db->db_blkid);
68369962b5Matthew Ahrens/*
68469962b5Matthew Ahrens * If we can't do 10 iops, something is wrong.  Let us go ahead
68569962b5Matthew Ahrens * and hit zfs_dirty_data_max.
68669962b5Matthew Ahrens */
68769962b5Matthew Ahrenshrtime_t zfs_delay_max_ns = MSEC2NSEC(100);
68869962b5Matthew Ahrensint zfs_delay_resolution_ns = 100 * 1000; /* 100 microseconds */
68969962b5Matthew Ahrens
69069962b5Matthew Ahrens/*
69169962b5Matthew Ahrens * We delay transactions when we've determined that the backend storage
69269962b5Matthew Ahrens * isn't able to accommodate the rate of incoming writes.
69369962b5Matthew Ahrens *
69469962b5Matthew Ahrens * If there is already a transaction waiting, we delay relative to when
69569962b5Matthew Ahrens * that transaction finishes waiting.  This way the calculated min_time
69669962b5Matthew Ahrens * is independent of the number of threads concurrently executing
69769962b5Matthew Ahrens * transactions.
69869962b5Matthew Ahrens *
69969962b5Matthew Ahrens * If we are the only waiter, wait relative to when the transaction
70069962b5Matthew Ahrens * started, rather than the current time.  This credits the transaction for
70169962b5Matthew Ahrens * "time already served", e.g. reading indirect blocks.
70269962b5Matthew Ahrens *
70369962b5Matthew Ahrens * The minimum time for a transaction to take is calculated as:
70469962b5Matthew Ahrens *     min_time = scale * (dirty - min) / (max - dirty)
70569962b5Matthew Ahrens *     min_time is then capped at zfs_delay_max_ns.
70669962b5Matthew Ahrens *
70769962b5Matthew Ahrens * The delay has two degrees of freedom that can be adjusted via tunables.
70869962b5Matthew Ahrens * The percentage of dirty data at which we start to delay is defined by
70969962b5Matthew Ahrens * zfs_delay_min_dirty_percent. This should typically be at or above
71069962b5Matthew Ahrens * zfs_vdev_async_write_active_max_dirty_percent so that we only start to
71169962b5Matthew Ahrens * delay after writing at full speed has failed to keep up with the incoming
71269962b5Matthew Ahrens * write rate. The scale of the curve is defined by zfs_delay_scale. Roughly
71369962b5Matthew Ahrens * speaking, this variable determines the amount of delay at the midpoint of
71469962b5Matthew Ahrens * the curve.
71569962b5Matthew Ahrens *
71669962b5Matthew Ahrens * delay
71769962b5Matthew Ahrens *  10ms +-------------------------------------------------------------*+
71869962b5Matthew Ahrens *       |                                                             *|
71969962b5Matthew Ahrens *   9ms +                                                             *+
72069962b5Matthew Ahrens *       |                                                             *|
72169962b5Matthew Ahrens *   8ms +                                                             *+
72269962b5Matthew Ahrens *       |                                                            * |
72369962b5Matthew Ahrens *   7ms +                                                            * +
72469962b5Matthew Ahrens *       |                                                            * |
72569962b5Matthew Ahrens *   6ms +                                                            * +
72669962b5Matthew Ahrens *       |                                                            * |
72769962b5Matthew Ahrens *   5ms +                                                           *  +
72869962b5Matthew Ahrens *       |                                                           *  |
72969962b5Matthew Ahrens *   4ms +                                                           *  +
73069962b5Matthew Ahrens *       |                                                           *  |
73169962b5Matthew Ahrens *   3ms +                                                          *   +
73269962b5Matthew Ahrens *       |                                                          *   |
73369962b5Matthew Ahrens *   2ms +                                              (midpoint) *    +
73469962b5Matthew Ahrens *       |                                                  |    **     |
73569962b5Matthew Ahrens *   1ms +                                                  v ***       +
73669962b5Matthew Ahrens *       |             zfs_delay_scale ---------->     ********         |
73769962b5Matthew Ahrens *     0 +-------------------------------------*********----------------+
73869962b5Matthew Ahrens *       0%                    <- zfs_dirty_data_max ->               100%
73969962b5Matthew Ahrens *
74069962b5Matthew Ahrens * Note that since the delay is added to the outstanding time remaining on the
74169962b5Matthew Ahrens * most recent transaction, the delay is effectively the inverse of IOPS.
74269962b5Matthew Ahrens * Here the midpoint of 500us translates to 2000 IOPS. The shape of the curve
74369962b5Matthew Ahrens * was chosen such that small changes in the amount of accumulated dirty data
74469962b5Matthew Ahrens * in the first 3/4 of the curve yield relatively small differences in the
74569962b5Matthew Ahrens * amount of delay.
74669962b5Matthew Ahrens *
74769962b5Matthew Ahrens * The effects can be easier to understand when the amount of delay is
74869962b5Matthew Ahrens * represented on a log scale:
74969962b5Matthew Ahrens *
75069962b5Matthew Ahrens * delay
75169962b5Matthew Ahrens * 100ms +-------------------------------------------------------------++
75269962b5Matthew Ahrens *       +                                                              +
75369962b5Matthew Ahrens *       |                                                              |
75469962b5Matthew Ahrens *       +                                                             *+
75569962b5Matthew Ahrens *  10ms +                                                             *+
75669962b5Matthew Ahrens *       +                                                           ** +
75769962b5Matthew Ahrens *       |                                              (midpoint)  **  |
75869962b5Matthew Ahrens *       +                                                  |     **    +
75969962b5Matthew Ahrens *   1ms +                                                  v ****      +
76069962b5Matthew Ahrens *       +             zfs_delay_scale ---------->        *****         +
76169962b5Matthew Ahrens *       |                                             ****             |
76269962b5Matthew Ahrens *       +                                          ****                +
76369962b5Matthew Ahrens * 100us +                                        **                    +
76469962b5Matthew Ahrens *       +                                       *                      +
76569962b5Matthew Ahrens *       |                                      *                       |
76669962b5Matthew Ahrens *       +                                     *                        +
76769962b5Matthew Ahrens *  10us +                                     *                        +
76869962b5Matthew Ahrens *       +                                                              +
76969962b5Matthew Ahrens *       |                                                              |
77069962b5Matthew Ahrens *       +                                                              +
77169962b5Matthew Ahrens *       +--------------------------------------------------------------+
77269962b5Matthew Ahrens *       0%                    <- zfs_dirty_data_max ->               100%
77369962b5Matthew Ahrens *
77469962b5Matthew Ahrens * Note here that only as the amount of dirty data approaches its limit does
77569962b5Matthew Ahrens * the delay start to increase rapidly. The goal of a properly tuned system
77669962b5Matthew Ahrens * should be to keep the amount of dirty data out of that range by first
77769962b5Matthew Ahrens * ensuring that the appropriate limits are set for the I/O scheduler to reach
77869962b5Matthew Ahrens * optimal throughput on the backend storage, and then by changing the value
77969962b5Matthew Ahrens * of zfs_delay_scale to increase the steepness of the curve.
78069962b5Matthew Ahrens */
78169962b5Matthew Ahrensstatic void
78269962b5Matthew Ahrensdmu_tx_delay(dmu_tx_t *tx, uint64_t dirty)
78369962b5Matthew Ahrens{
78469962b5Matthew Ahrens	dsl_pool_t *dp = tx->tx_pool;
78569962b5Matthew Ahrens	uint64_t delay_min_bytes =
78669962b5Matthew Ahrens	    zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100;
78769962b5Matthew Ahrens	hrtime_t wakeup, min_tx_time, now;
78869962b5Matthew Ahrens
78969962b5Matthew Ahrens	if (dirty <= delay_min_bytes)
79069962b5Matthew Ahrens		return;
79169962b5Matthew Ahrens
79269962b5Matthew Ahrens	/*
79369962b5Matthew Ahrens	 * The caller has already waited until we are under the max.
79469962b5Matthew Ahrens	 * We make them pass us the amount of dirty data so we don't
79569962b5Matthew Ahrens	 * have to handle the case of it being >= the max, which could
79669962b5Matthew Ahrens	 * cause a divide-by-zero if it's == the max.
79769962b5Matthew Ahrens	 */
79869962b5Matthew Ahrens	ASSERT3U(dirty, <, zfs_dirty_data_max);
79969962b5Matthew Ahrens
80069962b5Matthew Ahrens	now = gethrtime();
80169962b5Matthew Ahrens	min_tx_time = zfs_delay_scale *
80269962b5Matthew Ahrens	    (dirty - delay_min_bytes) / (zfs_dirty_data_max - dirty);
80369962b5Matthew Ahrens	if (now > tx->tx_start + min_tx_time)