1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5f65e61c0Sahrens * Common Development and Distribution License (the "License"). 6f65e61c0Sahrens * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2201025c89SJohn Harres * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 239dccfd2aSAlbert Lee * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24*b7b2590dSMatthew Ahrens * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 25c3d26abcSMatthew Ahrens * Copyright (c) 2014 Integros [integros.com] 269dccfd2aSAlbert Lee */ 27fa9e4066Sahrens 28fa9e4066Sahrens #include <sys/dmu.h> 29fa9e4066Sahrens #include <sys/dmu_impl.h> 30fa9e4066Sahrens #include <sys/dbuf.h> 31fa9e4066Sahrens #include <sys/dmu_tx.h> 32fa9e4066Sahrens #include <sys/dmu_objset.h> 3361e255ceSMatthew Ahrens #include <sys/dsl_dataset.h> 3461e255ceSMatthew Ahrens #include <sys/dsl_dir.h> 35fa9e4066Sahrens #include <sys/dsl_pool.h> 3661e255ceSMatthew Ahrens #include <sys/zap_impl.h> 37fa9e4066Sahrens #include <sys/spa.h> 380a586ceaSMark Shellenbaum #include <sys/sa.h> 390a586ceaSMark Shellenbaum #include <sys/sa_impl.h> 40fa9e4066Sahrens #include <sys/zfs_context.h> 410a586ceaSMark Shellenbaum #include <sys/varargs.h> 42fa9e4066Sahrens 43ea8dc4b6Seschrock typedef void (*dmu_tx_hold_func_t)(dmu_tx_t *tx, struct dnode *dn, 44ea8dc4b6Seschrock uint64_t arg1, uint64_t arg2); 45ea8dc4b6Seschrock 46fa9e4066Sahrens 47fa9e4066Sahrens dmu_tx_t * 481d452cf5Sahrens dmu_tx_create_dd(dsl_dir_t *dd) 49fa9e4066Sahrens { 50fa9e4066Sahrens dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_SLEEP); 51fa9e4066Sahrens tx->tx_dir = dd; 524445fffbSMatthew Ahrens if (dd != NULL) 53fa9e4066Sahrens tx->tx_pool = dd->dd_pool; 54fa9e4066Sahrens list_create(&tx->tx_holds, sizeof (dmu_tx_hold_t), 558a2f1b91Sahrens offsetof(dmu_tx_hold_t, txh_node)); 56d20e665cSRicardo M. Correia list_create(&tx->tx_callbacks, sizeof (dmu_tx_callback_t), 57d20e665cSRicardo M. Correia offsetof(dmu_tx_callback_t, dcb_node)); 5869962b56SMatthew Ahrens tx->tx_start = gethrtime(); 59fa9e4066Sahrens return (tx); 60fa9e4066Sahrens } 61fa9e4066Sahrens 62fa9e4066Sahrens dmu_tx_t * 63fa9e4066Sahrens dmu_tx_create(objset_t *os) 64fa9e4066Sahrens { 65503ad85cSMatthew Ahrens dmu_tx_t *tx = dmu_tx_create_dd(os->os_dsl_dataset->ds_dir); 66fa9e4066Sahrens tx->tx_objset = os; 67fa9e4066Sahrens return (tx); 68fa9e4066Sahrens } 69fa9e4066Sahrens 70fa9e4066Sahrens dmu_tx_t * 71fa9e4066Sahrens dmu_tx_create_assigned(struct dsl_pool *dp, uint64_t txg) 72fa9e4066Sahrens { 731d452cf5Sahrens dmu_tx_t *tx = dmu_tx_create_dd(NULL); 74fa9e4066Sahrens 75*b7b2590dSMatthew Ahrens txg_verify(dp->dp_spa, txg); 76fa9e4066Sahrens tx->tx_pool = dp; 77fa9e4066Sahrens tx->tx_txg = txg; 78fa9e4066Sahrens tx->tx_anyobj = TRUE; 79fa9e4066Sahrens 80fa9e4066Sahrens return (tx); 81fa9e4066Sahrens } 82fa9e4066Sahrens 83fa9e4066Sahrens int 84fa9e4066Sahrens dmu_tx_is_syncing(dmu_tx_t *tx) 85fa9e4066Sahrens { 86fa9e4066Sahrens return (tx->tx_anyobj); 87fa9e4066Sahrens } 88fa9e4066Sahrens 89fa9e4066Sahrens int 90fa9e4066Sahrens dmu_tx_private_ok(dmu_tx_t *tx) 91fa9e4066Sahrens { 92ea8dc4b6Seschrock return (tx->tx_anyobj); 93fa9e4066Sahrens } 94fa9e4066Sahrens 958a2f1b91Sahrens static dmu_tx_hold_t * 96b0c42cd4Sbzzz dmu_tx_hold_dnode_impl(dmu_tx_t *tx, dnode_t *dn, enum dmu_tx_hold_type type, 97b0c42cd4Sbzzz uint64_t arg1, uint64_t arg2) 98fa9e4066Sahrens { 998a2f1b91Sahrens dmu_tx_hold_t *txh; 100fa9e4066Sahrens 101b0c42cd4Sbzzz if (dn != NULL) { 102b0c42cd4Sbzzz (void) refcount_add(&dn->dn_holds, tx); 103b0c42cd4Sbzzz if (tx->tx_txg != 0) { 104fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 105fa9e4066Sahrens /* 106fa9e4066Sahrens * dn->dn_assigned_txg == tx->tx_txg doesn't pose a 107fa9e4066Sahrens * problem, but there's no way for it to happen (for 108fa9e4066Sahrens * now, at least). 109fa9e4066Sahrens */ 110fa9e4066Sahrens ASSERT(dn->dn_assigned_txg == 0); 111fa9e4066Sahrens dn->dn_assigned_txg = tx->tx_txg; 112fa9e4066Sahrens (void) refcount_add(&dn->dn_tx_holds, tx); 113fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 114fa9e4066Sahrens } 115fa9e4066Sahrens } 116fa9e4066Sahrens 1178a2f1b91Sahrens txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_SLEEP); 1188a2f1b91Sahrens txh->txh_tx = tx; 1198a2f1b91Sahrens txh->txh_dnode = dn; 1200c779ad4SMatthew Ahrens refcount_create(&txh->txh_space_towrite); 1210c779ad4SMatthew Ahrens refcount_create(&txh->txh_memory_tohold); 1228a2f1b91Sahrens txh->txh_type = type; 1238a2f1b91Sahrens txh->txh_arg1 = arg1; 1248a2f1b91Sahrens txh->txh_arg2 = arg2; 1258a2f1b91Sahrens list_insert_tail(&tx->tx_holds, txh); 126ea8dc4b6Seschrock 1278a2f1b91Sahrens return (txh); 128fa9e4066Sahrens } 129fa9e4066Sahrens 130b0c42cd4Sbzzz static dmu_tx_hold_t * 131b0c42cd4Sbzzz dmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object, 132b0c42cd4Sbzzz enum dmu_tx_hold_type type, uint64_t arg1, uint64_t arg2) 133b0c42cd4Sbzzz { 134b0c42cd4Sbzzz dnode_t *dn = NULL; 135b0c42cd4Sbzzz dmu_tx_hold_t *txh; 136b0c42cd4Sbzzz int err; 137b0c42cd4Sbzzz 138b0c42cd4Sbzzz if (object != DMU_NEW_OBJECT) { 139b0c42cd4Sbzzz err = dnode_hold(os, object, FTAG, &dn); 140b0c42cd4Sbzzz if (err != 0) { 141b0c42cd4Sbzzz tx->tx_err = err; 142b0c42cd4Sbzzz return (NULL); 143b0c42cd4Sbzzz } 144b0c42cd4Sbzzz } 145b0c42cd4Sbzzz txh = dmu_tx_hold_dnode_impl(tx, dn, type, arg1, arg2); 146b0c42cd4Sbzzz if (dn != NULL) 147b0c42cd4Sbzzz dnode_rele(dn, FTAG); 148b0c42cd4Sbzzz return (txh); 149b0c42cd4Sbzzz } 150b0c42cd4Sbzzz 151fa9e4066Sahrens void 152b0c42cd4Sbzzz dmu_tx_add_new_object(dmu_tx_t *tx, dnode_t *dn) 153fa9e4066Sahrens { 154fa9e4066Sahrens /* 155fa9e4066Sahrens * If we're syncing, they can manipulate any object anyhow, and 156fa9e4066Sahrens * the hold on the dnode_t can cause problems. 157fa9e4066Sahrens */ 158b0c42cd4Sbzzz if (!dmu_tx_is_syncing(tx)) 159b0c42cd4Sbzzz (void) dmu_tx_hold_dnode_impl(tx, dn, THT_NEWOBJECT, 0, 0); 160fa9e4066Sahrens } 161fa9e4066Sahrens 16261e255ceSMatthew Ahrens /* 16361e255ceSMatthew Ahrens * This function reads specified data from disk. The specified data will 16461e255ceSMatthew Ahrens * be needed to perform the transaction -- i.e, it will be read after 16561e255ceSMatthew Ahrens * we do dmu_tx_assign(). There are two reasons that we read the data now 16661e255ceSMatthew Ahrens * (before dmu_tx_assign()): 16761e255ceSMatthew Ahrens * 16861e255ceSMatthew Ahrens * 1. Reading it now has potentially better performance. The transaction 16961e255ceSMatthew Ahrens * has not yet been assigned, so the TXG is not held open, and also the 17061e255ceSMatthew Ahrens * caller typically has less locks held when calling dmu_tx_hold_*() than 17161e255ceSMatthew Ahrens * after the transaction has been assigned. This reduces the lock (and txg) 17261e255ceSMatthew Ahrens * hold times, thus reducing lock contention. 17361e255ceSMatthew Ahrens * 17461e255ceSMatthew Ahrens * 2. It is easier for callers (primarily the ZPL) to handle i/o errors 17561e255ceSMatthew Ahrens * that are detected before they start making changes to the DMU state 17661e255ceSMatthew Ahrens * (i.e. now). Once the transaction has been assigned, and some DMU 17761e255ceSMatthew Ahrens * state has been changed, it can be difficult to recover from an i/o 17861e255ceSMatthew Ahrens * error (e.g. to undo the changes already made in memory at the DMU 17961e255ceSMatthew Ahrens * layer). Typically code to do so does not exist in the caller -- it 18061e255ceSMatthew Ahrens * assumes that the data has already been cached and thus i/o errors are 18161e255ceSMatthew Ahrens * not possible. 18261e255ceSMatthew Ahrens * 18361e255ceSMatthew Ahrens * It has been observed that the i/o initiated here can be a performance 18461e255ceSMatthew Ahrens * problem, and it appears to be optional, because we don't look at the 18561e255ceSMatthew Ahrens * data which is read. However, removing this read would only serve to 18661e255ceSMatthew Ahrens * move the work elsewhere (after the dmu_tx_assign()), where it may 18761e255ceSMatthew Ahrens * have a greater impact on performance (in addition to the impact on 18861e255ceSMatthew Ahrens * fault tolerance noted above). 18961e255ceSMatthew Ahrens */ 190ea8dc4b6Seschrock static int 191ea8dc4b6Seschrock dmu_tx_check_ioerr(zio_t *zio, dnode_t *dn, int level, uint64_t blkid) 192ea8dc4b6Seschrock { 193ea8dc4b6Seschrock int err; 194ea8dc4b6Seschrock dmu_buf_impl_t *db; 195ea8dc4b6Seschrock 196ea8dc4b6Seschrock rw_enter(&dn->dn_struct_rwlock, RW_READER); 197ea8dc4b6Seschrock db = dbuf_hold_level(dn, level, blkid, FTAG); 198ea8dc4b6Seschrock rw_exit(&dn->dn_struct_rwlock); 199ea8dc4b6Seschrock if (db == NULL) 200be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 2011ab7f2deSmaybee err = dbuf_read(db, zio, DB_RF_CANFAIL | DB_RF_NOPREFETCH); 202ea8dc4b6Seschrock dbuf_rele(db, FTAG); 203ea8dc4b6Seschrock return (err); 204ea8dc4b6Seschrock } 205ea8dc4b6Seschrock 206fa9e4066Sahrens /* ARGSUSED */ 207fa9e4066Sahrens static void 2088a2f1b91Sahrens dmu_tx_count_write(dmu_tx_hold_t *txh, uint64_t off, uint64_t len) 209fa9e4066Sahrens { 2108a2f1b91Sahrens dnode_t *dn = txh->txh_dnode; 2118a2f1b91Sahrens int err = 0; 212fa9e4066Sahrens 213fa9e4066Sahrens if (len == 0) 214fa9e4066Sahrens return; 215fa9e4066Sahrens 21661e255ceSMatthew Ahrens (void) refcount_add_many(&txh->txh_space_towrite, len, FTAG); 2174a7f2a75SMark Maybee 21861e255ceSMatthew Ahrens if (refcount_count(&txh->txh_space_towrite) > 2 * DMU_MAX_ACCESS) 21961e255ceSMatthew Ahrens err = SET_ERROR(EFBIG); 220ea8dc4b6Seschrock 22161e255ceSMatthew Ahrens if (dn == NULL) 22261e255ceSMatthew Ahrens return; 223ea8dc4b6Seschrock 22461e255ceSMatthew Ahrens /* 22561e255ceSMatthew Ahrens * For i/o error checking, read the blocks that will be needed 22661e255ceSMatthew Ahrens * to perform the write: the first and last level-0 blocks (if 22761e255ceSMatthew Ahrens * they are not aligned, i.e. if they are partial-block writes), 22861e255ceSMatthew Ahrens * and all the level-1 blocks. 22961e255ceSMatthew Ahrens */ 23061e255ceSMatthew Ahrens if (dn->dn_maxblkid == 0) { 23161e255ceSMatthew Ahrens if (off < dn->dn_datablksz && 23261e255ceSMatthew Ahrens (off > 0 || len < dn->dn_datablksz)) { 23361e255ceSMatthew Ahrens err = dmu_tx_check_ioerr(NULL, dn, 0, 0); 23461e255ceSMatthew Ahrens if (err != 0) { 23561e255ceSMatthew Ahrens txh->txh_tx->tx_err = err; 236ea8dc4b6Seschrock } 2374a7f2a75SMark Maybee } 23861e255ceSMatthew Ahrens } else { 23961e255ceSMatthew Ahrens zio_t *zio = zio_root(dn->dn_objset->os_spa, 24061e255ceSMatthew Ahrens NULL, NULL, ZIO_FLAG_CANFAIL); 2414a7f2a75SMark Maybee 24261e255ceSMatthew Ahrens /* first level-0 block */ 24361e255ceSMatthew Ahrens uint64_t start = off >> dn->dn_datablkshift; 24461e255ceSMatthew Ahrens if (P2PHASE(off, dn->dn_datablksz) || len < dn->dn_datablksz) { 24561e255ceSMatthew Ahrens err = dmu_tx_check_ioerr(zio, dn, 0, start); 24661e255ceSMatthew Ahrens if (err != 0) { 24761e255ceSMatthew Ahrens txh->txh_tx->tx_err = err; 24861e255ceSMatthew Ahrens } 249b24ab676SJeff Bonwick } 2504a7f2a75SMark Maybee 25161e255ceSMatthew Ahrens /* last level-0 block */ 25261e255ceSMatthew Ahrens uint64_t end = (off + len - 1) >> dn->dn_datablkshift; 25361e255ceSMatthew Ahrens if (end != start && end <= dn->dn_maxblkid && 25461e255ceSMatthew Ahrens P2PHASE(off + len, dn->dn_datablksz)) { 25561e255ceSMatthew Ahrens err = dmu_tx_check_ioerr(zio, dn, 0, end); 25661e255ceSMatthew Ahrens if (err != 0) { 25701025c89SJohn Harres txh->txh_tx->tx_err = err; 25801025c89SJohn Harres } 25961e255ceSMatthew Ahrens } 26001025c89SJohn Harres 26161e255ceSMatthew Ahrens /* level-1 blocks */ 26261e255ceSMatthew Ahrens if (dn->dn_nlevels > 1) { 26361e255ceSMatthew Ahrens int shft = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 26461e255ceSMatthew Ahrens for (uint64_t i = (start >> shft) + 1; 26561e255ceSMatthew Ahrens i < end >> shft; i++) { 26661e255ceSMatthew Ahrens err = dmu_tx_check_ioerr(zio, dn, 1, i); 26761e255ceSMatthew Ahrens if (err != 0) { 26861e255ceSMatthew Ahrens txh->txh_tx->tx_err = err; 2690c779ad4SMatthew Ahrens } 2704a7f2a75SMark Maybee } 2714a7f2a75SMark Maybee } 272fa9e4066Sahrens 27361e255ceSMatthew Ahrens err = zio_wait(zio); 27461e255ceSMatthew Ahrens if (err != 0) { 27561e255ceSMatthew Ahrens txh->txh_tx->tx_err = err; 2764a7f2a75SMark Maybee } 277fa9e4066Sahrens } 278fa9e4066Sahrens } 279fa9e4066Sahrens 280fa9e4066Sahrens static void 2818a2f1b91Sahrens dmu_tx_count_dnode(dmu_tx_hold_t *txh) 282fa9e4066Sahrens { 28361e255ceSMatthew Ahrens (void) refcount_add_many(&txh->txh_space_towrite, DNODE_SIZE, FTAG); 284fa9e4066Sahrens } 285fa9e4066Sahrens 286fa9e4066Sahrens void 287fa9e4066Sahrens dmu_tx_hold_write(dmu_tx_t *tx, uint64_t object, uint64_t off, int len) 288fa9e4066Sahrens { 2898a2f1b91Sahrens dmu_tx_hold_t *txh; 2908a2f1b91Sahrens 29161e255ceSMatthew Ahrens ASSERT0(tx->tx_txg); 29261e255ceSMatthew Ahrens ASSERT3U(len, <=, DMU_MAX_ACCESS); 293dd6ef538Smaybee ASSERT(len == 0 || UINT64_MAX - off >= len - 1); 294fa9e4066Sahrens 2958a2f1b91Sahrens txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, 2968a2f1b91Sahrens object, THT_WRITE, off, len); 297b0c42cd4Sbzzz if (txh != NULL) { 298b0c42cd4Sbzzz dmu_tx_count_write(txh, off, len); 299b0c42cd4Sbzzz dmu_tx_count_dnode(txh); 300b0c42cd4Sbzzz } 301b0c42cd4Sbzzz } 3028a2f1b91Sahrens 303b0c42cd4Sbzzz void 304b0c42cd4Sbzzz dmu_tx_hold_write_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, int len) 305b0c42cd4Sbzzz { 306b0c42cd4Sbzzz dmu_tx_hold_t *txh; 307b0c42cd4Sbzzz 308b0c42cd4Sbzzz ASSERT0(tx->tx_txg); 309b0c42cd4Sbzzz ASSERT3U(len, <=, DMU_MAX_ACCESS); 310b0c42cd4Sbzzz ASSERT(len == 0 || UINT64_MAX - off >= len - 1); 311b0c42cd4Sbzzz 312b0c42cd4Sbzzz txh = dmu_tx_hold_dnode_impl(tx, dn, THT_WRITE, off, len); 313b0c42cd4Sbzzz if (txh != NULL) { 314b0c42cd4Sbzzz dmu_tx_count_write(txh, off, len); 315b0c42cd4Sbzzz dmu_tx_count_dnode(txh); 316b0c42cd4Sbzzz } 317fa9e4066Sahrens } 318fa9e4066Sahrens 3194bb73804SMatthew Ahrens /* 3204bb73804SMatthew Ahrens * This function marks the transaction as being a "net free". The end 3214bb73804SMatthew Ahrens * result is that refquotas will be disabled for this transaction, and 3224bb73804SMatthew Ahrens * this transaction will be able to use half of the pool space overhead 3234bb73804SMatthew Ahrens * (see dsl_pool_adjustedsize()). Therefore this function should only 3244bb73804SMatthew Ahrens * be called for transactions that we expect will not cause a net increase 3254bb73804SMatthew Ahrens * in the amount of space used (but it's OK if that is occasionally not true). 3264bb73804SMatthew Ahrens */ 3274bb73804SMatthew Ahrens void 3284bb73804SMatthew Ahrens dmu_tx_mark_netfree(dmu_tx_t *tx) 3294bb73804SMatthew Ahrens { 33061e255ceSMatthew Ahrens tx->tx_netfree = B_TRUE; 3314bb73804SMatthew Ahrens } 3324bb73804SMatthew Ahrens 333b0c42cd4Sbzzz static void 334b0c42cd4Sbzzz dmu_tx_hold_free_impl(dmu_tx_hold_t *txh, uint64_t off, uint64_t len) 335fa9e4066Sahrens { 336b0c42cd4Sbzzz dmu_tx_t *tx; 337b0c42cd4Sbzzz dnode_t *dn; 3382f3d8780SMatthew Ahrens int err; 339fa9e4066Sahrens 340b0c42cd4Sbzzz tx = txh->txh_tx; 3418a2f1b91Sahrens ASSERT(tx->tx_txg == 0); 3428a2f1b91Sahrens 343b0c42cd4Sbzzz dn = txh->txh_dnode; 34469962b56SMatthew Ahrens dmu_tx_count_dnode(txh); 3458a2f1b91Sahrens 34661e255ceSMatthew Ahrens if (off >= (dn->dn_maxblkid + 1) * dn->dn_datablksz) 347fa9e4066Sahrens return; 348fa9e4066Sahrens if (len == DMU_OBJECT_END) 34961e255ceSMatthew Ahrens len = (dn->dn_maxblkid + 1) * dn->dn_datablksz - off; 350fa9e4066Sahrens 351ea8dc4b6Seschrock /* 3522f3d8780SMatthew Ahrens * For i/o error checking, we read the first and last level-0 3532f3d8780SMatthew Ahrens * blocks if they are not aligned, and all the level-1 blocks. 3542f3d8780SMatthew Ahrens * 3552f3d8780SMatthew Ahrens * Note: dbuf_free_range() assumes that we have not instantiated 3562f3d8780SMatthew Ahrens * any level-0 dbufs that will be completely freed. Therefore we must 3572f3d8780SMatthew Ahrens * exercise care to not read or count the first and last blocks 3582f3d8780SMatthew Ahrens * if they are blocksize-aligned. 3592f3d8780SMatthew Ahrens */ 3602f3d8780SMatthew Ahrens if (dn->dn_datablkshift == 0) { 361713d6c20SMatthew Ahrens if (off != 0 || len < dn->dn_datablksz) 3625253393bSMatthew Ahrens dmu_tx_count_write(txh, 0, dn->dn_datablksz); 3632f3d8780SMatthew Ahrens } else { 3642f3d8780SMatthew Ahrens /* first block will be modified if it is not aligned */ 3652f3d8780SMatthew Ahrens if (!IS_P2ALIGNED(off, 1 << dn->dn_datablkshift)) 3662f3d8780SMatthew Ahrens dmu_tx_count_write(txh, off, 1); 3672f3d8780SMatthew Ahrens /* last block will be modified if it is not aligned */ 3682f3d8780SMatthew Ahrens if (!IS_P2ALIGNED(off + len, 1 << dn->dn_datablkshift)) 36961e255ceSMatthew Ahrens dmu_tx_count_write(txh, off + len, 1); 3702f3d8780SMatthew Ahrens } 3712f3d8780SMatthew Ahrens 3722f3d8780SMatthew Ahrens /* 3732f3d8780SMatthew Ahrens * Check level-1 blocks. 374ea8dc4b6Seschrock */ 37598572ac1Sahrens if (dn->dn_nlevels > 1) { 3762f3d8780SMatthew Ahrens int shift = dn->dn_datablkshift + dn->dn_indblkshift - 37798572ac1Sahrens SPA_BLKPTRSHIFT; 3782f3d8780SMatthew Ahrens uint64_t start = off >> shift; 3792f3d8780SMatthew Ahrens uint64_t end = (off + len) >> shift; 3802f3d8780SMatthew Ahrens 3812f3d8780SMatthew Ahrens ASSERT(dn->dn_indblkshift != 0); 38298572ac1Sahrens 383bb411a08SMatthew Ahrens /* 384bb411a08SMatthew Ahrens * dnode_reallocate() can result in an object with indirect 385bb411a08SMatthew Ahrens * blocks having an odd data block size. In this case, 386bb411a08SMatthew Ahrens * just check the single block. 387bb411a08SMatthew Ahrens */ 388bb411a08SMatthew Ahrens if (dn->dn_datablkshift == 0) 389bb411a08SMatthew Ahrens start = end = 0; 390bb411a08SMatthew Ahrens 39161e255ceSMatthew Ahrens zio_t *zio = zio_root(tx->tx_pool->dp_spa, 39298572ac1Sahrens NULL, NULL, ZIO_FLAG_CANFAIL); 3932f3d8780SMatthew Ahrens for (uint64_t i = start; i <= end; i++) { 39498572ac1Sahrens uint64_t ibyte = i << shift; 395cdb0ab79Smaybee err = dnode_next_offset(dn, 0, &ibyte, 2, 1, 0); 39698572ac1Sahrens i = ibyte >> shift; 39746e1baa6SMatthew Ahrens if (err == ESRCH || i > end) 39898572ac1Sahrens break; 39961e255ceSMatthew Ahrens if (err != 0) { 40098572ac1Sahrens tx->tx_err = err; 40161e255ceSMatthew Ahrens (void) zio_wait(zio); 40298572ac1Sahrens return; 40398572ac1Sahrens } 404ea8dc4b6Seschrock 40561e255ceSMatthew Ahrens (void) refcount_add_many(&txh->txh_memory_tohold, 40661e255ceSMatthew Ahrens 1 << dn->dn_indblkshift, FTAG); 40761e255ceSMatthew Ahrens 40898572ac1Sahrens err = dmu_tx_check_ioerr(zio, dn, 1, i); 40961e255ceSMatthew Ahrens if (err != 0) { 41098572ac1Sahrens tx->tx_err = err; 41161e255ceSMatthew Ahrens (void) zio_wait(zio); 41298572ac1Sahrens return; 41398572ac1Sahrens } 41498572ac1Sahrens } 41598572ac1Sahrens err = zio_wait(zio); 41661e255ceSMatthew Ahrens if (err != 0) { 417ea8dc4b6Seschrock tx->tx_err = err; 418ea8dc4b6Seschrock return; 419ea8dc4b6Seschrock } 420ea8dc4b6Seschrock } 421fa9e4066Sahrens } 422fa9e4066Sahrens 423fa9e4066Sahrens void 424b0c42cd4Sbzzz dmu_tx_hold_free(dmu_tx_t *tx, uint64_t object, uint64_t off, uint64_t len) 425b0c42cd4Sbzzz { 426b0c42cd4Sbzzz dmu_tx_hold_t *txh; 427b0c42cd4Sbzzz 428b0c42cd4Sbzzz txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, 429b0c42cd4Sbzzz object, THT_FREE, off, len); 430b0c42cd4Sbzzz if (txh != NULL) 431b0c42cd4Sbzzz (void) dmu_tx_hold_free_impl(txh, off, len); 432b0c42cd4Sbzzz } 433b0c42cd4Sbzzz 434b0c42cd4Sbzzz void 435b0c42cd4Sbzzz dmu_tx_hold_free_by_dnode(dmu_tx_t *tx, dnode_t *dn, uint64_t off, uint64_t len) 436b0c42cd4Sbzzz { 437b0c42cd4Sbzzz dmu_tx_hold_t *txh; 438b0c42cd4Sbzzz 439b0c42cd4Sbzzz txh = dmu_tx_hold_dnode_impl(tx, dn, THT_FREE, off, len); 440b0c42cd4Sbzzz if (txh != NULL) 441b0c42cd4Sbzzz (void) dmu_tx_hold_free_impl(txh, off, len); 442b0c42cd4Sbzzz } 443b0c42cd4Sbzzz 444b0c42cd4Sbzzz static void 445411be58aSMatthew Ahrens dmu_tx_hold_zap_impl(dmu_tx_hold_t *txh, const char *name) 446fa9e4066Sahrens { 447b0c42cd4Sbzzz dmu_tx_t *tx = txh->txh_tx; 448b0c42cd4Sbzzz dnode_t *dn; 4490c779ad4SMatthew Ahrens int err; 450fa9e4066Sahrens 4518a2f1b91Sahrens ASSERT(tx->tx_txg == 0); 4528a2f1b91Sahrens 453b0c42cd4Sbzzz dn = txh->txh_dnode; 4548a2f1b91Sahrens 4558a2f1b91Sahrens dmu_tx_count_dnode(txh); 456fa9e4066Sahrens 45761e255ceSMatthew Ahrens /* 45861e255ceSMatthew Ahrens * Modifying a almost-full microzap is around the worst case (128KB) 45961e255ceSMatthew Ahrens * 46061e255ceSMatthew Ahrens * If it is a fat zap, the worst case would be 7*16KB=112KB: 46161e255ceSMatthew Ahrens * - 3 blocks overwritten: target leaf, ptrtbl block, header block 46261e255ceSMatthew Ahrens * - 4 new blocks written if adding: 46361e255ceSMatthew Ahrens * - 2 blocks for possibly split leaves, 46461e255ceSMatthew Ahrens * - 2 grown ptrtbl blocks 46561e255ceSMatthew Ahrens */ 46661e255ceSMatthew Ahrens (void) refcount_add_many(&txh->txh_space_towrite, 46761e255ceSMatthew Ahrens MZAP_MAX_BLKSZ, FTAG); 46861e255ceSMatthew Ahrens 46961e255ceSMatthew Ahrens if (dn == NULL) 470fa9e4066Sahrens return; 471fa9e4066Sahrens 472ad135b5dSChristopher Siden ASSERT3P(DMU_OT_BYTESWAP(dn->dn_type), ==, DMU_BSWAP_ZAP); 473fa9e4066Sahrens 47461e255ceSMatthew Ahrens if (dn->dn_maxblkid == 0 || name == NULL) { 475fa9e4066Sahrens /* 47661e255ceSMatthew Ahrens * This is a microzap (only one block), or we don't know 47761e255ceSMatthew Ahrens * the name. Check the first block for i/o errors. 478fa9e4066Sahrens */ 479ea8dc4b6Seschrock err = dmu_tx_check_ioerr(NULL, dn, 0, 0); 48061e255ceSMatthew Ahrens if (err != 0) { 481ea8dc4b6Seschrock tx->tx_err = err; 4820c779ad4SMatthew Ahrens } 48361e255ceSMatthew Ahrens } else { 484ea8dc4b6Seschrock /* 48561e255ceSMatthew Ahrens * Access the name so that we'll check for i/o errors to 48661e255ceSMatthew Ahrens * the leaf blocks, etc. We ignore ENOENT, as this name 48761e255ceSMatthew Ahrens * may not yet exist. 488ea8dc4b6Seschrock */ 48979d72832SMatthew Ahrens err = zap_lookup_by_dnode(dn, name, 8, 0, NULL); 49061e255ceSMatthew Ahrens if (err == EIO || err == ECKSUM || err == ENXIO) { 491ea8dc4b6Seschrock tx->tx_err = err; 4920c779ad4SMatthew Ahrens } 4930c779ad4SMatthew Ahrens } 494fa9e4066Sahrens } 495fa9e4066Sahrens 496b0c42cd4Sbzzz void 497b0c42cd4Sbzzz dmu_tx_hold_zap(dmu_tx_t *tx, uint64_t object, int add, const char *name) 498b0c42cd4Sbzzz { 499b0c42cd4Sbzzz dmu_tx_hold_t *txh; 500b0c42cd4Sbzzz 501b0c42cd4Sbzzz ASSERT0(tx->tx_txg); 502b0c42cd4Sbzzz 503b0c42cd4Sbzzz txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, 504b0c42cd4Sbzzz object, THT_ZAP, add, (uintptr_t)name); 505b0c42cd4Sbzzz if (txh != NULL) 506411be58aSMatthew Ahrens dmu_tx_hold_zap_impl(txh, name); 507b0c42cd4Sbzzz } 508b0c42cd4Sbzzz 509b0c42cd4Sbzzz void 510b0c42cd4Sbzzz dmu_tx_hold_zap_by_dnode(dmu_tx_t *tx, dnode_t *dn, int add, const char *name) 511b0c42cd4Sbzzz { 512b0c42cd4Sbzzz dmu_tx_hold_t *txh; 513b0c42cd4Sbzzz 514b0c42cd4Sbzzz ASSERT0(tx->tx_txg); 515b0c42cd4Sbzzz ASSERT(dn != NULL); 516b0c42cd4Sbzzz 517b0c42cd4Sbzzz txh = dmu_tx_hold_dnode_impl(tx, dn, THT_ZAP, add, (uintptr_t)name); 518b0c42cd4Sbzzz if (txh != NULL) 519411be58aSMatthew Ahrens dmu_tx_hold_zap_impl(txh, name); 520b0c42cd4Sbzzz } 521b0c42cd4Sbzzz 522fa9e4066Sahrens void 523fa9e4066Sahrens dmu_tx_hold_bonus(dmu_tx_t *tx, uint64_t object) 524fa9e4066Sahrens { 5258a2f1b91Sahrens dmu_tx_hold_t *txh; 526fa9e4066Sahrens 5278a2f1b91Sahrens ASSERT(tx->tx_txg == 0); 528fa9e4066Sahrens 5298a2f1b91Sahrens txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, 5308a2f1b91Sahrens object, THT_BONUS, 0, 0); 5318a2f1b91Sahrens if (txh) 5328a2f1b91Sahrens dmu_tx_count_dnode(txh); 533fa9e4066Sahrens } 534fa9e4066Sahrens 535b0c42cd4Sbzzz void 536b0c42cd4Sbzzz dmu_tx_hold_bonus_by_dnode(dmu_tx_t *tx, dnode_t *dn) 537b0c42cd4Sbzzz { 538b0c42cd4Sbzzz dmu_tx_hold_t *txh; 539b0c42cd4Sbzzz 540b0c42cd4Sbzzz ASSERT0(tx->tx_txg); 541b0c42cd4Sbzzz 542b0c42cd4Sbzzz txh = dmu_tx_hold_dnode_impl(tx, dn, THT_BONUS, 0, 0); 543b0c42cd4Sbzzz if (txh) 544b0c42cd4Sbzzz dmu_tx_count_dnode(txh); 545b0c42cd4Sbzzz } 546b0c42cd4Sbzzz 547fa9e4066Sahrens void 548fa9e4066Sahrens dmu_tx_hold_space(dmu_tx_t *tx, uint64_t space) 549fa9e4066Sahrens { 5508a2f1b91Sahrens dmu_tx_hold_t *txh; 551fa9e4066Sahrens ASSERT(tx->tx_txg == 0); 552fa9e4066Sahrens 5538a2f1b91Sahrens txh = dmu_tx_hold_object_impl(tx, tx->tx_objset, 5548a2f1b91Sahrens DMU_NEW_OBJECT, THT_SPACE, space, 0); 5558a2f1b91Sahrens 5560c779ad4SMatthew Ahrens (void) refcount_add_many(&txh->txh_space_towrite, space, FTAG); 557fa9e4066Sahrens } 558fa9e4066Sahrens 5599c9dc39aSek #ifdef ZFS_DEBUG 560fa9e4066Sahrens void 561fa9e4066Sahrens dmu_tx_dirty_buf(dmu_tx_t *tx, dmu_buf_impl_t *db) 562fa9e4066Sahrens { 56361e255ceSMatthew Ahrens boolean_t match_object = B_FALSE; 56461e255ceSMatthew Ahrens boolean_t match_offset = B_FALSE; 565fa9e4066Sahrens 566744947dcSTom Erickson DB_DNODE_ENTER(db); 56761e255ceSMatthew Ahrens dnode_t *dn = DB_DNODE(db); 568fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 569503ad85cSMatthew Ahrens ASSERT(tx->tx_objset == NULL || dn->dn_objset == tx->tx_objset); 570fa9e4066Sahrens ASSERT3U(dn->dn_object, ==, db->db.db_object); 571fa9e4066Sahrens 572744947dcSTom Erickson if (tx->tx_anyobj) { 573744947dcSTom Erickson DB_DNODE_EXIT(db); 574fa9e4066Sahrens return; 575744947dcSTom Erickson } 576fa9e4066Sahrens 577fa9e4066Sahrens /* XXX No checking on the meta dnode for now */ 578744947dcSTom Erickson if (db->db.db_object == DMU_META_DNODE_OBJECT) { 579744947dcSTom Erickson DB_DNODE_EXIT(db); 580fa9e4066Sahrens return; 581744947dcSTom Erickson } 582fa9e4066Sahrens 58361e255ceSMatthew Ahrens for (dmu_tx_hold_t *txh = list_head(&tx->tx_holds); txh != NULL; 5848a2f1b91Sahrens txh = list_next(&tx->tx_holds, txh)) { 585fa9e4066Sahrens ASSERT(dn == NULL || dn->dn_assigned_txg == tx->tx_txg); 5868a2f1b91Sahrens if (txh->txh_dnode == dn && txh->txh_type != THT_NEWOBJECT) 587fa9e4066Sahrens match_object = TRUE; 5888a2f1b91Sahrens if (txh->txh_dnode == NULL || txh->txh_dnode == dn) { 589fa9e4066Sahrens int datablkshift = dn->dn_datablkshift ? 590fa9e4066Sahrens dn->dn_datablkshift : SPA_MAXBLOCKSHIFT; 591fa9e4066Sahrens int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 592fa9e4066Sahrens int shift = datablkshift + epbs * db->db_level; 593fa9e4066Sahrens uint64_t beginblk = shift >= 64 ? 0 : 5948a2f1b91Sahrens (txh->txh_arg1 >> shift); 595fa9e4066Sahrens uint64_t endblk = shift >= 64 ? 0 : 5968a2f1b91Sahrens ((txh->txh_arg1 + txh->txh_arg2 - 1) >> shift); 597fa9e4066Sahrens uint64_t blkid = db->db_blkid; 598fa9e4066Sahrens 5998a2f1b91Sahrens /* XXX txh_arg2 better not be zero... */ 600fa9e4066Sahrens 6018a2f1b91Sahrens dprintf("found txh type %x beginblk=%llx endblk=%llx\n", 6028a2f1b91Sahrens txh->txh_type, beginblk, endblk); 603fa9e4066Sahrens 6048a2f1b91Sahrens switch (txh->txh_type) { 605fa9e4066Sahrens case THT_WRITE: 606fa9e4066Sahrens if (blkid >= beginblk && blkid <= endblk) 607fa9e4066Sahrens match_offset = TRUE; 608fa9e4066Sahrens /* 609fa9e4066Sahrens * We will let this hold work for the bonus 6100a586ceaSMark Shellenbaum * or spill buffer so that we don't need to 6110a586ceaSMark Shellenbaum * hold it when creating a new object. 612fa9e4066Sahrens */ 6130a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID || 6140a586ceaSMark Shellenbaum blkid == DMU_SPILL_BLKID) 615fa9e4066Sahrens match_offset = TRUE; 616fa9e4066Sahrens /* 617fa9e4066Sahrens * They might have to increase nlevels, 618fa9e4066Sahrens * thus dirtying the new TLIBs. Or the 619fa9e4066Sahrens * might have to change the block size, 620fa9e4066Sahrens * thus dirying the new lvl=0 blk=0. 621fa9e4066Sahrens */ 622fa9e4066Sahrens if (blkid == 0) 623fa9e4066Sahrens match_offset = TRUE; 624fa9e4066Sahrens break; 625fa9e4066Sahrens case THT_FREE: 626cdb0ab79Smaybee /* 627cdb0ab79Smaybee * We will dirty all the level 1 blocks in 628cdb0ab79Smaybee * the free range and perhaps the first and 629cdb0ab79Smaybee * last level 0 block. 630cdb0ab79Smaybee */ 631cdb0ab79Smaybee if (blkid >= beginblk && (blkid <= endblk || 632cdb0ab79Smaybee txh->txh_arg2 == DMU_OBJECT_END)) 633fa9e4066Sahrens match_offset = TRUE; 634fa9e4066Sahrens break; 6350a586ceaSMark Shellenbaum case THT_SPILL: 6360a586ceaSMark Shellenbaum if (blkid == DMU_SPILL_BLKID) 6370a586ceaSMark Shellenbaum match_offset = TRUE; 6380a586ceaSMark Shellenbaum break; 639fa9e4066Sahrens case THT_BONUS: 6400a586ceaSMark Shellenbaum if (blkid == DMU_BONUS_BLKID) 641fa9e4066Sahrens match_offset = TRUE; 642fa9e4066Sahrens break; 643fa9e4066Sahrens case THT_ZAP: 644fa9e4066Sahrens match_offset = TRUE; 645fa9e4066Sahrens break; 646fa9e4066Sahrens case THT_NEWOBJECT: 647fa9e4066Sahrens match_object = TRUE; 648fa9e4066Sahrens break; 649fa9e4066Sahrens default: 6508a2f1b91Sahrens ASSERT(!"bad txh_type"); 651fa9e4066Sahrens } 652fa9e4066Sahrens } 653744947dcSTom Erickson if (match_object && match_offset) { 654744947dcSTom Erickson DB_DNODE_EXIT(db); 655fa9e4066Sahrens return; 656744947dcSTom Erickson } 657fa9e4066Sahrens } 658744947dcSTom Erickson DB_DNODE_EXIT(db); 659fa9e4066Sahrens panic("dirtying dbuf obj=%llx lvl=%u blkid=%llx but not tx_held\n", 660fa9e4066Sahrens (u_longlong_t)db->db.db_object, db->db_level, 661fa9e4066Sahrens (u_longlong_t)db->db_blkid); 662fa9e4066Sahrens } 6639c9dc39aSek #endif 664fa9e4066Sahrens 66569962b56SMatthew Ahrens /* 66669962b56SMatthew Ahrens * If we can't do 10 iops, something is wrong. Let us go ahead 66769962b56SMatthew Ahrens * and hit zfs_dirty_data_max. 66869962b56SMatthew Ahrens */ 66969962b56SMatthew Ahrens hrtime_t zfs_delay_max_ns = MSEC2NSEC(100); 67069962b56SMatthew Ahrens int zfs_delay_resolution_ns = 100 * 1000; /* 100 microseconds */ 67169962b56SMatthew Ahrens 67269962b56SMatthew Ahrens /* 67369962b56SMatthew Ahrens * We delay transactions when we've determined that the backend storage 67469962b56SMatthew Ahrens * isn't able to accommodate the rate of incoming writes. 67569962b56SMatthew Ahrens * 67669962b56SMatthew Ahrens * If there is already a transaction waiting, we delay relative to when 67769962b56SMatthew Ahrens * that transaction finishes waiting. This way the calculated min_time 67869962b56SMatthew Ahrens * is independent of the number of threads concurrently executing 67969962b56SMatthew Ahrens * transactions. 68069962b56SMatthew Ahrens * 68169962b56SMatthew Ahrens * If we are the only waiter, wait relative to when the transaction 68269962b56SMatthew Ahrens * started, rather than the current time. This credits the transaction for 68369962b56SMatthew Ahrens * "time already served", e.g. reading indirect blocks. 68469962b56SMatthew Ahrens * 68569962b56SMatthew Ahrens * The minimum time for a transaction to take is calculated as: 68669962b56SMatthew Ahrens * min_time = scale * (dirty - min) / (max - dirty) 68769962b56SMatthew Ahrens * min_time is then capped at zfs_delay_max_ns. 68869962b56SMatthew Ahrens * 68969962b56SMatthew Ahrens * The delay has two degrees of freedom that can be adjusted via tunables. 69069962b56SMatthew Ahrens * The percentage of dirty data at which we start to delay is defined by 69169962b56SMatthew Ahrens * zfs_delay_min_dirty_percent. This should typically be at or above 69269962b56SMatthew Ahrens * zfs_vdev_async_write_active_max_dirty_percent so that we only start to 69369962b56SMatthew Ahrens * delay after writing at full speed has failed to keep up with the incoming 69469962b56SMatthew Ahrens * write rate. The scale of the curve is defined by zfs_delay_scale. Roughly 69569962b56SMatthew Ahrens * speaking, this variable determines the amount of delay at the midpoint of 69669962b56SMatthew Ahrens * the curve. 69769962b56SMatthew Ahrens * 69869962b56SMatthew Ahrens * delay 69969962b56SMatthew Ahrens * 10ms +-------------------------------------------------------------*+ 70069962b56SMatthew Ahrens * | *| 70169962b56SMatthew Ahrens * 9ms + *+ 70269962b56SMatthew Ahrens * | *| 70369962b56SMatthew Ahrens * 8ms + *+ 70469962b56SMatthew Ahrens * | * | 70569962b56SMatthew Ahrens * 7ms + * + 70669962b56SMatthew Ahrens * | * | 70769962b56SMatthew Ahrens * 6ms + * + 70869962b56SMatthew Ahrens * | * | 70969962b56SMatthew Ahrens * 5ms + * + 71069962b56SMatthew Ahrens * | * | 71169962b56SMatthew Ahrens * 4ms + * + 71269962b56SMatthew Ahrens * | * | 71369962b56SMatthew Ahrens * 3ms + * + 71469962b56SMatthew Ahrens * | * | 71569962b56SMatthew Ahrens * 2ms + (midpoint) * + 71669962b56SMatthew Ahrens * | | ** | 71769962b56SMatthew Ahrens * 1ms + v *** + 71869962b56SMatthew Ahrens * | zfs_delay_scale ----------> ******** | 71969962b56SMatthew Ahrens * 0 +-------------------------------------*********----------------+ 72069962b56SMatthew Ahrens * 0% <- zfs_dirty_data_max -> 100% 72169962b56SMatthew Ahrens * 72269962b56SMatthew Ahrens * Note that since the delay is added to the outstanding time remaining on the 72369962b56SMatthew Ahrens * most recent transaction, the delay is effectively the inverse of IOPS. 72469962b56SMatthew Ahrens * Here the midpoint of 500us translates to 2000 IOPS. The shape of the curve 72569962b56SMatthew Ahrens * was chosen such that small changes in the amount of accumulated dirty data 72669962b56SMatthew Ahrens * in the first 3/4 of the curve yield relatively small differences in the 72769962b56SMatthew Ahrens * amount of delay. 72869962b56SMatthew Ahrens * 72969962b56SMatthew Ahrens * The effects can be easier to understand when the amount of delay is 73069962b56SMatthew Ahrens * represented on a log scale: 73169962b56SMatthew Ahrens * 73269962b56SMatthew Ahrens * delay 73369962b56SMatthew Ahrens * 100ms +-------------------------------------------------------------++ 73469962b56SMatthew Ahrens * + + 73569962b56SMatthew Ahrens * | | 73669962b56SMatthew Ahrens * + *+ 73769962b56SMatthew Ahrens * 10ms + *+ 73869962b56SMatthew Ahrens * + ** + 73969962b56SMatthew Ahrens * | (midpoint) ** | 74069962b56SMatthew Ahrens * + | ** + 74169962b56SMatthew Ahrens * 1ms + v **** + 74269962b56SMatthew Ahrens * + zfs_delay_scale ----------> ***** + 74369962b56SMatthew Ahrens * | **** | 74469962b56SMatthew Ahrens * + **** + 74569962b56SMatthew Ahrens * 100us + ** + 74669962b56SMatthew Ahrens * + * + 74769962b56SMatthew Ahrens * | * | 74869962b56SMatthew Ahrens * + * + 74969962b56SMatthew Ahrens * 10us + * + 75069962b56SMatthew Ahrens * + + 75169962b56SMatthew Ahrens * | | 75269962b56SMatthew Ahrens * + + 75369962b56SMatthew Ahrens * +--------------------------------------------------------------+ 75469962b56SMatthew Ahrens * 0% <- zfs_dirty_data_max -> 100% 75569962b56SMatthew Ahrens * 75669962b56SMatthew Ahrens * Note here that only as the amount of dirty data approaches its limit does 75769962b56SMatthew Ahrens * the delay start to increase rapidly. The goal of a properly tuned system 75869962b56SMatthew Ahrens * should be to keep the amount of dirty data out of that range by first 75969962b56SMatthew Ahrens * ensuring that the appropriate limits are set for the I/O scheduler to reach 76069962b56SMatthew Ahrens * optimal throughput on the backend storage, and then by changing the value 76169962b56SMatthew Ahrens * of zfs_delay_scale to increase the steepness of the curve. 76269962b56SMatthew Ahrens */ 76369962b56SMatthew Ahrens static void 76469962b56SMatthew Ahrens dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty) 76569962b56SMatthew Ahrens { 76669962b56SMatthew Ahrens dsl_pool_t *dp = tx->tx_pool; 76769962b56SMatthew Ahrens uint64_t delay_min_bytes = 76869962b56SMatthew Ahrens zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100; 76969962b56SMatthew Ahrens hrtime_t wakeup, min_tx_time, now; 77069962b56SMatthew Ahrens 77169962b56SMatthew Ahrens if (dirty <= delay_min_bytes) 77269962b56SMatthew Ahrens return; 77369962b56SMatthew Ahrens 77469962b56SMatthew Ahrens /* 77569962b56SMatthew Ahrens * The caller has already waited until we are under the max. 77669962b56SMatthew Ahrens * We make them pass us the amount of dirty data so we don't 77769962b56SMatthew Ahrens * have to handle the case of it being >= the max, which could 77869962b56SMatthew Ahrens * cause a divide-by-zero if it's == the max. 77969962b56SMatthew Ahrens */ 78069962b56SMatthew Ahrens ASSERT3U(dirty, <, zfs_dirty_data_max); 78169962b56SMatthew Ahrens 78269962b56SMatthew Ahrens now = gethrtime(); 78369962b56SMatthew Ahrens min_tx_time = zfs_delay_scale * 78469962b56SMatthew Ahrens (dirty - delay_min_bytes) / (zfs_dirty_data_max - dirty); 78569962b56SMatthew Ahrens if (now > tx->tx_start + min_tx_time) 78669962b56SMatthew Ahrens return; 78769962b56SMatthew Ahrens 78869962b56SMatthew Ahrens min_tx_time = MIN(min_tx_time, zfs_delay_max_ns); 78969962b56SMatthew Ahrens 79069962b56SMatthew Ahrens DTRACE_PROBE3(delay__mintime, dmu_tx_t *, tx, uint64_t, dirty, 79169962b56SMatthew Ahrens uint64_t, min_tx_time); 79269962b56SMatthew Ahrens 79369962b56SMatthew Ahrens mutex_enter(&dp->dp_lock); 79469962b56SMatthew Ahrens wakeup = MAX(tx->tx_start + min_tx_time, 79569962b56SMatthew Ahrens dp->dp_last_wakeup + min_tx_time); 79669962b56SMatthew Ahrens dp->dp_last_wakeup = wakeup; 79769962b56SMatthew Ahrens mutex_exit(&dp->dp_lock); 79869962b56SMatthew Ahrens 79969962b56SMatthew Ahrens #ifdef _KERNEL 80069962b56SMatthew Ahrens mutex_enter(&curthread->t_delay_lock); 80169962b56SMatthew Ahrens while (cv_timedwait_hires(&curthread->t_delay_cv, 80269962b56SMatthew Ahrens &curthread->t_delay_lock, wakeup, zfs_delay_resolution_ns, 80369962b56SMatthew Ahrens CALLOUT_FLAG_ABSOLUTE | CALLOUT_FLAG_ROUNDUP) > 0) 80469962b56SMatthew Ahrens continue; 80569962b56SMatthew Ahrens mutex_exit(&curthread->t_delay_lock); 80669962b56SMatthew Ahrens #else 80769962b56SMatthew Ahrens hrtime_t delta = wakeup - gethrtime(); 80869962b56SMatthew Ahrens struct timespec ts; 80969962b56SMatthew Ahrens ts.tv_sec = delta / NANOSEC; 81069962b56SMatthew Ahrens ts.tv_nsec = delta % NANOSEC; 81169962b56SMatthew Ahrens (void) nanosleep(&ts, NULL); 81269962b56SMatthew Ahrens #endif 81369962b56SMatthew Ahrens } 81469962b56SMatthew Ahrens 81561e255ceSMatthew Ahrens /* 81661e255ceSMatthew Ahrens * This routine attempts to assign the transaction to a transaction group. 81761e255ceSMatthew Ahrens * To do so, we must determine if there is sufficient free space on disk. 81861e255ceSMatthew Ahrens * 81961e255ceSMatthew Ahrens * If this is a "netfree" transaction (i.e. we called dmu_tx_mark_netfree() 82061e255ceSMatthew Ahrens * on it), then it is assumed that there is sufficient free space, 82161e255ceSMatthew Ahrens * unless there's insufficient slop space in the pool (see the comment 82261e255ceSMatthew Ahrens * above spa_slop_shift in spa_misc.c). 82361e255ceSMatthew Ahrens * 82461e255ceSMatthew Ahrens * If it is not a "netfree" transaction, then if the data already on disk 82561e255ceSMatthew Ahrens * is over the allowed usage (e.g. quota), this will fail with EDQUOT or 82661e255ceSMatthew Ahrens * ENOSPC. Otherwise, if the current rough estimate of pending changes, 82761e255ceSMatthew Ahrens * plus the rough estimate of this transaction's changes, may exceed the 82861e255ceSMatthew Ahrens * allowed usage, then this will fail with ERESTART, which will cause the 82961e255ceSMatthew Ahrens * caller to wait for the pending changes to be written to disk (by waiting 83061e255ceSMatthew Ahrens * for the next TXG to open), and then check the space usage again. 83161e255ceSMatthew Ahrens * 83261e255ceSMatthew Ahrens * The rough estimate of pending changes is comprised of the sum of: 83361e255ceSMatthew Ahrens * 83461e255ceSMatthew Ahrens * - this transaction's holds' txh_space_towrite 83561e255ceSMatthew Ahrens * 83661e255ceSMatthew Ahrens * - dd_tempreserved[], which is the sum of in-flight transactions' 83761e255ceSMatthew Ahrens * holds' txh_space_towrite (i.e. those transactions that have called 83861e255ceSMatthew Ahrens * dmu_tx_assign() but not yet called dmu_tx_commit()). 83961e255ceSMatthew Ahrens * 84061e255ceSMatthew Ahrens * - dd_space_towrite[], which is the amount of dirtied dbufs. 84161e255ceSMatthew Ahrens * 84261e255ceSMatthew Ahrens * Note that all of these values are inflated by spa_get_worst_case_asize(), 84361e255ceSMatthew Ahrens * which means that we may get ERESTART well before we are actually in danger 84461e255ceSMatthew Ahrens * of running out of space, but this also mitigates any small inaccuracies 84561e255ceSMatthew Ahrens * in the rough estimate (e.g. txh_space_towrite doesn't take into account 84661e255ceSMatthew Ahrens * indirect blocks, and dd_space_towrite[] doesn't take into account changes 84761e255ceSMatthew Ahrens * to the MOS). 84861e255ceSMatthew Ahrens * 84961e255ceSMatthew Ahrens * Note that due to this algorithm, it is possible to exceed the allowed 85061e255ceSMatthew Ahrens * usage by one transaction. Also, as we approach the allowed usage, 85161e255ceSMatthew Ahrens * we will allow a very limited amount of changes into each TXG, thus 85261e255ceSMatthew Ahrens * decreasing performance. 85361e255ceSMatthew Ahrens */ 854fa9e4066Sahrens static int 8553b2aab18SMatthew Ahrens dmu_tx_try_assign(dmu_tx_t *tx, txg_how_t txg_how) 856fa9e4066Sahrens { 8570a4e9518Sgw spa_t *spa = tx->tx_pool->dp_spa; 858fa9e4066Sahrens 859fb09f5aaSMadhav Suresh ASSERT0(tx->tx_txg); 8600a4e9518Sgw 8618a2f1b91Sahrens if (tx->tx_err) 8628a2f1b91Sahrens return (tx->tx_err); 863fa9e4066Sahrens 864e14bb325SJeff Bonwick if (spa_suspended(spa)) { 8650a4e9518Sgw /* 8660a4e9518Sgw * If the user has indicated a blocking failure mode 8670a4e9518Sgw * then return ERESTART which will block in dmu_tx_wait(). 8680a4e9518Sgw * Otherwise, return EIO so that an error can get 8690a4e9518Sgw * propagated back to the VOP calls. 8700a4e9518Sgw * 8710a4e9518Sgw * Note that we always honor the txg_how flag regardless 8720a4e9518Sgw * of the failuremode setting. 8730a4e9518Sgw */ 8740a4e9518Sgw if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_CONTINUE && 8750a4e9518Sgw txg_how != TXG_WAIT) 876be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 8770a4e9518Sgw 878be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 8790a4e9518Sgw } 8800a4e9518Sgw 88169962b56SMatthew Ahrens if (!tx->tx_waited && 88269962b56SMatthew Ahrens dsl_pool_need_dirty_delay(tx->tx_pool)) { 88369962b56SMatthew Ahrens tx->tx_wait_dirty = B_TRUE; 88469962b56SMatthew Ahrens return (SET_ERROR(ERESTART)); 88569962b56SMatthew Ahrens } 88669962b56SMatthew Ahrens 887fa9e4066Sahrens tx->tx_txg = txg_hold_open(tx->tx_pool, &tx->tx_txgh); 8888a2f1b91Sahrens tx->tx_needassign_txh = NULL; 889fa9e4066Sahrens 8908a2f1b91Sahrens /* 8918a2f1b91Sahrens * NB: No error returns are allowed after txg_hold_open, but 8928a2f1b91Sahrens * before processing the dnode holds, due to the 8938a2f1b91Sahrens * dmu_tx_unassign() logic. 8948a2f1b91Sahrens */ 895fa9e4066Sahrens 89661e255ceSMatthew Ahrens uint64_t towrite = 0; 89761e255ceSMatthew Ahrens uint64_t tohold = 0; 89861e255ceSMatthew Ahrens for (dmu_tx_hold_t *txh = list_head(&tx->tx_holds); txh != NULL; 8998a2f1b91Sahrens txh = list_next(&tx->tx_holds, txh)) { 9008a2f1b91Sahrens dnode_t *dn = txh->txh_dnode; 901fa9e4066Sahrens if (dn != NULL) { 902fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 9038a2f1b91Sahrens if (dn->dn_assigned_txg == tx->tx_txg - 1) { 9048a2f1b91Sahrens mutex_exit(&dn->dn_mtx); 9058a2f1b91Sahrens tx->tx_needassign_txh = txh; 906be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 907fa9e4066Sahrens } 9088a2f1b91Sahrens if (dn->dn_assigned_txg == 0) 909fa9e4066Sahrens dn->dn_assigned_txg = tx->tx_txg; 9108a2f1b91Sahrens ASSERT3U(dn->dn_assigned_txg, ==, tx->tx_txg); 911fa9e4066Sahrens (void) refcount_add(&dn->dn_tx_holds, tx); 912fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 913fa9e4066Sahrens } 9140c779ad4SMatthew Ahrens towrite += refcount_count(&txh->txh_space_towrite); 9150c779ad4SMatthew Ahrens tohold += refcount_count(&txh->txh_memory_tohold); 916fa9e4066Sahrens } 917fa9e4066Sahrens 918cdb0ab79Smaybee /* needed allocation: worst-case estimate of write space */ 91961e255ceSMatthew Ahrens uint64_t asize = spa_get_worst_case_asize(tx->tx_pool->dp_spa, towrite); 920cdb0ab79Smaybee /* calculate memory footprint estimate */ 92161e255ceSMatthew Ahrens uint64_t memory = towrite + tohold; 922fa9e4066Sahrens 92361e255ceSMatthew Ahrens if (tx->tx_dir != NULL && asize != 0) { 924cdb0ab79Smaybee int err = dsl_dir_tempreserve_space(tx->tx_dir, memory, 92561e255ceSMatthew Ahrens asize, tx->tx_netfree, &tx->tx_tempreserve_cookie, tx); 92661e255ceSMatthew Ahrens if (err != 0) 927fa9e4066Sahrens return (err); 928fa9e4066Sahrens } 929fa9e4066Sahrens 930fa9e4066Sahrens return (0); 931fa9e4066Sahrens } 932fa9e4066Sahrens 9338a2f1b91Sahrens static void 9348a2f1b91Sahrens dmu_tx_unassign(dmu_tx_t *tx) 935fa9e4066Sahrens { 9368a2f1b91Sahrens if (tx->tx_txg == 0) 9378a2f1b91Sahrens return; 938fa9e4066Sahrens 939fa9e4066Sahrens txg_rele_to_quiesce(&tx->tx_txgh); 940fa9e4066Sahrens 9413e30c24aSWill Andrews /* 9423e30c24aSWill Andrews * Walk the transaction's hold list, removing the hold on the 9433e30c24aSWill Andrews * associated dnode, and notifying waiters if the refcount drops to 0. 9443e30c24aSWill Andrews */ 94561e255ceSMatthew Ahrens for (dmu_tx_hold_t *txh = list_head(&tx->tx_holds); 94661e255ceSMatthew Ahrens txh != tx->tx_needassign_txh; 9478a2f1b91Sahrens txh = list_next(&tx->tx_holds, txh)) { 9488a2f1b91Sahrens dnode_t *dn = txh->txh_dnode; 949fa9e4066Sahrens 950fa9e4066Sahrens if (dn == NULL) 951fa9e4066Sahrens continue; 952fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 9538a2f1b91Sahrens ASSERT3U(dn->dn_assigned_txg, ==, tx->tx_txg); 954fa9e4066Sahrens 955fa9e4066Sahrens if (refcount_remove(&dn->dn_tx_holds, tx) == 0) { 956fa9e4066Sahrens dn->dn_assigned_txg = 0; 957fa9e4066Sahrens cv_broadcast(&dn->dn_notxholds); 958fa9e4066Sahrens } 959fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 960fa9e4066Sahrens } 961fa9e4066Sahrens 962fa9e4066Sahrens txg_rele_to_sync(&tx->tx_txgh); 963fa9e4066Sahrens 9648a2f1b91Sahrens tx->tx_lasttried_txg = tx->tx_txg; 965fa9e4066Sahrens tx->tx_txg = 0; 966fa9e4066Sahrens } 967fa9e4066Sahrens 968fa9e4066Sahrens /* 969fa9e4066Sahrens * Assign tx to a transaction group. txg_how can be one of: 970fa9e4066Sahrens * 971fa9e4066Sahrens * (1) TXG_WAIT. If the current open txg is full, waits until there's 972fa9e4066Sahrens * a new one. This should be used when you're not holding locks. 9733b2aab18SMatthew Ahrens * It will only fail if we're truly out of space (or over quota). 974fa9e4066Sahrens * 975fa9e4066Sahrens * (2) TXG_NOWAIT. If we can't assign into the current open txg without 976fa9e4066Sahrens * blocking, returns immediately with ERESTART. This should be used 977fa9e4066Sahrens * whenever you're holding locks. On an ERESTART error, the caller 9788a2f1b91Sahrens * should drop locks, do a dmu_tx_wait(tx), and try again. 97969962b56SMatthew Ahrens * 98069962b56SMatthew Ahrens * (3) TXG_WAITED. Like TXG_NOWAIT, but indicates that dmu_tx_wait() 98169962b56SMatthew Ahrens * has already been called on behalf of this operation (though 98269962b56SMatthew Ahrens * most likely on a different tx). 983fa9e4066Sahrens */ 984fa9e4066Sahrens int 9853b2aab18SMatthew Ahrens dmu_tx_assign(dmu_tx_t *tx, txg_how_t txg_how) 986fa9e4066Sahrens { 987fa9e4066Sahrens int err; 988fa9e4066Sahrens 989fa9e4066Sahrens ASSERT(tx->tx_txg == 0); 99069962b56SMatthew Ahrens ASSERT(txg_how == TXG_WAIT || txg_how == TXG_NOWAIT || 99169962b56SMatthew Ahrens txg_how == TXG_WAITED); 992fa9e4066Sahrens ASSERT(!dsl_pool_sync_context(tx->tx_pool)); 993fa9e4066Sahrens 9943b2aab18SMatthew Ahrens /* If we might wait, we must not hold the config lock. */ 9953b2aab18SMatthew Ahrens ASSERT(txg_how != TXG_WAIT || !dsl_pool_config_held(tx->tx_pool)); 9963b2aab18SMatthew Ahrens 99769962b56SMatthew Ahrens if (txg_how == TXG_WAITED) 99869962b56SMatthew Ahrens tx->tx_waited = B_TRUE; 99969962b56SMatthew Ahrens 10008a2f1b91Sahrens while ((err = dmu_tx_try_assign(tx, txg_how)) != 0) { 10018a2f1b91Sahrens dmu_tx_unassign(tx); 1002fa9e4066Sahrens 1003fa9e4066Sahrens if (err != ERESTART || txg_how != TXG_WAIT) 1004fa9e4066Sahrens return (err); 1005fa9e4066Sahrens 10068a2f1b91Sahrens dmu_tx_wait(tx); 1007fa9e4066Sahrens } 1008fa9e4066Sahrens 1009fa9e4066Sahrens txg_rele_to_quiesce(&tx->tx_txgh); 1010fa9e4066Sahrens 1011fa9e4066Sahrens return (0); 1012fa9e4066Sahrens } 1013fa9e4066Sahrens 10148a2f1b91Sahrens void 10158a2f1b91Sahrens dmu_tx_wait(dmu_tx_t *tx) 10168a2f1b91Sahrens { 10170a4e9518Sgw spa_t *spa = tx->tx_pool->dp_spa; 101869962b56SMatthew Ahrens dsl_pool_t *dp = tx->tx_pool; 10190a4e9518Sgw 10208a2f1b91Sahrens ASSERT(tx->tx_txg == 0); 10213b2aab18SMatthew Ahrens ASSERT(!dsl_pool_config_held(tx->tx_pool)); 10228a2f1b91Sahrens 102369962b56SMatthew Ahrens if (tx->tx_wait_dirty) { 102469962b56SMatthew Ahrens /* 102569962b56SMatthew Ahrens * dmu_tx_try_assign() has determined that we need to wait 102669962b56SMatthew Ahrens * because we've consumed much or all of the dirty buffer 102769962b56SMatthew Ahrens * space. 102869962b56SMatthew Ahrens */ 102969962b56SMatthew Ahrens mutex_enter(&dp->dp_lock); 103069962b56SMatthew Ahrens while (dp->dp_dirty_total >= zfs_dirty_data_max) 103169962b56SMatthew Ahrens cv_wait(&dp->dp_spaceavail_cv, &dp->dp_lock); 103269962b56SMatthew Ahrens uint64_t dirty = dp->dp_dirty_total; 103369962b56SMatthew Ahrens mutex_exit(&dp->dp_lock); 103469962b56SMatthew Ahrens 103569962b56SMatthew Ahrens dmu_tx_delay(tx, dirty); 103669962b56SMatthew Ahrens 103769962b56SMatthew Ahrens tx->tx_wait_dirty = B_FALSE; 103869962b56SMatthew Ahrens 103969962b56SMatthew Ahrens /* 104069962b56SMatthew Ahrens * Note: setting tx_waited only has effect if the caller 104169962b56SMatthew Ahrens * used TX_WAIT. Otherwise they are going to destroy 104269962b56SMatthew Ahrens * this tx and try again. The common case, zfs_write(), 104369962b56SMatthew Ahrens * uses TX_WAIT. 104469962b56SMatthew Ahrens */ 104569962b56SMatthew Ahrens tx->tx_waited = B_TRUE; 104669962b56SMatthew Ahrens } else if (spa_suspended(spa) || tx->tx_lasttried_txg == 0) { 104769962b56SMatthew Ahrens /* 104869962b56SMatthew Ahrens * If the pool is suspended we need to wait until it 104969962b56SMatthew Ahrens * is resumed. Note that it's possible that the pool 105069962b56SMatthew Ahrens * has become active after this thread has tried to 105169962b56SMatthew Ahrens * obtain a tx. If that's the case then tx_lasttried_txg 105269962b56SMatthew Ahrens * would not have been set. 105369962b56SMatthew Ahrens */ 105469962b56SMatthew Ahrens txg_wait_synced(dp, spa_last_synced_txg(spa) + 1); 10550a4e9518Sgw } else if (tx->tx_needassign_txh) { 105669962b56SMatthew Ahrens /* 105769962b56SMatthew Ahrens * A dnode is assigned to the quiescing txg. Wait for its 105869962b56SMatthew Ahrens * transaction to complete. 105969962b56SMatthew Ahrens */ 10608a2f1b91Sahrens dnode_t *dn = tx->tx_needassign_txh->txh_dnode; 10618a2f1b91Sahrens 10628a2f1b91Sahrens mutex_enter(&dn->dn_mtx); 10638a2f1b91Sahrens while (dn->dn_assigned_txg == tx->tx_lasttried_txg - 1) 10648a2f1b91Sahrens cv_wait(&dn->dn_notxholds, &dn->dn_mtx); 10658a2f1b91Sahrens mutex_exit(&dn->dn_mtx); 10668a2f1b91Sahrens tx->tx_needassign_txh = NULL; 10678a2f1b91Sahrens } else { 10688a2f1b91Sahrens txg_wait_open(tx->tx_pool, tx->tx_lasttried_txg + 1); 10698a2f1b91Sahrens } 10708a2f1b91Sahrens } 10718a2f1b91Sahrens 10720c779ad4SMatthew Ahrens static void 10730c779ad4SMatthew Ahrens dmu_tx_destroy(dmu_tx_t *tx) 1074fa9e4066Sahrens { 10758a2f1b91Sahrens dmu_tx_hold_t *txh; 1076fa9e4066Sahrens 10770c779ad4SMatthew Ahrens while ((txh = list_head(&tx->tx_holds)) != NULL) { 10780c779ad4SMatthew Ahrens dnode_t *dn = txh->txh_dnode; 10790c779ad4SMatthew Ahrens 10800c779ad4SMatthew Ahrens list_remove(&tx->tx_holds, txh); 10810c779ad4SMatthew Ahrens refcount_destroy_many(&txh->txh_space_towrite, 10820c779ad4SMatthew Ahrens refcount_count(&txh->txh_space_towrite)); 10830c779ad4SMatthew Ahrens refcount_destroy_many(&txh->txh_memory_tohold, 10840c779ad4SMatthew Ahrens refcount_count(&txh->txh_memory_tohold)); 10850c779ad4SMatthew Ahrens kmem_free(txh, sizeof (dmu_tx_hold_t)); 10860c779ad4SMatthew Ahrens if (dn != NULL) 10870c779ad4SMatthew Ahrens dnode_rele(dn, tx); 10880c779ad4SMatthew Ahrens } 10890c779ad4SMatthew Ahrens 10900c779ad4SMatthew Ahrens list_destroy(&tx->tx_callbacks); 10910c779ad4SMatthew Ahrens list_destroy(&tx->tx_holds); 10920c779ad4SMatthew Ahrens kmem_free(tx, sizeof (dmu_tx_t)); 10930c779ad4SMatthew Ahrens } 10940c779ad4SMatthew Ahrens 10950c779ad4SMatthew Ahrens void 10960c779ad4SMatthew Ahrens dmu_tx_commit(dmu_tx_t *tx) 10970c779ad4SMatthew Ahrens { 1098fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1099fa9e4066Sahrens 11003e30c24aSWill Andrews /* 11013e30c24aSWill Andrews * Go through the transaction's hold list and remove holds on 11023e30c24aSWill Andrews * associated dnodes, notifying waiters if no holds remain. 11033e30c24aSWill Andrews */ 11040c779ad4SMatthew Ahrens for (dmu_tx_hold_t *txh = list_head(&tx->tx_holds); txh != NULL; 11050c779ad4SMatthew Ahrens txh = list_next(&tx->tx_holds, txh)) { 11068a2f1b91Sahrens dnode_t *dn = txh->txh_dnode; 1107fa9e4066Sahrens 1108fa9e4066Sahrens if (dn == NULL) 1109fa9e4066Sahrens continue; 11100c779ad4SMatthew Ahrens 1111fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1112fa9e4066Sahrens ASSERT3U(dn->dn_assigned_txg, ==, tx->tx_txg); 1113fa9e4066Sahrens 1114fa9e4066Sahrens if (refcount_remove(&dn->dn_tx_holds, tx) == 0) { 1115fa9e4066Sahrens dn->dn_assigned_txg = 0; 1116fa9e4066Sahrens cv_broadcast(&dn->dn_notxholds); 1117fa9e4066Sahrens } 1118fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1119fa9e4066Sahrens } 1120fa9e4066Sahrens 11218a2f1b91Sahrens if (tx->tx_tempreserve_cookie) 1122fa9e4066Sahrens dsl_dir_tempreserve_clear(tx->tx_tempreserve_cookie, tx); 1123fa9e4066Sahrens 1124d20e665cSRicardo M. Correia if (!list_is_empty(&tx->tx_callbacks)) 1125d20e665cSRicardo M. Correia txg_register_callbacks(&tx->tx_txgh, &tx->tx_callbacks); 1126d20e665cSRicardo M. Correia 1127fa9e4066Sahrens if (tx->tx_anyobj == FALSE) 1128fa9e4066Sahrens txg_rele_to_sync(&tx->tx_txgh); 1129d20e665cSRicardo M. Correia 11300c779ad4SMatthew Ahrens dmu_tx_destroy(tx); 1131fa9e4066Sahrens } 1132fa9e4066Sahrens 1133fa9e4066Sahrens void 1134fa9e4066Sahrens dmu_tx_abort(dmu_tx_t *tx) 1135fa9e4066Sahrens { 1136fa9e4066Sahrens ASSERT(tx->tx_txg == 0); 1137fa9e4066Sahrens 1138d20e665cSRicardo M. Correia /* 1139d20e665cSRicardo M. Correia * Call any registered callbacks with an error code. 1140d20e665cSRicardo M. Correia */ 1141d20e665cSRicardo M. Correia if (!list_is_empty(&tx->tx_callbacks)) 1142d20e665cSRicardo M. Correia dmu_tx_do_callbacks(&tx->tx_callbacks, ECANCELED); 1143d20e665cSRicardo M. Correia 11440c779ad4SMatthew Ahrens dmu_tx_destroy(tx); 1145fa9e4066Sahrens } 1146fa9e4066Sahrens 1147fa9e4066Sahrens uint64_t 1148fa9e4066Sahrens dmu_tx_get_txg(dmu_tx_t *tx) 1149fa9e4066Sahrens { 1150fa9e4066Sahrens ASSERT(tx->tx_txg != 0); 1151fa9e4066Sahrens return (tx->tx_txg); 1152fa9e4066Sahrens } 1153d20e665cSRicardo M. Correia 11543b2aab18SMatthew Ahrens dsl_pool_t * 11553b2aab18SMatthew Ahrens dmu_tx_pool(dmu_tx_t *tx) 11563b2aab18SMatthew Ahrens { 11573b2aab18SMatthew Ahrens ASSERT(tx->tx_pool != NULL); 11583b2aab18SMatthew Ahrens return (tx->tx_pool); 11593b2aab18SMatthew Ahrens } 11603b2aab18SMatthew Ahrens 1161d20e665cSRicardo M. Correia void 1162d20e665cSRicardo M. Correia dmu_tx_callback_register(dmu_tx_t *tx, dmu_tx_callback_func_t *func, void *data) 1163d20e665cSRicardo M. Correia { 1164d20e665cSRicardo M. Correia dmu_tx_callback_t *dcb; 1165d20e665cSRicardo M. Correia 1166d20e665cSRicardo M. Correia dcb = kmem_alloc(sizeof (dmu_tx_callback_t), KM_SLEEP); 1167d20e665cSRicardo M. Correia 1168d20e665cSRicardo M. Correia dcb->dcb_func = func; 1169d20e665cSRicardo M. Correia dcb->dcb_data = data; 1170d20e665cSRicardo M. Correia 1171d20e665cSRicardo M. Correia list_insert_tail(&tx->tx_callbacks, dcb); 1172d20e665cSRicardo M. Correia } 1173d20e665cSRicardo M. Correia 1174d20e665cSRicardo M. Correia /* 1175d20e665cSRicardo M. Correia * Call all the commit callbacks on a list, with a given error code. 1176d20e665cSRicardo M. Correia */ 1177d20e665cSRicardo M. Correia void 1178d20e665cSRicardo M. Correia dmu_tx_do_callbacks(list_t *cb_list, int error) 1179d20e665cSRicardo M. Correia { 1180d20e665cSRicardo M. Correia dmu_tx_callback_t *dcb; 1181d20e665cSRicardo M. Correia 11820c779ad4SMatthew Ahrens while ((dcb = list_head(cb_list)) != NULL) { 1183d20e665cSRicardo M. Correia list_remove(cb_list, dcb); 1184d20e665cSRicardo M. Correia dcb->dcb_func(dcb->dcb_data, error); 1185d20e665cSRicardo M. Correia kmem_free(dcb, sizeof (dmu_tx_callback_t)); 1186d20e665cSRicardo M. Correia } 1187d20e665cSRicardo M. Correia } 11880a586ceaSMark Shellenbaum 11890a586ceaSMark Shellenbaum /* 11900a586ceaSMark Shellenbaum * Interface to hold a bunch of attributes. 11910a586ceaSMark Shellenbaum * used for creating new files. 11920a586ceaSMark Shellenbaum * attrsize is the total size of all attributes 11930a586ceaSMark Shellenbaum * to be added during object creation 11940a586ceaSMark Shellenbaum * 11950a586ceaSMark Shellenbaum * For updating/adding a single attribute dmu_tx_hold_sa() should be used. 11960a586ceaSMark Shellenbaum */ 11970a586ceaSMark Shellenbaum 11980a586ceaSMark Shellenbaum /* 11990a586ceaSMark Shellenbaum * hold necessary attribute name for attribute registration. 12000a586ceaSMark Shellenbaum * should be a very rare case where this is needed. If it does 12010a586ceaSMark Shellenbaum * happen it would only happen on the first write to the file system. 12020a586ceaSMark Shellenbaum */ 12030a586ceaSMark Shellenbaum static void 12040a586ceaSMark Shellenbaum dmu_tx_sa_registration_hold(sa_os_t *sa, dmu_tx_t *tx) 12050a586ceaSMark Shellenbaum { 12060a586ceaSMark Shellenbaum if (!sa->sa_need_attr_registration) 12070a586ceaSMark Shellenbaum return; 12080a586ceaSMark Shellenbaum 120961e255ceSMatthew Ahrens for (int i = 0; i != sa->sa_num_attrs; i++) { 12100a586ceaSMark Shellenbaum if (!sa->sa_attr_table[i].sa_registered) { 12110a586ceaSMark Shellenbaum if (sa->sa_reg_attr_obj) 12120a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_reg_attr_obj, 12130a586ceaSMark Shellenbaum B_TRUE, sa->sa_attr_table[i].sa_name); 12140a586ceaSMark Shellenbaum else 12150a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, 12160a586ceaSMark Shellenbaum B_TRUE, sa->sa_attr_table[i].sa_name); 12170a586ceaSMark Shellenbaum } 12180a586ceaSMark Shellenbaum } 12190a586ceaSMark Shellenbaum } 12200a586ceaSMark Shellenbaum 12210a586ceaSMark Shellenbaum void 12220a586ceaSMark Shellenbaum dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object) 12230a586ceaSMark Shellenbaum { 122461e255ceSMatthew Ahrens dmu_tx_hold_t *txh = dmu_tx_hold_object_impl(tx, 122561e255ceSMatthew Ahrens tx->tx_objset, object, THT_SPILL, 0, 0); 12260a586ceaSMark Shellenbaum 122761e255ceSMatthew Ahrens (void) refcount_add_many(&txh->txh_space_towrite, 122861e255ceSMatthew Ahrens SPA_OLD_MAXBLOCKSIZE, FTAG); 12290a586ceaSMark Shellenbaum } 12300a586ceaSMark Shellenbaum 12310a586ceaSMark Shellenbaum void 12320a586ceaSMark Shellenbaum dmu_tx_hold_sa_create(dmu_tx_t *tx, int attrsize) 12330a586ceaSMark Shellenbaum { 12340a586ceaSMark Shellenbaum sa_os_t *sa = tx->tx_objset->os_sa; 12350a586ceaSMark Shellenbaum 12360a586ceaSMark Shellenbaum dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT); 12370a586ceaSMark Shellenbaum 12380a586ceaSMark Shellenbaum if (tx->tx_objset->os_sa->sa_master_obj == 0) 12390a586ceaSMark Shellenbaum return; 12400a586ceaSMark Shellenbaum 124161e255ceSMatthew Ahrens if (tx->tx_objset->os_sa->sa_layout_attr_obj) { 12420a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_layout_attr_obj, B_TRUE, NULL); 124361e255ceSMatthew Ahrens } else { 12440a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_master_obj, B_TRUE, SA_LAYOUTS); 12450a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_master_obj, B_TRUE, SA_REGISTRY); 12460a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL); 12470a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL); 12480a586ceaSMark Shellenbaum } 12490a586ceaSMark Shellenbaum 12500a586ceaSMark Shellenbaum dmu_tx_sa_registration_hold(sa, tx); 12510a586ceaSMark Shellenbaum 12520a586ceaSMark Shellenbaum if (attrsize <= DN_MAX_BONUSLEN && !sa->sa_force_spill) 12530a586ceaSMark Shellenbaum return; 12540a586ceaSMark Shellenbaum 12550a586ceaSMark Shellenbaum (void) dmu_tx_hold_object_impl(tx, tx->tx_objset, DMU_NEW_OBJECT, 12560a586ceaSMark Shellenbaum THT_SPILL, 0, 0); 12570a586ceaSMark Shellenbaum } 12580a586ceaSMark Shellenbaum 12590a586ceaSMark Shellenbaum /* 12600a586ceaSMark Shellenbaum * Hold SA attribute 12610a586ceaSMark Shellenbaum * 12620a586ceaSMark Shellenbaum * dmu_tx_hold_sa(dmu_tx_t *tx, sa_handle_t *, attribute, add, size) 12630a586ceaSMark Shellenbaum * 12640a586ceaSMark Shellenbaum * variable_size is the total size of all variable sized attributes 12650a586ceaSMark Shellenbaum * passed to this function. It is not the total size of all 12660a586ceaSMark Shellenbaum * variable size attributes that *may* exist on this object. 12670a586ceaSMark Shellenbaum */ 12680a586ceaSMark Shellenbaum void 12690a586ceaSMark Shellenbaum dmu_tx_hold_sa(dmu_tx_t *tx, sa_handle_t *hdl, boolean_t may_grow) 12700a586ceaSMark Shellenbaum { 12710a586ceaSMark Shellenbaum uint64_t object; 12720a586ceaSMark Shellenbaum sa_os_t *sa = tx->tx_objset->os_sa; 12730a586ceaSMark Shellenbaum 12740a586ceaSMark Shellenbaum ASSERT(hdl != NULL); 12750a586ceaSMark Shellenbaum 12760a586ceaSMark Shellenbaum object = sa_handle_object(hdl); 12770a586ceaSMark Shellenbaum 12780a586ceaSMark Shellenbaum dmu_tx_hold_bonus(tx, object); 12790a586ceaSMark Shellenbaum 12800a586ceaSMark Shellenbaum if (tx->tx_objset->os_sa->sa_master_obj == 0) 12810a586ceaSMark Shellenbaum return; 12820a586ceaSMark Shellenbaum 12830a586ceaSMark Shellenbaum if (tx->tx_objset->os_sa->sa_reg_attr_obj == 0 || 12840a586ceaSMark Shellenbaum tx->tx_objset->os_sa->sa_layout_attr_obj == 0) { 12850a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_master_obj, B_TRUE, SA_LAYOUTS); 12860a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_master_obj, B_TRUE, SA_REGISTRY); 12870a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL); 12880a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, B_TRUE, NULL); 12890a586ceaSMark Shellenbaum } 12900a586ceaSMark Shellenbaum 12910a586ceaSMark Shellenbaum dmu_tx_sa_registration_hold(sa, tx); 12920a586ceaSMark Shellenbaum 12930a586ceaSMark Shellenbaum if (may_grow && tx->tx_objset->os_sa->sa_layout_attr_obj) 12940a586ceaSMark Shellenbaum dmu_tx_hold_zap(tx, sa->sa_layout_attr_obj, B_TRUE, NULL); 12950a586ceaSMark Shellenbaum 1296744947dcSTom Erickson if (sa->sa_force_spill || may_grow || hdl->sa_spill) { 12970a586ceaSMark Shellenbaum ASSERT(tx->tx_txg == 0); 12980a586ceaSMark Shellenbaum dmu_tx_hold_spill(tx, object); 1299744947dcSTom Erickson } else { 1300744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus; 1301744947dcSTom Erickson dnode_t *dn; 1302744947dcSTom Erickson 1303744947dcSTom Erickson DB_DNODE_ENTER(db); 1304744947dcSTom Erickson dn = DB_DNODE(db); 1305744947dcSTom Erickson if (dn->dn_have_spill) { 1306744947dcSTom Erickson ASSERT(tx->tx_txg == 0); 1307744947dcSTom Erickson dmu_tx_hold_spill(tx, object); 1308744947dcSTom Erickson } 1309744947dcSTom Erickson DB_DNODE_EXIT(db); 13100a586ceaSMark Shellenbaum } 13110a586ceaSMark Shellenbaum } 1312