1fa9e4066Sahrens /* 2fa9e4066Sahrens * CDDL HEADER START 3fa9e4066Sahrens * 4fa9e4066Sahrens * The contents of this file are subject to the terms of the 5ea8dc4b6Seschrock * Common Development and Distribution License (the "License"). 6ea8dc4b6Seschrock * You may not use this file except in compliance with the License. 7fa9e4066Sahrens * 8fa9e4066Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fa9e4066Sahrens * or http://www.opensolaris.org/os/licensing. 10fa9e4066Sahrens * See the License for the specific language governing permissions 11fa9e4066Sahrens * and limitations under the License. 12fa9e4066Sahrens * 13fa9e4066Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14fa9e4066Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fa9e4066Sahrens * If applicable, add the following below this CDDL HEADER, with the 16fa9e4066Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17fa9e4066Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18fa9e4066Sahrens * 19fa9e4066Sahrens * CDDL HEADER END 20fa9e4066Sahrens */ 21fa9e4066Sahrens /* 2294d1a210STim Haley * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 238df0bcf0SPaul Dagnelie * Copyright (c) 2011, 2016 by Delphix. All rights reserved. 24fa9e4066Sahrens */ 25aad02571SSaso Kiselkov /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */ 26810e43b2SBill Pijewski /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */ 27b8289d24SDaniil Lunev /* Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved. */ 28aad02571SSaso Kiselkov 29fa9e4066Sahrens #include <sys/dmu.h> 30fa9e4066Sahrens #include <sys/dmu_impl.h> 31fa9e4066Sahrens #include <sys/dmu_tx.h> 32fa9e4066Sahrens #include <sys/dbuf.h> 33fa9e4066Sahrens #include <sys/dnode.h> 34fa9e4066Sahrens #include <sys/zfs_context.h> 35fa9e4066Sahrens #include <sys/dmu_objset.h> 36fa9e4066Sahrens #include <sys/dmu_traverse.h> 37fa9e4066Sahrens #include <sys/dsl_dataset.h> 38fa9e4066Sahrens #include <sys/dsl_dir.h> 39fa9e4066Sahrens #include <sys/dsl_pool.h> 401d452cf5Sahrens #include <sys/dsl_synctask.h> 41a2eea2e1Sahrens #include <sys/dsl_prop.h> 42fa9e4066Sahrens #include <sys/dmu_zfetch.h> 43fa9e4066Sahrens #include <sys/zfs_ioctl.h> 44fa9e4066Sahrens #include <sys/zap.h> 45ea8dc4b6Seschrock #include <sys/zio_checksum.h> 4680901aeaSGeorge Wilson #include <sys/zio_compress.h> 470a586ceaSMark Shellenbaum #include <sys/sa.h> 48b8289d24SDaniil Lunev #include <sys/zfeature.h> 4944eda4d7Smaybee #ifdef _KERNEL 5044eda4d7Smaybee #include <sys/vmsystm.h> 510fab61baSJonathan W Adams #include <sys/zfs_znode.h> 5244eda4d7Smaybee #endif 53fa9e4066Sahrens 5480901aeaSGeorge Wilson /* 5580901aeaSGeorge Wilson * Enable/disable nopwrite feature. 5680901aeaSGeorge Wilson */ 5780901aeaSGeorge Wilson int zfs_nopwrite_enabled = 1; 5880901aeaSGeorge Wilson 59fa9e4066Sahrens const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = { 60ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "unallocated" }, 61ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "object directory" }, 62ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "object array" }, 63ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "packed nvlist" }, 64ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "packed nvlist size" }, 65ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "bpobj" }, 66ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "bpobj header" }, 67ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "SPA space map header" }, 68ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "SPA space map" }, 69ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "ZIL intent log" }, 70ad135b5dSChristopher Siden { DMU_BSWAP_DNODE, TRUE, "DMU dnode" }, 71ad135b5dSChristopher Siden { DMU_BSWAP_OBJSET, TRUE, "DMU objset" }, 72ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "DSL directory" }, 73ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL directory child map"}, 74ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL dataset snap map" }, 75ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL props" }, 76ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "DSL dataset" }, 77ad135b5dSChristopher Siden { DMU_BSWAP_ZNODE, TRUE, "ZFS znode" }, 78ad135b5dSChristopher Siden { DMU_BSWAP_OLDACL, TRUE, "ZFS V0 ACL" }, 79ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, FALSE, "ZFS plain file" }, 80ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "ZFS directory" }, 81ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "ZFS master node" }, 82ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "ZFS delete queue" }, 83ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, FALSE, "zvol object" }, 84ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "zvol prop" }, 85ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, FALSE, "other uint8[]" }, 86ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, FALSE, "other uint64[]" }, 87ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "other ZAP" }, 88ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "persistent error log" }, 89ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "SPA history" }, 90ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "SPA history offsets" }, 91ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "Pool properties" }, 92ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL permissions" }, 93ad135b5dSChristopher Siden { DMU_BSWAP_ACL, TRUE, "ZFS ACL" }, 94ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "ZFS SYSACL" }, 95ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "FUID table" }, 96ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "FUID table size" }, 97ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL dataset next clones"}, 98ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "scan work queue" }, 99ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "ZFS user/group used" }, 100ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "ZFS user/group quota" }, 101ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "snapshot refcount tags"}, 102ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DDT ZAP algorithm" }, 103ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DDT statistics" }, 104ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, TRUE, "System attributes" }, 105ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "SA master node" }, 106ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "SA attr registration" }, 107ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "SA attr layouts" }, 108ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "scan translations" }, 109ad135b5dSChristopher Siden { DMU_BSWAP_UINT8, FALSE, "deduplicated block" }, 110ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL deadlist map" }, 111ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "DSL deadlist map hdr" }, 112ad135b5dSChristopher Siden { DMU_BSWAP_ZAP, TRUE, "DSL dir clones" }, 113ad135b5dSChristopher Siden { DMU_BSWAP_UINT64, TRUE, "bpobj subobj" } 114ad135b5dSChristopher Siden }; 115ad135b5dSChristopher Siden 116ad135b5dSChristopher Siden const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = { 117ad135b5dSChristopher Siden { byteswap_uint8_array, "uint8" }, 118ad135b5dSChristopher Siden { byteswap_uint16_array, "uint16" }, 119ad135b5dSChristopher Siden { byteswap_uint32_array, "uint32" }, 120ad135b5dSChristopher Siden { byteswap_uint64_array, "uint64" }, 121ad135b5dSChristopher Siden { zap_byteswap, "zap" }, 122ad135b5dSChristopher Siden { dnode_buf_byteswap, "dnode" }, 123ad135b5dSChristopher Siden { dmu_objset_byteswap, "objset" }, 124ad135b5dSChristopher Siden { zfs_znode_byteswap, "znode" }, 125ad135b5dSChristopher Siden { zfs_oldacl_byteswap, "oldacl" }, 126ad135b5dSChristopher Siden { zfs_acl_byteswap, "acl" } 1273f9d6ad7SLin Ling }; 128fa9e4066Sahrens 129*79d72832SMatthew Ahrens int 130*79d72832SMatthew Ahrens dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset, 131*79d72832SMatthew Ahrens void *tag, dmu_buf_t **dbp) 132*79d72832SMatthew Ahrens { 133*79d72832SMatthew Ahrens uint64_t blkid; 134*79d72832SMatthew Ahrens dmu_buf_impl_t *db; 135*79d72832SMatthew Ahrens 136*79d72832SMatthew Ahrens blkid = dbuf_whichblock(dn, 0, offset); 137*79d72832SMatthew Ahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 138*79d72832SMatthew Ahrens db = dbuf_hold(dn, blkid, tag); 139*79d72832SMatthew Ahrens rw_exit(&dn->dn_struct_rwlock); 140*79d72832SMatthew Ahrens 141*79d72832SMatthew Ahrens if (db == NULL) { 142*79d72832SMatthew Ahrens *dbp = NULL; 143*79d72832SMatthew Ahrens return (SET_ERROR(EIO)); 144*79d72832SMatthew Ahrens } 145*79d72832SMatthew Ahrens 146*79d72832SMatthew Ahrens *dbp = &db->db; 147*79d72832SMatthew Ahrens return (0); 148*79d72832SMatthew Ahrens } 149fa9e4066Sahrens int 1505d7b4d43SMatthew Ahrens dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset, 1515d7b4d43SMatthew Ahrens void *tag, dmu_buf_t **dbp) 152fa9e4066Sahrens { 153fa9e4066Sahrens dnode_t *dn; 154fa9e4066Sahrens uint64_t blkid; 155fa9e4066Sahrens dmu_buf_impl_t *db; 156ea8dc4b6Seschrock int err; 157fa9e4066Sahrens 158503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 159ea8dc4b6Seschrock if (err) 160ea8dc4b6Seschrock return (err); 161a2cdcdd2SPaul Dagnelie blkid = dbuf_whichblock(dn, 0, offset); 162fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 163ea8dc4b6Seschrock db = dbuf_hold(dn, blkid, tag); 164fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1655d7b4d43SMatthew Ahrens dnode_rele(dn, FTAG); 1665d7b4d43SMatthew Ahrens 167ea8dc4b6Seschrock if (db == NULL) { 1685d7b4d43SMatthew Ahrens *dbp = NULL; 1695d7b4d43SMatthew Ahrens return (SET_ERROR(EIO)); 1705d7b4d43SMatthew Ahrens } 1715d7b4d43SMatthew Ahrens 1725d7b4d43SMatthew Ahrens *dbp = &db->db; 1735d7b4d43SMatthew Ahrens return (err); 1745d7b4d43SMatthew Ahrens } 1755d7b4d43SMatthew Ahrens 176*79d72832SMatthew Ahrens int 177*79d72832SMatthew Ahrens dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset, 178*79d72832SMatthew Ahrens void *tag, dmu_buf_t **dbp, int flags) 179*79d72832SMatthew Ahrens { 180*79d72832SMatthew Ahrens int err; 181*79d72832SMatthew Ahrens int db_flags = DB_RF_CANFAIL; 182*79d72832SMatthew Ahrens 183*79d72832SMatthew Ahrens if (flags & DMU_READ_NO_PREFETCH) 184*79d72832SMatthew Ahrens db_flags |= DB_RF_NOPREFETCH; 185*79d72832SMatthew Ahrens 186*79d72832SMatthew Ahrens err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp); 187*79d72832SMatthew Ahrens if (err == 0) { 188*79d72832SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 189*79d72832SMatthew Ahrens err = dbuf_read(db, NULL, db_flags); 190*79d72832SMatthew Ahrens if (err != 0) { 191*79d72832SMatthew Ahrens dbuf_rele(db, tag); 192*79d72832SMatthew Ahrens *dbp = NULL; 193*79d72832SMatthew Ahrens } 194*79d72832SMatthew Ahrens } 195*79d72832SMatthew Ahrens 196*79d72832SMatthew Ahrens return (err); 197*79d72832SMatthew Ahrens } 198*79d72832SMatthew Ahrens 1995d7b4d43SMatthew Ahrens int 2005d7b4d43SMatthew Ahrens dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, 2015d7b4d43SMatthew Ahrens void *tag, dmu_buf_t **dbp, int flags) 2025d7b4d43SMatthew Ahrens { 2035d7b4d43SMatthew Ahrens int err; 2045d7b4d43SMatthew Ahrens int db_flags = DB_RF_CANFAIL; 2055d7b4d43SMatthew Ahrens 2065d7b4d43SMatthew Ahrens if (flags & DMU_READ_NO_PREFETCH) 2075d7b4d43SMatthew Ahrens db_flags |= DB_RF_NOPREFETCH; 2085d7b4d43SMatthew Ahrens 2095d7b4d43SMatthew Ahrens err = dmu_buf_hold_noread(os, object, offset, tag, dbp); 2105d7b4d43SMatthew Ahrens if (err == 0) { 2115d7b4d43SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 21247cb52daSJeff Bonwick err = dbuf_read(db, NULL, db_flags); 2135d7b4d43SMatthew Ahrens if (err != 0) { 214ea8dc4b6Seschrock dbuf_rele(db, tag); 2155d7b4d43SMatthew Ahrens *dbp = NULL; 216ea8dc4b6Seschrock } 217ea8dc4b6Seschrock } 218fa9e4066Sahrens 219ea8dc4b6Seschrock return (err); 220fa9e4066Sahrens } 221fa9e4066Sahrens 222fa9e4066Sahrens int 223fa9e4066Sahrens dmu_bonus_max(void) 224fa9e4066Sahrens { 225fa9e4066Sahrens return (DN_MAX_BONUSLEN); 226fa9e4066Sahrens } 227fa9e4066Sahrens 2281934e92fSmaybee int 229744947dcSTom Erickson dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx) 2301934e92fSmaybee { 231744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 232744947dcSTom Erickson dnode_t *dn; 233744947dcSTom Erickson int error; 2341934e92fSmaybee 235744947dcSTom Erickson DB_DNODE_ENTER(db); 236744947dcSTom Erickson dn = DB_DNODE(db); 237744947dcSTom Erickson 238744947dcSTom Erickson if (dn->dn_bonus != db) { 239be6fd75aSMatthew Ahrens error = SET_ERROR(EINVAL); 240744947dcSTom Erickson } else if (newsize < 0 || newsize > db_fake->db_size) { 241be6fd75aSMatthew Ahrens error = SET_ERROR(EINVAL); 242744947dcSTom Erickson } else { 243744947dcSTom Erickson dnode_setbonuslen(dn, newsize, tx); 244744947dcSTom Erickson error = 0; 245744947dcSTom Erickson } 246744947dcSTom Erickson 247744947dcSTom Erickson DB_DNODE_EXIT(db); 248744947dcSTom Erickson return (error); 2491934e92fSmaybee } 2501934e92fSmaybee 2510a586ceaSMark Shellenbaum int 252744947dcSTom Erickson dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx) 2530a586ceaSMark Shellenbaum { 254744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 255744947dcSTom Erickson dnode_t *dn; 256744947dcSTom Erickson int error; 2570a586ceaSMark Shellenbaum 258744947dcSTom Erickson DB_DNODE_ENTER(db); 259744947dcSTom Erickson dn = DB_DNODE(db); 260744947dcSTom Erickson 261ad135b5dSChristopher Siden if (!DMU_OT_IS_VALID(type)) { 262be6fd75aSMatthew Ahrens error = SET_ERROR(EINVAL); 263744947dcSTom Erickson } else if (dn->dn_bonus != db) { 264be6fd75aSMatthew Ahrens error = SET_ERROR(EINVAL); 265744947dcSTom Erickson } else { 266744947dcSTom Erickson dnode_setbonus_type(dn, type, tx); 267744947dcSTom Erickson error = 0; 268744947dcSTom Erickson } 2690a586ceaSMark Shellenbaum 270744947dcSTom Erickson DB_DNODE_EXIT(db); 271744947dcSTom Erickson return (error); 272744947dcSTom Erickson } 2730a586ceaSMark Shellenbaum 274744947dcSTom Erickson dmu_object_type_t 275744947dcSTom Erickson dmu_get_bonustype(dmu_buf_t *db_fake) 276744947dcSTom Erickson { 277744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 278744947dcSTom Erickson dnode_t *dn; 279744947dcSTom Erickson dmu_object_type_t type; 280744947dcSTom Erickson 281744947dcSTom Erickson DB_DNODE_ENTER(db); 282744947dcSTom Erickson dn = DB_DNODE(db); 283744947dcSTom Erickson type = dn->dn_bonustype; 284744947dcSTom Erickson DB_DNODE_EXIT(db); 285744947dcSTom Erickson 286744947dcSTom Erickson return (type); 2870a586ceaSMark Shellenbaum } 2880a586ceaSMark Shellenbaum 2890a586ceaSMark Shellenbaum int 2900a586ceaSMark Shellenbaum dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx) 2910a586ceaSMark Shellenbaum { 2920a586ceaSMark Shellenbaum dnode_t *dn; 2930a586ceaSMark Shellenbaum int error; 2940a586ceaSMark Shellenbaum 2950a586ceaSMark Shellenbaum error = dnode_hold(os, object, FTAG, &dn); 2960a586ceaSMark Shellenbaum dbuf_rm_spill(dn, tx); 29706e0070dSMark Shellenbaum rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 29806e0070dSMark Shellenbaum dnode_rm_spill(dn, tx); 29906e0070dSMark Shellenbaum rw_exit(&dn->dn_struct_rwlock); 3000a586ceaSMark Shellenbaum dnode_rele(dn, FTAG); 3010a586ceaSMark Shellenbaum return (error); 3020a586ceaSMark Shellenbaum } 3030a586ceaSMark Shellenbaum 304fa9e4066Sahrens /* 305ea8dc4b6Seschrock * returns ENOENT, EIO, or 0. 306fa9e4066Sahrens */ 307ea8dc4b6Seschrock int 308ea8dc4b6Seschrock dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp) 309fa9e4066Sahrens { 310ea8dc4b6Seschrock dnode_t *dn; 311fa9e4066Sahrens dmu_buf_impl_t *db; 3121934e92fSmaybee int error; 313fa9e4066Sahrens 314503ad85cSMatthew Ahrens error = dnode_hold(os, object, FTAG, &dn); 3151934e92fSmaybee if (error) 3161934e92fSmaybee return (error); 317fa9e4066Sahrens 318ea8dc4b6Seschrock rw_enter(&dn->dn_struct_rwlock, RW_READER); 319ea8dc4b6Seschrock if (dn->dn_bonus == NULL) { 320fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 321ea8dc4b6Seschrock rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 322ea8dc4b6Seschrock if (dn->dn_bonus == NULL) 3231934e92fSmaybee dbuf_create_bonus(dn); 324fa9e4066Sahrens } 325ea8dc4b6Seschrock db = dn->dn_bonus; 3261934e92fSmaybee 3271934e92fSmaybee /* as long as the bonus buf is held, the dnode will be held */ 328744947dcSTom Erickson if (refcount_add(&db->db_holds, tag) == 1) { 3291934e92fSmaybee VERIFY(dnode_add_ref(dn, db)); 330640c1670SJosef 'Jeff' Sipek atomic_inc_32(&dn->dn_dbufs_count); 331744947dcSTom Erickson } 332744947dcSTom Erickson 333744947dcSTom Erickson /* 334744947dcSTom Erickson * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's 335744947dcSTom Erickson * hold and incrementing the dbuf count to ensure that dnode_move() sees 336744947dcSTom Erickson * a dnode hold for every dbuf. 337744947dcSTom Erickson */ 338744947dcSTom Erickson rw_exit(&dn->dn_struct_rwlock); 3391934e92fSmaybee 340fa9e4066Sahrens dnode_rele(dn, FTAG); 341ea8dc4b6Seschrock 34247cb52daSJeff Bonwick VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH)); 343ea8dc4b6Seschrock 344ea8dc4b6Seschrock *dbp = &db->db; 345ea8dc4b6Seschrock return (0); 346fa9e4066Sahrens } 347fa9e4066Sahrens 3480a586ceaSMark Shellenbaum /* 3490a586ceaSMark Shellenbaum * returns ENOENT, EIO, or 0. 3500a586ceaSMark Shellenbaum * 3510a586ceaSMark Shellenbaum * This interface will allocate a blank spill dbuf when a spill blk 3520a586ceaSMark Shellenbaum * doesn't already exist on the dnode. 3530a586ceaSMark Shellenbaum * 3540a586ceaSMark Shellenbaum * if you only want to find an already existing spill db, then 3550a586ceaSMark Shellenbaum * dmu_spill_hold_existing() should be used. 3560a586ceaSMark Shellenbaum */ 3570a586ceaSMark Shellenbaum int 3580a586ceaSMark Shellenbaum dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp) 3590a586ceaSMark Shellenbaum { 3600a586ceaSMark Shellenbaum dmu_buf_impl_t *db = NULL; 3610a586ceaSMark Shellenbaum int err; 3620a586ceaSMark Shellenbaum 3630a586ceaSMark Shellenbaum if ((flags & DB_RF_HAVESTRUCT) == 0) 3640a586ceaSMark Shellenbaum rw_enter(&dn->dn_struct_rwlock, RW_READER); 3650a586ceaSMark Shellenbaum 3660a586ceaSMark Shellenbaum db = dbuf_hold(dn, DMU_SPILL_BLKID, tag); 3670a586ceaSMark Shellenbaum 3680a586ceaSMark Shellenbaum if ((flags & DB_RF_HAVESTRUCT) == 0) 3690a586ceaSMark Shellenbaum rw_exit(&dn->dn_struct_rwlock); 3700a586ceaSMark Shellenbaum 3710a586ceaSMark Shellenbaum ASSERT(db != NULL); 3721d8ccc7bSMark Shellenbaum err = dbuf_read(db, NULL, flags); 3731d8ccc7bSMark Shellenbaum if (err == 0) 3741d8ccc7bSMark Shellenbaum *dbp = &db->db; 3751d8ccc7bSMark Shellenbaum else 3761d8ccc7bSMark Shellenbaum dbuf_rele(db, tag); 3770a586ceaSMark Shellenbaum return (err); 3780a586ceaSMark Shellenbaum } 3790a586ceaSMark Shellenbaum 3800a586ceaSMark Shellenbaum int 3810a586ceaSMark Shellenbaum dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp) 3820a586ceaSMark Shellenbaum { 383744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 384744947dcSTom Erickson dnode_t *dn; 3850a586ceaSMark Shellenbaum int err; 3860a586ceaSMark Shellenbaum 387744947dcSTom Erickson DB_DNODE_ENTER(db); 388744947dcSTom Erickson dn = DB_DNODE(db); 389744947dcSTom Erickson 390744947dcSTom Erickson if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) { 391be6fd75aSMatthew Ahrens err = SET_ERROR(EINVAL); 392744947dcSTom Erickson } else { 393744947dcSTom Erickson rw_enter(&dn->dn_struct_rwlock, RW_READER); 394744947dcSTom Erickson 395744947dcSTom Erickson if (!dn->dn_have_spill) { 396be6fd75aSMatthew Ahrens err = SET_ERROR(ENOENT); 397744947dcSTom Erickson } else { 398744947dcSTom Erickson err = dmu_spill_hold_by_dnode(dn, 399744947dcSTom Erickson DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp); 400744947dcSTom Erickson } 4010a586ceaSMark Shellenbaum 4020a586ceaSMark Shellenbaum rw_exit(&dn->dn_struct_rwlock); 4030a586ceaSMark Shellenbaum } 404744947dcSTom Erickson 405744947dcSTom Erickson DB_DNODE_EXIT(db); 4060a586ceaSMark Shellenbaum return (err); 4070a586ceaSMark Shellenbaum } 4080a586ceaSMark Shellenbaum 4090a586ceaSMark Shellenbaum int 4100a586ceaSMark Shellenbaum dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp) 4110a586ceaSMark Shellenbaum { 412744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 413744947dcSTom Erickson dnode_t *dn; 414744947dcSTom Erickson int err; 415744947dcSTom Erickson 416744947dcSTom Erickson DB_DNODE_ENTER(db); 417744947dcSTom Erickson dn = DB_DNODE(db); 418744947dcSTom Erickson err = dmu_spill_hold_by_dnode(dn, DB_RF_CANFAIL, tag, dbp); 419744947dcSTom Erickson DB_DNODE_EXIT(db); 420744947dcSTom Erickson 421744947dcSTom Erickson return (err); 4220a586ceaSMark Shellenbaum } 4230a586ceaSMark Shellenbaum 42413506d1eSmaybee /* 42513506d1eSmaybee * Note: longer-term, we should modify all of the dmu_buf_*() interfaces 42613506d1eSmaybee * to take a held dnode rather than <os, object> -- the lookup is wasteful, 42713506d1eSmaybee * and can induce severe lock contention when writing to several files 42813506d1eSmaybee * whose dnodes are in the same block. 42913506d1eSmaybee */ 43013506d1eSmaybee static int 4317bfdf011SNeil Perrin dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, 432cf6106c8SMatthew Ahrens boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags) 433fa9e4066Sahrens { 434fa9e4066Sahrens dmu_buf_t **dbp; 435fa9e4066Sahrens uint64_t blkid, nblks, i; 4367bfdf011SNeil Perrin uint32_t dbuf_flags; 437ea8dc4b6Seschrock int err; 438ea8dc4b6Seschrock zio_t *zio; 439ea8dc4b6Seschrock 440ea8dc4b6Seschrock ASSERT(length <= DMU_MAX_ACCESS); 441fa9e4066Sahrens 442cf6106c8SMatthew Ahrens /* 443cf6106c8SMatthew Ahrens * Note: We directly notify the prefetch code of this read, so that 444cf6106c8SMatthew Ahrens * we can tell it about the multi-block read. dbuf_read() only knows 445cf6106c8SMatthew Ahrens * about the one block it is accessing. 446cf6106c8SMatthew Ahrens */ 447cf6106c8SMatthew Ahrens dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT | 448cf6106c8SMatthew Ahrens DB_RF_NOPREFETCH; 449ea8dc4b6Seschrock 450fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 451fa9e4066Sahrens if (dn->dn_datablkshift) { 452fa9e4066Sahrens int blkshift = dn->dn_datablkshift; 453cf6106c8SMatthew Ahrens nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) - 454cf6106c8SMatthew Ahrens P2ALIGN(offset, 1ULL << blkshift)) >> blkshift; 455fa9e4066Sahrens } else { 4560125049cSahrens if (offset + length > dn->dn_datablksz) { 4570125049cSahrens zfs_panic_recover("zfs: accessing past end of object " 4580125049cSahrens "%llx/%llx (size=%u access=%llu+%llu)", 4590125049cSahrens (longlong_t)dn->dn_objset-> 4600125049cSahrens os_dsl_dataset->ds_object, 4610125049cSahrens (longlong_t)dn->dn_object, dn->dn_datablksz, 4620125049cSahrens (longlong_t)offset, (longlong_t)length); 463c87b8fc5SMark J Musante rw_exit(&dn->dn_struct_rwlock); 464be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 4650125049cSahrens } 466fa9e4066Sahrens nblks = 1; 467fa9e4066Sahrens } 468ea8dc4b6Seschrock dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP); 469fa9e4066Sahrens 470e14bb325SJeff Bonwick zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL); 471a2cdcdd2SPaul Dagnelie blkid = dbuf_whichblock(dn, 0, offset); 472fa9e4066Sahrens for (i = 0; i < nblks; i++) { 473cf6106c8SMatthew Ahrens dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag); 474ea8dc4b6Seschrock if (db == NULL) { 475ea8dc4b6Seschrock rw_exit(&dn->dn_struct_rwlock); 476ea8dc4b6Seschrock dmu_buf_rele_array(dbp, nblks, tag); 477ea8dc4b6Seschrock zio_nowait(zio); 478be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 479ea8dc4b6Seschrock } 480cf6106c8SMatthew Ahrens 481ea8dc4b6Seschrock /* initiate async i/o */ 482cf6106c8SMatthew Ahrens if (read) 4837bfdf011SNeil Perrin (void) dbuf_read(db, zio, dbuf_flags); 484ea8dc4b6Seschrock dbp[i] = &db->db; 485fa9e4066Sahrens } 486cf6106c8SMatthew Ahrens 487cb92f413SAlexander Motin if ((flags & DMU_READ_NO_PREFETCH) == 0 && 488cb92f413SAlexander Motin DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) { 489cb92f413SAlexander Motin dmu_zfetch(&dn->dn_zfetch, blkid, nblks, 490cb92f413SAlexander Motin read && DNODE_IS_CACHEABLE(dn)); 491cf6106c8SMatthew Ahrens } 492fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 493fa9e4066Sahrens 494ea8dc4b6Seschrock /* wait for async i/o */ 495ea8dc4b6Seschrock err = zio_wait(zio); 496ea8dc4b6Seschrock if (err) { 497ea8dc4b6Seschrock dmu_buf_rele_array(dbp, nblks, tag); 498ea8dc4b6Seschrock return (err); 499ea8dc4b6Seschrock } 500fa9e4066Sahrens 501ea8dc4b6Seschrock /* wait for other io to complete */ 502ea8dc4b6Seschrock if (read) { 503ea8dc4b6Seschrock for (i = 0; i < nblks; i++) { 504ea8dc4b6Seschrock dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; 505ea8dc4b6Seschrock mutex_enter(&db->db_mtx); 506ea8dc4b6Seschrock while (db->db_state == DB_READ || 507ea8dc4b6Seschrock db->db_state == DB_FILL) 508ea8dc4b6Seschrock cv_wait(&db->db_changed, &db->db_mtx); 509ea8dc4b6Seschrock if (db->db_state == DB_UNCACHED) 510be6fd75aSMatthew Ahrens err = SET_ERROR(EIO); 511ea8dc4b6Seschrock mutex_exit(&db->db_mtx); 512ea8dc4b6Seschrock if (err) { 513ea8dc4b6Seschrock dmu_buf_rele_array(dbp, nblks, tag); 514ea8dc4b6Seschrock return (err); 515ea8dc4b6Seschrock } 516ea8dc4b6Seschrock } 517ea8dc4b6Seschrock } 518fa9e4066Sahrens 519ea8dc4b6Seschrock *numbufsp = nblks; 520ea8dc4b6Seschrock *dbpp = dbp; 521ea8dc4b6Seschrock return (0); 522fa9e4066Sahrens } 523fa9e4066Sahrens 524a2eea2e1Sahrens static int 52513506d1eSmaybee dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset, 52613506d1eSmaybee uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp) 52713506d1eSmaybee { 52813506d1eSmaybee dnode_t *dn; 52913506d1eSmaybee int err; 53013506d1eSmaybee 531503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 53213506d1eSmaybee if (err) 53313506d1eSmaybee return (err); 53413506d1eSmaybee 53513506d1eSmaybee err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, 5367bfdf011SNeil Perrin numbufsp, dbpp, DMU_READ_PREFETCH); 53713506d1eSmaybee 53813506d1eSmaybee dnode_rele(dn, FTAG); 53913506d1eSmaybee 54013506d1eSmaybee return (err); 54113506d1eSmaybee } 54213506d1eSmaybee 54313506d1eSmaybee int 544744947dcSTom Erickson dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset, 545cf6106c8SMatthew Ahrens uint64_t length, boolean_t read, void *tag, int *numbufsp, 546cf6106c8SMatthew Ahrens dmu_buf_t ***dbpp) 54713506d1eSmaybee { 548744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 549744947dcSTom Erickson dnode_t *dn; 55013506d1eSmaybee int err; 55113506d1eSmaybee 552744947dcSTom Erickson DB_DNODE_ENTER(db); 553744947dcSTom Erickson dn = DB_DNODE(db); 55413506d1eSmaybee err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, 5557bfdf011SNeil Perrin numbufsp, dbpp, DMU_READ_PREFETCH); 556744947dcSTom Erickson DB_DNODE_EXIT(db); 55713506d1eSmaybee 55813506d1eSmaybee return (err); 55913506d1eSmaybee } 56013506d1eSmaybee 561fa9e4066Sahrens void 562ea8dc4b6Seschrock dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag) 563fa9e4066Sahrens { 564fa9e4066Sahrens int i; 565fa9e4066Sahrens dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake; 566fa9e4066Sahrens 567fa9e4066Sahrens if (numbufs == 0) 568fa9e4066Sahrens return; 569fa9e4066Sahrens 570ea8dc4b6Seschrock for (i = 0; i < numbufs; i++) { 571ea8dc4b6Seschrock if (dbp[i]) 572ea8dc4b6Seschrock dbuf_rele(dbp[i], tag); 573ea8dc4b6Seschrock } 574fa9e4066Sahrens 575fa9e4066Sahrens kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs); 576fa9e4066Sahrens } 577fa9e4066Sahrens 57869962b56SMatthew Ahrens /* 579a2cdcdd2SPaul Dagnelie * Issue prefetch i/os for the given blocks. If level is greater than 0, the 580a2cdcdd2SPaul Dagnelie * indirect blocks prefeteched will be those that point to the blocks containing 581a2cdcdd2SPaul Dagnelie * the data starting at offset, and continuing to offset + len. 58269962b56SMatthew Ahrens * 583a2cdcdd2SPaul Dagnelie * Note that if the indirect blocks above the blocks being prefetched are not in 584a2cdcdd2SPaul Dagnelie * cache, they will be asychronously read in. 58569962b56SMatthew Ahrens */ 586fa9e4066Sahrens void 587a2cdcdd2SPaul Dagnelie dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset, 588a2cdcdd2SPaul Dagnelie uint64_t len, zio_priority_t pri) 589fa9e4066Sahrens { 590fa9e4066Sahrens dnode_t *dn; 591fa9e4066Sahrens uint64_t blkid; 59269962b56SMatthew Ahrens int nblks, err; 593fa9e4066Sahrens 594fa9e4066Sahrens if (len == 0) { /* they're interested in the bonus buffer */ 595744947dcSTom Erickson dn = DMU_META_DNODE(os); 596fa9e4066Sahrens 597fa9e4066Sahrens if (object == 0 || object >= DN_MAX_OBJECT) 598fa9e4066Sahrens return; 599fa9e4066Sahrens 600fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 601a2cdcdd2SPaul Dagnelie blkid = dbuf_whichblock(dn, level, 602a2cdcdd2SPaul Dagnelie object * sizeof (dnode_phys_t)); 603a2cdcdd2SPaul Dagnelie dbuf_prefetch(dn, level, blkid, pri, 0); 604fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 605fa9e4066Sahrens return; 606fa9e4066Sahrens } 607fa9e4066Sahrens 608fa9e4066Sahrens /* 609fa9e4066Sahrens * XXX - Note, if the dnode for the requested object is not 610fa9e4066Sahrens * already cached, we will do a *synchronous* read in the 611fa9e4066Sahrens * dnode_hold() call. The same is true for any indirects. 612fa9e4066Sahrens */ 613503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 614ea8dc4b6Seschrock if (err != 0) 615fa9e4066Sahrens return; 616fa9e4066Sahrens 617fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 618a2cdcdd2SPaul Dagnelie /* 619a2cdcdd2SPaul Dagnelie * offset + len - 1 is the last byte we want to prefetch for, and offset 620a2cdcdd2SPaul Dagnelie * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the 621a2cdcdd2SPaul Dagnelie * last block we want to prefetch, and dbuf_whichblock(dn, level, 622a2cdcdd2SPaul Dagnelie * offset) is the first. Then the number we need to prefetch is the 623a2cdcdd2SPaul Dagnelie * last - first + 1. 624a2cdcdd2SPaul Dagnelie */ 625a2cdcdd2SPaul Dagnelie if (level > 0 || dn->dn_datablkshift != 0) { 626a2cdcdd2SPaul Dagnelie nblks = dbuf_whichblock(dn, level, offset + len - 1) - 627a2cdcdd2SPaul Dagnelie dbuf_whichblock(dn, level, offset) + 1; 628fa9e4066Sahrens } else { 629fa9e4066Sahrens nblks = (offset < dn->dn_datablksz); 630fa9e4066Sahrens } 631fa9e4066Sahrens 632fa9e4066Sahrens if (nblks != 0) { 633a2cdcdd2SPaul Dagnelie blkid = dbuf_whichblock(dn, level, offset); 63469962b56SMatthew Ahrens for (int i = 0; i < nblks; i++) 635a2cdcdd2SPaul Dagnelie dbuf_prefetch(dn, level, blkid + i, pri, 0); 636fa9e4066Sahrens } 637fa9e4066Sahrens 638fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 639fa9e4066Sahrens 640fa9e4066Sahrens dnode_rele(dn, FTAG); 641fa9e4066Sahrens } 642fa9e4066Sahrens 64376256205SMark Maybee /* 64476256205SMark Maybee * Get the next "chunk" of file data to free. We traverse the file from 64576256205SMark Maybee * the end so that the file gets shorter over time (if we crashes in the 64676256205SMark Maybee * middle, this will leave us in a better state). We find allocated file 64776256205SMark Maybee * data by simply searching the allocated level 1 indirects. 648713d6c20SMatthew Ahrens * 649713d6c20SMatthew Ahrens * On input, *start should be the first offset that does not need to be 650713d6c20SMatthew Ahrens * freed (e.g. "offset + length"). On return, *start will be the first 651713d6c20SMatthew Ahrens * offset that should be freed. 65276256205SMark Maybee */ 653cdb0ab79Smaybee static int 654713d6c20SMatthew Ahrens get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum) 655cdb0ab79Smaybee { 656713d6c20SMatthew Ahrens uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1); 657713d6c20SMatthew Ahrens /* bytes of data covered by a level-1 indirect block */ 65876256205SMark Maybee uint64_t iblkrange = 6591c8564a7SMark Maybee dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT); 660cdb0ab79Smaybee 661713d6c20SMatthew Ahrens ASSERT3U(minimum, <=, *start); 662cdb0ab79Smaybee 663713d6c20SMatthew Ahrens if (*start - minimum <= iblkrange * maxblks) { 664713d6c20SMatthew Ahrens *start = minimum; 665cdb0ab79Smaybee return (0); 666cdb0ab79Smaybee } 66776256205SMark Maybee ASSERT(ISP2(iblkrange)); 668cdb0ab79Smaybee 669713d6c20SMatthew Ahrens for (uint64_t blks = 0; *start > minimum && blks < maxblks; blks++) { 6701c8564a7SMark Maybee int err; 671cdb0ab79Smaybee 672713d6c20SMatthew Ahrens /* 673713d6c20SMatthew Ahrens * dnode_next_offset(BACKWARDS) will find an allocated L1 674713d6c20SMatthew Ahrens * indirect block at or before the input offset. We must 675713d6c20SMatthew Ahrens * decrement *start so that it is at the end of the region 676713d6c20SMatthew Ahrens * to search. 677713d6c20SMatthew Ahrens */ 678713d6c20SMatthew Ahrens (*start)--; 679cdb0ab79Smaybee err = dnode_next_offset(dn, 68076256205SMark Maybee DNODE_FIND_BACKWARDS, start, 2, 1, 0); 681cdb0ab79Smaybee 682713d6c20SMatthew Ahrens /* if there are no indirect blocks before start, we are done */ 68376256205SMark Maybee if (err == ESRCH) { 684713d6c20SMatthew Ahrens *start = minimum; 685713d6c20SMatthew Ahrens break; 686713d6c20SMatthew Ahrens } else if (err != 0) { 687cdb0ab79Smaybee return (err); 68876256205SMark Maybee } 68976256205SMark Maybee 690713d6c20SMatthew Ahrens /* set start to the beginning of this L1 indirect */ 69176256205SMark Maybee *start = P2ALIGN(*start, iblkrange); 692cdb0ab79Smaybee } 693713d6c20SMatthew Ahrens if (*start < minimum) 694713d6c20SMatthew Ahrens *start = minimum; 695cdb0ab79Smaybee return (0); 696cdb0ab79Smaybee } 697cdb0ab79Smaybee 698cdb0ab79Smaybee static int 699cdb0ab79Smaybee dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset, 700713d6c20SMatthew Ahrens uint64_t length) 701cdb0ab79Smaybee { 702713d6c20SMatthew Ahrens uint64_t object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 703713d6c20SMatthew Ahrens int err; 704713d6c20SMatthew Ahrens 705713d6c20SMatthew Ahrens if (offset >= object_size) 706cdb0ab79Smaybee return (0); 707cdb0ab79Smaybee 708713d6c20SMatthew Ahrens if (length == DMU_OBJECT_END || offset + length > object_size) 709713d6c20SMatthew Ahrens length = object_size - offset; 710713d6c20SMatthew Ahrens 711713d6c20SMatthew Ahrens while (length != 0) { 712713d6c20SMatthew Ahrens uint64_t chunk_end, chunk_begin; 713713d6c20SMatthew Ahrens 714713d6c20SMatthew Ahrens chunk_end = chunk_begin = offset + length; 715713d6c20SMatthew Ahrens 716713d6c20SMatthew Ahrens /* move chunk_begin backwards to the beginning of this chunk */ 717713d6c20SMatthew Ahrens err = get_next_chunk(dn, &chunk_begin, offset); 718cdb0ab79Smaybee if (err) 719cdb0ab79Smaybee return (err); 720713d6c20SMatthew Ahrens ASSERT3U(chunk_begin, >=, offset); 721713d6c20SMatthew Ahrens ASSERT3U(chunk_begin, <=, chunk_end); 722cdb0ab79Smaybee 723713d6c20SMatthew Ahrens dmu_tx_t *tx = dmu_tx_create(os); 724713d6c20SMatthew Ahrens dmu_tx_hold_free(tx, dn->dn_object, 725713d6c20SMatthew Ahrens chunk_begin, chunk_end - chunk_begin); 7264bb73804SMatthew Ahrens 7274bb73804SMatthew Ahrens /* 7284bb73804SMatthew Ahrens * Mark this transaction as typically resulting in a net 7294bb73804SMatthew Ahrens * reduction in space used. 7304bb73804SMatthew Ahrens */ 7314bb73804SMatthew Ahrens dmu_tx_mark_netfree(tx); 732cdb0ab79Smaybee err = dmu_tx_assign(tx, TXG_WAIT); 733cdb0ab79Smaybee if (err) { 734cdb0ab79Smaybee dmu_tx_abort(tx); 735cdb0ab79Smaybee return (err); 736cdb0ab79Smaybee } 737713d6c20SMatthew Ahrens dnode_free_range(dn, chunk_begin, chunk_end - chunk_begin, tx); 738cdb0ab79Smaybee dmu_tx_commit(tx); 739713d6c20SMatthew Ahrens 740713d6c20SMatthew Ahrens length -= chunk_end - chunk_begin; 741cdb0ab79Smaybee } 742cdb0ab79Smaybee return (0); 743cdb0ab79Smaybee } 744cdb0ab79Smaybee 745cdb0ab79Smaybee int 746cdb0ab79Smaybee dmu_free_long_range(objset_t *os, uint64_t object, 747cdb0ab79Smaybee uint64_t offset, uint64_t length) 748cdb0ab79Smaybee { 749cdb0ab79Smaybee dnode_t *dn; 750cdb0ab79Smaybee int err; 751cdb0ab79Smaybee 752503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 753cdb0ab79Smaybee if (err != 0) 754cdb0ab79Smaybee return (err); 755713d6c20SMatthew Ahrens err = dmu_free_long_range_impl(os, dn, offset, length); 7565253393bSMatthew Ahrens 7575253393bSMatthew Ahrens /* 7585253393bSMatthew Ahrens * It is important to zero out the maxblkid when freeing the entire 7595253393bSMatthew Ahrens * file, so that (a) subsequent calls to dmu_free_long_range_impl() 7605253393bSMatthew Ahrens * will take the fast path, and (b) dnode_reallocate() can verify 7615253393bSMatthew Ahrens * that the entire file has been freed. 7625253393bSMatthew Ahrens */ 76343466aaeSMax Grossman if (err == 0 && offset == 0 && length == DMU_OBJECT_END) 7645253393bSMatthew Ahrens dn->dn_maxblkid = 0; 7655253393bSMatthew Ahrens 766cdb0ab79Smaybee dnode_rele(dn, FTAG); 767cdb0ab79Smaybee return (err); 768cdb0ab79Smaybee } 769cdb0ab79Smaybee 770cdb0ab79Smaybee int 771713d6c20SMatthew Ahrens dmu_free_long_object(objset_t *os, uint64_t object) 772cdb0ab79Smaybee { 773cdb0ab79Smaybee dmu_tx_t *tx; 774cdb0ab79Smaybee int err; 775cdb0ab79Smaybee 776713d6c20SMatthew Ahrens err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END); 777cdb0ab79Smaybee if (err != 0) 778cdb0ab79Smaybee return (err); 779713d6c20SMatthew Ahrens 780713d6c20SMatthew Ahrens tx = dmu_tx_create(os); 781713d6c20SMatthew Ahrens dmu_tx_hold_bonus(tx, object); 782713d6c20SMatthew Ahrens dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END); 7834bb73804SMatthew Ahrens dmu_tx_mark_netfree(tx); 784713d6c20SMatthew Ahrens err = dmu_tx_assign(tx, TXG_WAIT); 785713d6c20SMatthew Ahrens if (err == 0) { 786713d6c20SMatthew Ahrens err = dmu_object_free(os, object, tx); 787713d6c20SMatthew Ahrens dmu_tx_commit(tx); 788cdb0ab79Smaybee } else { 789713d6c20SMatthew Ahrens dmu_tx_abort(tx); 790cdb0ab79Smaybee } 791713d6c20SMatthew Ahrens 792cdb0ab79Smaybee return (err); 793cdb0ab79Smaybee } 794cdb0ab79Smaybee 795ea8dc4b6Seschrock int 796fa9e4066Sahrens dmu_free_range(objset_t *os, uint64_t object, uint64_t offset, 797fa9e4066Sahrens uint64_t size, dmu_tx_t *tx) 798fa9e4066Sahrens { 799ea8dc4b6Seschrock dnode_t *dn; 800503ad85cSMatthew Ahrens int err = dnode_hold(os, object, FTAG, &dn); 801ea8dc4b6Seschrock if (err) 802ea8dc4b6Seschrock return (err); 803fa9e4066Sahrens ASSERT(offset < UINT64_MAX); 804fa9e4066Sahrens ASSERT(size == -1ULL || size <= UINT64_MAX - offset); 805fa9e4066Sahrens dnode_free_range(dn, offset, size, tx); 806fa9e4066Sahrens dnode_rele(dn, FTAG); 807ea8dc4b6Seschrock return (0); 808fa9e4066Sahrens } 809fa9e4066Sahrens 810ea8dc4b6Seschrock int 811ea8dc4b6Seschrock dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 8127bfdf011SNeil Perrin void *buf, uint32_t flags) 813fa9e4066Sahrens { 814fa9e4066Sahrens dnode_t *dn; 815fa9e4066Sahrens dmu_buf_t **dbp; 816c87b8fc5SMark J Musante int numbufs, err; 817fa9e4066Sahrens 818503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 819ea8dc4b6Seschrock if (err) 820ea8dc4b6Seschrock return (err); 821feb08c6bSbillm 822feb08c6bSbillm /* 823feb08c6bSbillm * Deal with odd block sizes, where there can't be data past the first 824feb08c6bSbillm * block. If we ever do the tail block optimization, we will need to 825feb08c6bSbillm * handle that here as well. 826feb08c6bSbillm */ 827c87b8fc5SMark J Musante if (dn->dn_maxblkid == 0) { 828fa9e4066Sahrens int newsz = offset > dn->dn_datablksz ? 0 : 829fa9e4066Sahrens MIN(size, dn->dn_datablksz - offset); 830fa9e4066Sahrens bzero((char *)buf + newsz, size - newsz); 831fa9e4066Sahrens size = newsz; 832fa9e4066Sahrens } 833fa9e4066Sahrens 834fa9e4066Sahrens while (size > 0) { 835fa9e4066Sahrens uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2); 836c87b8fc5SMark J Musante int i; 837fa9e4066Sahrens 838fa9e4066Sahrens /* 839fa9e4066Sahrens * NB: we could do this block-at-a-time, but it's nice 840fa9e4066Sahrens * to be reading in parallel. 841fa9e4066Sahrens */ 842a2eea2e1Sahrens err = dmu_buf_hold_array_by_dnode(dn, offset, mylen, 8437bfdf011SNeil Perrin TRUE, FTAG, &numbufs, &dbp, flags); 844ea8dc4b6Seschrock if (err) 8451934e92fSmaybee break; 846fa9e4066Sahrens 847fa9e4066Sahrens for (i = 0; i < numbufs; i++) { 848fa9e4066Sahrens int tocpy; 849fa9e4066Sahrens int bufoff; 850fa9e4066Sahrens dmu_buf_t *db = dbp[i]; 851fa9e4066Sahrens 852fa9e4066Sahrens ASSERT(size > 0); 853fa9e4066Sahrens 854fa9e4066Sahrens bufoff = offset - db->db_offset; 855fa9e4066Sahrens tocpy = (int)MIN(db->db_size - bufoff, size); 856fa9e4066Sahrens 857fa9e4066Sahrens bcopy((char *)db->db_data + bufoff, buf, tocpy); 858fa9e4066Sahrens 859fa9e4066Sahrens offset += tocpy; 860fa9e4066Sahrens size -= tocpy; 861fa9e4066Sahrens buf = (char *)buf + tocpy; 862fa9e4066Sahrens } 863ea8dc4b6Seschrock dmu_buf_rele_array(dbp, numbufs, FTAG); 864fa9e4066Sahrens } 865a2eea2e1Sahrens dnode_rele(dn, FTAG); 8661934e92fSmaybee return (err); 867fa9e4066Sahrens } 868fa9e4066Sahrens 869fa9e4066Sahrens void 870fa9e4066Sahrens dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 871fa9e4066Sahrens const void *buf, dmu_tx_t *tx) 872fa9e4066Sahrens { 873fa9e4066Sahrens dmu_buf_t **dbp; 874fa9e4066Sahrens int numbufs, i; 875fa9e4066Sahrens 87613506d1eSmaybee if (size == 0) 87713506d1eSmaybee return; 87813506d1eSmaybee 879ea8dc4b6Seschrock VERIFY(0 == dmu_buf_hold_array(os, object, offset, size, 880ea8dc4b6Seschrock FALSE, FTAG, &numbufs, &dbp)); 881fa9e4066Sahrens 882fa9e4066Sahrens for (i = 0; i < numbufs; i++) { 883fa9e4066Sahrens int tocpy; 884fa9e4066Sahrens int bufoff; 885fa9e4066Sahrens dmu_buf_t *db = dbp[i]; 886fa9e4066Sahrens 887fa9e4066Sahrens ASSERT(size > 0); 888fa9e4066Sahrens 889fa9e4066Sahrens bufoff = offset - db->db_offset; 890fa9e4066Sahrens tocpy = (int)MIN(db->db_size - bufoff, size); 891fa9e4066Sahrens 892fa9e4066Sahrens ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 893fa9e4066Sahrens 894fa9e4066Sahrens if (tocpy == db->db_size) 895fa9e4066Sahrens dmu_buf_will_fill(db, tx); 896fa9e4066Sahrens else 897fa9e4066Sahrens dmu_buf_will_dirty(db, tx); 898fa9e4066Sahrens 899fa9e4066Sahrens bcopy(buf, (char *)db->db_data + bufoff, tocpy); 900fa9e4066Sahrens 901fa9e4066Sahrens if (tocpy == db->db_size) 902fa9e4066Sahrens dmu_buf_fill_done(db, tx); 903fa9e4066Sahrens 904fa9e4066Sahrens offset += tocpy; 905fa9e4066Sahrens size -= tocpy; 906fa9e4066Sahrens buf = (char *)buf + tocpy; 907fa9e4066Sahrens } 908ea8dc4b6Seschrock dmu_buf_rele_array(dbp, numbufs, FTAG); 909fa9e4066Sahrens } 910fa9e4066Sahrens 91182c9918fSTim Haley void 91282c9918fSTim Haley dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 91382c9918fSTim Haley dmu_tx_t *tx) 91482c9918fSTim Haley { 91582c9918fSTim Haley dmu_buf_t **dbp; 91682c9918fSTim Haley int numbufs, i; 91782c9918fSTim Haley 91882c9918fSTim Haley if (size == 0) 91982c9918fSTim Haley return; 92082c9918fSTim Haley 92182c9918fSTim Haley VERIFY(0 == dmu_buf_hold_array(os, object, offset, size, 92282c9918fSTim Haley FALSE, FTAG, &numbufs, &dbp)); 92382c9918fSTim Haley 92482c9918fSTim Haley for (i = 0; i < numbufs; i++) { 92582c9918fSTim Haley dmu_buf_t *db = dbp[i]; 92682c9918fSTim Haley 92782c9918fSTim Haley dmu_buf_will_not_fill(db, tx); 92882c9918fSTim Haley } 92982c9918fSTim Haley dmu_buf_rele_array(dbp, numbufs, FTAG); 93082c9918fSTim Haley } 93182c9918fSTim Haley 9325d7b4d43SMatthew Ahrens void 9335d7b4d43SMatthew Ahrens dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset, 9345d7b4d43SMatthew Ahrens void *data, uint8_t etype, uint8_t comp, int uncompressed_size, 9355d7b4d43SMatthew Ahrens int compressed_size, int byteorder, dmu_tx_t *tx) 9365d7b4d43SMatthew Ahrens { 9375d7b4d43SMatthew Ahrens dmu_buf_t *db; 9385d7b4d43SMatthew Ahrens 9395d7b4d43SMatthew Ahrens ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES); 9405d7b4d43SMatthew Ahrens ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS); 9415d7b4d43SMatthew Ahrens VERIFY0(dmu_buf_hold_noread(os, object, offset, 9425d7b4d43SMatthew Ahrens FTAG, &db)); 9435d7b4d43SMatthew Ahrens 9445d7b4d43SMatthew Ahrens dmu_buf_write_embedded(db, 9455d7b4d43SMatthew Ahrens data, (bp_embedded_type_t)etype, (enum zio_compress)comp, 9465d7b4d43SMatthew Ahrens uncompressed_size, compressed_size, byteorder, tx); 9475d7b4d43SMatthew Ahrens 9485d7b4d43SMatthew Ahrens dmu_buf_rele(db, FTAG); 9495d7b4d43SMatthew Ahrens } 9505d7b4d43SMatthew Ahrens 951c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 952c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * DMU support for xuio 953c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 954c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kstat_t *xuio_ksp = NULL; 955c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 956c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int 957c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_init(xuio_t *xuio, int nblk) 958c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 959c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv; 960c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio_t *uio = &xuio->xu_uio; 961c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 962c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio->uio_iovcnt = nblk; 963c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP); 964c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 965c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP); 966c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv->cnt = nblk; 967c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP); 968c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv->iovp = uio->uio_iov; 969c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIO_XUZC_PRIV(xuio) = priv; 970c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 971c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (XUIO_XUZC_RW(xuio) == UIO_READ) 972c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk); 973c242f9a0Schunli zhang - Sun Microsystems - Irvine United States else 974c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk); 975c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 976c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (0); 977c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 978c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 979c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 980c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_fini(xuio_t *xuio) 981c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 982c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio); 983c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int nblk = priv->cnt; 984c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 985c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kmem_free(priv->iovp, nblk * sizeof (iovec_t)); 986c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *)); 987c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kmem_free(priv, sizeof (dmu_xuio_t)); 988c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 989c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (XUIO_XUZC_RW(xuio) == UIO_READ) 990c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk); 991c242f9a0Schunli zhang - Sun Microsystems - Irvine United States else 992c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk); 993c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 994c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 995c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /* 996c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf } 997c242f9a0Schunli zhang - Sun Microsystems - Irvine United States * and increase priv->next by 1. 998c242f9a0Schunli zhang - Sun Microsystems - Irvine United States */ 999c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int 1000c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n) 1001c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1002c242f9a0Schunli zhang - Sun Microsystems - Irvine United States struct iovec *iov; 1003c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio_t *uio = &xuio->xu_uio; 1004c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio); 1005c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int i = priv->next++; 1006c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1007c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(i < priv->cnt); 1008c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(off + n <= arc_buf_size(abuf)); 1009c242f9a0Schunli zhang - Sun Microsystems - Irvine United States iov = uio->uio_iov + i; 1010c242f9a0Schunli zhang - Sun Microsystems - Irvine United States iov->iov_base = (char *)abuf->b_data + off; 1011c242f9a0Schunli zhang - Sun Microsystems - Irvine United States iov->iov_len = n; 1012c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv->bufs[i] = abuf; 1013c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (0); 1014c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1015c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1016c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int 1017c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_cnt(xuio_t *xuio) 1018c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1019c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio); 1020c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (priv->cnt); 1021c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1022c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1023c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t * 1024c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_arcbuf(xuio_t *xuio, int i) 1025c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1026c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio); 1027c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1028c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(i < priv->cnt); 1029c242f9a0Schunli zhang - Sun Microsystems - Irvine United States return (priv->bufs[i]); 1030c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1031c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1032c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 1033c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_clear(xuio_t *xuio, int i) 1034c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1035c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio); 1036c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1037c242f9a0Schunli zhang - Sun Microsystems - Irvine United States ASSERT(i < priv->cnt); 1038c242f9a0Schunli zhang - Sun Microsystems - Irvine United States priv->bufs[i] = NULL; 1039c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1040c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1041c242f9a0Schunli zhang - Sun Microsystems - Irvine United States static void 1042c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_init(void) 1043c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1044c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc", 1045c242f9a0Schunli zhang - Sun Microsystems - Irvine United States KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t), 1046c242f9a0Schunli zhang - Sun Microsystems - Irvine United States KSTAT_FLAG_VIRTUAL); 1047c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (xuio_ksp != NULL) { 1048c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_ksp->ks_data = &xuio_stats; 1049c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kstat_install(xuio_ksp); 1050c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1051c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1052c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1053c242f9a0Schunli zhang - Sun Microsystems - Irvine United States static void 1054c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_fini(void) 1055c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1056c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (xuio_ksp != NULL) { 1057c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kstat_delete(xuio_ksp); 1058c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_ksp = NULL; 1059c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1060c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1061c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1062c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 1063c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_copied() 1064c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1065c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_BUMP(xuiostat_wbuf_copied); 1066c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1067c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1068c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void 1069c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_wbuf_nocopy() 1070c242f9a0Schunli zhang - Sun Microsystems - Irvine United States { 1071c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_BUMP(xuiostat_wbuf_nocopy); 1072c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1073c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1074fa9e4066Sahrens #ifdef _KERNEL 1075f8554bb9SMatthew Ahrens static int 1076f8554bb9SMatthew Ahrens dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size) 1077feb08c6bSbillm { 1078feb08c6bSbillm dmu_buf_t **dbp; 1079feb08c6bSbillm int numbufs, i, err; 1080c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_t *xuio = NULL; 1081feb08c6bSbillm 1082feb08c6bSbillm /* 1083feb08c6bSbillm * NB: we could do this block-at-a-time, but it's nice 1084feb08c6bSbillm * to be reading in parallel. 1085feb08c6bSbillm */ 1086f8554bb9SMatthew Ahrens err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size, 1087f8554bb9SMatthew Ahrens TRUE, FTAG, &numbufs, &dbp, 0); 1088feb08c6bSbillm if (err) 1089feb08c6bSbillm return (err); 1090feb08c6bSbillm 1091c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (uio->uio_extflg == UIO_XUIO) 1092c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio = (xuio_t *)uio; 1093c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1094feb08c6bSbillm for (i = 0; i < numbufs; i++) { 1095feb08c6bSbillm int tocpy; 1096feb08c6bSbillm int bufoff; 1097feb08c6bSbillm dmu_buf_t *db = dbp[i]; 1098feb08c6bSbillm 1099feb08c6bSbillm ASSERT(size > 0); 1100feb08c6bSbillm 1101feb08c6bSbillm bufoff = uio->uio_loffset - db->db_offset; 1102feb08c6bSbillm tocpy = (int)MIN(db->db_size - bufoff, size); 1103feb08c6bSbillm 1104c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (xuio) { 1105c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 1106c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *dbuf_abuf = dbi->db_buf; 1107c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *abuf = dbuf_loan_arcbuf(dbi); 1108c242f9a0Schunli zhang - Sun Microsystems - Irvine United States err = dmu_xuio_add(xuio, abuf, bufoff, tocpy); 1109c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (!err) { 1110c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio->uio_resid -= tocpy; 1111c242f9a0Schunli zhang - Sun Microsystems - Irvine United States uio->uio_loffset += tocpy; 1112c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1113c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 1114c242f9a0Schunli zhang - Sun Microsystems - Irvine United States if (abuf == dbuf_abuf) 1115c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_BUMP(xuiostat_rbuf_nocopy); 1116c242f9a0Schunli zhang - Sun Microsystems - Irvine United States else 1117c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_BUMP(xuiostat_rbuf_copied); 1118c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } else { 1119c242f9a0Schunli zhang - Sun Microsystems - Irvine United States err = uiomove((char *)db->db_data + bufoff, tocpy, 1120c242f9a0Schunli zhang - Sun Microsystems - Irvine United States UIO_READ, uio); 1121c242f9a0Schunli zhang - Sun Microsystems - Irvine United States } 1122feb08c6bSbillm if (err) 1123feb08c6bSbillm break; 1124feb08c6bSbillm 1125feb08c6bSbillm size -= tocpy; 1126feb08c6bSbillm } 1127feb08c6bSbillm dmu_buf_rele_array(dbp, numbufs, FTAG); 1128feb08c6bSbillm 1129feb08c6bSbillm return (err); 1130feb08c6bSbillm } 1131feb08c6bSbillm 1132f8554bb9SMatthew Ahrens /* 1133f8554bb9SMatthew Ahrens * Read 'size' bytes into the uio buffer. 1134f8554bb9SMatthew Ahrens * From object zdb->db_object. 1135f8554bb9SMatthew Ahrens * Starting at offset uio->uio_loffset. 1136f8554bb9SMatthew Ahrens * 1137f8554bb9SMatthew Ahrens * If the caller already has a dbuf in the target object 1138f8554bb9SMatthew Ahrens * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(), 1139f8554bb9SMatthew Ahrens * because we don't have to find the dnode_t for the object. 1140f8554bb9SMatthew Ahrens */ 1141f8554bb9SMatthew Ahrens int 1142f8554bb9SMatthew Ahrens dmu_read_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size) 1143f8554bb9SMatthew Ahrens { 1144f8554bb9SMatthew Ahrens dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1145f8554bb9SMatthew Ahrens dnode_t *dn; 1146f8554bb9SMatthew Ahrens int err; 1147f8554bb9SMatthew Ahrens 1148f8554bb9SMatthew Ahrens if (size == 0) 1149f8554bb9SMatthew Ahrens return (0); 1150f8554bb9SMatthew Ahrens 1151f8554bb9SMatthew Ahrens DB_DNODE_ENTER(db); 1152f8554bb9SMatthew Ahrens dn = DB_DNODE(db); 1153f8554bb9SMatthew Ahrens err = dmu_read_uio_dnode(dn, uio, size); 1154f8554bb9SMatthew Ahrens DB_DNODE_EXIT(db); 1155f8554bb9SMatthew Ahrens 1156f8554bb9SMatthew Ahrens return (err); 1157f8554bb9SMatthew Ahrens } 1158f8554bb9SMatthew Ahrens 1159f8554bb9SMatthew Ahrens /* 1160f8554bb9SMatthew Ahrens * Read 'size' bytes into the uio buffer. 1161f8554bb9SMatthew Ahrens * From the specified object 1162f8554bb9SMatthew Ahrens * Starting at offset uio->uio_loffset. 1163f8554bb9SMatthew Ahrens */ 1164f8554bb9SMatthew Ahrens int 1165f8554bb9SMatthew Ahrens dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size) 1166f8554bb9SMatthew Ahrens { 1167f8554bb9SMatthew Ahrens dnode_t *dn; 1168f8554bb9SMatthew Ahrens int err; 1169f8554bb9SMatthew Ahrens 1170f8554bb9SMatthew Ahrens if (size == 0) 1171f8554bb9SMatthew Ahrens return (0); 1172f8554bb9SMatthew Ahrens 1173f8554bb9SMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 1174f8554bb9SMatthew Ahrens if (err) 1175f8554bb9SMatthew Ahrens return (err); 1176f8554bb9SMatthew Ahrens 1177f8554bb9SMatthew Ahrens err = dmu_read_uio_dnode(dn, uio, size); 1178f8554bb9SMatthew Ahrens 1179f8554bb9SMatthew Ahrens dnode_rele(dn, FTAG); 1180f8554bb9SMatthew Ahrens 1181f8554bb9SMatthew Ahrens return (err); 1182f8554bb9SMatthew Ahrens } 1183f8554bb9SMatthew Ahrens 118494d1a210STim Haley static int 118594d1a210STim Haley dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx) 1186fa9e4066Sahrens { 1187fa9e4066Sahrens dmu_buf_t **dbp; 118894d1a210STim Haley int numbufs; 1189fa9e4066Sahrens int err = 0; 119094d1a210STim Haley int i; 1191fa9e4066Sahrens 119294d1a210STim Haley err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size, 119394d1a210STim Haley FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH); 1194ea8dc4b6Seschrock if (err) 1195ea8dc4b6Seschrock return (err); 1196fa9e4066Sahrens 1197fa9e4066Sahrens for (i = 0; i < numbufs; i++) { 1198fa9e4066Sahrens int tocpy; 1199fa9e4066Sahrens int bufoff; 1200fa9e4066Sahrens dmu_buf_t *db = dbp[i]; 1201fa9e4066Sahrens 1202fa9e4066Sahrens ASSERT(size > 0); 1203fa9e4066Sahrens 1204feb08c6bSbillm bufoff = uio->uio_loffset - db->db_offset; 1205fa9e4066Sahrens tocpy = (int)MIN(db->db_size - bufoff, size); 1206fa9e4066Sahrens 1207fa9e4066Sahrens ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 1208fa9e4066Sahrens 1209fa9e4066Sahrens if (tocpy == db->db_size) 1210fa9e4066Sahrens dmu_buf_will_fill(db, tx); 1211fa9e4066Sahrens else 1212fa9e4066Sahrens dmu_buf_will_dirty(db, tx); 1213fa9e4066Sahrens 1214fa9e4066Sahrens /* 1215fa9e4066Sahrens * XXX uiomove could block forever (eg. nfs-backed 1216fa9e4066Sahrens * pages). There needs to be a uiolockdown() function 1217fa9e4066Sahrens * to lock the pages in memory, so that uiomove won't 1218fa9e4066Sahrens * block. 1219fa9e4066Sahrens */ 1220fa9e4066Sahrens err = uiomove((char *)db->db_data + bufoff, tocpy, 1221fa9e4066Sahrens UIO_WRITE, uio); 1222fa9e4066Sahrens 1223fa9e4066Sahrens if (tocpy == db->db_size) 1224fa9e4066Sahrens dmu_buf_fill_done(db, tx); 1225fa9e4066Sahrens 1226fa9e4066Sahrens if (err) 1227fa9e4066Sahrens break; 1228fa9e4066Sahrens 1229fa9e4066Sahrens size -= tocpy; 1230fa9e4066Sahrens } 123194d1a210STim Haley 1232ea8dc4b6Seschrock dmu_buf_rele_array(dbp, numbufs, FTAG); 1233fa9e4066Sahrens return (err); 1234fa9e4066Sahrens } 123544eda4d7Smaybee 1236f8554bb9SMatthew Ahrens /* 1237f8554bb9SMatthew Ahrens * Write 'size' bytes from the uio buffer. 1238f8554bb9SMatthew Ahrens * To object zdb->db_object. 1239f8554bb9SMatthew Ahrens * Starting at offset uio->uio_loffset. 1240f8554bb9SMatthew Ahrens * 1241f8554bb9SMatthew Ahrens * If the caller already has a dbuf in the target object 1242f8554bb9SMatthew Ahrens * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(), 1243f8554bb9SMatthew Ahrens * because we don't have to find the dnode_t for the object. 1244f8554bb9SMatthew Ahrens */ 124594d1a210STim Haley int 124694d1a210STim Haley dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size, 124794d1a210STim Haley dmu_tx_t *tx) 124894d1a210STim Haley { 1249744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1250744947dcSTom Erickson dnode_t *dn; 1251744947dcSTom Erickson int err; 1252744947dcSTom Erickson 125394d1a210STim Haley if (size == 0) 125494d1a210STim Haley return (0); 125594d1a210STim Haley 1256744947dcSTom Erickson DB_DNODE_ENTER(db); 1257744947dcSTom Erickson dn = DB_DNODE(db); 1258744947dcSTom Erickson err = dmu_write_uio_dnode(dn, uio, size, tx); 1259744947dcSTom Erickson DB_DNODE_EXIT(db); 1260744947dcSTom Erickson 1261744947dcSTom Erickson return (err); 126294d1a210STim Haley } 126394d1a210STim Haley 1264f8554bb9SMatthew Ahrens /* 1265f8554bb9SMatthew Ahrens * Write 'size' bytes from the uio buffer. 1266f8554bb9SMatthew Ahrens * To the specified object. 1267f8554bb9SMatthew Ahrens * Starting at offset uio->uio_loffset. 1268f8554bb9SMatthew Ahrens */ 126994d1a210STim Haley int 127094d1a210STim Haley dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size, 127194d1a210STim Haley dmu_tx_t *tx) 127294d1a210STim Haley { 127394d1a210STim Haley dnode_t *dn; 127494d1a210STim Haley int err; 127594d1a210STim Haley 127694d1a210STim Haley if (size == 0) 127794d1a210STim Haley return (0); 127894d1a210STim Haley 127994d1a210STim Haley err = dnode_hold(os, object, FTAG, &dn); 128094d1a210STim Haley if (err) 128194d1a210STim Haley return (err); 128294d1a210STim Haley 128394d1a210STim Haley err = dmu_write_uio_dnode(dn, uio, size, tx); 128494d1a210STim Haley 128594d1a210STim Haley dnode_rele(dn, FTAG); 128694d1a210STim Haley 128794d1a210STim Haley return (err); 128894d1a210STim Haley } 128994d1a210STim Haley 129044eda4d7Smaybee int 129144eda4d7Smaybee dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 129244eda4d7Smaybee page_t *pp, dmu_tx_t *tx) 129344eda4d7Smaybee { 129444eda4d7Smaybee dmu_buf_t **dbp; 129544eda4d7Smaybee int numbufs, i; 129644eda4d7Smaybee int err; 129744eda4d7Smaybee 129844eda4d7Smaybee if (size == 0) 129944eda4d7Smaybee return (0); 130044eda4d7Smaybee 130144eda4d7Smaybee err = dmu_buf_hold_array(os, object, offset, size, 130244eda4d7Smaybee FALSE, FTAG, &numbufs, &dbp); 130344eda4d7Smaybee if (err) 130444eda4d7Smaybee return (err); 130544eda4d7Smaybee 130644eda4d7Smaybee for (i = 0; i < numbufs; i++) { 130744eda4d7Smaybee int tocpy, copied, thiscpy; 130844eda4d7Smaybee int bufoff; 130944eda4d7Smaybee dmu_buf_t *db = dbp[i]; 131044eda4d7Smaybee caddr_t va; 131144eda4d7Smaybee 131244eda4d7Smaybee ASSERT(size > 0); 131344eda4d7Smaybee ASSERT3U(db->db_size, >=, PAGESIZE); 131444eda4d7Smaybee 131544eda4d7Smaybee bufoff = offset - db->db_offset; 131644eda4d7Smaybee tocpy = (int)MIN(db->db_size - bufoff, size); 131744eda4d7Smaybee 131844eda4d7Smaybee ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 131944eda4d7Smaybee 132044eda4d7Smaybee if (tocpy == db->db_size) 132144eda4d7Smaybee dmu_buf_will_fill(db, tx); 132244eda4d7Smaybee else 132344eda4d7Smaybee dmu_buf_will_dirty(db, tx); 132444eda4d7Smaybee 132544eda4d7Smaybee for (copied = 0; copied < tocpy; copied += PAGESIZE) { 132644eda4d7Smaybee ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff); 132744eda4d7Smaybee thiscpy = MIN(PAGESIZE, tocpy - copied); 13280fab61baSJonathan W Adams va = zfs_map_page(pp, S_READ); 132944eda4d7Smaybee bcopy(va, (char *)db->db_data + bufoff, thiscpy); 13300fab61baSJonathan W Adams zfs_unmap_page(pp, va); 133144eda4d7Smaybee pp = pp->p_next; 133244eda4d7Smaybee bufoff += PAGESIZE; 133344eda4d7Smaybee } 133444eda4d7Smaybee 133544eda4d7Smaybee if (tocpy == db->db_size) 133644eda4d7Smaybee dmu_buf_fill_done(db, tx); 133744eda4d7Smaybee 133844eda4d7Smaybee offset += tocpy; 133944eda4d7Smaybee size -= tocpy; 134044eda4d7Smaybee } 134144eda4d7Smaybee dmu_buf_rele_array(dbp, numbufs, FTAG); 134244eda4d7Smaybee return (err); 134344eda4d7Smaybee } 1344fa9e4066Sahrens #endif 1345fa9e4066Sahrens 13462fdbea25SAleksandr Guzovskiy /* 13472fdbea25SAleksandr Guzovskiy * Allocate a loaned anonymous arc buffer. 13482fdbea25SAleksandr Guzovskiy */ 13492fdbea25SAleksandr Guzovskiy arc_buf_t * 13502fdbea25SAleksandr Guzovskiy dmu_request_arcbuf(dmu_buf_t *handle, int size) 13512fdbea25SAleksandr Guzovskiy { 1352744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle; 13532fdbea25SAleksandr Guzovskiy 135443466aaeSMax Grossman return (arc_loan_buf(db->db_objset->os_spa, size)); 13552fdbea25SAleksandr Guzovskiy } 13562fdbea25SAleksandr Guzovskiy 13572fdbea25SAleksandr Guzovskiy /* 13582fdbea25SAleksandr Guzovskiy * Free a loaned arc buffer. 13592fdbea25SAleksandr Guzovskiy */ 13602fdbea25SAleksandr Guzovskiy void 13612fdbea25SAleksandr Guzovskiy dmu_return_arcbuf(arc_buf_t *buf) 13622fdbea25SAleksandr Guzovskiy { 13632fdbea25SAleksandr Guzovskiy arc_return_buf(buf, FTAG); 1364dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, FTAG); 13652fdbea25SAleksandr Guzovskiy } 13662fdbea25SAleksandr Guzovskiy 13672fdbea25SAleksandr Guzovskiy /* 13682fdbea25SAleksandr Guzovskiy * When possible directly assign passed loaned arc buffer to a dbuf. 13692fdbea25SAleksandr Guzovskiy * If this is not possible copy the contents of passed arc buf via 13702fdbea25SAleksandr Guzovskiy * dmu_write(). 13712fdbea25SAleksandr Guzovskiy */ 13722fdbea25SAleksandr Guzovskiy void 13732fdbea25SAleksandr Guzovskiy dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf, 13742fdbea25SAleksandr Guzovskiy dmu_tx_t *tx) 13752fdbea25SAleksandr Guzovskiy { 1376744947dcSTom Erickson dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle; 1377744947dcSTom Erickson dnode_t *dn; 13782fdbea25SAleksandr Guzovskiy dmu_buf_impl_t *db; 13792fdbea25SAleksandr Guzovskiy uint32_t blksz = (uint32_t)arc_buf_size(buf); 13802fdbea25SAleksandr Guzovskiy uint64_t blkid; 13812fdbea25SAleksandr Guzovskiy 1382744947dcSTom Erickson DB_DNODE_ENTER(dbuf); 1383744947dcSTom Erickson dn = DB_DNODE(dbuf); 13842fdbea25SAleksandr Guzovskiy rw_enter(&dn->dn_struct_rwlock, RW_READER); 1385a2cdcdd2SPaul Dagnelie blkid = dbuf_whichblock(dn, 0, offset); 13862fdbea25SAleksandr Guzovskiy VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL); 13872fdbea25SAleksandr Guzovskiy rw_exit(&dn->dn_struct_rwlock); 1388744947dcSTom Erickson DB_DNODE_EXIT(dbuf); 13892fdbea25SAleksandr Guzovskiy 13908a904709SMatthew Ahrens /* 13918a904709SMatthew Ahrens * We can only assign if the offset is aligned, the arc buf is the 13928a904709SMatthew Ahrens * same size as the dbuf, and the dbuf is not metadata. It 13938a904709SMatthew Ahrens * can't be metadata because the loaned arc buf comes from the 13948a904709SMatthew Ahrens * user-data kmem arena. 13958a904709SMatthew Ahrens */ 13968a904709SMatthew Ahrens if (offset == db->db.db_offset && blksz == db->db.db_size && 13978a904709SMatthew Ahrens DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA) { 13982fdbea25SAleksandr Guzovskiy dbuf_assign_arcbuf(db, buf, tx); 13992fdbea25SAleksandr Guzovskiy dbuf_rele(db, FTAG); 14002fdbea25SAleksandr Guzovskiy } else { 1401744947dcSTom Erickson objset_t *os; 1402744947dcSTom Erickson uint64_t object; 1403744947dcSTom Erickson 1404744947dcSTom Erickson DB_DNODE_ENTER(dbuf); 1405744947dcSTom Erickson dn = DB_DNODE(dbuf); 1406744947dcSTom Erickson os = dn->dn_objset; 1407744947dcSTom Erickson object = dn->dn_object; 1408744947dcSTom Erickson DB_DNODE_EXIT(dbuf); 1409744947dcSTom Erickson 14102fdbea25SAleksandr Guzovskiy dbuf_rele(db, FTAG); 1411744947dcSTom Erickson dmu_write(os, object, offset, blksz, buf->b_data, tx); 14122fdbea25SAleksandr Guzovskiy dmu_return_arcbuf(buf); 1413c242f9a0Schunli zhang - Sun Microsystems - Irvine United States XUIOSTAT_BUMP(xuiostat_wbuf_copied); 14142fdbea25SAleksandr Guzovskiy } 14152fdbea25SAleksandr Guzovskiy } 14162fdbea25SAleksandr Guzovskiy 1417c5c6ffa0Smaybee typedef struct { 1418b24ab676SJeff Bonwick dbuf_dirty_record_t *dsa_dr; 1419b24ab676SJeff Bonwick dmu_sync_cb_t *dsa_done; 1420b24ab676SJeff Bonwick zgd_t *dsa_zgd; 1421b24ab676SJeff Bonwick dmu_tx_t *dsa_tx; 1422c717a561Smaybee } dmu_sync_arg_t; 1423c5c6ffa0Smaybee 1424e14bb325SJeff Bonwick /* ARGSUSED */ 1425e14bb325SJeff Bonwick static void 1426e14bb325SJeff Bonwick dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg) 1427e14bb325SJeff Bonwick { 1428b24ab676SJeff Bonwick dmu_sync_arg_t *dsa = varg; 1429b24ab676SJeff Bonwick dmu_buf_t *db = dsa->dsa_zgd->zgd_db; 1430e14bb325SJeff Bonwick blkptr_t *bp = zio->io_bp; 1431975c32a0SNeil Perrin 1432b24ab676SJeff Bonwick if (zio->io_error == 0) { 1433b24ab676SJeff Bonwick if (BP_IS_HOLE(bp)) { 1434b24ab676SJeff Bonwick /* 1435b24ab676SJeff Bonwick * A block of zeros may compress to a hole, but the 1436b24ab676SJeff Bonwick * block size still needs to be known for replay. 1437b24ab676SJeff Bonwick */ 1438b24ab676SJeff Bonwick BP_SET_LSIZE(bp, db->db_size); 14395d7b4d43SMatthew Ahrens } else if (!BP_IS_EMBEDDED(bp)) { 1440b24ab676SJeff Bonwick ASSERT(BP_GET_LEVEL(bp) == 0); 1441b24ab676SJeff Bonwick bp->blk_fill = 1; 1442b24ab676SJeff Bonwick } 1443e14bb325SJeff Bonwick } 1444e14bb325SJeff Bonwick } 1445e14bb325SJeff Bonwick 1446b24ab676SJeff Bonwick static void 1447b24ab676SJeff Bonwick dmu_sync_late_arrival_ready(zio_t *zio) 1448b24ab676SJeff Bonwick { 1449b24ab676SJeff Bonwick dmu_sync_ready(zio, NULL, zio->io_private); 1450b24ab676SJeff Bonwick } 1451b24ab676SJeff Bonwick 1452c5c6ffa0Smaybee /* ARGSUSED */ 1453c5c6ffa0Smaybee static void 1454c5c6ffa0Smaybee dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg) 1455c5c6ffa0Smaybee { 1456b24ab676SJeff Bonwick dmu_sync_arg_t *dsa = varg; 1457b24ab676SJeff Bonwick dbuf_dirty_record_t *dr = dsa->dsa_dr; 1458c717a561Smaybee dmu_buf_impl_t *db = dr->dr_dbuf; 1459c5c6ffa0Smaybee 1460b50a0fe0SNeil Perrin mutex_enter(&db->db_mtx); 1461b50a0fe0SNeil Perrin ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC); 1462b24ab676SJeff Bonwick if (zio->io_error == 0) { 146380901aeaSGeorge Wilson dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE); 146480901aeaSGeorge Wilson if (dr->dt.dl.dr_nopwrite) { 146580901aeaSGeorge Wilson blkptr_t *bp = zio->io_bp; 146680901aeaSGeorge Wilson blkptr_t *bp_orig = &zio->io_bp_orig; 146780901aeaSGeorge Wilson uint8_t chksum = BP_GET_CHECKSUM(bp_orig); 146880901aeaSGeorge Wilson 146980901aeaSGeorge Wilson ASSERT(BP_EQUAL(bp, bp_orig)); 147080901aeaSGeorge Wilson ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF); 147145818ee1SMatthew Ahrens ASSERT(zio_checksum_table[chksum].ci_flags & 147245818ee1SMatthew Ahrens ZCHECKSUM_FLAG_NOPWRITE); 147380901aeaSGeorge Wilson } 1474b24ab676SJeff Bonwick dr->dt.dl.dr_overridden_by = *zio->io_bp; 1475b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_OVERRIDDEN; 1476b24ab676SJeff Bonwick dr->dt.dl.dr_copies = zio->io_prop.zp_copies; 147770163ac5SPrakash Surya 147870163ac5SPrakash Surya /* 147970163ac5SPrakash Surya * Old style holes are filled with all zeros, whereas 148070163ac5SPrakash Surya * new-style holes maintain their lsize, type, level, 148170163ac5SPrakash Surya * and birth time (see zio_write_compress). While we 148270163ac5SPrakash Surya * need to reset the BP_SET_LSIZE() call that happened 148370163ac5SPrakash Surya * in dmu_sync_ready for old style holes, we do *not* 148470163ac5SPrakash Surya * want to wipe out the information contained in new 148570163ac5SPrakash Surya * style holes. Thus, only zero out the block pointer if 148670163ac5SPrakash Surya * it's an old style hole. 148770163ac5SPrakash Surya */ 148870163ac5SPrakash Surya if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) && 148970163ac5SPrakash Surya dr->dt.dl.dr_overridden_by.blk_birth == 0) 1490b24ab676SJeff Bonwick BP_ZERO(&dr->dt.dl.dr_overridden_by); 1491b24ab676SJeff Bonwick } else { 1492b24ab676SJeff Bonwick dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1493b24ab676SJeff Bonwick } 1494c5c6ffa0Smaybee cv_broadcast(&db->db_changed); 1495b50a0fe0SNeil Perrin mutex_exit(&db->db_mtx); 1496b50a0fe0SNeil Perrin 1497b24ab676SJeff Bonwick dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1498b24ab676SJeff Bonwick 1499b24ab676SJeff Bonwick kmem_free(dsa, sizeof (*dsa)); 1500b24ab676SJeff Bonwick } 1501b24ab676SJeff Bonwick 1502b24ab676SJeff Bonwick static void 1503b24ab676SJeff Bonwick dmu_sync_late_arrival_done(zio_t *zio) 1504b24ab676SJeff Bonwick { 1505b24ab676SJeff Bonwick blkptr_t *bp = zio->io_bp; 1506b24ab676SJeff Bonwick dmu_sync_arg_t *dsa = zio->io_private; 150780901aeaSGeorge Wilson blkptr_t *bp_orig = &zio->io_bp_orig; 1508b24ab676SJeff Bonwick 1509b24ab676SJeff Bonwick if (zio->io_error == 0 && !BP_IS_HOLE(bp)) { 151080901aeaSGeorge Wilson /* 151180901aeaSGeorge Wilson * If we didn't allocate a new block (i.e. ZIO_FLAG_NOPWRITE) 151280901aeaSGeorge Wilson * then there is nothing to do here. Otherwise, free the 151380901aeaSGeorge Wilson * newly allocated block in this txg. 151480901aeaSGeorge Wilson */ 151580901aeaSGeorge Wilson if (zio->io_flags & ZIO_FLAG_NOPWRITE) { 151680901aeaSGeorge Wilson ASSERT(BP_EQUAL(bp, bp_orig)); 151780901aeaSGeorge Wilson } else { 151880901aeaSGeorge Wilson ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig)); 151980901aeaSGeorge Wilson ASSERT(zio->io_bp->blk_birth == zio->io_txg); 152080901aeaSGeorge Wilson ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa)); 152180901aeaSGeorge Wilson zio_free(zio->io_spa, zio->io_txg, zio->io_bp); 152280901aeaSGeorge Wilson } 1523b24ab676SJeff Bonwick } 1524b24ab676SJeff Bonwick 1525b24ab676SJeff Bonwick dmu_tx_commit(dsa->dsa_tx); 1526b24ab676SJeff Bonwick 1527b24ab676SJeff Bonwick dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1528b24ab676SJeff Bonwick 1529b24ab676SJeff Bonwick kmem_free(dsa, sizeof (*dsa)); 1530b24ab676SJeff Bonwick } 1531b24ab676SJeff Bonwick 1532b24ab676SJeff Bonwick static int 1533b24ab676SJeff Bonwick dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, 15347802d7bfSMatthew Ahrens zio_prop_t *zp, zbookmark_phys_t *zb) 1535b24ab676SJeff Bonwick { 1536b24ab676SJeff Bonwick dmu_sync_arg_t *dsa; 1537b24ab676SJeff Bonwick dmu_tx_t *tx; 1538b24ab676SJeff Bonwick 1539b24ab676SJeff Bonwick tx = dmu_tx_create(os); 1540b24ab676SJeff Bonwick dmu_tx_hold_space(tx, zgd->zgd_db->db_size); 15416e1f5caaSNeil Perrin if (dmu_tx_assign(tx, TXG_WAIT) != 0) { 1542b24ab676SJeff Bonwick dmu_tx_abort(tx); 1543be6fd75aSMatthew Ahrens /* Make zl_get_data do txg_waited_synced() */ 1544be6fd75aSMatthew Ahrens return (SET_ERROR(EIO)); 1545b24ab676SJeff Bonwick } 1546b24ab676SJeff Bonwick 1547b24ab676SJeff Bonwick dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 1548b24ab676SJeff Bonwick dsa->dsa_dr = NULL; 1549b24ab676SJeff Bonwick dsa->dsa_done = done; 1550b24ab676SJeff Bonwick dsa->dsa_zgd = zgd; 1551b24ab676SJeff Bonwick dsa->dsa_tx = tx; 1552c717a561Smaybee 15538df0bcf0SPaul Dagnelie zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), 15548df0bcf0SPaul Dagnelie zgd->zgd_bp, zgd->zgd_db->db_data, zgd->zgd_db->db_size, 15558df0bcf0SPaul Dagnelie zp, dmu_sync_late_arrival_ready, NULL, 15568df0bcf0SPaul Dagnelie NULL, dmu_sync_late_arrival_done, dsa, ZIO_PRIORITY_SYNC_WRITE, 15578df0bcf0SPaul Dagnelie ZIO_FLAG_CANFAIL, zb)); 1558b24ab676SJeff Bonwick 1559b24ab676SJeff Bonwick return (0); 1560c5c6ffa0Smaybee } 1561c5c6ffa0Smaybee 1562fa9e4066Sahrens /* 1563c5c6ffa0Smaybee * Intent log support: sync the block associated with db to disk. 1564c5c6ffa0Smaybee * N.B. and XXX: the caller is responsible for making sure that the 1565c5c6ffa0Smaybee * data isn't changing while dmu_sync() is writing it. 1566fa9e4066Sahrens * 1567fa9e4066Sahrens * Return values: 1568fa9e4066Sahrens * 156980901aeaSGeorge Wilson * EEXIST: this txg has already been synced, so there's nothing to do. 1570fa9e4066Sahrens * The caller should not log the write. 1571fa9e4066Sahrens * 1572fa9e4066Sahrens * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do. 1573fa9e4066Sahrens * The caller should not log the write. 1574fa9e4066Sahrens * 1575c5c6ffa0Smaybee * EALREADY: this block is already in the process of being synced. 1576c5c6ffa0Smaybee * The caller should track its progress (somehow). 1577fa9e4066Sahrens * 1578b24ab676SJeff Bonwick * EIO: could not do the I/O. 1579b24ab676SJeff Bonwick * The caller should do a txg_wait_synced(). 1580fa9e4066Sahrens * 1581b24ab676SJeff Bonwick * 0: the I/O has been initiated. 1582b24ab676SJeff Bonwick * The caller should log this blkptr in the done callback. 1583b24ab676SJeff Bonwick * It is possible that the I/O will fail, in which case 1584b24ab676SJeff Bonwick * the error will be reported to the done callback and 1585b24ab676SJeff Bonwick * propagated to pio from zio_done(). 1586fa9e4066Sahrens */ 1587fa9e4066Sahrens int 1588b24ab676SJeff Bonwick dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) 1589fa9e4066Sahrens { 1590b24ab676SJeff Bonwick blkptr_t *bp = zgd->zgd_bp; 1591b24ab676SJeff Bonwick dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db; 1592503ad85cSMatthew Ahrens objset_t *os = db->db_objset; 1593b24ab676SJeff Bonwick dsl_dataset_t *ds = os->os_dsl_dataset; 1594c717a561Smaybee dbuf_dirty_record_t *dr; 1595b24ab676SJeff Bonwick dmu_sync_arg_t *dsa; 15967802d7bfSMatthew Ahrens zbookmark_phys_t zb; 1597b24ab676SJeff Bonwick zio_prop_t zp; 1598744947dcSTom Erickson dnode_t *dn; 1599fa9e4066Sahrens 1600b24ab676SJeff Bonwick ASSERT(pio != NULL); 1601fa9e4066Sahrens ASSERT(txg != 0); 1602fa9e4066Sahrens 1603b24ab676SJeff Bonwick SET_BOOKMARK(&zb, ds->ds_object, 1604b24ab676SJeff Bonwick db->db.db_object, db->db_level, db->db_blkid); 1605b24ab676SJeff Bonwick 1606744947dcSTom Erickson DB_DNODE_ENTER(db); 1607744947dcSTom Erickson dn = DB_DNODE(db); 1608744947dcSTom Erickson dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp); 1609744947dcSTom Erickson DB_DNODE_EXIT(db); 1610fa9e4066Sahrens 1611ea8dc4b6Seschrock /* 1612b24ab676SJeff Bonwick * If we're frozen (running ziltest), we always need to generate a bp. 1613ea8dc4b6Seschrock */ 1614b24ab676SJeff Bonwick if (txg > spa_freeze_txg(os->os_spa)) 1615b24ab676SJeff Bonwick return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 1616ea8dc4b6Seschrock 1617fa9e4066Sahrens /* 1618b24ab676SJeff Bonwick * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf() 1619b24ab676SJeff Bonwick * and us. If we determine that this txg is not yet syncing, 1620b24ab676SJeff Bonwick * but it begins to sync a moment later, that's OK because the 1621b24ab676SJeff Bonwick * sync thread will block in dbuf_sync_leaf() until we drop db_mtx. 1622fa9e4066Sahrens */ 1623b24ab676SJeff Bonwick mutex_enter(&db->db_mtx); 1624b24ab676SJeff Bonwick 1625b24ab676SJeff Bonwick if (txg <= spa_last_synced_txg(os->os_spa)) { 1626fa9e4066Sahrens /* 1627b24ab676SJeff Bonwick * This txg has already synced. There's nothing to do. 1628fa9e4066Sahrens */ 1629b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 1630be6fd75aSMatthew Ahrens return (SET_ERROR(EEXIST)); 1631fa9e4066Sahrens } 1632fa9e4066Sahrens 1633b24ab676SJeff Bonwick if (txg <= spa_syncing_txg(os->os_spa)) { 1634b24ab676SJeff Bonwick /* 1635b24ab676SJeff Bonwick * This txg is currently syncing, so we can't mess with 1636b24ab676SJeff Bonwick * the dirty record anymore; just write a new log block. 1637b24ab676SJeff Bonwick */ 1638b24ab676SJeff Bonwick mutex_exit(&db->db_mtx); 1639b24ab676SJeff Bonwick return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 1640fa9e4066Sahrens } 1641fa9e4066Sahrens 1642c717a561Smaybee dr = db->db_last_dirty; 1643b24ab676SJeff Bonwick while (dr && dr->dr_txg != txg) 1644c717a561Smaybee dr = dr->dr_next; 1645b24ab676SJeff Bonwick 1646b24ab676SJeff Bonwick if (dr == NULL) { 1647c5c6ffa0Smaybee /* 1648b24ab676SJeff Bonwick * There's no dr for this dbuf, so it must have been freed. 1649c5c6ffa0Smaybee * There's no need to log writes to freed blocks, so we're done. 1650c5c6ffa0Smaybee */ 1651c5c6ffa0Smaybee mutex_exit(&db->db_mtx); 1652be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 1653c5c6ffa0Smaybee } 1654c5c6ffa0Smaybee 165580901aeaSGeorge Wilson ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg); 165680901aeaSGeorge Wilson 165780901aeaSGeorge Wilson /* 165834e8acefSMatthew Ahrens * Assume the on-disk data is X, the current syncing data (in 165934e8acefSMatthew Ahrens * txg - 1) is Y, and the current in-memory data is Z (currently 166034e8acefSMatthew Ahrens * in dmu_sync). 166134e8acefSMatthew Ahrens * 166234e8acefSMatthew Ahrens * We usually want to perform a nopwrite if X and Z are the 166334e8acefSMatthew Ahrens * same. However, if Y is different (i.e. the BP is going to 166434e8acefSMatthew Ahrens * change before this write takes effect), then a nopwrite will 166534e8acefSMatthew Ahrens * be incorrect - we would override with X, which could have 166634e8acefSMatthew Ahrens * been freed when Y was written. 166734e8acefSMatthew Ahrens * 166834e8acefSMatthew Ahrens * (Note that this is not a concern when we are nop-writing from 166934e8acefSMatthew Ahrens * syncing context, because X and Y must be identical, because 167034e8acefSMatthew Ahrens * all previous txgs have been synced.) 167134e8acefSMatthew Ahrens * 167234e8acefSMatthew Ahrens * Therefore, we disable nopwrite if the current BP could change 167334e8acefSMatthew Ahrens * before this TXG. There are two ways it could change: by 167434e8acefSMatthew Ahrens * being dirty (dr_next is non-NULL), or by being freed 167534e8acefSMatthew Ahrens * (dnode_block_freed()). This behavior is verified by 167634e8acefSMatthew Ahrens * zio_done(), which VERIFYs that the override BP is identical 167734e8acefSMatthew Ahrens * to the on-disk BP. 167880901aeaSGeorge Wilson */ 167934e8acefSMatthew Ahrens DB_DNODE_ENTER(db); 168034e8acefSMatthew Ahrens dn = DB_DNODE(db); 168134e8acefSMatthew Ahrens if (dr->dr_next != NULL || dnode_block_freed(dn, db->db_blkid)) 168280901aeaSGeorge Wilson zp.zp_nopwrite = B_FALSE; 168334e8acefSMatthew Ahrens DB_DNODE_EXIT(db); 168480901aeaSGeorge Wilson 1685c717a561Smaybee ASSERT(dr->dr_txg == txg); 1686b24ab676SJeff Bonwick if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC || 1687b24ab676SJeff Bonwick dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 1688c717a561Smaybee /* 1689b24ab676SJeff Bonwick * We have already issued a sync write for this buffer, 1690b24ab676SJeff Bonwick * or this buffer has already been synced. It could not 1691c717a561Smaybee * have been dirtied since, or we would have cleared the state. 1692c717a561Smaybee */ 1693c717a561Smaybee mutex_exit(&db->db_mtx); 1694be6fd75aSMatthew Ahrens return (SET_ERROR(EALREADY)); 1695c717a561Smaybee } 1696c717a561Smaybee 1697b24ab676SJeff Bonwick ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 1698c717a561Smaybee dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; 1699fa9e4066Sahrens mutex_exit(&db->db_mtx); 1700e14bb325SJeff Bonwick 1701b24ab676SJeff Bonwick dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 1702b24ab676SJeff Bonwick dsa->dsa_dr = dr; 1703b24ab676SJeff Bonwick dsa->dsa_done = done; 1704b24ab676SJeff Bonwick dsa->dsa_zgd = zgd; 1705b24ab676SJeff Bonwick dsa->dsa_tx = NULL; 1706e14bb325SJeff Bonwick 1707b24ab676SJeff Bonwick zio_nowait(arc_write(pio, os->os_spa, txg, 1708aad02571SSaso Kiselkov bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db), 1709dcbf3bd6SGeorge Wilson &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa, 17108df0bcf0SPaul Dagnelie ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb)); 1711e14bb325SJeff Bonwick 1712b24ab676SJeff Bonwick return (0); 1713fa9e4066Sahrens } 1714fa9e4066Sahrens 1715fa9e4066Sahrens int 1716fa9e4066Sahrens dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs, 17179a686fbcSPaul Dagnelie dmu_tx_t *tx) 1718fa9e4066Sahrens { 1719ea8dc4b6Seschrock dnode_t *dn; 1720ea8dc4b6Seschrock int err; 1721ea8dc4b6Seschrock 1722503ad85cSMatthew Ahrens err = dnode_hold(os, object, FTAG, &dn); 1723ea8dc4b6Seschrock if (err) 1724ea8dc4b6Seschrock return (err); 1725ea8dc4b6Seschrock err = dnode_set_blksz(dn, size, ibs, tx); 1726fa9e4066Sahrens dnode_rele(dn, FTAG); 1727fa9e4066Sahrens return (err); 1728fa9e4066Sahrens } 1729fa9e4066Sahrens 1730fa9e4066Sahrens void 1731fa9e4066Sahrens dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum, 17329a686fbcSPaul Dagnelie dmu_tx_t *tx) 1733fa9e4066Sahrens { 1734ea8dc4b6Seschrock dnode_t *dn; 1735ea8dc4b6Seschrock 17365d7b4d43SMatthew Ahrens /* 17375d7b4d43SMatthew Ahrens * Send streams include each object's checksum function. This 17385d7b4d43SMatthew Ahrens * check ensures that the receiving system can understand the 17395d7b4d43SMatthew Ahrens * checksum function transmitted. 17405d7b4d43SMatthew Ahrens */ 17415d7b4d43SMatthew Ahrens ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS); 17425d7b4d43SMatthew Ahrens 17435d7b4d43SMatthew Ahrens VERIFY0(dnode_hold(os, object, FTAG, &dn)); 17445d7b4d43SMatthew Ahrens ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS); 1745fa9e4066Sahrens dn->dn_checksum = checksum; 1746fa9e4066Sahrens dnode_setdirty(dn, tx); 1747fa9e4066Sahrens dnode_rele(dn, FTAG); 1748fa9e4066Sahrens } 1749fa9e4066Sahrens 1750fa9e4066Sahrens void 1751fa9e4066Sahrens dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress, 17529a686fbcSPaul Dagnelie dmu_tx_t *tx) 1753fa9e4066Sahrens { 1754ea8dc4b6Seschrock dnode_t *dn; 1755ea8dc4b6Seschrock 17565d7b4d43SMatthew Ahrens /* 17575d7b4d43SMatthew Ahrens * Send streams include each object's compression function. This 17585d7b4d43SMatthew Ahrens * check ensures that the receiving system can understand the 17595d7b4d43SMatthew Ahrens * compression function transmitted. 17605d7b4d43SMatthew Ahrens */ 17615d7b4d43SMatthew Ahrens ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS); 17625d7b4d43SMatthew Ahrens 17635d7b4d43SMatthew Ahrens VERIFY0(dnode_hold(os, object, FTAG, &dn)); 1764fa9e4066Sahrens dn->dn_compress = compress; 1765fa9e4066Sahrens dnode_setdirty(dn, tx); 1766fa9e4066Sahrens dnode_rele(dn, FTAG); 1767fa9e4066Sahrens } 1768fa9e4066Sahrens 1769b24ab676SJeff Bonwick int zfs_mdcomp_disable = 0; 1770b24ab676SJeff Bonwick 1771edf345e6SMatthew Ahrens /* 1772edf345e6SMatthew Ahrens * When the "redundant_metadata" property is set to "most", only indirect 1773edf345e6SMatthew Ahrens * blocks of this level and higher will have an additional ditto block. 1774edf345e6SMatthew Ahrens */ 1775edf345e6SMatthew Ahrens int zfs_redundant_metadata_most_ditto_level = 2; 1776edf345e6SMatthew Ahrens 1777b24ab676SJeff Bonwick void 1778b24ab676SJeff Bonwick dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp) 1779b24ab676SJeff Bonwick { 1780b24ab676SJeff Bonwick dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET; 1781ad135b5dSChristopher Siden boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) || 17821d8ccc7bSMark Shellenbaum (wp & WP_SPILL)); 1783b24ab676SJeff Bonwick enum zio_checksum checksum = os->os_checksum; 1784b24ab676SJeff Bonwick enum zio_compress compress = os->os_compress; 1785b24ab676SJeff Bonwick enum zio_checksum dedup_checksum = os->os_dedup_checksum; 17867540df39SGeorge Wilson boolean_t dedup = B_FALSE; 17877540df39SGeorge Wilson boolean_t nopwrite = B_FALSE; 1788b24ab676SJeff Bonwick boolean_t dedup_verify = os->os_dedup_verify; 1789b24ab676SJeff Bonwick int copies = os->os_copies; 1790b24ab676SJeff Bonwick 1791b24ab676SJeff Bonwick /* 179280901aeaSGeorge Wilson * We maintain different write policies for each of the following 179380901aeaSGeorge Wilson * types of data: 179480901aeaSGeorge Wilson * 1. metadata 179580901aeaSGeorge Wilson * 2. preallocated blocks (i.e. level-0 blocks of a dump device) 179680901aeaSGeorge Wilson * 3. all other level 0 blocks 1797b24ab676SJeff Bonwick */ 1798b24ab676SJeff Bonwick if (ismd) { 1799b8289d24SDaniil Lunev if (zfs_mdcomp_disable) { 1800b8289d24SDaniil Lunev compress = ZIO_COMPRESS_EMPTY; 1801b8289d24SDaniil Lunev } else { 1802db1741f5SJustin T. Gibbs /* 1803db1741f5SJustin T. Gibbs * XXX -- we should design a compression algorithm 1804db1741f5SJustin T. Gibbs * that specializes in arrays of bps. 1805db1741f5SJustin T. Gibbs */ 1806db1741f5SJustin T. Gibbs compress = zio_compress_select(os->os_spa, 1807db1741f5SJustin T. Gibbs ZIO_COMPRESS_ON, ZIO_COMPRESS_ON); 1808b8289d24SDaniil Lunev } 180980901aeaSGeorge Wilson 1810b24ab676SJeff Bonwick /* 1811b24ab676SJeff Bonwick * Metadata always gets checksummed. If the data 1812b24ab676SJeff Bonwick * checksum is multi-bit correctable, and it's not a 1813b24ab676SJeff Bonwick * ZBT-style checksum, then it's suitable for metadata 1814b24ab676SJeff Bonwick * as well. Otherwise, the metadata checksum defaults 1815b24ab676SJeff Bonwick * to fletcher4. 1816b24ab676SJeff Bonwick */ 181745818ee1SMatthew Ahrens if (!(zio_checksum_table[checksum].ci_flags & 181845818ee1SMatthew Ahrens ZCHECKSUM_FLAG_METADATA) || 181945818ee1SMatthew Ahrens (zio_checksum_table[checksum].ci_flags & 182045818ee1SMatthew Ahrens ZCHECKSUM_FLAG_EMBEDDED)) 1821b24ab676SJeff Bonwick checksum = ZIO_CHECKSUM_FLETCHER_4; 1822edf345e6SMatthew Ahrens 1823edf345e6SMatthew Ahrens if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL || 1824edf345e6SMatthew Ahrens (os->os_redundant_metadata == 1825edf345e6SMatthew Ahrens ZFS_REDUNDANT_METADATA_MOST && 1826edf345e6SMatthew Ahrens (level >= zfs_redundant_metadata_most_ditto_level || 1827edf345e6SMatthew Ahrens DMU_OT_IS_METADATA(type) || (wp & WP_SPILL)))) 1828edf345e6SMatthew Ahrens copies++; 182980901aeaSGeorge Wilson } else if (wp & WP_NOFILL) { 183080901aeaSGeorge Wilson ASSERT(level == 0); 183180901aeaSGeorge Wilson 1832b24ab676SJeff Bonwick /* 183380901aeaSGeorge Wilson * If we're writing preallocated blocks, we aren't actually 183480901aeaSGeorge Wilson * writing them so don't set any policy properties. These 183580901aeaSGeorge Wilson * blocks are currently only used by an external subsystem 183680901aeaSGeorge Wilson * outside of zfs (i.e. dump) and not written by the zio 183780901aeaSGeorge Wilson * pipeline. 1838b24ab676SJeff Bonwick */ 183980901aeaSGeorge Wilson compress = ZIO_COMPRESS_OFF; 1840810e43b2SBill Pijewski checksum = ZIO_CHECKSUM_NOPARITY; 1841b24ab676SJeff Bonwick } else { 1842db1741f5SJustin T. Gibbs compress = zio_compress_select(os->os_spa, dn->dn_compress, 1843db1741f5SJustin T. Gibbs compress); 1844b24ab676SJeff Bonwick 184580901aeaSGeorge Wilson checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ? 184680901aeaSGeorge Wilson zio_checksum_select(dn->dn_checksum, checksum) : 184780901aeaSGeorge Wilson dedup_checksum; 1848b24ab676SJeff Bonwick 184980901aeaSGeorge Wilson /* 185080901aeaSGeorge Wilson * Determine dedup setting. If we are in dmu_sync(), 185180901aeaSGeorge Wilson * we won't actually dedup now because that's all 185280901aeaSGeorge Wilson * done in syncing context; but we do want to use the 185380901aeaSGeorge Wilson * dedup checkum. If the checksum is not strong 185480901aeaSGeorge Wilson * enough to ensure unique signatures, force 185580901aeaSGeorge Wilson * dedup_verify. 185680901aeaSGeorge Wilson */ 185780901aeaSGeorge Wilson if (dedup_checksum != ZIO_CHECKSUM_OFF) { 185880901aeaSGeorge Wilson dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE; 185945818ee1SMatthew Ahrens if (!(zio_checksum_table[checksum].ci_flags & 186045818ee1SMatthew Ahrens ZCHECKSUM_FLAG_DEDUP)) 186180901aeaSGeorge Wilson dedup_verify = B_TRUE; 186280901aeaSGeorge Wilson } 1863b24ab676SJeff Bonwick 186480901aeaSGeorge Wilson /* 186545818ee1SMatthew Ahrens * Enable nopwrite if we have secure enough checksum 186645818ee1SMatthew Ahrens * algorithm (see comment in zio_nop_write) and 186745818ee1SMatthew Ahrens * compression is enabled. We don't enable nopwrite if 186845818ee1SMatthew Ahrens * dedup is enabled as the two features are mutually 186945818ee1SMatthew Ahrens * exclusive. 187080901aeaSGeorge Wilson */ 187145818ee1SMatthew Ahrens nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags & 187245818ee1SMatthew Ahrens ZCHECKSUM_FLAG_NOPWRITE) && 187380901aeaSGeorge Wilson compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled); 1874b24ab676SJeff Bonwick } 1875b24ab676SJeff Bonwick 1876b24ab676SJeff Bonwick zp->zp_checksum = checksum; 1877b24ab676SJeff Bonwick zp->zp_compress = compress; 18780a586ceaSMark Shellenbaum zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type; 1879b24ab676SJeff Bonwick zp->zp_level = level; 1880edf345e6SMatthew Ahrens zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa)); 1881b24ab676SJeff Bonwick zp->zp_dedup = dedup; 1882b24ab676SJeff Bonwick zp->zp_dedup_verify = dedup && dedup_verify; 188380901aeaSGeorge Wilson zp->zp_nopwrite = nopwrite; 1884b24ab676SJeff Bonwick } 1885b24ab676SJeff Bonwick 1886fa9e4066Sahrens int 1887fa9e4066Sahrens dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off) 1888fa9e4066Sahrens { 1889fa9e4066Sahrens dnode_t *dn; 18902bcf0248SMax Grossman int err; 1891fa9e4066Sahrens 1892fa9e4066Sahrens /* 1893fa9e4066Sahrens * Sync any current changes before 1894fa9e4066Sahrens * we go trundling through the block pointers. 1895fa9e4066Sahrens */ 18962bcf0248SMax Grossman err = dmu_object_wait_synced(os, object); 18972bcf0248SMax Grossman if (err) { 18982bcf0248SMax Grossman return (err); 1899fa9e4066Sahrens } 19002bcf0248SMax Grossman 19012bcf0248SMax Grossman err = dnode_hold(os, object, FTAG, &dn); 19022bcf0248SMax Grossman if (err) { 19032bcf0248SMax Grossman return (err); 1904fa9e4066Sahrens } 1905fa9e4066Sahrens 1906cdb0ab79Smaybee err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0); 1907fa9e4066Sahrens dnode_rele(dn, FTAG); 1908fa9e4066Sahrens 1909fa9e4066Sahrens return (err); 1910fa9e4066Sahrens } 1911fa9e4066Sahrens 19122bcf0248SMax Grossman /* 19132bcf0248SMax Grossman * Given the ZFS object, if it contains any dirty nodes 19142bcf0248SMax Grossman * this function flushes all dirty blocks to disk. This 19152bcf0248SMax Grossman * ensures the DMU object info is updated. A more efficient 19162bcf0248SMax Grossman * future version might just find the TXG with the maximum 19172bcf0248SMax Grossman * ID and wait for that to be synced. 19182bcf0248SMax Grossman */ 19192bcf0248SMax Grossman int 19209a686fbcSPaul Dagnelie dmu_object_wait_synced(objset_t *os, uint64_t object) 19219a686fbcSPaul Dagnelie { 19222bcf0248SMax Grossman dnode_t *dn; 19232bcf0248SMax Grossman int error, i; 19242bcf0248SMax Grossman 19252bcf0248SMax Grossman error = dnode_hold(os, object, FTAG, &dn); 19262bcf0248SMax Grossman if (error) { 19272bcf0248SMax Grossman return (error); 19282bcf0248SMax Grossman } 19292bcf0248SMax Grossman 19302bcf0248SMax Grossman for (i = 0; i < TXG_SIZE; i++) { 19312bcf0248SMax Grossman if (list_link_active(&dn->dn_dirty_link[i])) { 19322bcf0248SMax Grossman break; 19332bcf0248SMax Grossman } 19342bcf0248SMax Grossman } 19352bcf0248SMax Grossman dnode_rele(dn, FTAG); 19362bcf0248SMax Grossman if (i != TXG_SIZE) { 19372bcf0248SMax Grossman txg_wait_synced(dmu_objset_pool(os), 0); 19382bcf0248SMax Grossman } 19392bcf0248SMax Grossman 19402bcf0248SMax Grossman return (0); 19412bcf0248SMax Grossman } 19422bcf0248SMax Grossman 1943fa9e4066Sahrens void 1944fa9e4066Sahrens dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) 1945fa9e4066Sahrens { 1946b24ab676SJeff Bonwick dnode_phys_t *dnp; 1947b24ab676SJeff Bonwick 1948fa9e4066Sahrens rw_enter(&dn->dn_struct_rwlock, RW_READER); 1949fa9e4066Sahrens mutex_enter(&dn->dn_mtx); 1950fa9e4066Sahrens 1951b24ab676SJeff Bonwick dnp = dn->dn_phys; 1952b24ab676SJeff Bonwick 1953fa9e4066Sahrens doi->doi_data_block_size = dn->dn_datablksz; 1954fa9e4066Sahrens doi->doi_metadata_block_size = dn->dn_indblkshift ? 1955fa9e4066Sahrens 1ULL << dn->dn_indblkshift : 0; 1956b24ab676SJeff Bonwick doi->doi_type = dn->dn_type; 1957b24ab676SJeff Bonwick doi->doi_bonus_type = dn->dn_bonustype; 1958b24ab676SJeff Bonwick doi->doi_bonus_size = dn->dn_bonuslen; 1959fa9e4066Sahrens doi->doi_indirection = dn->dn_nlevels; 1960fa9e4066Sahrens doi->doi_checksum = dn->dn_checksum; 1961fa9e4066Sahrens doi->doi_compress = dn->dn_compress; 1962e77d42eaSMatthew Ahrens doi->doi_nblkptr = dn->dn_nblkptr; 1963b24ab676SJeff Bonwick doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9; 1964d0475637SMatthew Ahrens doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 1965b24ab676SJeff Bonwick doi->doi_fill_count = 0; 1966b24ab676SJeff Bonwick for (int i = 0; i < dnp->dn_nblkptr; i++) 19675d7b4d43SMatthew Ahrens doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]); 1968fa9e4066Sahrens 1969fa9e4066Sahrens mutex_exit(&dn->dn_mtx); 1970fa9e4066Sahrens rw_exit(&dn->dn_struct_rwlock); 1971fa9e4066Sahrens } 1972fa9e4066Sahrens 1973fa9e4066Sahrens /* 1974fa9e4066Sahrens * Get information on a DMU object. 1975fa9e4066Sahrens * If doi is NULL, just indicates whether the object exists. 1976fa9e4066Sahrens */ 1977fa9e4066Sahrens int 1978fa9e4066Sahrens dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi) 1979fa9e4066Sahrens { 1980ea8dc4b6Seschrock dnode_t *dn; 1981503ad85cSMatthew Ahrens int err = dnode_hold(os, object, FTAG, &dn); 1982fa9e4066Sahrens 1983ea8dc4b6Seschrock if (err) 1984ea8dc4b6Seschrock return (err); 1985fa9e4066Sahrens 1986fa9e4066Sahrens if (doi != NULL) 1987fa9e4066Sahrens dmu_object_info_from_dnode(dn, doi); 1988fa9e4066Sahrens 1989fa9e4066Sahrens dnode_rele(dn, FTAG); 1990fa9e4066Sahrens return (0); 1991fa9e4066Sahrens } 1992fa9e4066Sahrens 1993fa9e4066Sahrens /* 1994fa9e4066Sahrens * As above, but faster; can be used when you have a held dbuf in hand. 1995fa9e4066Sahrens */ 1996fa9e4066Sahrens void 1997744947dcSTom Erickson dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi) 1998fa9e4066Sahrens { 1999744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2000744947dcSTom Erickson 2001744947dcSTom Erickson DB_DNODE_ENTER(db); 2002744947dcSTom Erickson dmu_object_info_from_dnode(DB_DNODE(db), doi); 2003744947dcSTom Erickson DB_DNODE_EXIT(db); 2004fa9e4066Sahrens } 2005fa9e4066Sahrens 2006fa9e4066Sahrens /* 2007fa9e4066Sahrens * Faster still when you only care about the size. 2008fa9e4066Sahrens * This is specifically optimized for zfs_getattr(). 2009fa9e4066Sahrens */ 2010fa9e4066Sahrens void 2011744947dcSTom Erickson dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize, 2012744947dcSTom Erickson u_longlong_t *nblk512) 2013fa9e4066Sahrens { 2014744947dcSTom Erickson dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2015744947dcSTom Erickson dnode_t *dn; 2016744947dcSTom Erickson 2017744947dcSTom Erickson DB_DNODE_ENTER(db); 2018744947dcSTom Erickson dn = DB_DNODE(db); 2019fa9e4066Sahrens 2020fa9e4066Sahrens *blksize = dn->dn_datablksz; 202199653d4eSeschrock /* add 1 for dnode space */ 202299653d4eSeschrock *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >> 202399653d4eSeschrock SPA_MINBLOCKSHIFT) + 1; 2024744947dcSTom Erickson DB_DNODE_EXIT(db); 2025fa9e4066Sahrens } 2026fa9e4066Sahrens 2027fa9e4066Sahrens void 2028fa9e4066Sahrens byteswap_uint64_array(void *vbuf, size_t size) 2029fa9e4066Sahrens { 2030fa9e4066Sahrens uint64_t *buf = vbuf; 2031fa9e4066Sahrens size_t count = size >> 3; 2032fa9e4066Sahrens int i; 2033fa9e4066Sahrens 2034fa9e4066Sahrens ASSERT((size & 7) == 0); 2035fa9e4066Sahrens 2036fa9e4066Sahrens for (i = 0; i < count; i++) 2037fa9e4066Sahrens buf[i] = BSWAP_64(buf[i]); 2038fa9e4066Sahrens } 2039fa9e4066Sahrens 2040fa9e4066Sahrens void 2041fa9e4066Sahrens byteswap_uint32_array(void *vbuf, size_t size) 2042fa9e4066Sahrens { 2043fa9e4066Sahrens uint32_t *buf = vbuf; 2044fa9e4066Sahrens size_t count = size >> 2; 2045fa9e4066Sahrens int i; 2046fa9e4066Sahrens 2047fa9e4066Sahrens ASSERT((size & 3) == 0); 2048fa9e4066Sahrens 2049fa9e4066Sahrens for (i = 0; i < count; i++) 2050fa9e4066Sahrens buf[i] = BSWAP_32(buf[i]); 2051fa9e4066Sahrens } 2052fa9e4066Sahrens 2053fa9e4066Sahrens void 2054fa9e4066Sahrens byteswap_uint16_array(void *vbuf, size_t size) 2055fa9e4066Sahrens { 2056fa9e4066Sahrens uint16_t *buf = vbuf; 2057fa9e4066Sahrens size_t count = size >> 1; 2058fa9e4066Sahrens int i; 2059fa9e4066Sahrens 2060fa9e4066Sahrens ASSERT((size & 1) == 0); 2061fa9e4066Sahrens 2062fa9e4066Sahrens for (i = 0; i < count; i++) 2063fa9e4066Sahrens buf[i] = BSWAP_16(buf[i]); 2064fa9e4066Sahrens } 2065fa9e4066Sahrens 2066fa9e4066Sahrens /* ARGSUSED */ 2067fa9e4066Sahrens void 2068fa9e4066Sahrens byteswap_uint8_array(void *vbuf, size_t size) 2069fa9e4066Sahrens { 2070fa9e4066Sahrens } 2071fa9e4066Sahrens 2072fa9e4066Sahrens void 2073fa9e4066Sahrens dmu_init(void) 2074fa9e4066Sahrens { 20753f9d6ad7SLin Ling zfs_dbgmsg_init(); 2076744947dcSTom Erickson sa_cache_init(); 2077744947dcSTom Erickson xuio_stat_init(); 2078744947dcSTom Erickson dmu_objset_init(); 2079fa9e4066Sahrens dnode_init(); 20807cbf8b43SRich Morris zfetch_init(); 2081fa94a07fSbrendan l2arc_init(); 2082ce636f8bSMatthew Ahrens arc_init(); 2083dcbf3bd6SGeorge Wilson dbuf_init(); 2084fa9e4066Sahrens } 2085fa9e4066Sahrens 2086fa9e4066Sahrens void 2087fa9e4066Sahrens dmu_fini(void) 2088fa9e4066Sahrens { 20893e30c24aSWill Andrews arc_fini(); /* arc depends on l2arc, so arc must go first */ 2090ce636f8bSMatthew Ahrens l2arc_fini(); 20917cbf8b43SRich Morris zfetch_fini(); 2092fa9e4066Sahrens dbuf_fini(); 2093744947dcSTom Erickson dnode_fini(); 2094744947dcSTom Erickson dmu_objset_fini(); 2095c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_fini(); 20960a586ceaSMark Shellenbaum sa_cache_fini(); 20973f9d6ad7SLin Ling zfs_dbgmsg_fini(); 2098fa9e4066Sahrens } 2099