xref: /illumos-gate/usr/src/uts/common/fs/zfs/dmu.c (revision ff5177ee)
1fa9e4066Sahrens /*
2fa9e4066Sahrens  * CDDL HEADER START
3fa9e4066Sahrens  *
4fa9e4066Sahrens  * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock  * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock  * You may not use this file except in compliance with the License.
7fa9e4066Sahrens  *
8fa9e4066Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens  * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens  * See the License for the specific language governing permissions
11fa9e4066Sahrens  * and limitations under the License.
12fa9e4066Sahrens  *
13fa9e4066Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens  *
19fa9e4066Sahrens  * CDDL HEADER END
20fa9e4066Sahrens  */
21fa9e4066Sahrens /*
2294d1a210STim Haley  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
238df0bcf0SPaul Dagnelie  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24fa9e4066Sahrens  */
25aad02571SSaso Kiselkov /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */
26810e43b2SBill Pijewski /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */
27eb721827SAlek Pinchuk /* Copyright 2016 Nexenta Systems, Inc. All rights reserved. */
28aad02571SSaso Kiselkov 
29fa9e4066Sahrens #include <sys/dmu.h>
30fa9e4066Sahrens #include <sys/dmu_impl.h>
31fa9e4066Sahrens #include <sys/dmu_tx.h>
32fa9e4066Sahrens #include <sys/dbuf.h>
33fa9e4066Sahrens #include <sys/dnode.h>
34fa9e4066Sahrens #include <sys/zfs_context.h>
35fa9e4066Sahrens #include <sys/dmu_objset.h>
36fa9e4066Sahrens #include <sys/dmu_traverse.h>
37fa9e4066Sahrens #include <sys/dsl_dataset.h>
38fa9e4066Sahrens #include <sys/dsl_dir.h>
39fa9e4066Sahrens #include <sys/dsl_pool.h>
401d452cf5Sahrens #include <sys/dsl_synctask.h>
41a2eea2e1Sahrens #include <sys/dsl_prop.h>
42fa9e4066Sahrens #include <sys/dmu_zfetch.h>
43fa9e4066Sahrens #include <sys/zfs_ioctl.h>
44fa9e4066Sahrens #include <sys/zap.h>
45ea8dc4b6Seschrock #include <sys/zio_checksum.h>
4680901aeaSGeorge Wilson #include <sys/zio_compress.h>
470a586ceaSMark Shellenbaum #include <sys/sa.h>
48b8289d24SDaniil Lunev #include <sys/zfeature.h>
4944eda4d7Smaybee #ifdef _KERNEL
5044eda4d7Smaybee #include <sys/vmsystm.h>
510fab61baSJonathan W Adams #include <sys/zfs_znode.h>
5244eda4d7Smaybee #endif
53fa9e4066Sahrens 
5480901aeaSGeorge Wilson /*
5580901aeaSGeorge Wilson  * Enable/disable nopwrite feature.
5680901aeaSGeorge Wilson  */
5780901aeaSGeorge Wilson int zfs_nopwrite_enabled = 1;
5880901aeaSGeorge Wilson 
59*ff5177eeSAlek Pinchuk /*
60*ff5177eeSAlek Pinchuk  * Tunable to control percentage of dirtied blocks from frees in one TXG.
61*ff5177eeSAlek Pinchuk  * After this threshold is crossed, additional dirty blocks from frees
62*ff5177eeSAlek Pinchuk  * wait until the next TXG.
63*ff5177eeSAlek Pinchuk  * A value of zero will disable this throttle.
64*ff5177eeSAlek Pinchuk  */
65*ff5177eeSAlek Pinchuk uint32_t zfs_per_txg_dirty_frees_percent = 30;
66*ff5177eeSAlek Pinchuk 
67fa9e4066Sahrens const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
68ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"unallocated"		},
69ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"object directory"	},
70ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"object array"		},
71ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"packed nvlist"		},
72ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"packed nvlist size"	},
73ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"bpobj"			},
74ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"bpobj header"		},
75ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"SPA space map header"	},
76ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"SPA space map"		},
77ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"ZIL intent log"	},
78ad135b5dSChristopher Siden 	{	DMU_BSWAP_DNODE,	TRUE,	"DMU dnode"		},
79ad135b5dSChristopher Siden 	{	DMU_BSWAP_OBJSET,	TRUE,	"DMU objset"		},
80ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"DSL directory"		},
81ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL directory child map"},
82ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL dataset snap map"	},
83ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL props"		},
84ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"DSL dataset"		},
85ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZNODE,	TRUE,	"ZFS znode"		},
86ad135b5dSChristopher Siden 	{	DMU_BSWAP_OLDACL,	TRUE,	"ZFS V0 ACL"		},
87ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	FALSE,	"ZFS plain file"	},
88ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"ZFS directory"		},
89ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"ZFS master node"	},
90ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"ZFS delete queue"	},
91ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	FALSE,	"zvol object"		},
92ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"zvol prop"		},
93ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	FALSE,	"other uint8[]"		},
94ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	FALSE,	"other uint64[]"	},
95ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"other ZAP"		},
96ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"persistent error log"	},
97ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"SPA history"		},
98ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"SPA history offsets"	},
99ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"Pool properties"	},
100ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL permissions"	},
101ad135b5dSChristopher Siden 	{	DMU_BSWAP_ACL,		TRUE,	"ZFS ACL"		},
102ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"ZFS SYSACL"		},
103ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"FUID table"		},
104ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"FUID table size"	},
105ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL dataset next clones"},
106ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"scan work queue"	},
107ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"ZFS user/group used"	},
108ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"ZFS user/group quota"	},
109ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"snapshot refcount tags"},
110ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DDT ZAP algorithm"	},
111ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DDT statistics"	},
112ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	TRUE,	"System attributes"	},
113ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"SA master node"	},
114ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"SA attr registration"	},
115ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"SA attr layouts"	},
116ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"scan translations"	},
117ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT8,	FALSE,	"deduplicated block"	},
118ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL deadlist map"	},
119ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"DSL deadlist map hdr"	},
120ad135b5dSChristopher Siden 	{	DMU_BSWAP_ZAP,		TRUE,	"DSL dir clones"	},
121ad135b5dSChristopher Siden 	{	DMU_BSWAP_UINT64,	TRUE,	"bpobj subobj"		}
122ad135b5dSChristopher Siden };
123ad135b5dSChristopher Siden 
124ad135b5dSChristopher Siden const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
125ad135b5dSChristopher Siden 	{	byteswap_uint8_array,	"uint8"		},
126ad135b5dSChristopher Siden 	{	byteswap_uint16_array,	"uint16"	},
127ad135b5dSChristopher Siden 	{	byteswap_uint32_array,	"uint32"	},
128ad135b5dSChristopher Siden 	{	byteswap_uint64_array,	"uint64"	},
129ad135b5dSChristopher Siden 	{	zap_byteswap,		"zap"		},
130ad135b5dSChristopher Siden 	{	dnode_buf_byteswap,	"dnode"		},
131ad135b5dSChristopher Siden 	{	dmu_objset_byteswap,	"objset"	},
132ad135b5dSChristopher Siden 	{	zfs_znode_byteswap,	"znode"		},
133ad135b5dSChristopher Siden 	{	zfs_oldacl_byteswap,	"oldacl"	},
134ad135b5dSChristopher Siden 	{	zfs_acl_byteswap,	"acl"		}
1353f9d6ad7SLin Ling };
136fa9e4066Sahrens 
13779d72832SMatthew Ahrens int
13879d72832SMatthew Ahrens dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset,
13979d72832SMatthew Ahrens     void *tag, dmu_buf_t **dbp)
14079d72832SMatthew Ahrens {
14179d72832SMatthew Ahrens 	uint64_t blkid;
14279d72832SMatthew Ahrens 	dmu_buf_impl_t *db;
14379d72832SMatthew Ahrens 
14479d72832SMatthew Ahrens 	blkid = dbuf_whichblock(dn, 0, offset);
14579d72832SMatthew Ahrens 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
14679d72832SMatthew Ahrens 	db = dbuf_hold(dn, blkid, tag);
14779d72832SMatthew Ahrens 	rw_exit(&dn->dn_struct_rwlock);
14879d72832SMatthew Ahrens 
14979d72832SMatthew Ahrens 	if (db == NULL) {
15079d72832SMatthew Ahrens 		*dbp = NULL;
15179d72832SMatthew Ahrens 		return (SET_ERROR(EIO));
15279d72832SMatthew Ahrens 	}
15379d72832SMatthew Ahrens 
15479d72832SMatthew Ahrens 	*dbp = &db->db;
15579d72832SMatthew Ahrens 	return (0);
15679d72832SMatthew Ahrens }
157fa9e4066Sahrens int
1585d7b4d43SMatthew Ahrens dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
1595d7b4d43SMatthew Ahrens     void *tag, dmu_buf_t **dbp)
160fa9e4066Sahrens {
161fa9e4066Sahrens 	dnode_t *dn;
162fa9e4066Sahrens 	uint64_t blkid;
163fa9e4066Sahrens 	dmu_buf_impl_t *db;
164ea8dc4b6Seschrock 	int err;
165fa9e4066Sahrens 
166503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
167ea8dc4b6Seschrock 	if (err)
168ea8dc4b6Seschrock 		return (err);
169a2cdcdd2SPaul Dagnelie 	blkid = dbuf_whichblock(dn, 0, offset);
170fa9e4066Sahrens 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
171ea8dc4b6Seschrock 	db = dbuf_hold(dn, blkid, tag);
172fa9e4066Sahrens 	rw_exit(&dn->dn_struct_rwlock);
1735d7b4d43SMatthew Ahrens 	dnode_rele(dn, FTAG);
1745d7b4d43SMatthew Ahrens 
175ea8dc4b6Seschrock 	if (db == NULL) {
1765d7b4d43SMatthew Ahrens 		*dbp = NULL;
1775d7b4d43SMatthew Ahrens 		return (SET_ERROR(EIO));
1785d7b4d43SMatthew Ahrens 	}
1795d7b4d43SMatthew Ahrens 
1805d7b4d43SMatthew Ahrens 	*dbp = &db->db;
1815d7b4d43SMatthew Ahrens 	return (err);
1825d7b4d43SMatthew Ahrens }
1835d7b4d43SMatthew Ahrens 
18479d72832SMatthew Ahrens int
18579d72832SMatthew Ahrens dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
18679d72832SMatthew Ahrens     void *tag, dmu_buf_t **dbp, int flags)
18779d72832SMatthew Ahrens {
18879d72832SMatthew Ahrens 	int err;
18979d72832SMatthew Ahrens 	int db_flags = DB_RF_CANFAIL;
19079d72832SMatthew Ahrens 
19179d72832SMatthew Ahrens 	if (flags & DMU_READ_NO_PREFETCH)
19279d72832SMatthew Ahrens 		db_flags |= DB_RF_NOPREFETCH;
19379d72832SMatthew Ahrens 
19479d72832SMatthew Ahrens 	err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp);
19579d72832SMatthew Ahrens 	if (err == 0) {
19679d72832SMatthew Ahrens 		dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
19779d72832SMatthew Ahrens 		err = dbuf_read(db, NULL, db_flags);
19879d72832SMatthew Ahrens 		if (err != 0) {
19979d72832SMatthew Ahrens 			dbuf_rele(db, tag);
20079d72832SMatthew Ahrens 			*dbp = NULL;
20179d72832SMatthew Ahrens 		}
20279d72832SMatthew Ahrens 	}
20379d72832SMatthew Ahrens 
20479d72832SMatthew Ahrens 	return (err);
20579d72832SMatthew Ahrens }
20679d72832SMatthew Ahrens 
2075d7b4d43SMatthew Ahrens int
2085d7b4d43SMatthew Ahrens dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
2095d7b4d43SMatthew Ahrens     void *tag, dmu_buf_t **dbp, int flags)
2105d7b4d43SMatthew Ahrens {
2115d7b4d43SMatthew Ahrens 	int err;
2125d7b4d43SMatthew Ahrens 	int db_flags = DB_RF_CANFAIL;
2135d7b4d43SMatthew Ahrens 
2145d7b4d43SMatthew Ahrens 	if (flags & DMU_READ_NO_PREFETCH)
2155d7b4d43SMatthew Ahrens 		db_flags |= DB_RF_NOPREFETCH;
2165d7b4d43SMatthew Ahrens 
2175d7b4d43SMatthew Ahrens 	err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
2185d7b4d43SMatthew Ahrens 	if (err == 0) {
2195d7b4d43SMatthew Ahrens 		dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
22047cb52daSJeff Bonwick 		err = dbuf_read(db, NULL, db_flags);
2215d7b4d43SMatthew Ahrens 		if (err != 0) {
222ea8dc4b6Seschrock 			dbuf_rele(db, tag);
2235d7b4d43SMatthew Ahrens 			*dbp = NULL;
224ea8dc4b6Seschrock 		}
225ea8dc4b6Seschrock 	}
226fa9e4066Sahrens 
227ea8dc4b6Seschrock 	return (err);
228fa9e4066Sahrens }
229fa9e4066Sahrens 
230fa9e4066Sahrens int
231fa9e4066Sahrens dmu_bonus_max(void)
232fa9e4066Sahrens {
233fa9e4066Sahrens 	return (DN_MAX_BONUSLEN);
234fa9e4066Sahrens }
235fa9e4066Sahrens 
2361934e92fSmaybee int
237744947dcSTom Erickson dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
2381934e92fSmaybee {
239744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
240744947dcSTom Erickson 	dnode_t *dn;
241744947dcSTom Erickson 	int error;
2421934e92fSmaybee 
243744947dcSTom Erickson 	DB_DNODE_ENTER(db);
244744947dcSTom Erickson 	dn = DB_DNODE(db);
245744947dcSTom Erickson 
246744947dcSTom Erickson 	if (dn->dn_bonus != db) {
247be6fd75aSMatthew Ahrens 		error = SET_ERROR(EINVAL);
248744947dcSTom Erickson 	} else if (newsize < 0 || newsize > db_fake->db_size) {
249be6fd75aSMatthew Ahrens 		error = SET_ERROR(EINVAL);
250744947dcSTom Erickson 	} else {
251744947dcSTom Erickson 		dnode_setbonuslen(dn, newsize, tx);
252744947dcSTom Erickson 		error = 0;
253744947dcSTom Erickson 	}
254744947dcSTom Erickson 
255744947dcSTom Erickson 	DB_DNODE_EXIT(db);
256744947dcSTom Erickson 	return (error);
2571934e92fSmaybee }
2581934e92fSmaybee 
2590a586ceaSMark Shellenbaum int
260744947dcSTom Erickson dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
2610a586ceaSMark Shellenbaum {
262744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
263744947dcSTom Erickson 	dnode_t *dn;
264744947dcSTom Erickson 	int error;
2650a586ceaSMark Shellenbaum 
266744947dcSTom Erickson 	DB_DNODE_ENTER(db);
267744947dcSTom Erickson 	dn = DB_DNODE(db);
268744947dcSTom Erickson 
269ad135b5dSChristopher Siden 	if (!DMU_OT_IS_VALID(type)) {
270be6fd75aSMatthew Ahrens 		error = SET_ERROR(EINVAL);
271744947dcSTom Erickson 	} else if (dn->dn_bonus != db) {
272be6fd75aSMatthew Ahrens 		error = SET_ERROR(EINVAL);
273744947dcSTom Erickson 	} else {
274744947dcSTom Erickson 		dnode_setbonus_type(dn, type, tx);
275744947dcSTom Erickson 		error = 0;
276744947dcSTom Erickson 	}
2770a586ceaSMark Shellenbaum 
278744947dcSTom Erickson 	DB_DNODE_EXIT(db);
279744947dcSTom Erickson 	return (error);
280744947dcSTom Erickson }
2810a586ceaSMark Shellenbaum 
282744947dcSTom Erickson dmu_object_type_t
283744947dcSTom Erickson dmu_get_bonustype(dmu_buf_t *db_fake)
284744947dcSTom Erickson {
285744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
286744947dcSTom Erickson 	dnode_t *dn;
287744947dcSTom Erickson 	dmu_object_type_t type;
288744947dcSTom Erickson 
289744947dcSTom Erickson 	DB_DNODE_ENTER(db);
290744947dcSTom Erickson 	dn = DB_DNODE(db);
291744947dcSTom Erickson 	type = dn->dn_bonustype;
292744947dcSTom Erickson 	DB_DNODE_EXIT(db);
293744947dcSTom Erickson 
294744947dcSTom Erickson 	return (type);
2950a586ceaSMark Shellenbaum }
2960a586ceaSMark Shellenbaum 
2970a586ceaSMark Shellenbaum int
2980a586ceaSMark Shellenbaum dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
2990a586ceaSMark Shellenbaum {
3000a586ceaSMark Shellenbaum 	dnode_t *dn;
3010a586ceaSMark Shellenbaum 	int error;
3020a586ceaSMark Shellenbaum 
3030a586ceaSMark Shellenbaum 	error = dnode_hold(os, object, FTAG, &dn);
3040a586ceaSMark Shellenbaum 	dbuf_rm_spill(dn, tx);
30506e0070dSMark Shellenbaum 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
30606e0070dSMark Shellenbaum 	dnode_rm_spill(dn, tx);
30706e0070dSMark Shellenbaum 	rw_exit(&dn->dn_struct_rwlock);
3080a586ceaSMark Shellenbaum 	dnode_rele(dn, FTAG);
3090a586ceaSMark Shellenbaum 	return (error);
3100a586ceaSMark Shellenbaum }
3110a586ceaSMark Shellenbaum 
312fa9e4066Sahrens /*
313ea8dc4b6Seschrock  * returns ENOENT, EIO, or 0.
314fa9e4066Sahrens  */
315ea8dc4b6Seschrock int
316ea8dc4b6Seschrock dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
317fa9e4066Sahrens {
318ea8dc4b6Seschrock 	dnode_t *dn;
319fa9e4066Sahrens 	dmu_buf_impl_t *db;
3201934e92fSmaybee 	int error;
321fa9e4066Sahrens 
322503ad85cSMatthew Ahrens 	error = dnode_hold(os, object, FTAG, &dn);
3231934e92fSmaybee 	if (error)
3241934e92fSmaybee 		return (error);
325fa9e4066Sahrens 
326ea8dc4b6Seschrock 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
327ea8dc4b6Seschrock 	if (dn->dn_bonus == NULL) {
328fa9e4066Sahrens 		rw_exit(&dn->dn_struct_rwlock);
329ea8dc4b6Seschrock 		rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
330ea8dc4b6Seschrock 		if (dn->dn_bonus == NULL)
3311934e92fSmaybee 			dbuf_create_bonus(dn);
332fa9e4066Sahrens 	}
333ea8dc4b6Seschrock 	db = dn->dn_bonus;
3341934e92fSmaybee 
3351934e92fSmaybee 	/* as long as the bonus buf is held, the dnode will be held */
336744947dcSTom Erickson 	if (refcount_add(&db->db_holds, tag) == 1) {
3371934e92fSmaybee 		VERIFY(dnode_add_ref(dn, db));
338640c1670SJosef 'Jeff' Sipek 		atomic_inc_32(&dn->dn_dbufs_count);
339744947dcSTom Erickson 	}
340744947dcSTom Erickson 
341744947dcSTom Erickson 	/*
342744947dcSTom Erickson 	 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
343744947dcSTom Erickson 	 * hold and incrementing the dbuf count to ensure that dnode_move() sees
344744947dcSTom Erickson 	 * a dnode hold for every dbuf.
345744947dcSTom Erickson 	 */
346744947dcSTom Erickson 	rw_exit(&dn->dn_struct_rwlock);
3471934e92fSmaybee 
348fa9e4066Sahrens 	dnode_rele(dn, FTAG);
349ea8dc4b6Seschrock 
35047cb52daSJeff Bonwick 	VERIFY(0 == dbuf_read(db, NULL, DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH));
351ea8dc4b6Seschrock 
352ea8dc4b6Seschrock 	*dbp = &db->db;
353ea8dc4b6Seschrock 	return (0);
354fa9e4066Sahrens }
355fa9e4066Sahrens 
3560a586ceaSMark Shellenbaum /*
3570a586ceaSMark Shellenbaum  * returns ENOENT, EIO, or 0.
3580a586ceaSMark Shellenbaum  *
3590a586ceaSMark Shellenbaum  * This interface will allocate a blank spill dbuf when a spill blk
3600a586ceaSMark Shellenbaum  * doesn't already exist on the dnode.
3610a586ceaSMark Shellenbaum  *
3620a586ceaSMark Shellenbaum  * if you only want to find an already existing spill db, then
3630a586ceaSMark Shellenbaum  * dmu_spill_hold_existing() should be used.
3640a586ceaSMark Shellenbaum  */
3650a586ceaSMark Shellenbaum int
3660a586ceaSMark Shellenbaum dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
3670a586ceaSMark Shellenbaum {
3680a586ceaSMark Shellenbaum 	dmu_buf_impl_t *db = NULL;
3690a586ceaSMark Shellenbaum 	int err;
3700a586ceaSMark Shellenbaum 
3710a586ceaSMark Shellenbaum 	if ((flags & DB_RF_HAVESTRUCT) == 0)
3720a586ceaSMark Shellenbaum 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
3730a586ceaSMark Shellenbaum 
3740a586ceaSMark Shellenbaum 	db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
3750a586ceaSMark Shellenbaum 
3760a586ceaSMark Shellenbaum 	if ((flags & DB_RF_HAVESTRUCT) == 0)
3770a586ceaSMark Shellenbaum 		rw_exit(&dn->dn_struct_rwlock);
3780a586ceaSMark Shellenbaum 
3790a586ceaSMark Shellenbaum 	ASSERT(db != NULL);
3801d8ccc7bSMark Shellenbaum 	err = dbuf_read(db, NULL, flags);
3811d8ccc7bSMark Shellenbaum 	if (err == 0)
3821d8ccc7bSMark Shellenbaum 		*dbp = &db->db;
3831d8ccc7bSMark Shellenbaum 	else
3841d8ccc7bSMark Shellenbaum 		dbuf_rele(db, tag);
3850a586ceaSMark Shellenbaum 	return (err);
3860a586ceaSMark Shellenbaum }
3870a586ceaSMark Shellenbaum 
3880a586ceaSMark Shellenbaum int
3890a586ceaSMark Shellenbaum dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
3900a586ceaSMark Shellenbaum {
391744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
392744947dcSTom Erickson 	dnode_t *dn;
3930a586ceaSMark Shellenbaum 	int err;
3940a586ceaSMark Shellenbaum 
395744947dcSTom Erickson 	DB_DNODE_ENTER(db);
396744947dcSTom Erickson 	dn = DB_DNODE(db);
397744947dcSTom Erickson 
398744947dcSTom Erickson 	if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
399be6fd75aSMatthew Ahrens 		err = SET_ERROR(EINVAL);
400744947dcSTom Erickson 	} else {
401744947dcSTom Erickson 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
402744947dcSTom Erickson 
403744947dcSTom Erickson 		if (!dn->dn_have_spill) {
404be6fd75aSMatthew Ahrens 			err = SET_ERROR(ENOENT);
405744947dcSTom Erickson 		} else {
406744947dcSTom Erickson 			err = dmu_spill_hold_by_dnode(dn,
407744947dcSTom Erickson 			    DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
408744947dcSTom Erickson 		}
4090a586ceaSMark Shellenbaum 
4100a586ceaSMark Shellenbaum 		rw_exit(&dn->dn_struct_rwlock);
4110a586ceaSMark Shellenbaum 	}
412744947dcSTom Erickson 
413744947dcSTom Erickson 	DB_DNODE_EXIT(db);
4140a586ceaSMark Shellenbaum 	return (err);
4150a586ceaSMark Shellenbaum }
4160a586ceaSMark Shellenbaum 
4170a586ceaSMark Shellenbaum int
4180a586ceaSMark Shellenbaum dmu_spill_hold_by_bonus(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
4190a586ceaSMark Shellenbaum {
420744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
421744947dcSTom Erickson 	dnode_t *dn;
422744947dcSTom Erickson 	int err;
423744947dcSTom Erickson 
424744947dcSTom Erickson 	DB_DNODE_ENTER(db);
425744947dcSTom Erickson 	dn = DB_DNODE(db);
426744947dcSTom Erickson 	err = dmu_spill_hold_by_dnode(dn, DB_RF_CANFAIL, tag, dbp);
427744947dcSTom Erickson 	DB_DNODE_EXIT(db);
428744947dcSTom Erickson 
429744947dcSTom Erickson 	return (err);
4300a586ceaSMark Shellenbaum }
4310a586ceaSMark Shellenbaum 
43213506d1eSmaybee /*
43313506d1eSmaybee  * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
43413506d1eSmaybee  * to take a held dnode rather than <os, object> -- the lookup is wasteful,
43513506d1eSmaybee  * and can induce severe lock contention when writing to several files
43613506d1eSmaybee  * whose dnodes are in the same block.
43713506d1eSmaybee  */
43813506d1eSmaybee static int
4397bfdf011SNeil Perrin dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
440cf6106c8SMatthew Ahrens     boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
441fa9e4066Sahrens {
442fa9e4066Sahrens 	dmu_buf_t **dbp;
443fa9e4066Sahrens 	uint64_t blkid, nblks, i;
4447bfdf011SNeil Perrin 	uint32_t dbuf_flags;
445ea8dc4b6Seschrock 	int err;
446ea8dc4b6Seschrock 	zio_t *zio;
447ea8dc4b6Seschrock 
448ea8dc4b6Seschrock 	ASSERT(length <= DMU_MAX_ACCESS);
449fa9e4066Sahrens 
450cf6106c8SMatthew Ahrens 	/*
451cf6106c8SMatthew Ahrens 	 * Note: We directly notify the prefetch code of this read, so that
452cf6106c8SMatthew Ahrens 	 * we can tell it about the multi-block read.  dbuf_read() only knows
453cf6106c8SMatthew Ahrens 	 * about the one block it is accessing.
454cf6106c8SMatthew Ahrens 	 */
455cf6106c8SMatthew Ahrens 	dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT |
456cf6106c8SMatthew Ahrens 	    DB_RF_NOPREFETCH;
457ea8dc4b6Seschrock 
458fa9e4066Sahrens 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
459fa9e4066Sahrens 	if (dn->dn_datablkshift) {
460fa9e4066Sahrens 		int blkshift = dn->dn_datablkshift;
461cf6106c8SMatthew Ahrens 		nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
462cf6106c8SMatthew Ahrens 		    P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
463fa9e4066Sahrens 	} else {
4640125049cSahrens 		if (offset + length > dn->dn_datablksz) {
4650125049cSahrens 			zfs_panic_recover("zfs: accessing past end of object "
4660125049cSahrens 			    "%llx/%llx (size=%u access=%llu+%llu)",
4670125049cSahrens 			    (longlong_t)dn->dn_objset->
4680125049cSahrens 			    os_dsl_dataset->ds_object,
4690125049cSahrens 			    (longlong_t)dn->dn_object, dn->dn_datablksz,
4700125049cSahrens 			    (longlong_t)offset, (longlong_t)length);
471c87b8fc5SMark J Musante 			rw_exit(&dn->dn_struct_rwlock);
472be6fd75aSMatthew Ahrens 			return (SET_ERROR(EIO));
4730125049cSahrens 		}
474fa9e4066Sahrens 		nblks = 1;
475fa9e4066Sahrens 	}
476ea8dc4b6Seschrock 	dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
477fa9e4066Sahrens 
478e14bb325SJeff Bonwick 	zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
479a2cdcdd2SPaul Dagnelie 	blkid = dbuf_whichblock(dn, 0, offset);
480fa9e4066Sahrens 	for (i = 0; i < nblks; i++) {
481cf6106c8SMatthew Ahrens 		dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
482ea8dc4b6Seschrock 		if (db == NULL) {
483ea8dc4b6Seschrock 			rw_exit(&dn->dn_struct_rwlock);
484ea8dc4b6Seschrock 			dmu_buf_rele_array(dbp, nblks, tag);
485ea8dc4b6Seschrock 			zio_nowait(zio);
486be6fd75aSMatthew Ahrens 			return (SET_ERROR(EIO));
487ea8dc4b6Seschrock 		}
488cf6106c8SMatthew Ahrens 
489ea8dc4b6Seschrock 		/* initiate async i/o */
490cf6106c8SMatthew Ahrens 		if (read)
4917bfdf011SNeil Perrin 			(void) dbuf_read(db, zio, dbuf_flags);
492ea8dc4b6Seschrock 		dbp[i] = &db->db;
493fa9e4066Sahrens 	}
494cf6106c8SMatthew Ahrens 
495cb92f413SAlexander Motin 	if ((flags & DMU_READ_NO_PREFETCH) == 0 &&
496cb92f413SAlexander Motin 	    DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) {
497cb92f413SAlexander Motin 		dmu_zfetch(&dn->dn_zfetch, blkid, nblks,
498cb92f413SAlexander Motin 		    read && DNODE_IS_CACHEABLE(dn));
499cf6106c8SMatthew Ahrens 	}
500fa9e4066Sahrens 	rw_exit(&dn->dn_struct_rwlock);
501fa9e4066Sahrens 
502ea8dc4b6Seschrock 	/* wait for async i/o */
503ea8dc4b6Seschrock 	err = zio_wait(zio);
504ea8dc4b6Seschrock 	if (err) {
505ea8dc4b6Seschrock 		dmu_buf_rele_array(dbp, nblks, tag);
506ea8dc4b6Seschrock 		return (err);
507ea8dc4b6Seschrock 	}
508fa9e4066Sahrens 
509ea8dc4b6Seschrock 	/* wait for other io to complete */
510ea8dc4b6Seschrock 	if (read) {
511ea8dc4b6Seschrock 		for (i = 0; i < nblks; i++) {
512ea8dc4b6Seschrock 			dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
513ea8dc4b6Seschrock 			mutex_enter(&db->db_mtx);
514ea8dc4b6Seschrock 			while (db->db_state == DB_READ ||
515ea8dc4b6Seschrock 			    db->db_state == DB_FILL)
516ea8dc4b6Seschrock 				cv_wait(&db->db_changed, &db->db_mtx);
517ea8dc4b6Seschrock 			if (db->db_state == DB_UNCACHED)
518be6fd75aSMatthew Ahrens 				err = SET_ERROR(EIO);
519ea8dc4b6Seschrock 			mutex_exit(&db->db_mtx);
520ea8dc4b6Seschrock 			if (err) {
521ea8dc4b6Seschrock 				dmu_buf_rele_array(dbp, nblks, tag);
522ea8dc4b6Seschrock 				return (err);
523ea8dc4b6Seschrock 			}
524ea8dc4b6Seschrock 		}
525ea8dc4b6Seschrock 	}
526fa9e4066Sahrens 
527ea8dc4b6Seschrock 	*numbufsp = nblks;
528ea8dc4b6Seschrock 	*dbpp = dbp;
529ea8dc4b6Seschrock 	return (0);
530fa9e4066Sahrens }
531fa9e4066Sahrens 
532a2eea2e1Sahrens static int
53313506d1eSmaybee dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
53413506d1eSmaybee     uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
53513506d1eSmaybee {
53613506d1eSmaybee 	dnode_t *dn;
53713506d1eSmaybee 	int err;
53813506d1eSmaybee 
539503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
54013506d1eSmaybee 	if (err)
54113506d1eSmaybee 		return (err);
54213506d1eSmaybee 
54313506d1eSmaybee 	err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
5447bfdf011SNeil Perrin 	    numbufsp, dbpp, DMU_READ_PREFETCH);
54513506d1eSmaybee 
54613506d1eSmaybee 	dnode_rele(dn, FTAG);
54713506d1eSmaybee 
54813506d1eSmaybee 	return (err);
54913506d1eSmaybee }
55013506d1eSmaybee 
55113506d1eSmaybee int
552744947dcSTom Erickson dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
553cf6106c8SMatthew Ahrens     uint64_t length, boolean_t read, void *tag, int *numbufsp,
554cf6106c8SMatthew Ahrens     dmu_buf_t ***dbpp)
55513506d1eSmaybee {
556744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
557744947dcSTom Erickson 	dnode_t *dn;
55813506d1eSmaybee 	int err;
55913506d1eSmaybee 
560744947dcSTom Erickson 	DB_DNODE_ENTER(db);
561744947dcSTom Erickson 	dn = DB_DNODE(db);
56213506d1eSmaybee 	err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
5637bfdf011SNeil Perrin 	    numbufsp, dbpp, DMU_READ_PREFETCH);
564744947dcSTom Erickson 	DB_DNODE_EXIT(db);
56513506d1eSmaybee 
56613506d1eSmaybee 	return (err);
56713506d1eSmaybee }
56813506d1eSmaybee 
569fa9e4066Sahrens void
570ea8dc4b6Seschrock dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
571fa9e4066Sahrens {
572fa9e4066Sahrens 	int i;
573fa9e4066Sahrens 	dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
574fa9e4066Sahrens 
575fa9e4066Sahrens 	if (numbufs == 0)
576fa9e4066Sahrens 		return;
577fa9e4066Sahrens 
578ea8dc4b6Seschrock 	for (i = 0; i < numbufs; i++) {
579ea8dc4b6Seschrock 		if (dbp[i])
580ea8dc4b6Seschrock 			dbuf_rele(dbp[i], tag);
581ea8dc4b6Seschrock 	}
582fa9e4066Sahrens 
583fa9e4066Sahrens 	kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
584fa9e4066Sahrens }
585fa9e4066Sahrens 
58669962b56SMatthew Ahrens /*
587a2cdcdd2SPaul Dagnelie  * Issue prefetch i/os for the given blocks.  If level is greater than 0, the
588a2cdcdd2SPaul Dagnelie  * indirect blocks prefeteched will be those that point to the blocks containing
589a2cdcdd2SPaul Dagnelie  * the data starting at offset, and continuing to offset + len.
59069962b56SMatthew Ahrens  *
591a2cdcdd2SPaul Dagnelie  * Note that if the indirect blocks above the blocks being prefetched are not in
592a2cdcdd2SPaul Dagnelie  * cache, they will be asychronously read in.
59369962b56SMatthew Ahrens  */
594fa9e4066Sahrens void
595a2cdcdd2SPaul Dagnelie dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
596a2cdcdd2SPaul Dagnelie     uint64_t len, zio_priority_t pri)
597fa9e4066Sahrens {
598fa9e4066Sahrens 	dnode_t *dn;
599fa9e4066Sahrens 	uint64_t blkid;
60069962b56SMatthew Ahrens 	int nblks, err;
601fa9e4066Sahrens 
602fa9e4066Sahrens 	if (len == 0) {  /* they're interested in the bonus buffer */
603744947dcSTom Erickson 		dn = DMU_META_DNODE(os);
604fa9e4066Sahrens 
605fa9e4066Sahrens 		if (object == 0 || object >= DN_MAX_OBJECT)
606fa9e4066Sahrens 			return;
607fa9e4066Sahrens 
608fa9e4066Sahrens 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
609a2cdcdd2SPaul Dagnelie 		blkid = dbuf_whichblock(dn, level,
610a2cdcdd2SPaul Dagnelie 		    object * sizeof (dnode_phys_t));
611a2cdcdd2SPaul Dagnelie 		dbuf_prefetch(dn, level, blkid, pri, 0);
612fa9e4066Sahrens 		rw_exit(&dn->dn_struct_rwlock);
613fa9e4066Sahrens 		return;
614fa9e4066Sahrens 	}
615fa9e4066Sahrens 
616fa9e4066Sahrens 	/*
617fa9e4066Sahrens 	 * XXX - Note, if the dnode for the requested object is not
618fa9e4066Sahrens 	 * already cached, we will do a *synchronous* read in the
619fa9e4066Sahrens 	 * dnode_hold() call.  The same is true for any indirects.
620fa9e4066Sahrens 	 */
621503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
622ea8dc4b6Seschrock 	if (err != 0)
623fa9e4066Sahrens 		return;
624fa9e4066Sahrens 
625fa9e4066Sahrens 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
626a2cdcdd2SPaul Dagnelie 	/*
627a2cdcdd2SPaul Dagnelie 	 * offset + len - 1 is the last byte we want to prefetch for, and offset
628a2cdcdd2SPaul Dagnelie 	 * is the first.  Then dbuf_whichblk(dn, level, off + len - 1) is the
629a2cdcdd2SPaul Dagnelie 	 * last block we want to prefetch, and dbuf_whichblock(dn, level,
630a2cdcdd2SPaul Dagnelie 	 * offset)  is the first.  Then the number we need to prefetch is the
631a2cdcdd2SPaul Dagnelie 	 * last - first + 1.
632a2cdcdd2SPaul Dagnelie 	 */
633a2cdcdd2SPaul Dagnelie 	if (level > 0 || dn->dn_datablkshift != 0) {
634a2cdcdd2SPaul Dagnelie 		nblks = dbuf_whichblock(dn, level, offset + len - 1) -
635a2cdcdd2SPaul Dagnelie 		    dbuf_whichblock(dn, level, offset) + 1;
636fa9e4066Sahrens 	} else {
637fa9e4066Sahrens 		nblks = (offset < dn->dn_datablksz);
638fa9e4066Sahrens 	}
639fa9e4066Sahrens 
640fa9e4066Sahrens 	if (nblks != 0) {
641a2cdcdd2SPaul Dagnelie 		blkid = dbuf_whichblock(dn, level, offset);
64269962b56SMatthew Ahrens 		for (int i = 0; i < nblks; i++)
643a2cdcdd2SPaul Dagnelie 			dbuf_prefetch(dn, level, blkid + i, pri, 0);
644fa9e4066Sahrens 	}
645fa9e4066Sahrens 
646fa9e4066Sahrens 	rw_exit(&dn->dn_struct_rwlock);
647fa9e4066Sahrens 
648fa9e4066Sahrens 	dnode_rele(dn, FTAG);
649fa9e4066Sahrens }
650fa9e4066Sahrens 
65176256205SMark Maybee /*
65276256205SMark Maybee  * Get the next "chunk" of file data to free.  We traverse the file from
65376256205SMark Maybee  * the end so that the file gets shorter over time (if we crashes in the
65476256205SMark Maybee  * middle, this will leave us in a better state).  We find allocated file
65576256205SMark Maybee  * data by simply searching the allocated level 1 indirects.
656713d6c20SMatthew Ahrens  *
657713d6c20SMatthew Ahrens  * On input, *start should be the first offset that does not need to be
658713d6c20SMatthew Ahrens  * freed (e.g. "offset + length").  On return, *start will be the first
659713d6c20SMatthew Ahrens  * offset that should be freed.
66076256205SMark Maybee  */
661cdb0ab79Smaybee static int
662713d6c20SMatthew Ahrens get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum)
663cdb0ab79Smaybee {
664713d6c20SMatthew Ahrens 	uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
665713d6c20SMatthew Ahrens 	/* bytes of data covered by a level-1 indirect block */
66676256205SMark Maybee 	uint64_t iblkrange =
6671c8564a7SMark Maybee 	    dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
668cdb0ab79Smaybee 
669713d6c20SMatthew Ahrens 	ASSERT3U(minimum, <=, *start);
670cdb0ab79Smaybee 
671713d6c20SMatthew Ahrens 	if (*start - minimum <= iblkrange * maxblks) {
672713d6c20SMatthew Ahrens 		*start = minimum;
673cdb0ab79Smaybee 		return (0);
674cdb0ab79Smaybee 	}
67576256205SMark Maybee 	ASSERT(ISP2(iblkrange));
676cdb0ab79Smaybee 
677713d6c20SMatthew Ahrens 	for (uint64_t blks = 0; *start > minimum && blks < maxblks; blks++) {
6781c8564a7SMark Maybee 		int err;
679cdb0ab79Smaybee 
680713d6c20SMatthew Ahrens 		/*
681713d6c20SMatthew Ahrens 		 * dnode_next_offset(BACKWARDS) will find an allocated L1
682713d6c20SMatthew Ahrens 		 * indirect block at or before the input offset.  We must
683713d6c20SMatthew Ahrens 		 * decrement *start so that it is at the end of the region
684713d6c20SMatthew Ahrens 		 * to search.
685713d6c20SMatthew Ahrens 		 */
686713d6c20SMatthew Ahrens 		(*start)--;
687cdb0ab79Smaybee 		err = dnode_next_offset(dn,
68876256205SMark Maybee 		    DNODE_FIND_BACKWARDS, start, 2, 1, 0);
689cdb0ab79Smaybee 
690713d6c20SMatthew Ahrens 		/* if there are no indirect blocks before start, we are done */
69176256205SMark Maybee 		if (err == ESRCH) {
692713d6c20SMatthew Ahrens 			*start = minimum;
693713d6c20SMatthew Ahrens 			break;
694713d6c20SMatthew Ahrens 		} else if (err != 0) {
695cdb0ab79Smaybee 			return (err);
69676256205SMark Maybee 		}
69776256205SMark Maybee 
698713d6c20SMatthew Ahrens 		/* set start to the beginning of this L1 indirect */
69976256205SMark Maybee 		*start = P2ALIGN(*start, iblkrange);
700cdb0ab79Smaybee 	}
701713d6c20SMatthew Ahrens 	if (*start < minimum)
702713d6c20SMatthew Ahrens 		*start = minimum;
703cdb0ab79Smaybee 	return (0);
704cdb0ab79Smaybee }
705cdb0ab79Smaybee 
706eb721827SAlek Pinchuk /*
707eb721827SAlek Pinchuk  * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set,
708eb721827SAlek Pinchuk  * otherwise return false.
709eb721827SAlek Pinchuk  * Used below in dmu_free_long_range_impl() to enable abort when unmounting
710eb721827SAlek Pinchuk  */
711eb721827SAlek Pinchuk /*ARGSUSED*/
712eb721827SAlek Pinchuk static boolean_t
713eb721827SAlek Pinchuk dmu_objset_zfs_unmounting(objset_t *os)
714eb721827SAlek Pinchuk {
715eb721827SAlek Pinchuk #ifdef _KERNEL
716eb721827SAlek Pinchuk 	if (dmu_objset_type(os) == DMU_OST_ZFS)
717eb721827SAlek Pinchuk 		return (zfs_get_vfs_flag_unmounted(os));
718eb721827SAlek Pinchuk #endif
719eb721827SAlek Pinchuk 	return (B_FALSE);
720eb721827SAlek Pinchuk }
721eb721827SAlek Pinchuk 
722cdb0ab79Smaybee static int
723cdb0ab79Smaybee dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
724713d6c20SMatthew Ahrens     uint64_t length)
725cdb0ab79Smaybee {
726713d6c20SMatthew Ahrens 	uint64_t object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
727713d6c20SMatthew Ahrens 	int err;
728*ff5177eeSAlek Pinchuk 	uint64_t dirty_frees_threshold;
729*ff5177eeSAlek Pinchuk 	dsl_pool_t *dp = dmu_objset_pool(os);
730713d6c20SMatthew Ahrens 
731713d6c20SMatthew Ahrens 	if (offset >= object_size)
732cdb0ab79Smaybee 		return (0);
733cdb0ab79Smaybee 
734*ff5177eeSAlek Pinchuk 	if (zfs_per_txg_dirty_frees_percent <= 100)
735*ff5177eeSAlek Pinchuk 		dirty_frees_threshold =
736*ff5177eeSAlek Pinchuk 		    zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100;
737*ff5177eeSAlek Pinchuk 	else
738*ff5177eeSAlek Pinchuk 		dirty_frees_threshold = zfs_dirty_data_max / 4;
739*ff5177eeSAlek Pinchuk 
740713d6c20SMatthew Ahrens 	if (length == DMU_OBJECT_END || offset + length > object_size)
741713d6c20SMatthew Ahrens 		length = object_size - offset;
742713d6c20SMatthew Ahrens 
743713d6c20SMatthew Ahrens 	while (length != 0) {
744*ff5177eeSAlek Pinchuk 		uint64_t chunk_end, chunk_begin, chunk_len;
745*ff5177eeSAlek Pinchuk 		uint64_t long_free_dirty_all_txgs = 0;
746*ff5177eeSAlek Pinchuk 		dmu_tx_t *tx;
747713d6c20SMatthew Ahrens 
748eb721827SAlek Pinchuk 		if (dmu_objset_zfs_unmounting(dn->dn_objset))
749eb721827SAlek Pinchuk 			return (SET_ERROR(EINTR));
750eb721827SAlek Pinchuk 
751713d6c20SMatthew Ahrens 		chunk_end = chunk_begin = offset + length;
752713d6c20SMatthew Ahrens 
753713d6c20SMatthew Ahrens 		/* move chunk_begin backwards to the beginning of this chunk */
754713d6c20SMatthew Ahrens 		err = get_next_chunk(dn, &chunk_begin, offset);
755cdb0ab79Smaybee 		if (err)
756cdb0ab79Smaybee 			return (err);
757713d6c20SMatthew Ahrens 		ASSERT3U(chunk_begin, >=, offset);
758713d6c20SMatthew Ahrens 		ASSERT3U(chunk_begin, <=, chunk_end);
759cdb0ab79Smaybee 
760*ff5177eeSAlek Pinchuk 		chunk_len = chunk_end - chunk_begin;
761*ff5177eeSAlek Pinchuk 
762*ff5177eeSAlek Pinchuk 		mutex_enter(&dp->dp_lock);
763*ff5177eeSAlek Pinchuk 		for (int t = 0; t < TXG_SIZE; t++) {
764*ff5177eeSAlek Pinchuk 			long_free_dirty_all_txgs +=
765*ff5177eeSAlek Pinchuk 			    dp->dp_long_free_dirty_pertxg[t];
766*ff5177eeSAlek Pinchuk 		}
767*ff5177eeSAlek Pinchuk 		mutex_exit(&dp->dp_lock);
768*ff5177eeSAlek Pinchuk 
769*ff5177eeSAlek Pinchuk 		/*
770*ff5177eeSAlek Pinchuk 		 * To avoid filling up a TXG with just frees wait for
771*ff5177eeSAlek Pinchuk 		 * the next TXG to open before freeing more chunks if
772*ff5177eeSAlek Pinchuk 		 * we have reached the threshold of frees
773*ff5177eeSAlek Pinchuk 		 */
774*ff5177eeSAlek Pinchuk 		if (dirty_frees_threshold != 0 &&
775*ff5177eeSAlek Pinchuk 		    long_free_dirty_all_txgs >= dirty_frees_threshold) {
776*ff5177eeSAlek Pinchuk 			txg_wait_open(dp, 0);
777*ff5177eeSAlek Pinchuk 			continue;
778*ff5177eeSAlek Pinchuk 		}
779*ff5177eeSAlek Pinchuk 
780*ff5177eeSAlek Pinchuk 		tx = dmu_tx_create(os);
781*ff5177eeSAlek Pinchuk 		dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len);
7824bb73804SMatthew Ahrens 
7834bb73804SMatthew Ahrens 		/*
7844bb73804SMatthew Ahrens 		 * Mark this transaction as typically resulting in a net
7854bb73804SMatthew Ahrens 		 * reduction in space used.
7864bb73804SMatthew Ahrens 		 */
7874bb73804SMatthew Ahrens 		dmu_tx_mark_netfree(tx);
788cdb0ab79Smaybee 		err = dmu_tx_assign(tx, TXG_WAIT);
789cdb0ab79Smaybee 		if (err) {
790cdb0ab79Smaybee 			dmu_tx_abort(tx);
791cdb0ab79Smaybee 			return (err);
792cdb0ab79Smaybee 		}
793*ff5177eeSAlek Pinchuk 
794*ff5177eeSAlek Pinchuk 		mutex_enter(&dp->dp_lock);
795*ff5177eeSAlek Pinchuk 		dp->dp_long_free_dirty_pertxg[dmu_tx_get_txg(tx) & TXG_MASK] +=
796*ff5177eeSAlek Pinchuk 		    chunk_len;
797*ff5177eeSAlek Pinchuk 		mutex_exit(&dp->dp_lock);
798*ff5177eeSAlek Pinchuk 		DTRACE_PROBE3(free__long__range,
799*ff5177eeSAlek Pinchuk 		    uint64_t, long_free_dirty_all_txgs, uint64_t, chunk_len,
800*ff5177eeSAlek Pinchuk 		    uint64_t, dmu_tx_get_txg(tx));
801*ff5177eeSAlek Pinchuk 		dnode_free_range(dn, chunk_begin, chunk_len, tx);
802cdb0ab79Smaybee 		dmu_tx_commit(tx);
803713d6c20SMatthew Ahrens 
804*ff5177eeSAlek Pinchuk 		length -= chunk_len;
805cdb0ab79Smaybee 	}
806cdb0ab79Smaybee 	return (0);
807cdb0ab79Smaybee }
808cdb0ab79Smaybee 
809cdb0ab79Smaybee int
810cdb0ab79Smaybee dmu_free_long_range(objset_t *os, uint64_t object,
811cdb0ab79Smaybee     uint64_t offset, uint64_t length)
812cdb0ab79Smaybee {
813cdb0ab79Smaybee 	dnode_t *dn;
814cdb0ab79Smaybee 	int err;
815cdb0ab79Smaybee 
816503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
817cdb0ab79Smaybee 	if (err != 0)
818cdb0ab79Smaybee 		return (err);
819713d6c20SMatthew Ahrens 	err = dmu_free_long_range_impl(os, dn, offset, length);
8205253393bSMatthew Ahrens 
8215253393bSMatthew Ahrens 	/*
8225253393bSMatthew Ahrens 	 * It is important to zero out the maxblkid when freeing the entire
8235253393bSMatthew Ahrens 	 * file, so that (a) subsequent calls to dmu_free_long_range_impl()
8245253393bSMatthew Ahrens 	 * will take the fast path, and (b) dnode_reallocate() can verify
8255253393bSMatthew Ahrens 	 * that the entire file has been freed.
8265253393bSMatthew Ahrens 	 */
82743466aaeSMax Grossman 	if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
8285253393bSMatthew Ahrens 		dn->dn_maxblkid = 0;
8295253393bSMatthew Ahrens 
830cdb0ab79Smaybee 	dnode_rele(dn, FTAG);
831cdb0ab79Smaybee 	return (err);
832cdb0ab79Smaybee }
833cdb0ab79Smaybee 
834cdb0ab79Smaybee int
835713d6c20SMatthew Ahrens dmu_free_long_object(objset_t *os, uint64_t object)
836cdb0ab79Smaybee {
837cdb0ab79Smaybee 	dmu_tx_t *tx;
838cdb0ab79Smaybee 	int err;
839cdb0ab79Smaybee 
840713d6c20SMatthew Ahrens 	err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
841cdb0ab79Smaybee 	if (err != 0)
842cdb0ab79Smaybee 		return (err);
843713d6c20SMatthew Ahrens 
844713d6c20SMatthew Ahrens 	tx = dmu_tx_create(os);
845713d6c20SMatthew Ahrens 	dmu_tx_hold_bonus(tx, object);
846713d6c20SMatthew Ahrens 	dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
8474bb73804SMatthew Ahrens 	dmu_tx_mark_netfree(tx);
848713d6c20SMatthew Ahrens 	err = dmu_tx_assign(tx, TXG_WAIT);
849713d6c20SMatthew Ahrens 	if (err == 0) {
850713d6c20SMatthew Ahrens 		err = dmu_object_free(os, object, tx);
851713d6c20SMatthew Ahrens 		dmu_tx_commit(tx);
852cdb0ab79Smaybee 	} else {
853713d6c20SMatthew Ahrens 		dmu_tx_abort(tx);
854cdb0ab79Smaybee 	}
855713d6c20SMatthew Ahrens 
856cdb0ab79Smaybee 	return (err);
857cdb0ab79Smaybee }
858cdb0ab79Smaybee 
859ea8dc4b6Seschrock int
860fa9e4066Sahrens dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
861fa9e4066Sahrens     uint64_t size, dmu_tx_t *tx)
862fa9e4066Sahrens {
863ea8dc4b6Seschrock 	dnode_t *dn;
864503ad85cSMatthew Ahrens 	int err = dnode_hold(os, object, FTAG, &dn);
865ea8dc4b6Seschrock 	if (err)
866ea8dc4b6Seschrock 		return (err);
867fa9e4066Sahrens 	ASSERT(offset < UINT64_MAX);
868fa9e4066Sahrens 	ASSERT(size == -1ULL || size <= UINT64_MAX - offset);
869fa9e4066Sahrens 	dnode_free_range(dn, offset, size, tx);
870fa9e4066Sahrens 	dnode_rele(dn, FTAG);
871ea8dc4b6Seschrock 	return (0);
872fa9e4066Sahrens }
873fa9e4066Sahrens 
874ea8dc4b6Seschrock int
875ea8dc4b6Seschrock dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
8767bfdf011SNeil Perrin     void *buf, uint32_t flags)
877fa9e4066Sahrens {
878fa9e4066Sahrens 	dnode_t *dn;
879fa9e4066Sahrens 	dmu_buf_t **dbp;
880c87b8fc5SMark J Musante 	int numbufs, err;
881fa9e4066Sahrens 
882503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
883ea8dc4b6Seschrock 	if (err)
884ea8dc4b6Seschrock 		return (err);
885feb08c6bSbillm 
886feb08c6bSbillm 	/*
887feb08c6bSbillm 	 * Deal with odd block sizes, where there can't be data past the first
888feb08c6bSbillm 	 * block.  If we ever do the tail block optimization, we will need to
889feb08c6bSbillm 	 * handle that here as well.
890feb08c6bSbillm 	 */
891c87b8fc5SMark J Musante 	if (dn->dn_maxblkid == 0) {
892fa9e4066Sahrens 		int newsz = offset > dn->dn_datablksz ? 0 :
893fa9e4066Sahrens 		    MIN(size, dn->dn_datablksz - offset);
894fa9e4066Sahrens 		bzero((char *)buf + newsz, size - newsz);
895fa9e4066Sahrens 		size = newsz;
896fa9e4066Sahrens 	}
897fa9e4066Sahrens 
898fa9e4066Sahrens 	while (size > 0) {
899fa9e4066Sahrens 		uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
900c87b8fc5SMark J Musante 		int i;
901fa9e4066Sahrens 
902fa9e4066Sahrens 		/*
903fa9e4066Sahrens 		 * NB: we could do this block-at-a-time, but it's nice
904fa9e4066Sahrens 		 * to be reading in parallel.
905fa9e4066Sahrens 		 */
906a2eea2e1Sahrens 		err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
9077bfdf011SNeil Perrin 		    TRUE, FTAG, &numbufs, &dbp, flags);
908ea8dc4b6Seschrock 		if (err)
9091934e92fSmaybee 			break;
910fa9e4066Sahrens 
911fa9e4066Sahrens 		for (i = 0; i < numbufs; i++) {
912fa9e4066Sahrens 			int tocpy;
913fa9e4066Sahrens 			int bufoff;
914fa9e4066Sahrens 			dmu_buf_t *db = dbp[i];
915fa9e4066Sahrens 
916fa9e4066Sahrens 			ASSERT(size > 0);
917fa9e4066Sahrens 
918fa9e4066Sahrens 			bufoff = offset - db->db_offset;
919fa9e4066Sahrens 			tocpy = (int)MIN(db->db_size - bufoff, size);
920fa9e4066Sahrens 
921fa9e4066Sahrens 			bcopy((char *)db->db_data + bufoff, buf, tocpy);
922fa9e4066Sahrens 
923fa9e4066Sahrens 			offset += tocpy;
924fa9e4066Sahrens 			size -= tocpy;
925fa9e4066Sahrens 			buf = (char *)buf + tocpy;
926fa9e4066Sahrens 		}
927ea8dc4b6Seschrock 		dmu_buf_rele_array(dbp, numbufs, FTAG);
928fa9e4066Sahrens 	}
929a2eea2e1Sahrens 	dnode_rele(dn, FTAG);
9301934e92fSmaybee 	return (err);
931fa9e4066Sahrens }
932fa9e4066Sahrens 
933fa9e4066Sahrens void
934fa9e4066Sahrens dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
935fa9e4066Sahrens     const void *buf, dmu_tx_t *tx)
936fa9e4066Sahrens {
937fa9e4066Sahrens 	dmu_buf_t **dbp;
938fa9e4066Sahrens 	int numbufs, i;
939fa9e4066Sahrens 
94013506d1eSmaybee 	if (size == 0)
94113506d1eSmaybee 		return;
94213506d1eSmaybee 
943ea8dc4b6Seschrock 	VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
944ea8dc4b6Seschrock 	    FALSE, FTAG, &numbufs, &dbp));
945fa9e4066Sahrens 
946fa9e4066Sahrens 	for (i = 0; i < numbufs; i++) {
947fa9e4066Sahrens 		int tocpy;
948fa9e4066Sahrens 		int bufoff;
949fa9e4066Sahrens 		dmu_buf_t *db = dbp[i];
950fa9e4066Sahrens 
951fa9e4066Sahrens 		ASSERT(size > 0);
952fa9e4066Sahrens 
953fa9e4066Sahrens 		bufoff = offset - db->db_offset;
954fa9e4066Sahrens 		tocpy = (int)MIN(db->db_size - bufoff, size);
955fa9e4066Sahrens 
956fa9e4066Sahrens 		ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
957fa9e4066Sahrens 
958fa9e4066Sahrens 		if (tocpy == db->db_size)
959fa9e4066Sahrens 			dmu_buf_will_fill(db, tx);
960fa9e4066Sahrens 		else
961fa9e4066Sahrens 			dmu_buf_will_dirty(db, tx);
962fa9e4066Sahrens 
963fa9e4066Sahrens 		bcopy(buf, (char *)db->db_data + bufoff, tocpy);
964fa9e4066Sahrens 
965fa9e4066Sahrens 		if (tocpy == db->db_size)
966fa9e4066Sahrens 			dmu_buf_fill_done(db, tx);
967fa9e4066Sahrens 
968fa9e4066Sahrens 		offset += tocpy;
969fa9e4066Sahrens 		size -= tocpy;
970fa9e4066Sahrens 		buf = (char *)buf + tocpy;
971fa9e4066Sahrens 	}
972ea8dc4b6Seschrock 	dmu_buf_rele_array(dbp, numbufs, FTAG);
973fa9e4066Sahrens }
974fa9e4066Sahrens 
97582c9918fSTim Haley void
97682c9918fSTim Haley dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
97782c9918fSTim Haley     dmu_tx_t *tx)
97882c9918fSTim Haley {
97982c9918fSTim Haley 	dmu_buf_t **dbp;
98082c9918fSTim Haley 	int numbufs, i;
98182c9918fSTim Haley 
98282c9918fSTim Haley 	if (size == 0)
98382c9918fSTim Haley 		return;
98482c9918fSTim Haley 
98582c9918fSTim Haley 	VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
98682c9918fSTim Haley 	    FALSE, FTAG, &numbufs, &dbp));
98782c9918fSTim Haley 
98882c9918fSTim Haley 	for (i = 0; i < numbufs; i++) {
98982c9918fSTim Haley 		dmu_buf_t *db = dbp[i];
99082c9918fSTim Haley 
99182c9918fSTim Haley 		dmu_buf_will_not_fill(db, tx);
99282c9918fSTim Haley 	}
99382c9918fSTim Haley 	dmu_buf_rele_array(dbp, numbufs, FTAG);
99482c9918fSTim Haley }
99582c9918fSTim Haley 
9965d7b4d43SMatthew Ahrens void
9975d7b4d43SMatthew Ahrens dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
9985d7b4d43SMatthew Ahrens     void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
9995d7b4d43SMatthew Ahrens     int compressed_size, int byteorder, dmu_tx_t *tx)
10005d7b4d43SMatthew Ahrens {
10015d7b4d43SMatthew Ahrens 	dmu_buf_t *db;
10025d7b4d43SMatthew Ahrens 
10035d7b4d43SMatthew Ahrens 	ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES);
10045d7b4d43SMatthew Ahrens 	ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS);
10055d7b4d43SMatthew Ahrens 	VERIFY0(dmu_buf_hold_noread(os, object, offset,
10065d7b4d43SMatthew Ahrens 	    FTAG, &db));
10075d7b4d43SMatthew Ahrens 
10085d7b4d43SMatthew Ahrens 	dmu_buf_write_embedded(db,
10095d7b4d43SMatthew Ahrens 	    data, (bp_embedded_type_t)etype, (enum zio_compress)comp,
10105d7b4d43SMatthew Ahrens 	    uncompressed_size, compressed_size, byteorder, tx);
10115d7b4d43SMatthew Ahrens 
10125d7b4d43SMatthew Ahrens 	dmu_buf_rele(db, FTAG);
10135d7b4d43SMatthew Ahrens }
10145d7b4d43SMatthew Ahrens 
1015c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /*
1016c242f9a0Schunli zhang - Sun Microsystems - Irvine United States  * DMU support for xuio
1017c242f9a0Schunli zhang - Sun Microsystems - Irvine United States  */
1018c242f9a0Schunli zhang - Sun Microsystems - Irvine United States kstat_t *xuio_ksp = NULL;
1019c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1020c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int
1021c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_init(xuio_t *xuio, int nblk)
1022c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1023c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv;
1024c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	uio_t *uio = &xuio->xu_uio;
1025c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1026c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	uio->uio_iovcnt = nblk;
1027c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
1028c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1029c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
1030c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv->cnt = nblk;
1031c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
1032c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv->iovp = uio->uio_iov;
1033c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	XUIO_XUZC_PRIV(xuio) = priv;
1034c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1035c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	if (XUIO_XUZC_RW(xuio) == UIO_READ)
1036c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk);
1037c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	else
1038c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk);
1039c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1040c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	return (0);
1041c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1042c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1043c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void
1044c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_fini(xuio_t *xuio)
1045c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1046c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1047c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	int nblk = priv->cnt;
1048c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1049c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	kmem_free(priv->iovp, nblk * sizeof (iovec_t));
1050c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *));
1051c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	kmem_free(priv, sizeof (dmu_xuio_t));
1052c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1053c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	if (XUIO_XUZC_RW(xuio) == UIO_READ)
1054c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk);
1055c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	else
1056c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk);
1057c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1058c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1059c242f9a0Schunli zhang - Sun Microsystems - Irvine United States /*
1060c242f9a0Schunli zhang - Sun Microsystems - Irvine United States  * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf }
1061c242f9a0Schunli zhang - Sun Microsystems - Irvine United States  * and increase priv->next by 1.
1062c242f9a0Schunli zhang - Sun Microsystems - Irvine United States  */
1063c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int
1064c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n)
1065c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1066c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	struct iovec *iov;
1067c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	uio_t *uio = &xuio->xu_uio;
1068c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1069c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	int i = priv->next++;
1070c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1071c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	ASSERT(i < priv->cnt);
10725602294fSDan Kimmel 	ASSERT(off + n <= arc_buf_lsize(abuf));
1073c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	iov = uio->uio_iov + i;
1074c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	iov->iov_base = (char *)abuf->b_data + off;
1075c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	iov->iov_len = n;
1076c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv->bufs[i] = abuf;
1077c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	return (0);
1078c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1079c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1080c242f9a0Schunli zhang - Sun Microsystems - Irvine United States int
1081c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_cnt(xuio_t *xuio)
1082c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1083c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1084c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	return (priv->cnt);
1085c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1086c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1087c242f9a0Schunli zhang - Sun Microsystems - Irvine United States arc_buf_t *
1088c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_arcbuf(xuio_t *xuio, int i)
1089c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1090c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1091c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1092c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	ASSERT(i < priv->cnt);
1093c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	return (priv->bufs[i]);
1094c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1095c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1096c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void
1097c242f9a0Schunli zhang - Sun Microsystems - Irvine United States dmu_xuio_clear(xuio_t *xuio, int i)
1098c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1099c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1100c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1101c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	ASSERT(i < priv->cnt);
1102c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	priv->bufs[i] = NULL;
1103c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1104c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1105c242f9a0Schunli zhang - Sun Microsystems - Irvine United States static void
1106c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_init(void)
1107c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1108c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc",
1109c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	    KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t),
1110c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	    KSTAT_FLAG_VIRTUAL);
1111c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	if (xuio_ksp != NULL) {
1112c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		xuio_ksp->ks_data = &xuio_stats;
1113c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		kstat_install(xuio_ksp);
1114c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	}
1115c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1116c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1117c242f9a0Schunli zhang - Sun Microsystems - Irvine United States static void
1118c242f9a0Schunli zhang - Sun Microsystems - Irvine United States xuio_stat_fini(void)
1119c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1120c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	if (xuio_ksp != NULL) {
1121c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		kstat_delete(xuio_ksp);
1122c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		xuio_ksp = NULL;
1123c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	}
1124c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1125c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1126c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void
112799aa8b55SPrashanth Sreenivasa xuio_stat_wbuf_copied(void)
1128c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1129c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1130c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1131c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1132c242f9a0Schunli zhang - Sun Microsystems - Irvine United States void
113399aa8b55SPrashanth Sreenivasa xuio_stat_wbuf_nocopy(void)
1134c242f9a0Schunli zhang - Sun Microsystems - Irvine United States {
1135c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	XUIOSTAT_BUMP(xuiostat_wbuf_nocopy);
1136c242f9a0Schunli zhang - Sun Microsystems - Irvine United States }
1137c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1138fa9e4066Sahrens #ifdef _KERNEL
1139f8554bb9SMatthew Ahrens static int
1140f8554bb9SMatthew Ahrens dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size)
1141feb08c6bSbillm {
1142feb08c6bSbillm 	dmu_buf_t **dbp;
1143feb08c6bSbillm 	int numbufs, i, err;
1144c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	xuio_t *xuio = NULL;
1145feb08c6bSbillm 
1146feb08c6bSbillm 	/*
1147feb08c6bSbillm 	 * NB: we could do this block-at-a-time, but it's nice
1148feb08c6bSbillm 	 * to be reading in parallel.
1149feb08c6bSbillm 	 */
1150f8554bb9SMatthew Ahrens 	err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1151f8554bb9SMatthew Ahrens 	    TRUE, FTAG, &numbufs, &dbp, 0);
1152feb08c6bSbillm 	if (err)
1153feb08c6bSbillm 		return (err);
1154feb08c6bSbillm 
1155c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	if (uio->uio_extflg == UIO_XUIO)
1156c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		xuio = (xuio_t *)uio;
1157c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1158feb08c6bSbillm 	for (i = 0; i < numbufs; i++) {
1159feb08c6bSbillm 		int tocpy;
1160feb08c6bSbillm 		int bufoff;
1161feb08c6bSbillm 		dmu_buf_t *db = dbp[i];
1162feb08c6bSbillm 
1163feb08c6bSbillm 		ASSERT(size > 0);
1164feb08c6bSbillm 
1165feb08c6bSbillm 		bufoff = uio->uio_loffset - db->db_offset;
1166feb08c6bSbillm 		tocpy = (int)MIN(db->db_size - bufoff, size);
1167feb08c6bSbillm 
1168c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		if (xuio) {
1169c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
1170c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			arc_buf_t *dbuf_abuf = dbi->db_buf;
1171c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
1172c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
1173c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			if (!err) {
1174c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 				uio->uio_resid -= tocpy;
1175c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 				uio->uio_loffset += tocpy;
1176c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			}
1177c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 
1178c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			if (abuf == dbuf_abuf)
1179c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 				XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
1180c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			else
1181c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 				XUIOSTAT_BUMP(xuiostat_rbuf_copied);
1182c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		} else {
1183c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			err = uiomove((char *)db->db_data + bufoff, tocpy,
1184c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 			    UIO_READ, uio);
1185c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		}
1186feb08c6bSbillm 		if (err)
1187feb08c6bSbillm 			break;
1188feb08c6bSbillm 
1189feb08c6bSbillm 		size -= tocpy;
1190feb08c6bSbillm 	}
1191feb08c6bSbillm 	dmu_buf_rele_array(dbp, numbufs, FTAG);
1192feb08c6bSbillm 
1193feb08c6bSbillm 	return (err);
1194feb08c6bSbillm }
1195feb08c6bSbillm 
1196f8554bb9SMatthew Ahrens /*
1197f8554bb9SMatthew Ahrens  * Read 'size' bytes into the uio buffer.
1198f8554bb9SMatthew Ahrens  * From object zdb->db_object.
1199f8554bb9SMatthew Ahrens  * Starting at offset uio->uio_loffset.
1200f8554bb9SMatthew Ahrens  *
1201f8554bb9SMatthew Ahrens  * If the caller already has a dbuf in the target object
1202f8554bb9SMatthew Ahrens  * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
1203f8554bb9SMatthew Ahrens  * because we don't have to find the dnode_t for the object.
1204f8554bb9SMatthew Ahrens  */
1205f8554bb9SMatthew Ahrens int
1206f8554bb9SMatthew Ahrens dmu_read_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size)
1207f8554bb9SMatthew Ahrens {
1208f8554bb9SMatthew Ahrens 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1209f8554bb9SMatthew Ahrens 	dnode_t *dn;
1210f8554bb9SMatthew Ahrens 	int err;
1211f8554bb9SMatthew Ahrens 
1212f8554bb9SMatthew Ahrens 	if (size == 0)
1213f8554bb9SMatthew Ahrens 		return (0);
1214f8554bb9SMatthew Ahrens 
1215f8554bb9SMatthew Ahrens 	DB_DNODE_ENTER(db);
1216f8554bb9SMatthew Ahrens 	dn = DB_DNODE(db);
1217f8554bb9SMatthew Ahrens 	err = dmu_read_uio_dnode(dn, uio, size);
1218f8554bb9SMatthew Ahrens 	DB_DNODE_EXIT(db);
1219f8554bb9SMatthew Ahrens 
1220f8554bb9SMatthew Ahrens 	return (err);
1221f8554bb9SMatthew Ahrens }
1222f8554bb9SMatthew Ahrens 
1223f8554bb9SMatthew Ahrens /*
1224f8554bb9SMatthew Ahrens  * Read 'size' bytes into the uio buffer.
1225f8554bb9SMatthew Ahrens  * From the specified object
1226f8554bb9SMatthew Ahrens  * Starting at offset uio->uio_loffset.
1227f8554bb9SMatthew Ahrens  */
1228f8554bb9SMatthew Ahrens int
1229f8554bb9SMatthew Ahrens dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
1230f8554bb9SMatthew Ahrens {
1231f8554bb9SMatthew Ahrens 	dnode_t *dn;
1232f8554bb9SMatthew Ahrens 	int err;
1233f8554bb9SMatthew Ahrens 
1234f8554bb9SMatthew Ahrens 	if (size == 0)
1235f8554bb9SMatthew Ahrens 		return (0);
1236f8554bb9SMatthew Ahrens 
1237f8554bb9SMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
1238f8554bb9SMatthew Ahrens 	if (err)
1239f8554bb9SMatthew Ahrens 		return (err);
1240f8554bb9SMatthew Ahrens 
1241f8554bb9SMatthew Ahrens 	err = dmu_read_uio_dnode(dn, uio, size);
1242f8554bb9SMatthew Ahrens 
1243f8554bb9SMatthew Ahrens 	dnode_rele(dn, FTAG);
1244f8554bb9SMatthew Ahrens 
1245f8554bb9SMatthew Ahrens 	return (err);
1246f8554bb9SMatthew Ahrens }
1247f8554bb9SMatthew Ahrens 
124894d1a210STim Haley static int
124994d1a210STim Haley dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
1250fa9e4066Sahrens {
1251fa9e4066Sahrens 	dmu_buf_t **dbp;
125294d1a210STim Haley 	int numbufs;
1253fa9e4066Sahrens 	int err = 0;
125494d1a210STim Haley 	int i;
1255fa9e4066Sahrens 
125694d1a210STim Haley 	err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
125794d1a210STim Haley 	    FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1258ea8dc4b6Seschrock 	if (err)
1259ea8dc4b6Seschrock 		return (err);
1260fa9e4066Sahrens 
1261fa9e4066Sahrens 	for (i = 0; i < numbufs; i++) {
1262fa9e4066Sahrens 		int tocpy;
1263fa9e4066Sahrens 		int bufoff;
1264fa9e4066Sahrens 		dmu_buf_t *db = dbp[i];
1265fa9e4066Sahrens 
1266fa9e4066Sahrens 		ASSERT(size > 0);
1267fa9e4066Sahrens 
1268feb08c6bSbillm 		bufoff = uio->uio_loffset - db->db_offset;
1269fa9e4066Sahrens 		tocpy = (int)MIN(db->db_size - bufoff, size);
1270fa9e4066Sahrens 
1271fa9e4066Sahrens 		ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1272fa9e4066Sahrens 
1273fa9e4066Sahrens 		if (tocpy == db->db_size)
1274fa9e4066Sahrens 			dmu_buf_will_fill(db, tx);
1275fa9e4066Sahrens 		else
1276fa9e4066Sahrens 			dmu_buf_will_dirty(db, tx);
1277fa9e4066Sahrens 
1278fa9e4066Sahrens 		/*
1279fa9e4066Sahrens 		 * XXX uiomove could block forever (eg. nfs-backed
1280fa9e4066Sahrens 		 * pages).  There needs to be a uiolockdown() function
1281fa9e4066Sahrens 		 * to lock the pages in memory, so that uiomove won't
1282fa9e4066Sahrens 		 * block.
1283fa9e4066Sahrens 		 */
1284fa9e4066Sahrens 		err = uiomove((char *)db->db_data + bufoff, tocpy,
1285fa9e4066Sahrens 		    UIO_WRITE, uio);
1286fa9e4066Sahrens 
1287fa9e4066Sahrens 		if (tocpy == db->db_size)
1288fa9e4066Sahrens 			dmu_buf_fill_done(db, tx);
1289fa9e4066Sahrens 
1290fa9e4066Sahrens 		if (err)
1291fa9e4066Sahrens 			break;
1292fa9e4066Sahrens 
1293fa9e4066Sahrens 		size -= tocpy;
1294fa9e4066Sahrens 	}
129594d1a210STim Haley 
1296ea8dc4b6Seschrock 	dmu_buf_rele_array(dbp, numbufs, FTAG);
1297fa9e4066Sahrens 	return (err);
1298fa9e4066Sahrens }
129944eda4d7Smaybee 
1300f8554bb9SMatthew Ahrens /*
1301f8554bb9SMatthew Ahrens  * Write 'size' bytes from the uio buffer.
1302f8554bb9SMatthew Ahrens  * To object zdb->db_object.
1303f8554bb9SMatthew Ahrens  * Starting at offset uio->uio_loffset.
1304f8554bb9SMatthew Ahrens  *
1305f8554bb9SMatthew Ahrens  * If the caller already has a dbuf in the target object
1306f8554bb9SMatthew Ahrens  * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
1307f8554bb9SMatthew Ahrens  * because we don't have to find the dnode_t for the object.
1308f8554bb9SMatthew Ahrens  */
130994d1a210STim Haley int
131094d1a210STim Haley dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
131194d1a210STim Haley     dmu_tx_t *tx)
131294d1a210STim Haley {
1313744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1314744947dcSTom Erickson 	dnode_t *dn;
1315744947dcSTom Erickson 	int err;
1316744947dcSTom Erickson 
131794d1a210STim Haley 	if (size == 0)
131894d1a210STim Haley 		return (0);
131994d1a210STim Haley 
1320744947dcSTom Erickson 	DB_DNODE_ENTER(db);
1321744947dcSTom Erickson 	dn = DB_DNODE(db);
1322744947dcSTom Erickson 	err = dmu_write_uio_dnode(dn, uio, size, tx);
1323744947dcSTom Erickson 	DB_DNODE_EXIT(db);
1324744947dcSTom Erickson 
1325744947dcSTom Erickson 	return (err);
132694d1a210STim Haley }
132794d1a210STim Haley 
1328f8554bb9SMatthew Ahrens /*
1329f8554bb9SMatthew Ahrens  * Write 'size' bytes from the uio buffer.
1330f8554bb9SMatthew Ahrens  * To the specified object.
1331f8554bb9SMatthew Ahrens  * Starting at offset uio->uio_loffset.
1332f8554bb9SMatthew Ahrens  */
133394d1a210STim Haley int
133494d1a210STim Haley dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
133594d1a210STim Haley     dmu_tx_t *tx)
133694d1a210STim Haley {
133794d1a210STim Haley 	dnode_t *dn;
133894d1a210STim Haley 	int err;
133994d1a210STim Haley 
134094d1a210STim Haley 	if (size == 0)
134194d1a210STim Haley 		return (0);
134294d1a210STim Haley 
134394d1a210STim Haley 	err = dnode_hold(os, object, FTAG, &dn);
134494d1a210STim Haley 	if (err)
134594d1a210STim Haley 		return (err);
134694d1a210STim Haley 
134794d1a210STim Haley 	err = dmu_write_uio_dnode(dn, uio, size, tx);
134894d1a210STim Haley 
134994d1a210STim Haley 	dnode_rele(dn, FTAG);
135094d1a210STim Haley 
135194d1a210STim Haley 	return (err);
135294d1a210STim Haley }
135394d1a210STim Haley 
135444eda4d7Smaybee int
135544eda4d7Smaybee dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
135644eda4d7Smaybee     page_t *pp, dmu_tx_t *tx)
135744eda4d7Smaybee {
135844eda4d7Smaybee 	dmu_buf_t **dbp;
135944eda4d7Smaybee 	int numbufs, i;
136044eda4d7Smaybee 	int err;
136144eda4d7Smaybee 
136244eda4d7Smaybee 	if (size == 0)
136344eda4d7Smaybee 		return (0);
136444eda4d7Smaybee 
136544eda4d7Smaybee 	err = dmu_buf_hold_array(os, object, offset, size,
136644eda4d7Smaybee 	    FALSE, FTAG, &numbufs, &dbp);
136744eda4d7Smaybee 	if (err)
136844eda4d7Smaybee 		return (err);
136944eda4d7Smaybee 
137044eda4d7Smaybee 	for (i = 0; i < numbufs; i++) {
137144eda4d7Smaybee 		int tocpy, copied, thiscpy;
137244eda4d7Smaybee 		int bufoff;
137344eda4d7Smaybee 		dmu_buf_t *db = dbp[i];
137444eda4d7Smaybee 		caddr_t va;
137544eda4d7Smaybee 
137644eda4d7Smaybee 		ASSERT(size > 0);
137744eda4d7Smaybee 		ASSERT3U(db->db_size, >=, PAGESIZE);
137844eda4d7Smaybee 
137944eda4d7Smaybee 		bufoff = offset - db->db_offset;
138044eda4d7Smaybee 		tocpy = (int)MIN(db->db_size - bufoff, size);
138144eda4d7Smaybee 
138244eda4d7Smaybee 		ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
138344eda4d7Smaybee 
138444eda4d7Smaybee 		if (tocpy == db->db_size)
138544eda4d7Smaybee 			dmu_buf_will_fill(db, tx);
138644eda4d7Smaybee 		else
138744eda4d7Smaybee 			dmu_buf_will_dirty(db, tx);
138844eda4d7Smaybee 
138944eda4d7Smaybee 		for (copied = 0; copied < tocpy; copied += PAGESIZE) {
139044eda4d7Smaybee 			ASSERT3U(pp->p_offset, ==, db->db_offset + bufoff);
139144eda4d7Smaybee 			thiscpy = MIN(PAGESIZE, tocpy - copied);
13920fab61baSJonathan W Adams 			va = zfs_map_page(pp, S_READ);
139344eda4d7Smaybee 			bcopy(va, (char *)db->db_data + bufoff, thiscpy);
13940fab61baSJonathan W Adams 			zfs_unmap_page(pp, va);
139544eda4d7Smaybee 			pp = pp->p_next;
139644eda4d7Smaybee 			bufoff += PAGESIZE;
139744eda4d7Smaybee 		}
139844eda4d7Smaybee 
139944eda4d7Smaybee 		if (tocpy == db->db_size)
140044eda4d7Smaybee 			dmu_buf_fill_done(db, tx);
140144eda4d7Smaybee 
140244eda4d7Smaybee 		offset += tocpy;
140344eda4d7Smaybee 		size -= tocpy;
140444eda4d7Smaybee 	}
140544eda4d7Smaybee 	dmu_buf_rele_array(dbp, numbufs, FTAG);
140644eda4d7Smaybee 	return (err);
140744eda4d7Smaybee }
1408fa9e4066Sahrens #endif
1409fa9e4066Sahrens 
14102fdbea25SAleksandr Guzovskiy /*
14112fdbea25SAleksandr Guzovskiy  * Allocate a loaned anonymous arc buffer.
14122fdbea25SAleksandr Guzovskiy  */
14132fdbea25SAleksandr Guzovskiy arc_buf_t *
14142fdbea25SAleksandr Guzovskiy dmu_request_arcbuf(dmu_buf_t *handle, int size)
14152fdbea25SAleksandr Guzovskiy {
1416744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
14172fdbea25SAleksandr Guzovskiy 
14185602294fSDan Kimmel 	return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size));
14192fdbea25SAleksandr Guzovskiy }
14202fdbea25SAleksandr Guzovskiy 
14212fdbea25SAleksandr Guzovskiy /*
14222fdbea25SAleksandr Guzovskiy  * Free a loaned arc buffer.
14232fdbea25SAleksandr Guzovskiy  */
14242fdbea25SAleksandr Guzovskiy void
14252fdbea25SAleksandr Guzovskiy dmu_return_arcbuf(arc_buf_t *buf)
14262fdbea25SAleksandr Guzovskiy {
14272fdbea25SAleksandr Guzovskiy 	arc_return_buf(buf, FTAG);
1428dcbf3bd6SGeorge Wilson 	arc_buf_destroy(buf, FTAG);
14292fdbea25SAleksandr Guzovskiy }
14302fdbea25SAleksandr Guzovskiy 
14312fdbea25SAleksandr Guzovskiy /*
14322fdbea25SAleksandr Guzovskiy  * When possible directly assign passed loaned arc buffer to a dbuf.
14332fdbea25SAleksandr Guzovskiy  * If this is not possible copy the contents of passed arc buf via
14342fdbea25SAleksandr Guzovskiy  * dmu_write().
14352fdbea25SAleksandr Guzovskiy  */
14362fdbea25SAleksandr Guzovskiy void
14372fdbea25SAleksandr Guzovskiy dmu_assign_arcbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
14382fdbea25SAleksandr Guzovskiy     dmu_tx_t *tx)
14392fdbea25SAleksandr Guzovskiy {
1440744947dcSTom Erickson 	dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1441744947dcSTom Erickson 	dnode_t *dn;
14422fdbea25SAleksandr Guzovskiy 	dmu_buf_impl_t *db;
14435602294fSDan Kimmel 	uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
14442fdbea25SAleksandr Guzovskiy 	uint64_t blkid;
14452fdbea25SAleksandr Guzovskiy 
1446744947dcSTom Erickson 	DB_DNODE_ENTER(dbuf);
1447744947dcSTom Erickson 	dn = DB_DNODE(dbuf);
14482fdbea25SAleksandr Guzovskiy 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
1449a2cdcdd2SPaul Dagnelie 	blkid = dbuf_whichblock(dn, 0, offset);
14502fdbea25SAleksandr Guzovskiy 	VERIFY((db = dbuf_hold(dn, blkid, FTAG)) != NULL);
14512fdbea25SAleksandr Guzovskiy 	rw_exit(&dn->dn_struct_rwlock);
1452744947dcSTom Erickson 	DB_DNODE_EXIT(dbuf);
14532fdbea25SAleksandr Guzovskiy 
14548a904709SMatthew Ahrens 	/*
14558a904709SMatthew Ahrens 	 * We can only assign if the offset is aligned, the arc buf is the
14565602294fSDan Kimmel 	 * same size as the dbuf, and the dbuf is not metadata.
14578a904709SMatthew Ahrens 	 */
14585602294fSDan Kimmel 	if (offset == db->db.db_offset && blksz == db->db.db_size) {
14592fdbea25SAleksandr Guzovskiy 		dbuf_assign_arcbuf(db, buf, tx);
14602fdbea25SAleksandr Guzovskiy 		dbuf_rele(db, FTAG);
14612fdbea25SAleksandr Guzovskiy 	} else {
1462744947dcSTom Erickson 		objset_t *os;
1463744947dcSTom Erickson 		uint64_t object;
1464744947dcSTom Erickson 
14655602294fSDan Kimmel 		/* compressed bufs must always be assignable to their dbuf */
14665602294fSDan Kimmel 		ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
14675602294fSDan Kimmel 		ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
14685602294fSDan Kimmel 
1469744947dcSTom Erickson 		DB_DNODE_ENTER(dbuf);
1470744947dcSTom Erickson 		dn = DB_DNODE(dbuf);
1471744947dcSTom Erickson 		os = dn->dn_objset;
1472744947dcSTom Erickson 		object = dn->dn_object;
1473744947dcSTom Erickson 		DB_DNODE_EXIT(dbuf);
1474744947dcSTom Erickson 
14752fdbea25SAleksandr Guzovskiy 		dbuf_rele(db, FTAG);
1476744947dcSTom Erickson 		dmu_write(os, object, offset, blksz, buf->b_data, tx);
14772fdbea25SAleksandr Guzovskiy 		dmu_return_arcbuf(buf);
1478c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 		XUIOSTAT_BUMP(xuiostat_wbuf_copied);
14792fdbea25SAleksandr Guzovskiy 	}
14802fdbea25SAleksandr Guzovskiy }
14812fdbea25SAleksandr Guzovskiy 
1482c5c6ffa0Smaybee typedef struct {
1483b24ab676SJeff Bonwick 	dbuf_dirty_record_t	*dsa_dr;
1484b24ab676SJeff Bonwick 	dmu_sync_cb_t		*dsa_done;
1485b24ab676SJeff Bonwick 	zgd_t			*dsa_zgd;
1486b24ab676SJeff Bonwick 	dmu_tx_t		*dsa_tx;
1487c717a561Smaybee } dmu_sync_arg_t;
1488c5c6ffa0Smaybee 
1489e14bb325SJeff Bonwick /* ARGSUSED */
1490e14bb325SJeff Bonwick static void
1491e14bb325SJeff Bonwick dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1492e14bb325SJeff Bonwick {
1493b24ab676SJeff Bonwick 	dmu_sync_arg_t *dsa = varg;
1494b24ab676SJeff Bonwick 	dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1495e14bb325SJeff Bonwick 	blkptr_t *bp = zio->io_bp;
1496975c32a0SNeil Perrin 
1497b24ab676SJeff Bonwick 	if (zio->io_error == 0) {
1498b24ab676SJeff Bonwick 		if (BP_IS_HOLE(bp)) {
1499b24ab676SJeff Bonwick 			/*
1500b24ab676SJeff Bonwick 			 * A block of zeros may compress to a hole, but the
1501b24ab676SJeff Bonwick 			 * block size still needs to be known for replay.
1502b24ab676SJeff Bonwick 			 */
1503b24ab676SJeff Bonwick 			BP_SET_LSIZE(bp, db->db_size);
15045d7b4d43SMatthew Ahrens 		} else if (!BP_IS_EMBEDDED(bp)) {
1505b24ab676SJeff Bonwick 			ASSERT(BP_GET_LEVEL(bp) == 0);
1506b24ab676SJeff Bonwick 			bp->blk_fill = 1;
1507b24ab676SJeff Bonwick 		}
1508e14bb325SJeff Bonwick 	}
1509e14bb325SJeff Bonwick }
1510e14bb325SJeff Bonwick 
1511b24ab676SJeff Bonwick static void
1512b24ab676SJeff Bonwick dmu_sync_late_arrival_ready(zio_t *zio)
1513b24ab676SJeff Bonwick {
1514b24ab676SJeff Bonwick 	dmu_sync_ready(zio, NULL, zio->io_private);
1515b24ab676SJeff Bonwick }
1516b24ab676SJeff Bonwick 
1517c5c6ffa0Smaybee /* ARGSUSED */
1518c5c6ffa0Smaybee static void
1519c5c6ffa0Smaybee dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1520c5c6ffa0Smaybee {
1521b24ab676SJeff Bonwick 	dmu_sync_arg_t *dsa = varg;
1522b24ab676SJeff Bonwick 	dbuf_dirty_record_t *dr = dsa->dsa_dr;
1523c717a561Smaybee 	dmu_buf_impl_t *db = dr->dr_dbuf;
1524c5c6ffa0Smaybee 
1525b50a0fe0SNeil Perrin 	mutex_enter(&db->db_mtx);
1526b50a0fe0SNeil Perrin 	ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1527b24ab676SJeff Bonwick 	if (zio->io_error == 0) {
152880901aeaSGeorge Wilson 		dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
152980901aeaSGeorge Wilson 		if (dr->dt.dl.dr_nopwrite) {
153080901aeaSGeorge Wilson 			blkptr_t *bp = zio->io_bp;
153180901aeaSGeorge Wilson 			blkptr_t *bp_orig = &zio->io_bp_orig;
153280901aeaSGeorge Wilson 			uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
153380901aeaSGeorge Wilson 
153480901aeaSGeorge Wilson 			ASSERT(BP_EQUAL(bp, bp_orig));
153580901aeaSGeorge Wilson 			ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
153645818ee1SMatthew Ahrens 			ASSERT(zio_checksum_table[chksum].ci_flags &
153745818ee1SMatthew Ahrens 			    ZCHECKSUM_FLAG_NOPWRITE);
153880901aeaSGeorge Wilson 		}
1539b24ab676SJeff Bonwick 		dr->dt.dl.dr_overridden_by = *zio->io_bp;
1540b24ab676SJeff Bonwick 		dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1541b24ab676SJeff Bonwick 		dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
154270163ac5SPrakash Surya 
154370163ac5SPrakash Surya 		/*
154470163ac5SPrakash Surya 		 * Old style holes are filled with all zeros, whereas
154570163ac5SPrakash Surya 		 * new-style holes maintain their lsize, type, level,
154670163ac5SPrakash Surya 		 * and birth time (see zio_write_compress). While we
154770163ac5SPrakash Surya 		 * need to reset the BP_SET_LSIZE() call that happened
154870163ac5SPrakash Surya 		 * in dmu_sync_ready for old style holes, we do *not*
154970163ac5SPrakash Surya 		 * want to wipe out the information contained in new
155070163ac5SPrakash Surya 		 * style holes. Thus, only zero out the block pointer if
155170163ac5SPrakash Surya 		 * it's an old style hole.
155270163ac5SPrakash Surya 		 */
155370163ac5SPrakash Surya 		if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
155470163ac5SPrakash Surya 		    dr->dt.dl.dr_overridden_by.blk_birth == 0)
1555b24ab676SJeff Bonwick 			BP_ZERO(&dr->dt.dl.dr_overridden_by);
1556b24ab676SJeff Bonwick 	} else {
1557b24ab676SJeff Bonwick 		dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1558b24ab676SJeff Bonwick 	}
1559c5c6ffa0Smaybee 	cv_broadcast(&db->db_changed);
1560b50a0fe0SNeil Perrin 	mutex_exit(&db->db_mtx);
1561b50a0fe0SNeil Perrin 
1562b24ab676SJeff Bonwick 	dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1563b24ab676SJeff Bonwick 
1564b24ab676SJeff Bonwick 	kmem_free(dsa, sizeof (*dsa));
1565b24ab676SJeff Bonwick }
1566b24ab676SJeff Bonwick 
1567b24ab676SJeff Bonwick static void
1568b24ab676SJeff Bonwick dmu_sync_late_arrival_done(zio_t *zio)
1569b24ab676SJeff Bonwick {
1570b24ab676SJeff Bonwick 	blkptr_t *bp = zio->io_bp;
1571b24ab676SJeff Bonwick 	dmu_sync_arg_t *dsa = zio->io_private;
157280901aeaSGeorge Wilson 	blkptr_t *bp_orig = &zio->io_bp_orig;
1573b24ab676SJeff Bonwick 
1574b24ab676SJeff Bonwick 	if (zio->io_error == 0 && !BP_IS_HOLE(bp)) {
157580901aeaSGeorge Wilson 		/*
157680901aeaSGeorge Wilson 		 * If we didn't allocate a new block (i.e. ZIO_FLAG_NOPWRITE)
157780901aeaSGeorge Wilson 		 * then there is nothing to do here. Otherwise, free the
157880901aeaSGeorge Wilson 		 * newly allocated block in this txg.
157980901aeaSGeorge Wilson 		 */
158080901aeaSGeorge Wilson 		if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
158180901aeaSGeorge Wilson 			ASSERT(BP_EQUAL(bp, bp_orig));
158280901aeaSGeorge Wilson 		} else {
158380901aeaSGeorge Wilson 			ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
158480901aeaSGeorge Wilson 			ASSERT(zio->io_bp->blk_birth == zio->io_txg);
158580901aeaSGeorge Wilson 			ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
158680901aeaSGeorge Wilson 			zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
158780901aeaSGeorge Wilson 		}
1588b24ab676SJeff Bonwick 	}
1589b24ab676SJeff Bonwick 
1590b24ab676SJeff Bonwick 	dmu_tx_commit(dsa->dsa_tx);
1591b24ab676SJeff Bonwick 
1592b24ab676SJeff Bonwick 	dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1593b24ab676SJeff Bonwick 
1594b24ab676SJeff Bonwick 	kmem_free(dsa, sizeof (*dsa));
1595b24ab676SJeff Bonwick }
1596b24ab676SJeff Bonwick 
1597b24ab676SJeff Bonwick static int
1598b24ab676SJeff Bonwick dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
15997802d7bfSMatthew Ahrens     zio_prop_t *zp, zbookmark_phys_t *zb)
1600b24ab676SJeff Bonwick {
1601b24ab676SJeff Bonwick 	dmu_sync_arg_t *dsa;
1602b24ab676SJeff Bonwick 	dmu_tx_t *tx;
1603b24ab676SJeff Bonwick 
1604b24ab676SJeff Bonwick 	tx = dmu_tx_create(os);
1605b24ab676SJeff Bonwick 	dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
16066e1f5caaSNeil Perrin 	if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1607b24ab676SJeff Bonwick 		dmu_tx_abort(tx);
1608be6fd75aSMatthew Ahrens 		/* Make zl_get_data do txg_waited_synced() */
1609be6fd75aSMatthew Ahrens 		return (SET_ERROR(EIO));
1610b24ab676SJeff Bonwick 	}
1611b24ab676SJeff Bonwick 
1612b24ab676SJeff Bonwick 	dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1613b24ab676SJeff Bonwick 	dsa->dsa_dr = NULL;
1614b24ab676SJeff Bonwick 	dsa->dsa_done = done;
1615b24ab676SJeff Bonwick 	dsa->dsa_zgd = zgd;
1616b24ab676SJeff Bonwick 	dsa->dsa_tx = tx;
1617c717a561Smaybee 
16185602294fSDan Kimmel 	zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
16195602294fSDan Kimmel 	    zgd->zgd_db->db_data, zgd->zgd_db->db_size, zgd->zgd_db->db_size,
16208df0bcf0SPaul Dagnelie 	    zp, dmu_sync_late_arrival_ready, NULL,
16218df0bcf0SPaul Dagnelie 	    NULL, dmu_sync_late_arrival_done, dsa, ZIO_PRIORITY_SYNC_WRITE,
16228df0bcf0SPaul Dagnelie 	    ZIO_FLAG_CANFAIL, zb));
1623b24ab676SJeff Bonwick 
1624b24ab676SJeff Bonwick 	return (0);
1625c5c6ffa0Smaybee }
1626c5c6ffa0Smaybee 
1627fa9e4066Sahrens /*
1628c5c6ffa0Smaybee  * Intent log support: sync the block associated with db to disk.
1629c5c6ffa0Smaybee  * N.B. and XXX: the caller is responsible for making sure that the
1630c5c6ffa0Smaybee  * data isn't changing while dmu_sync() is writing it.
1631fa9e4066Sahrens  *
1632fa9e4066Sahrens  * Return values:
1633fa9e4066Sahrens  *
163480901aeaSGeorge Wilson  *	EEXIST: this txg has already been synced, so there's nothing to do.
1635fa9e4066Sahrens  *		The caller should not log the write.
1636fa9e4066Sahrens  *
1637fa9e4066Sahrens  *	ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
1638fa9e4066Sahrens  *		The caller should not log the write.
1639fa9e4066Sahrens  *
1640c5c6ffa0Smaybee  *	EALREADY: this block is already in the process of being synced.
1641c5c6ffa0Smaybee  *		The caller should track its progress (somehow).
1642fa9e4066Sahrens  *
1643b24ab676SJeff Bonwick  *	EIO: could not do the I/O.
1644b24ab676SJeff Bonwick  *		The caller should do a txg_wait_synced().
1645fa9e4066Sahrens  *
1646b24ab676SJeff Bonwick  *	0: the I/O has been initiated.
1647b24ab676SJeff Bonwick  *		The caller should log this blkptr in the done callback.
1648b24ab676SJeff Bonwick  *		It is possible that the I/O will fail, in which case
1649b24ab676SJeff Bonwick  *		the error will be reported to the done callback and
1650b24ab676SJeff Bonwick  *		propagated to pio from zio_done().
1651fa9e4066Sahrens  */
1652fa9e4066Sahrens int
1653b24ab676SJeff Bonwick dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
1654fa9e4066Sahrens {
1655b24ab676SJeff Bonwick 	blkptr_t *bp = zgd->zgd_bp;
1656b24ab676SJeff Bonwick 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
1657503ad85cSMatthew Ahrens 	objset_t *os = db->db_objset;
1658b24ab676SJeff Bonwick 	dsl_dataset_t *ds = os->os_dsl_dataset;
1659c717a561Smaybee 	dbuf_dirty_record_t *dr;
1660b24ab676SJeff Bonwick 	dmu_sync_arg_t *dsa;
16617802d7bfSMatthew Ahrens 	zbookmark_phys_t zb;
1662b24ab676SJeff Bonwick 	zio_prop_t zp;
1663744947dcSTom Erickson 	dnode_t *dn;
1664fa9e4066Sahrens 
1665b24ab676SJeff Bonwick 	ASSERT(pio != NULL);
1666fa9e4066Sahrens 	ASSERT(txg != 0);
1667fa9e4066Sahrens 
1668b24ab676SJeff Bonwick 	SET_BOOKMARK(&zb, ds->ds_object,
1669b24ab676SJeff Bonwick 	    db->db.db_object, db->db_level, db->db_blkid);
1670b24ab676SJeff Bonwick 
1671744947dcSTom Erickson 	DB_DNODE_ENTER(db);
1672744947dcSTom Erickson 	dn = DB_DNODE(db);
16735602294fSDan Kimmel 	dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC,
16745602294fSDan Kimmel 	    ZIO_COMPRESS_INHERIT, &zp);
1675744947dcSTom Erickson 	DB_DNODE_EXIT(db);
1676fa9e4066Sahrens 
1677ea8dc4b6Seschrock 	/*
1678b24ab676SJeff Bonwick 	 * If we're frozen (running ziltest), we always need to generate a bp.
1679ea8dc4b6Seschrock 	 */
1680b24ab676SJeff Bonwick 	if (txg > spa_freeze_txg(os->os_spa))
1681b24ab676SJeff Bonwick 		return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1682ea8dc4b6Seschrock 
1683fa9e4066Sahrens 	/*
1684b24ab676SJeff Bonwick 	 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
1685b24ab676SJeff Bonwick 	 * and us.  If we determine that this txg is not yet syncing,
1686b24ab676SJeff Bonwick 	 * but it begins to sync a moment later, that's OK because the
1687b24ab676SJeff Bonwick 	 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
1688fa9e4066Sahrens 	 */
1689b24ab676SJeff Bonwick 	mutex_enter(&db->db_mtx);
1690b24ab676SJeff Bonwick 
1691b24ab676SJeff Bonwick 	if (txg <= spa_last_synced_txg(os->os_spa)) {
1692fa9e4066Sahrens 		/*
1693b24ab676SJeff Bonwick 		 * This txg has already synced.  There's nothing to do.
1694fa9e4066Sahrens 		 */
1695b24ab676SJeff Bonwick 		mutex_exit(&db->db_mtx);
1696be6fd75aSMatthew Ahrens 		return (SET_ERROR(EEXIST));
1697fa9e4066Sahrens 	}
1698fa9e4066Sahrens 
1699b24ab676SJeff Bonwick 	if (txg <= spa_syncing_txg(os->os_spa)) {
1700b24ab676SJeff Bonwick 		/*
1701b24ab676SJeff Bonwick 		 * This txg is currently syncing, so we can't mess with
1702b24ab676SJeff Bonwick 		 * the dirty record anymore; just write a new log block.
1703b24ab676SJeff Bonwick 		 */
1704b24ab676SJeff Bonwick 		mutex_exit(&db->db_mtx);
1705b24ab676SJeff Bonwick 		return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1706fa9e4066Sahrens 	}
1707fa9e4066Sahrens 
1708c717a561Smaybee 	dr = db->db_last_dirty;
1709b24ab676SJeff Bonwick 	while (dr && dr->dr_txg != txg)
1710c717a561Smaybee 		dr = dr->dr_next;
1711b24ab676SJeff Bonwick 
1712b24ab676SJeff Bonwick 	if (dr == NULL) {
1713c5c6ffa0Smaybee 		/*
1714b24ab676SJeff Bonwick 		 * There's no dr for this dbuf, so it must have been freed.
1715c5c6ffa0Smaybee 		 * There's no need to log writes to freed blocks, so we're done.
1716c5c6ffa0Smaybee 		 */
1717c5c6ffa0Smaybee 		mutex_exit(&db->db_mtx);
1718be6fd75aSMatthew Ahrens 		return (SET_ERROR(ENOENT));
1719c5c6ffa0Smaybee 	}
1720c5c6ffa0Smaybee 
172180901aeaSGeorge Wilson 	ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
172280901aeaSGeorge Wilson 
172380901aeaSGeorge Wilson 	/*
172434e8acefSMatthew Ahrens 	 * Assume the on-disk data is X, the current syncing data (in
172534e8acefSMatthew Ahrens 	 * txg - 1) is Y, and the current in-memory data is Z (currently
172634e8acefSMatthew Ahrens 	 * in dmu_sync).
172734e8acefSMatthew Ahrens 	 *
172834e8acefSMatthew Ahrens 	 * We usually want to perform a nopwrite if X and Z are the
172934e8acefSMatthew Ahrens 	 * same.  However, if Y is different (i.e. the BP is going to
173034e8acefSMatthew Ahrens 	 * change before this write takes effect), then a nopwrite will
173134e8acefSMatthew Ahrens 	 * be incorrect - we would override with X, which could have
173234e8acefSMatthew Ahrens 	 * been freed when Y was written.
173334e8acefSMatthew Ahrens 	 *
173434e8acefSMatthew Ahrens 	 * (Note that this is not a concern when we are nop-writing from
173534e8acefSMatthew Ahrens 	 * syncing context, because X and Y must be identical, because
173634e8acefSMatthew Ahrens 	 * all previous txgs have been synced.)
173734e8acefSMatthew Ahrens 	 *
173834e8acefSMatthew Ahrens 	 * Therefore, we disable nopwrite if the current BP could change
173934e8acefSMatthew Ahrens 	 * before this TXG.  There are two ways it could change: by
174034e8acefSMatthew Ahrens 	 * being dirty (dr_next is non-NULL), or by being freed
174134e8acefSMatthew Ahrens 	 * (dnode_block_freed()).  This behavior is verified by
174234e8acefSMatthew Ahrens 	 * zio_done(), which VERIFYs that the override BP is identical
174334e8acefSMatthew Ahrens 	 * to the on-disk BP.
174480901aeaSGeorge Wilson 	 */
174534e8acefSMatthew Ahrens 	DB_DNODE_ENTER(db);
174634e8acefSMatthew Ahrens 	dn = DB_DNODE(db);
174734e8acefSMatthew Ahrens 	if (dr->dr_next != NULL || dnode_block_freed(dn, db->db_blkid))
174880901aeaSGeorge Wilson 		zp.zp_nopwrite = B_FALSE;
174934e8acefSMatthew Ahrens 	DB_DNODE_EXIT(db);
175080901aeaSGeorge Wilson 
1751c717a561Smaybee 	ASSERT(dr->dr_txg == txg);
1752b24ab676SJeff Bonwick 	if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
1753b24ab676SJeff Bonwick 	    dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1754c717a561Smaybee 		/*
1755b24ab676SJeff Bonwick 		 * We have already issued a sync write for this buffer,
1756b24ab676SJeff Bonwick 		 * or this buffer has already been synced.  It could not
1757c717a561Smaybee 		 * have been dirtied since, or we would have cleared the state.
1758c717a561Smaybee 		 */
1759c717a561Smaybee 		mutex_exit(&db->db_mtx);
1760be6fd75aSMatthew Ahrens 		return (SET_ERROR(EALREADY));
1761c717a561Smaybee 	}
1762c717a561Smaybee 
1763b24ab676SJeff Bonwick 	ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
1764c717a561Smaybee 	dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
1765fa9e4066Sahrens 	mutex_exit(&db->db_mtx);
1766e14bb325SJeff Bonwick 
1767b24ab676SJeff Bonwick 	dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1768b24ab676SJeff Bonwick 	dsa->dsa_dr = dr;
1769b24ab676SJeff Bonwick 	dsa->dsa_done = done;
1770b24ab676SJeff Bonwick 	dsa->dsa_zgd = zgd;
1771b24ab676SJeff Bonwick 	dsa->dsa_tx = NULL;
1772e14bb325SJeff Bonwick 
1773b24ab676SJeff Bonwick 	zio_nowait(arc_write(pio, os->os_spa, txg,
1774aad02571SSaso Kiselkov 	    bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
1775dcbf3bd6SGeorge Wilson 	    &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa,
17768df0bcf0SPaul Dagnelie 	    ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
1777e14bb325SJeff Bonwick 
1778b24ab676SJeff Bonwick 	return (0);
1779fa9e4066Sahrens }
1780fa9e4066Sahrens 
1781fa9e4066Sahrens int
1782fa9e4066Sahrens dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
17839a686fbcSPaul Dagnelie     dmu_tx_t *tx)
1784fa9e4066Sahrens {
1785ea8dc4b6Seschrock 	dnode_t *dn;
1786ea8dc4b6Seschrock 	int err;
1787ea8dc4b6Seschrock 
1788503ad85cSMatthew Ahrens 	err = dnode_hold(os, object, FTAG, &dn);
1789ea8dc4b6Seschrock 	if (err)
1790ea8dc4b6Seschrock 		return (err);
1791ea8dc4b6Seschrock 	err = dnode_set_blksz(dn, size, ibs, tx);
1792fa9e4066Sahrens 	dnode_rele(dn, FTAG);
1793fa9e4066Sahrens 	return (err);
1794fa9e4066Sahrens }
1795fa9e4066Sahrens 
1796fa9e4066Sahrens void
1797fa9e4066Sahrens dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
17989a686fbcSPaul Dagnelie     dmu_tx_t *tx)
1799fa9e4066Sahrens {
1800ea8dc4b6Seschrock 	dnode_t *dn;
1801ea8dc4b6Seschrock 
18025d7b4d43SMatthew Ahrens 	/*
18035d7b4d43SMatthew Ahrens 	 * Send streams include each object's checksum function.  This
18045d7b4d43SMatthew Ahrens 	 * check ensures that the receiving system can understand the
18055d7b4d43SMatthew Ahrens 	 * checksum function transmitted.
18065d7b4d43SMatthew Ahrens 	 */
18075d7b4d43SMatthew Ahrens 	ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS);
18085d7b4d43SMatthew Ahrens 
18095d7b4d43SMatthew Ahrens 	VERIFY0(dnode_hold(os, object, FTAG, &dn));
18105d7b4d43SMatthew Ahrens 	ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS);
1811fa9e4066Sahrens 	dn->dn_checksum = checksum;
1812fa9e4066Sahrens 	dnode_setdirty(dn, tx);
1813fa9e4066Sahrens 	dnode_rele(dn, FTAG);
1814fa9e4066Sahrens }
1815fa9e4066Sahrens 
1816fa9e4066Sahrens void
1817fa9e4066Sahrens dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
18189a686fbcSPaul Dagnelie     dmu_tx_t *tx)
1819fa9e4066Sahrens {
1820ea8dc4b6Seschrock 	dnode_t *dn;
1821ea8dc4b6Seschrock 
18225d7b4d43SMatthew Ahrens 	/*
18235d7b4d43SMatthew Ahrens 	 * Send streams include each object's compression function.  This
18245d7b4d43SMatthew Ahrens 	 * check ensures that the receiving system can understand the
18255d7b4d43SMatthew Ahrens 	 * compression function transmitted.
18265d7b4d43SMatthew Ahrens 	 */
18275d7b4d43SMatthew Ahrens 	ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS);
18285d7b4d43SMatthew Ahrens 
18295d7b4d43SMatthew Ahrens 	VERIFY0(dnode_hold(os, object, FTAG, &dn));
1830fa9e4066Sahrens 	dn->dn_compress = compress;
1831fa9e4066Sahrens 	dnode_setdirty(dn, tx);
1832fa9e4066Sahrens 	dnode_rele(dn, FTAG);
1833fa9e4066Sahrens }
1834fa9e4066Sahrens 
1835b24ab676SJeff Bonwick int zfs_mdcomp_disable = 0;
1836b24ab676SJeff Bonwick 
1837edf345e6SMatthew Ahrens /*
1838edf345e6SMatthew Ahrens  * When the "redundant_metadata" property is set to "most", only indirect
1839edf345e6SMatthew Ahrens  * blocks of this level and higher will have an additional ditto block.
1840edf345e6SMatthew Ahrens  */
1841edf345e6SMatthew Ahrens int zfs_redundant_metadata_most_ditto_level = 2;
1842edf345e6SMatthew Ahrens 
1843b24ab676SJeff Bonwick void
18445602294fSDan Kimmel dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp,
18455602294fSDan Kimmel     enum zio_compress override_compress, zio_prop_t *zp)
1846b24ab676SJeff Bonwick {
1847b24ab676SJeff Bonwick 	dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
1848ad135b5dSChristopher Siden 	boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
18491d8ccc7bSMark Shellenbaum 	    (wp & WP_SPILL));
1850b24ab676SJeff Bonwick 	enum zio_checksum checksum = os->os_checksum;
1851b24ab676SJeff Bonwick 	enum zio_compress compress = os->os_compress;
1852b24ab676SJeff Bonwick 	enum zio_checksum dedup_checksum = os->os_dedup_checksum;
18537540df39SGeorge Wilson 	boolean_t dedup = B_FALSE;
18547540df39SGeorge Wilson 	boolean_t nopwrite = B_FALSE;
1855b24ab676SJeff Bonwick 	boolean_t dedup_verify = os->os_dedup_verify;
1856b24ab676SJeff Bonwick 	int copies = os->os_copies;
18575602294fSDan Kimmel 	boolean_t lz4_ac = spa_feature_is_active(os->os_spa,
18585602294fSDan Kimmel 	    SPA_FEATURE_LZ4_COMPRESS);
18595602294fSDan Kimmel 
18605602294fSDan Kimmel 	IMPLY(override_compress == ZIO_COMPRESS_LZ4, lz4_ac);
1861b24ab676SJeff Bonwick 
1862b24ab676SJeff Bonwick 	/*
186380901aeaSGeorge Wilson 	 * We maintain different write policies for each of the following
186480901aeaSGeorge Wilson 	 * types of data:
186580901aeaSGeorge Wilson 	 *	 1. metadata
186680901aeaSGeorge Wilson 	 *	 2. preallocated blocks (i.e. level-0 blocks of a dump device)
186780901aeaSGeorge Wilson 	 *	 3. all other level 0 blocks
1868b24ab676SJeff Bonwick 	 */
1869b24ab676SJeff Bonwick 	if (ismd) {
1870b8289d24SDaniil Lunev 		if (zfs_mdcomp_disable) {
1871b8289d24SDaniil Lunev 			compress = ZIO_COMPRESS_EMPTY;
1872b8289d24SDaniil Lunev 		} else {
1873db1741f5SJustin T. Gibbs 			/*
1874db1741f5SJustin T. Gibbs 			 * XXX -- we should design a compression algorithm
1875db1741f5SJustin T. Gibbs 			 * that specializes in arrays of bps.
1876db1741f5SJustin T. Gibbs 			 */
1877db1741f5SJustin T. Gibbs 			compress = zio_compress_select(os->os_spa,
1878db1741f5SJustin T. Gibbs 			    ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);
1879b8289d24SDaniil Lunev 		}
188080901aeaSGeorge Wilson 
1881b24ab676SJeff Bonwick 		/*
1882b24ab676SJeff Bonwick 		 * Metadata always gets checksummed.  If the data
1883b24ab676SJeff Bonwick 		 * checksum is multi-bit correctable, and it's not a
1884b24ab676SJeff Bonwick 		 * ZBT-style checksum, then it's suitable for metadata
1885b24ab676SJeff Bonwick 		 * as well.  Otherwise, the metadata checksum defaults
1886b24ab676SJeff Bonwick 		 * to fletcher4.
1887b24ab676SJeff Bonwick 		 */
188845818ee1SMatthew Ahrens 		if (!(zio_checksum_table[checksum].ci_flags &
188945818ee1SMatthew Ahrens 		    ZCHECKSUM_FLAG_METADATA) ||
189045818ee1SMatthew Ahrens 		    (zio_checksum_table[checksum].ci_flags &
189145818ee1SMatthew Ahrens 		    ZCHECKSUM_FLAG_EMBEDDED))
1892b24ab676SJeff Bonwick 			checksum = ZIO_CHECKSUM_FLETCHER_4;
1893edf345e6SMatthew Ahrens 
1894edf345e6SMatthew Ahrens 		if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL ||
1895edf345e6SMatthew Ahrens 		    (os->os_redundant_metadata ==
1896edf345e6SMatthew Ahrens 		    ZFS_REDUNDANT_METADATA_MOST &&
1897edf345e6SMatthew Ahrens 		    (level >= zfs_redundant_metadata_most_ditto_level ||
1898edf345e6SMatthew Ahrens 		    DMU_OT_IS_METADATA(type) || (wp & WP_SPILL))))
1899edf345e6SMatthew Ahrens 			copies++;
190080901aeaSGeorge Wilson 	} else if (wp & WP_NOFILL) {
190180901aeaSGeorge Wilson 		ASSERT(level == 0);
190280901aeaSGeorge Wilson 
1903b24ab676SJeff Bonwick 		/*
190480901aeaSGeorge Wilson 		 * If we're writing preallocated blocks, we aren't actually
190580901aeaSGeorge Wilson 		 * writing them so don't set any policy properties.  These
190680901aeaSGeorge Wilson 		 * blocks are currently only used by an external subsystem
190780901aeaSGeorge Wilson 		 * outside of zfs (i.e. dump) and not written by the zio
190880901aeaSGeorge Wilson 		 * pipeline.
1909b24ab676SJeff Bonwick 		 */
191080901aeaSGeorge Wilson 		compress = ZIO_COMPRESS_OFF;
1911810e43b2SBill Pijewski 		checksum = ZIO_CHECKSUM_NOPARITY;
1912b24ab676SJeff Bonwick 	} else {
1913db1741f5SJustin T. Gibbs 		compress = zio_compress_select(os->os_spa, dn->dn_compress,
1914db1741f5SJustin T. Gibbs 		    compress);
1915b24ab676SJeff Bonwick 
191680901aeaSGeorge Wilson 		checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
191780901aeaSGeorge Wilson 		    zio_checksum_select(dn->dn_checksum, checksum) :
191880901aeaSGeorge Wilson 		    dedup_checksum;
1919b24ab676SJeff Bonwick 
192080901aeaSGeorge Wilson 		/*
192180901aeaSGeorge Wilson 		 * Determine dedup setting.  If we are in dmu_sync(),
192280901aeaSGeorge Wilson 		 * we won't actually dedup now because that's all
192380901aeaSGeorge Wilson 		 * done in syncing context; but we do want to use the
192480901aeaSGeorge Wilson 		 * dedup checkum.  If the checksum is not strong
192580901aeaSGeorge Wilson 		 * enough to ensure unique signatures, force
192680901aeaSGeorge Wilson 		 * dedup_verify.
192780901aeaSGeorge Wilson 		 */
192880901aeaSGeorge Wilson 		if (dedup_checksum != ZIO_CHECKSUM_OFF) {
192980901aeaSGeorge Wilson 			dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
193045818ee1SMatthew Ahrens 			if (!(zio_checksum_table[checksum].ci_flags &
193145818ee1SMatthew Ahrens 			    ZCHECKSUM_FLAG_DEDUP))
193280901aeaSGeorge Wilson 				dedup_verify = B_TRUE;
193380901aeaSGeorge Wilson 		}
1934b24ab676SJeff Bonwick 
193580901aeaSGeorge Wilson 		/*
193645818ee1SMatthew Ahrens 		 * Enable nopwrite if we have secure enough checksum
193745818ee1SMatthew Ahrens 		 * algorithm (see comment in zio_nop_write) and
193845818ee1SMatthew Ahrens 		 * compression is enabled.  We don't enable nopwrite if
193945818ee1SMatthew Ahrens 		 * dedup is enabled as the two features are mutually
194045818ee1SMatthew Ahrens 		 * exclusive.
194180901aeaSGeorge Wilson 		 */
194245818ee1SMatthew Ahrens 		nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
194345818ee1SMatthew Ahrens 		    ZCHECKSUM_FLAG_NOPWRITE) &&
194480901aeaSGeorge Wilson 		    compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
1945b24ab676SJeff Bonwick 	}
1946b24ab676SJeff Bonwick 
1947b24ab676SJeff Bonwick 	zp->zp_checksum = checksum;
19485602294fSDan Kimmel 
19495602294fSDan Kimmel 	/*
19505602294fSDan Kimmel 	 * If we're writing a pre-compressed buffer, the compression type we use
19515602294fSDan Kimmel 	 * must match the data. If it hasn't been compressed yet, then we should
19525602294fSDan Kimmel 	 * use the value dictated by the policies above.
19535602294fSDan Kimmel 	 */
19545602294fSDan Kimmel 	zp->zp_compress = override_compress != ZIO_COMPRESS_INHERIT
19555602294fSDan Kimmel 	    ? override_compress : compress;
19565602294fSDan Kimmel 	ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT);
19575602294fSDan Kimmel 
19580a586ceaSMark Shellenbaum 	zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
1959b24ab676SJeff Bonwick 	zp->zp_level = level;
1960edf345e6SMatthew Ahrens 	zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa));
1961b24ab676SJeff Bonwick 	zp->zp_dedup = dedup;
1962b24ab676SJeff Bonwick 	zp->zp_dedup_verify = dedup && dedup_verify;
196380901aeaSGeorge Wilson 	zp->zp_nopwrite = nopwrite;
1964b24ab676SJeff Bonwick }
1965b24ab676SJeff Bonwick 
1966fa9e4066Sahrens int
1967fa9e4066Sahrens dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
1968fa9e4066Sahrens {
1969fa9e4066Sahrens 	dnode_t *dn;
19702bcf0248SMax Grossman 	int err;
1971fa9e4066Sahrens 
1972fa9e4066Sahrens 	/*
1973fa9e4066Sahrens 	 * Sync any current changes before
1974fa9e4066Sahrens 	 * we go trundling through the block pointers.
1975fa9e4066Sahrens 	 */
19762bcf0248SMax Grossman 	err = dmu_object_wait_synced(os, object);
19772bcf0248SMax Grossman 	if (err) {
19782bcf0248SMax Grossman 		return (err);
1979fa9e4066Sahrens 	}
19802bcf0248SMax Grossman 
19812bcf0248SMax Grossman 	err = dnode_hold(os, object, FTAG, &dn);
19822bcf0248SMax Grossman 	if (err) {
19832bcf0248SMax Grossman 		return (err);
1984fa9e4066Sahrens 	}
1985fa9e4066Sahrens 
1986cdb0ab79Smaybee 	err = dnode_next_offset(dn, (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
1987fa9e4066Sahrens 	dnode_rele(dn, FTAG);
1988fa9e4066Sahrens 
1989fa9e4066Sahrens 	return (err);
1990fa9e4066Sahrens }
1991fa9e4066Sahrens 
19922bcf0248SMax Grossman /*
19932bcf0248SMax Grossman  * Given the ZFS object, if it contains any dirty nodes
19942bcf0248SMax Grossman  * this function flushes all dirty blocks to disk. This
19952bcf0248SMax Grossman  * ensures the DMU object info is updated. A more efficient
19962bcf0248SMax Grossman  * future version might just find the TXG with the maximum
19972bcf0248SMax Grossman  * ID and wait for that to be synced.
19982bcf0248SMax Grossman  */
19992bcf0248SMax Grossman int
20009a686fbcSPaul Dagnelie dmu_object_wait_synced(objset_t *os, uint64_t object)
20019a686fbcSPaul Dagnelie {
20022bcf0248SMax Grossman 	dnode_t *dn;
20032bcf0248SMax Grossman 	int error, i;
20042bcf0248SMax Grossman 
20052bcf0248SMax Grossman 	error = dnode_hold(os, object, FTAG, &dn);
20062bcf0248SMax Grossman 	if (error) {
20072bcf0248SMax Grossman 		return (error);
20082bcf0248SMax Grossman 	}
20092bcf0248SMax Grossman 
20102bcf0248SMax Grossman 	for (i = 0; i < TXG_SIZE; i++) {
20112bcf0248SMax Grossman 		if (list_link_active(&dn->dn_dirty_link[i])) {
20122bcf0248SMax Grossman 			break;
20132bcf0248SMax Grossman 		}
20142bcf0248SMax Grossman 	}
20152bcf0248SMax Grossman 	dnode_rele(dn, FTAG);
20162bcf0248SMax Grossman 	if (i != TXG_SIZE) {
20172bcf0248SMax Grossman 		txg_wait_synced(dmu_objset_pool(os), 0);
20182bcf0248SMax Grossman 	}
20192bcf0248SMax Grossman 
20202bcf0248SMax Grossman 	return (0);
20212bcf0248SMax Grossman }
20222bcf0248SMax Grossman 
2023fa9e4066Sahrens void
2024fa9e4066Sahrens dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2025fa9e4066Sahrens {
2026b24ab676SJeff Bonwick 	dnode_phys_t *dnp;
2027b24ab676SJeff Bonwick 
2028fa9e4066Sahrens 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
2029fa9e4066Sahrens 	mutex_enter(&dn->dn_mtx);
2030fa9e4066Sahrens 
2031b24ab676SJeff Bonwick 	dnp = dn->dn_phys;
2032b24ab676SJeff Bonwick 
2033fa9e4066Sahrens 	doi->doi_data_block_size = dn->dn_datablksz;
2034fa9e4066Sahrens 	doi->doi_metadata_block_size = dn->dn_indblkshift ?
2035fa9e4066Sahrens 	    1ULL << dn->dn_indblkshift : 0;
2036b24ab676SJeff Bonwick 	doi->doi_type = dn->dn_type;
2037b24ab676SJeff Bonwick 	doi->doi_bonus_type = dn->dn_bonustype;
2038b24ab676SJeff Bonwick 	doi->doi_bonus_size = dn->dn_bonuslen;
2039fa9e4066Sahrens 	doi->doi_indirection = dn->dn_nlevels;
2040fa9e4066Sahrens 	doi->doi_checksum = dn->dn_checksum;
2041fa9e4066Sahrens 	doi->doi_compress = dn->dn_compress;
2042e77d42eaSMatthew Ahrens 	doi->doi_nblkptr = dn->dn_nblkptr;
2043b24ab676SJeff Bonwick 	doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
2044d0475637SMatthew Ahrens 	doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
2045b24ab676SJeff Bonwick 	doi->doi_fill_count = 0;
2046b24ab676SJeff Bonwick 	for (int i = 0; i < dnp->dn_nblkptr; i++)
20475d7b4d43SMatthew Ahrens 		doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]);
2048fa9e4066Sahrens 
2049fa9e4066Sahrens 	mutex_exit(&dn->dn_mtx);
2050fa9e4066Sahrens 	rw_exit(&dn->dn_struct_rwlock);
2051fa9e4066Sahrens }
2052fa9e4066Sahrens 
2053fa9e4066Sahrens /*
2054fa9e4066Sahrens  * Get information on a DMU object.
2055fa9e4066Sahrens  * If doi is NULL, just indicates whether the object exists.
2056fa9e4066Sahrens  */
2057fa9e4066Sahrens int
2058fa9e4066Sahrens dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
2059fa9e4066Sahrens {
2060ea8dc4b6Seschrock 	dnode_t *dn;
2061503ad85cSMatthew Ahrens 	int err = dnode_hold(os, object, FTAG, &dn);
2062fa9e4066Sahrens 
2063ea8dc4b6Seschrock 	if (err)
2064ea8dc4b6Seschrock 		return (err);
2065fa9e4066Sahrens 
2066fa9e4066Sahrens 	if (doi != NULL)
2067fa9e4066Sahrens 		dmu_object_info_from_dnode(dn, doi);
2068fa9e4066Sahrens 
2069fa9e4066Sahrens 	dnode_rele(dn, FTAG);
2070fa9e4066Sahrens 	return (0);
2071fa9e4066Sahrens }
2072fa9e4066Sahrens 
2073fa9e4066Sahrens /*
2074fa9e4066Sahrens  * As above, but faster; can be used when you have a held dbuf in hand.
2075fa9e4066Sahrens  */
2076fa9e4066Sahrens void
2077744947dcSTom Erickson dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
2078fa9e4066Sahrens {
2079744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2080744947dcSTom Erickson 
2081744947dcSTom Erickson 	DB_DNODE_ENTER(db);
2082744947dcSTom Erickson 	dmu_object_info_from_dnode(DB_DNODE(db), doi);
2083744947dcSTom Erickson 	DB_DNODE_EXIT(db);
2084fa9e4066Sahrens }
2085fa9e4066Sahrens 
2086fa9e4066Sahrens /*
2087fa9e4066Sahrens  * Faster still when you only care about the size.
2088fa9e4066Sahrens  * This is specifically optimized for zfs_getattr().
2089fa9e4066Sahrens  */
2090fa9e4066Sahrens void
2091744947dcSTom Erickson dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
2092744947dcSTom Erickson     u_longlong_t *nblk512)
2093fa9e4066Sahrens {
2094744947dcSTom Erickson 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2095744947dcSTom Erickson 	dnode_t *dn;
2096744947dcSTom Erickson 
2097744947dcSTom Erickson 	DB_DNODE_ENTER(db);
2098744947dcSTom Erickson 	dn = DB_DNODE(db);
2099fa9e4066Sahrens 
2100fa9e4066Sahrens 	*blksize = dn->dn_datablksz;
210199653d4eSeschrock 	/* add 1 for dnode space */
210299653d4eSeschrock 	*nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
210399653d4eSeschrock 	    SPA_MINBLOCKSHIFT) + 1;
2104744947dcSTom Erickson 	DB_DNODE_EXIT(db);
2105fa9e4066Sahrens }
2106fa9e4066Sahrens 
2107fa9e4066Sahrens void
2108fa9e4066Sahrens byteswap_uint64_array(void *vbuf, size_t size)
2109fa9e4066Sahrens {
2110fa9e4066Sahrens 	uint64_t *buf = vbuf;
2111fa9e4066Sahrens 	size_t count = size >> 3;
2112fa9e4066Sahrens 	int i;
2113fa9e4066Sahrens 
2114fa9e4066Sahrens 	ASSERT((size & 7) == 0);
2115fa9e4066Sahrens 
2116fa9e4066Sahrens 	for (i = 0; i < count; i++)
2117fa9e4066Sahrens 		buf[i] = BSWAP_64(buf[i]);
2118fa9e4066Sahrens }
2119fa9e4066Sahrens 
2120fa9e4066Sahrens void
2121fa9e4066Sahrens byteswap_uint32_array(void *vbuf, size_t size)
2122fa9e4066Sahrens {
2123fa9e4066Sahrens 	uint32_t *buf = vbuf;
2124fa9e4066Sahrens 	size_t count = size >> 2;
2125fa9e4066Sahrens 	int i;
2126fa9e4066Sahrens 
2127fa9e4066Sahrens 	ASSERT((size & 3) == 0);
2128fa9e4066Sahrens 
2129fa9e4066Sahrens 	for (i = 0; i < count; i++)
2130fa9e4066Sahrens 		buf[i] = BSWAP_32(buf[i]);
2131fa9e4066Sahrens }
2132fa9e4066Sahrens 
2133fa9e4066Sahrens void
2134fa9e4066Sahrens byteswap_uint16_array(void *vbuf, size_t size)
2135fa9e4066Sahrens {
2136fa9e4066Sahrens 	uint16_t *buf = vbuf;
2137fa9e4066Sahrens 	size_t count = size >> 1;
2138fa9e4066Sahrens 	int i;
2139fa9e4066Sahrens 
2140fa9e4066Sahrens 	ASSERT((size & 1) == 0);
2141fa9e4066Sahrens 
2142fa9e4066Sahrens 	for (i = 0; i < count; i++)
2143fa9e4066Sahrens 		buf[i] = BSWAP_16(buf[i]);
2144fa9e4066Sahrens }
2145fa9e4066Sahrens 
2146fa9e4066Sahrens /* ARGSUSED */
2147fa9e4066Sahrens void
2148fa9e4066Sahrens byteswap_uint8_array(void *vbuf, size_t size)
2149fa9e4066Sahrens {
2150fa9e4066Sahrens }
2151fa9e4066Sahrens 
2152fa9e4066Sahrens void
2153fa9e4066Sahrens dmu_init(void)
2154fa9e4066Sahrens {
21553f9d6ad7SLin Ling 	zfs_dbgmsg_init();
2156744947dcSTom Erickson 	sa_cache_init();
2157744947dcSTom Erickson 	xuio_stat_init();
2158744947dcSTom Erickson 	dmu_objset_init();
2159fa9e4066Sahrens 	dnode_init();
21607cbf8b43SRich Morris 	zfetch_init();
2161fa94a07fSbrendan 	l2arc_init();
2162ce636f8bSMatthew Ahrens 	arc_init();
2163dcbf3bd6SGeorge Wilson 	dbuf_init();
2164fa9e4066Sahrens }
2165fa9e4066Sahrens 
2166fa9e4066Sahrens void
2167fa9e4066Sahrens dmu_fini(void)
2168fa9e4066Sahrens {
21693e30c24aSWill Andrews 	arc_fini(); /* arc depends on l2arc, so arc must go first */
2170ce636f8bSMatthew Ahrens 	l2arc_fini();
21717cbf8b43SRich Morris 	zfetch_fini();
2172fa9e4066Sahrens 	dbuf_fini();
2173744947dcSTom Erickson 	dnode_fini();
2174744947dcSTom Erickson 	dmu_objset_fini();
2175c242f9a0Schunli zhang - Sun Microsystems - Irvine United States 	xuio_stat_fini();
21760a586ceaSMark Shellenbaum 	sa_cache_fini();
21773f9d6ad7SLin Ling 	zfs_dbgmsg_fini();
2178fa9e4066Sahrens }
2179