xref: /illumos-gate/usr/src/uts/common/fs/zfs/sa.c (revision c4ab0d3f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Portions Copyright 2011 iXsystems, Inc
25  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  * Copyright (c) 2014 Integros [integros.com]
28  */
29 
30 #include <sys/zfs_context.h>
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/sysmacros.h>
35 #include <sys/dmu.h>
36 #include <sys/dmu_impl.h>
37 #include <sys/dmu_objset.h>
38 #include <sys/dmu_tx.h>
39 #include <sys/dbuf.h>
40 #include <sys/dnode.h>
41 #include <sys/zap.h>
42 #include <sys/sa.h>
43 #include <sys/sunddi.h>
44 #include <sys/sa_impl.h>
45 #include <sys/dnode.h>
46 #include <sys/errno.h>
47 #include <sys/zfs_context.h>
48 
49 /*
50  * ZFS System attributes:
51  *
52  * A generic mechanism to allow for arbitrary attributes
53  * to be stored in a dnode.  The data will be stored in the bonus buffer of
54  * the dnode and if necessary a special "spill" block will be used to handle
55  * overflow situations.  The spill block will be sized to fit the data
56  * from 512 - 128K.  When a spill block is used the BP (blkptr_t) for the
57  * spill block is stored at the end of the current bonus buffer.  Any
58  * attributes that would be in the way of the blkptr_t will be relocated
59  * into the spill block.
60  *
61  * Attribute registration:
62  *
63  * Stored persistently on a per dataset basis
64  * a mapping between attribute "string" names and their actual attribute
65  * numeric values, length, and byteswap function.  The names are only used
66  * during registration.  All  attributes are known by their unique attribute
67  * id value.  If an attribute can have a variable size then the value
68  * 0 will be used to indicate this.
69  *
70  * Attribute Layout:
71  *
72  * Attribute layouts are a way to compactly store multiple attributes, but
73  * without taking the overhead associated with managing each attribute
74  * individually.  Since you will typically have the same set of attributes
75  * stored in the same order a single table will be used to represent that
76  * layout.  The ZPL for example will usually have only about 10 different
77  * layouts (regular files, device files, symlinks,
78  * regular files + scanstamp, files/dir with extended attributes, and then
79  * you have the possibility of all of those minus ACL, because it would
80  * be kicked out into the spill block)
81  *
82  * Layouts are simply an array of the attributes and their
83  * ordering i.e. [0, 1, 4, 5, 2]
84  *
85  * Each distinct layout is given a unique layout number and that is whats
86  * stored in the header at the beginning of the SA data buffer.
87  *
88  * A layout only covers a single dbuf (bonus or spill).  If a set of
89  * attributes is split up between the bonus buffer and a spill buffer then
90  * two different layouts will be used.  This allows us to byteswap the
91  * spill without looking at the bonus buffer and keeps the on disk format of
92  * the bonus and spill buffer the same.
93  *
94  * Adding a single attribute will cause the entire set of attributes to
95  * be rewritten and could result in a new layout number being constructed
96  * as part of the rewrite if no such layout exists for the new set of
97  * attribues.  The new attribute will be appended to the end of the already
98  * existing attributes.
99  *
100  * Both the attribute registration and attribute layout information are
101  * stored in normal ZAP attributes.  Their should be a small number of
102  * known layouts and the set of attributes is assumed to typically be quite
103  * small.
104  *
105  * The registered attributes and layout "table" information is maintained
106  * in core and a special "sa_os_t" is attached to the objset_t.
107  *
108  * A special interface is provided to allow for quickly applying
109  * a large set of attributes at once.  sa_replace_all_by_template() is
110  * used to set an array of attributes.  This is used by the ZPL when
111  * creating a brand new file.  The template that is passed into the function
112  * specifies the attribute, size for variable length attributes, location of
113  * data and special "data locator" function if the data isn't in a contiguous
114  * location.
115  *
116  * Byteswap implications:
117  *
118  * Since the SA attributes are not entirely self describing we can't do
119  * the normal byteswap processing.  The special ZAP layout attribute and
120  * attribute registration attributes define the byteswap function and the
121  * size of the attributes, unless it is variable sized.
122  * The normal ZFS byteswapping infrastructure assumes you don't need
123  * to read any objects in order to do the necessary byteswapping.  Whereas
124  * SA attributes can only be properly byteswapped if the dataset is opened
125  * and the layout/attribute ZAP attributes are available.  Because of this
126  * the SA attributes will be byteswapped when they are first accessed by
127  * the SA code that will read the SA data.
128  */
129 
130 typedef void (sa_iterfunc_t)(void *hdr, void *addr, sa_attr_type_t,
131     uint16_t length, int length_idx, boolean_t, void *userp);
132 
133 static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
134 static void sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab);
135 static sa_idx_tab_t *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype,
136     sa_hdr_phys_t *hdr);
137 static void sa_idx_tab_rele(objset_t *os, void *arg);
138 static void sa_copy_data(sa_data_locator_t *func, void *start, void *target,
139     int buflen);
140 static int sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
141     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
142     uint16_t buflen, dmu_tx_t *tx);
143 
144 arc_byteswap_func_t *sa_bswap_table[] = {
145 	byteswap_uint64_array,
146 	byteswap_uint32_array,
147 	byteswap_uint16_array,
148 	byteswap_uint8_array,
149 	zfs_acl_byteswap,
150 };
151 
152 #define	SA_COPY_DATA(f, s, t, l) \
153 	{ \
154 		if (f == NULL) { \
155 			if (l == 8) { \
156 				*(uint64_t *)t = *(uint64_t *)s; \
157 			} else if (l == 16) { \
158 				*(uint64_t *)t = *(uint64_t *)s; \
159 				*(uint64_t *)((uintptr_t)t + 8) = \
160 				    *(uint64_t *)((uintptr_t)s + 8); \
161 			} else { \
162 				bcopy(s, t, l); \
163 			} \
164 		} else \
165 			sa_copy_data(f, s, t, l); \
166 	}
167 
168 /*
169  * This table is fixed and cannot be changed.  Its purpose is to
170  * allow the SA code to work with both old/new ZPL file systems.
171  * It contains the list of legacy attributes.  These attributes aren't
172  * stored in the "attribute" registry zap objects, since older ZPL file systems
173  * won't have the registry.  Only objsets of type ZFS_TYPE_FILESYSTEM will
174  * use this static table.
175  */
176 sa_attr_reg_t sa_legacy_attrs[] = {
177 	{"ZPL_ATIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 0},
178 	{"ZPL_MTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 1},
179 	{"ZPL_CTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 2},
180 	{"ZPL_CRTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 3},
181 	{"ZPL_GEN", sizeof (uint64_t), SA_UINT64_ARRAY, 4},
182 	{"ZPL_MODE", sizeof (uint64_t), SA_UINT64_ARRAY, 5},
183 	{"ZPL_SIZE", sizeof (uint64_t), SA_UINT64_ARRAY, 6},
184 	{"ZPL_PARENT", sizeof (uint64_t), SA_UINT64_ARRAY, 7},
185 	{"ZPL_LINKS", sizeof (uint64_t), SA_UINT64_ARRAY, 8},
186 	{"ZPL_XATTR", sizeof (uint64_t), SA_UINT64_ARRAY, 9},
187 	{"ZPL_RDEV", sizeof (uint64_t), SA_UINT64_ARRAY, 10},
188 	{"ZPL_FLAGS", sizeof (uint64_t), SA_UINT64_ARRAY, 11},
189 	{"ZPL_UID", sizeof (uint64_t), SA_UINT64_ARRAY, 12},
190 	{"ZPL_GID", sizeof (uint64_t), SA_UINT64_ARRAY, 13},
191 	{"ZPL_PAD", sizeof (uint64_t) * 4, SA_UINT64_ARRAY, 14},
192 	{"ZPL_ZNODE_ACL", 88, SA_UINT8_ARRAY, 15},
193 };
194 
195 /*
196  * This is only used for objects of type DMU_OT_ZNODE
197  */
198 sa_attr_type_t sa_legacy_zpl_layout[] = {
199     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
200 };
201 
202 /*
203  * Special dummy layout used for buffers with no attributes.
204  */
205 sa_attr_type_t sa_dummy_zpl_layout[] = { 0 };
206 
207 static int sa_legacy_attr_count = 16;
208 static kmem_cache_t *sa_cache = NULL;
209 
210 /*ARGSUSED*/
211 static int
212 sa_cache_constructor(void *buf, void *unused, int kmflag)
213 {
214 	sa_handle_t *hdl = buf;
215 
216 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);
217 	return (0);
218 }
219 
220 /*ARGSUSED*/
221 static void
222 sa_cache_destructor(void *buf, void *unused)
223 {
224 	sa_handle_t *hdl = buf;
225 	mutex_destroy(&hdl->sa_lock);
226 }
227 
228 void
229 sa_cache_init(void)
230 {
231 	sa_cache = kmem_cache_create("sa_cache",
232 	    sizeof (sa_handle_t), 0, sa_cache_constructor,
233 	    sa_cache_destructor, NULL, NULL, NULL, 0);
234 }
235 
236 void
237 sa_cache_fini(void)
238 {
239 	if (sa_cache)
240 		kmem_cache_destroy(sa_cache);
241 }
242 
243 static int
244 layout_num_compare(const void *arg1, const void *arg2)
245 {
246 	const sa_lot_t *node1 = (const sa_lot_t *)arg1;
247 	const sa_lot_t *node2 = (const sa_lot_t *)arg2;
248 
249 	return (AVL_CMP(node1->lot_num, node2->lot_num));
250 }
251 
252 static int
253 layout_hash_compare(const void *arg1, const void *arg2)
254 {
255 	const sa_lot_t *node1 = (const sa_lot_t *)arg1;
256 	const sa_lot_t *node2 = (const sa_lot_t *)arg2;
257 
258 	int cmp = AVL_CMP(node1->lot_hash, node2->lot_hash);
259 	if (likely(cmp))
260 		return (cmp);
261 
262 	return (AVL_CMP(node1->lot_instance, node2->lot_instance));
263 }
264 
265 boolean_t
266 sa_layout_equal(sa_lot_t *tbf, sa_attr_type_t *attrs, int count)
267 {
268 	int i;
269 
270 	if (count != tbf->lot_attr_count)
271 		return (1);
272 
273 	for (i = 0; i != count; i++) {
274 		if (attrs[i] != tbf->lot_attrs[i])
275 			return (1);
276 	}
277 	return (0);
278 }
279 
280 #define	SA_ATTR_HASH(attr) (zfs_crc64_table[(-1ULL ^ attr) & 0xFF])
281 
282 static uint64_t
283 sa_layout_info_hash(sa_attr_type_t *attrs, int attr_count)
284 {
285 	int i;
286 	uint64_t crc = -1ULL;
287 
288 	for (i = 0; i != attr_count; i++)
289 		crc ^= SA_ATTR_HASH(attrs[i]);
290 
291 	return (crc);
292 }
293 
294 static int
295 sa_get_spill(sa_handle_t *hdl)
296 {
297 	int rc;
298 	if (hdl->sa_spill == NULL) {
299 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,
300 		    &hdl->sa_spill)) == 0)
301 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
302 	} else {
303 		rc = 0;
304 	}
305 
306 	return (rc);
307 }
308 
309 /*
310  * Main attribute lookup/update function
311  * returns 0 for success or non zero for failures
312  *
313  * Operates on bulk array, first failure will abort further processing
314  */
315 int
316 sa_attr_op(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
317     sa_data_op_t data_op, dmu_tx_t *tx)
318 {
319 	sa_os_t *sa = hdl->sa_os->os_sa;
320 	int i;
321 	int error = 0;
322 	sa_buf_type_t buftypes;
323 
324 	buftypes = 0;
325 
326 	ASSERT(count > 0);
327 	for (i = 0; i != count; i++) {
328 		ASSERT(bulk[i].sa_attr <= hdl->sa_os->os_sa->sa_num_attrs);
329 
330 		bulk[i].sa_addr = NULL;
331 		/* First check the bonus buffer */
332 
333 		if (hdl->sa_bonus_tab && TOC_ATTR_PRESENT(
334 		    hdl->sa_bonus_tab->sa_idx_tab[bulk[i].sa_attr])) {
335 			SA_ATTR_INFO(sa, hdl->sa_bonus_tab,
336 			    SA_GET_HDR(hdl, SA_BONUS),
337 			    bulk[i].sa_attr, bulk[i], SA_BONUS, hdl);
338 			if (tx && !(buftypes & SA_BONUS)) {
339 				dmu_buf_will_dirty(hdl->sa_bonus, tx);
340 				buftypes |= SA_BONUS;
341 			}
342 		}
343 		if (bulk[i].sa_addr == NULL &&
344 		    ((error = sa_get_spill(hdl)) == 0)) {
345 			if (TOC_ATTR_PRESENT(
346 			    hdl->sa_spill_tab->sa_idx_tab[bulk[i].sa_attr])) {
347 				SA_ATTR_INFO(sa, hdl->sa_spill_tab,
348 				    SA_GET_HDR(hdl, SA_SPILL),
349 				    bulk[i].sa_attr, bulk[i], SA_SPILL, hdl);
350 				if (tx && !(buftypes & SA_SPILL) &&
351 				    bulk[i].sa_size == bulk[i].sa_length) {
352 					dmu_buf_will_dirty(hdl->sa_spill, tx);
353 					buftypes |= SA_SPILL;
354 				}
355 			}
356 		}
357 		if (error && error != ENOENT) {
358 			return ((error == ECKSUM) ? EIO : error);
359 		}
360 
361 		switch (data_op) {
362 		case SA_LOOKUP:
363 			if (bulk[i].sa_addr == NULL)
364 				return (SET_ERROR(ENOENT));
365 			if (bulk[i].sa_data) {
366 				SA_COPY_DATA(bulk[i].sa_data_func,
367 				    bulk[i].sa_addr, bulk[i].sa_data,
368 				    bulk[i].sa_size);
369 			}
370 			continue;
371 
372 		case SA_UPDATE:
373 			/* existing rewrite of attr */
374 			if (bulk[i].sa_addr &&
375 			    bulk[i].sa_size == bulk[i].sa_length) {
376 				SA_COPY_DATA(bulk[i].sa_data_func,
377 				    bulk[i].sa_data, bulk[i].sa_addr,
378 				    bulk[i].sa_length);
379 				continue;
380 			} else if (bulk[i].sa_addr) { /* attr size change */
381 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
382 				    SA_REPLACE, bulk[i].sa_data_func,
383 				    bulk[i].sa_data, bulk[i].sa_length, tx);
384 			} else { /* adding new attribute */
385 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
386 				    SA_ADD, bulk[i].sa_data_func,
387 				    bulk[i].sa_data, bulk[i].sa_length, tx);
388 			}
389 			if (error)
390 				return (error);
391 			break;
392 		}
393 	}
394 	return (error);
395 }
396 
397 static sa_lot_t *
398 sa_add_layout_entry(objset_t *os, sa_attr_type_t *attrs, int attr_count,
399     uint64_t lot_num, uint64_t hash, boolean_t zapadd, dmu_tx_t *tx)
400 {
401 	sa_os_t *sa = os->os_sa;
402 	sa_lot_t *tb, *findtb;
403 	int i;
404 	avl_index_t loc;
405 
406 	ASSERT(MUTEX_HELD(&sa->sa_lock));
407 	tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP);
408 	tb->lot_attr_count = attr_count;
409 	tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
410 	    KM_SLEEP);
411 	bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count);
412 	tb->lot_num = lot_num;
413 	tb->lot_hash = hash;
414 	tb->lot_instance = 0;
415 
416 	if (zapadd) {
417 		char attr_name[8];
418 
419 		if (sa->sa_layout_attr_obj == 0) {
420 			sa->sa_layout_attr_obj = zap_create_link(os,
421 			    DMU_OT_SA_ATTR_LAYOUTS,
422 			    sa->sa_master_obj, SA_LAYOUTS, tx);
423 		}
424 
425 		(void) snprintf(attr_name, sizeof (attr_name),
426 		    "%d", (int)lot_num);
427 		VERIFY(0 == zap_update(os, os->os_sa->sa_layout_attr_obj,
428 		    attr_name, 2, attr_count, attrs, tx));
429 	}
430 
431 	list_create(&tb->lot_idx_tab, sizeof (sa_idx_tab_t),
432 	    offsetof(sa_idx_tab_t, sa_next));
433 
434 	for (i = 0; i != attr_count; i++) {
435 		if (sa->sa_attr_table[tb->lot_attrs[i]].sa_length == 0)
436 			tb->lot_var_sizes++;
437 	}
438 
439 	avl_add(&sa->sa_layout_num_tree, tb);
440 
441 	/* verify we don't have a hash collision */
442 	if ((findtb = avl_find(&sa->sa_layout_hash_tree, tb, &loc)) != NULL) {
443 		for (; findtb && findtb->lot_hash == hash;
444 		    findtb = AVL_NEXT(&sa->sa_layout_hash_tree, findtb)) {
445 			if (findtb->lot_instance != tb->lot_instance)
446 				break;
447 			tb->lot_instance++;
448 		}
449 	}
450 	avl_add(&sa->sa_layout_hash_tree, tb);
451 	return (tb);
452 }
453 
454 static void
455 sa_find_layout(objset_t *os, uint64_t hash, sa_attr_type_t *attrs,
456     int count, dmu_tx_t *tx, sa_lot_t **lot)
457 {
458 	sa_lot_t *tb, tbsearch;
459 	avl_index_t loc;
460 	sa_os_t *sa = os->os_sa;
461 	boolean_t found = B_FALSE;
462 
463 	mutex_enter(&sa->sa_lock);
464 	tbsearch.lot_hash = hash;
465 	tbsearch.lot_instance = 0;
466 	tb = avl_find(&sa->sa_layout_hash_tree, &tbsearch, &loc);
467 	if (tb) {
468 		for (; tb && tb->lot_hash == hash;
469 		    tb = AVL_NEXT(&sa->sa_layout_hash_tree, tb)) {
470 			if (sa_layout_equal(tb, attrs, count) == 0) {
471 				found = B_TRUE;
472 				break;
473 			}
474 		}
475 	}
476 	if (!found) {
477 		tb = sa_add_layout_entry(os, attrs, count,
478 		    avl_numnodes(&sa->sa_layout_num_tree), hash, B_TRUE, tx);
479 	}
480 	mutex_exit(&sa->sa_lock);
481 	*lot = tb;
482 }
483 
484 static int
485 sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)
486 {
487 	int error;
488 	uint32_t blocksize;
489 
490 	if (size == 0) {
491 		blocksize = SPA_MINBLOCKSIZE;
492 	} else if (size > SPA_OLD_MAXBLOCKSIZE) {
493 		ASSERT(0);
494 		return (SET_ERROR(EFBIG));
495 	} else {
496 		blocksize = P2ROUNDUP_TYPED(size, SPA_MINBLOCKSIZE, uint32_t);
497 	}
498 
499 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);
500 	ASSERT(error == 0);
501 	return (error);
502 }
503 
504 static void
505 sa_copy_data(sa_data_locator_t *func, void *datastart, void *target, int buflen)
506 {
507 	if (func == NULL) {
508 		bcopy(datastart, target, buflen);
509 	} else {
510 		boolean_t start;
511 		int bytes;
512 		void *dataptr;
513 		void *saptr = target;
514 		uint32_t length;
515 
516 		start = B_TRUE;
517 		bytes = 0;
518 		while (bytes < buflen) {
519 			func(&dataptr, &length, buflen, start, datastart);
520 			bcopy(dataptr, saptr, length);
521 			saptr = (void *)((caddr_t)saptr + length);
522 			bytes += length;
523 			start = B_FALSE;
524 		}
525 	}
526 }
527 
528 /*
529  * Determine several different sizes
530  * first the sa header size
531  * the number of bytes to be stored
532  * if spill would occur the index in the attribute array is returned
533  *
534  * the boolean will_spill will be set when spilling is necessary.  It
535  * is only set when the buftype is SA_BONUS
536  */
537 static int
538 sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count,
539     dmu_buf_t *db, sa_buf_type_t buftype, int full_space, int *index,
540     int *total, boolean_t *will_spill)
541 {
542 	int var_size = 0;
543 	int i;
544 	int hdrsize;
545 	int extra_hdrsize;
546 
547 	if (buftype == SA_BONUS && sa->sa_force_spill) {
548 		*total = 0;
549 		*index = 0;
550 		*will_spill = B_TRUE;
551 		return (0);
552 	}
553 
554 	*index = -1;
555 	*total = 0;
556 	*will_spill = B_FALSE;
557 
558 	extra_hdrsize = 0;
559 	hdrsize = (SA_BONUSTYPE_FROM_DB(db) == DMU_OT_ZNODE) ? 0 :
560 	    sizeof (sa_hdr_phys_t);
561 
562 	ASSERT(IS_P2ALIGNED(full_space, 8));
563 
564 	for (i = 0; i != attr_count; i++) {
565 		boolean_t is_var_sz;
566 
567 		*total = P2ROUNDUP(*total, 8);
568 		*total += attr_desc[i].sa_length;
569 		if (*will_spill)
570 			continue;
571 
572 		is_var_sz = (SA_REGISTERED_LEN(sa, attr_desc[i].sa_attr) == 0);
573 		if (is_var_sz) {
574 			var_size++;
575 		}
576 
577 		if (is_var_sz && var_size > 1) {
578 			/*
579 			 * Don't worry that the spill block might overflow.
580 			 * It will be resized if needed in sa_build_layouts().
581 			 */
582 			if (buftype == SA_SPILL ||
583 			    P2ROUNDUP(hdrsize + sizeof (uint16_t), 8) +
584 			    *total < full_space) {
585 				/*
586 				 * Account for header space used by array of
587 				 * optional sizes of variable-length attributes.
588 				 * Record the extra header size in case this
589 				 * increase needs to be reversed due to
590 				 * spill-over.
591 				 */
592 				hdrsize += sizeof (uint16_t);
593 				if (*index != -1)
594 					extra_hdrsize += sizeof (uint16_t);
595 			} else {
596 				ASSERT(buftype == SA_BONUS);
597 				if (*index == -1)
598 					*index = i;
599 				*will_spill = B_TRUE;
600 				continue;
601 			}
602 		}
603 
604 		/*
605 		 * find index of where spill *could* occur.
606 		 * Then continue to count of remainder attribute
607 		 * space.  The sum is used later for sizing bonus
608 		 * and spill buffer.
609 		 */
610 		if (buftype == SA_BONUS && *index == -1 &&
611 		    *total + P2ROUNDUP(hdrsize, 8) >
612 		    (full_space - sizeof (blkptr_t))) {
613 			*index = i;
614 		}
615 
616 		if (*total + P2ROUNDUP(hdrsize, 8) > full_space &&
617 		    buftype == SA_BONUS)
618 			*will_spill = B_TRUE;
619 	}
620 
621 	if (*will_spill)
622 		hdrsize -= extra_hdrsize;
623 
624 	hdrsize = P2ROUNDUP(hdrsize, 8);
625 	return (hdrsize);
626 }
627 
628 #define	BUF_SPACE_NEEDED(total, header) (total + header)
629 
630 /*
631  * Find layout that corresponds to ordering of attributes
632  * If not found a new layout number is created and added to
633  * persistent layout tables.
634  */
635 static int
636 sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
637     dmu_tx_t *tx)
638 {
639 	sa_os_t *sa = hdl->sa_os->os_sa;
640 	uint64_t hash;
641 	sa_buf_type_t buftype;
642 	sa_hdr_phys_t *sahdr;
643 	void *data_start;
644 	int buf_space;
645 	sa_attr_type_t *attrs, *attrs_start;
646 	int i, lot_count;
647 	int dnodesize;
648 	int hdrsize;
649 	int spillhdrsize = 0;
650 	int used;
651 	dmu_object_type_t bonustype;
652 	sa_lot_t *lot;
653 	int len_idx;
654 	int spill_used;
655 	int bonuslen;
656 	boolean_t spilling;
657 
658 	dmu_buf_will_dirty(hdl->sa_bonus, tx);
659 	bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);
660 
661 	dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize);
662 	bonuslen = DN_BONUS_SIZE(dnodesize);
663 
664 	/* first determine bonus header size and sum of all attributes */
665 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,
666 	    SA_BONUS, bonuslen, &i, &used, &spilling);
667 
668 	if (used > SPA_OLD_MAXBLOCKSIZE)
669 		return (SET_ERROR(EFBIG));
670 
671 	VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ?
672 	    MIN(bonuslen - sizeof (blkptr_t), used + hdrsize) :
673 	    used + hdrsize, tx));
674 
675 	ASSERT((bonustype == DMU_OT_ZNODE && spilling == 0) ||
676 	    bonustype == DMU_OT_SA);
677 
678 	/* setup and size spill buffer when needed */
679 	if (spilling) {
680 		boolean_t dummy;
681 
682 		if (hdl->sa_spill == NULL) {
683 			VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, NULL,
684 			    &hdl->sa_spill) == 0);
685 		}
686 		dmu_buf_will_dirty(hdl->sa_spill, tx);
687 
688 		spillhdrsize = sa_find_sizes(sa, &attr_desc[i],
689 		    attr_count - i, hdl->sa_spill, SA_SPILL,
690 		    hdl->sa_spill->db_size, &i, &spill_used, &dummy);
691 
692 		if (spill_used > SPA_OLD_MAXBLOCKSIZE)
693 			return (SET_ERROR(EFBIG));
694 
695 		buf_space = hdl->sa_spill->db_size - spillhdrsize;
696 		if (BUF_SPACE_NEEDED(spill_used, spillhdrsize) >
697 		    hdl->sa_spill->db_size)
698 			VERIFY(0 == sa_resize_spill(hdl,
699 			    BUF_SPACE_NEEDED(spill_used, spillhdrsize), tx));
700 	}
701 
702 	/* setup starting pointers to lay down data */
703 	data_start = (void *)((uintptr_t)hdl->sa_bonus->db_data + hdrsize);
704 	sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;
705 	buftype = SA_BONUS;
706 
707 	if (spilling)
708 		buf_space = (sa->sa_force_spill) ?
709 		    0 : SA_BLKPTR_SPACE - hdrsize;
710 	else
711 		buf_space = hdl->sa_bonus->db_size - hdrsize;
712 
713 	attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
714 	    KM_SLEEP);
715 	lot_count = 0;
716 
717 	for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) {
718 		uint16_t length;
719 
720 		ASSERT(IS_P2ALIGNED(data_start, 8));
721 		ASSERT(IS_P2ALIGNED(buf_space, 8));
722 		attrs[i] = attr_desc[i].sa_attr;
723 		length = SA_REGISTERED_LEN(sa, attrs[i]);
724 		if (length == 0)
725 			length = attr_desc[i].sa_length;
726 
727 		if (buf_space < length) {  /* switch to spill buffer */
728 			VERIFY(spilling);
729 			VERIFY(bonustype == DMU_OT_SA);
730 			if (buftype == SA_BONUS && !sa->sa_force_spill) {
731 				sa_find_layout(hdl->sa_os, hash, attrs_start,
732 				    lot_count, tx, &lot);
733 				SA_SET_HDR(sahdr, lot->lot_num, hdrsize);
734 			}
735 
736 			buftype = SA_SPILL;
737 			hash = -1ULL;
738 			len_idx = 0;
739 
740 			sahdr = (sa_hdr_phys_t *)hdl->sa_spill->db_data;
741 			sahdr->sa_magic = SA_MAGIC;
742 			data_start = (void *)((uintptr_t)sahdr +
743 			    spillhdrsize);
744 			attrs_start = &attrs[i];
745 			buf_space = hdl->sa_spill->db_size - spillhdrsize;
746 			lot_count = 0;
747 		}
748 		hash ^= SA_ATTR_HASH(attrs[i]);
749 		attr_desc[i].sa_addr = data_start;
750 		attr_desc[i].sa_size = length;
751 		SA_COPY_DATA(attr_desc[i].sa_data_func, attr_desc[i].sa_data,
752 		    data_start, length);
753 		if (sa->sa_attr_table[attrs[i]].sa_length == 0) {
754 			sahdr->sa_lengths[len_idx++] = length;
755 		}
756 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
757 		    length), 8);
758 		buf_space -= P2ROUNDUP(length, 8);
759 		lot_count++;
760 	}
761 
762 	sa_find_layout(hdl->sa_os, hash, attrs_start, lot_count, tx, &lot);
763 
764 	/*
765 	 * Verify that old znodes always have layout number 0.
766 	 * Must be DMU_OT_SA for arbitrary layouts
767 	 */
768 	VERIFY((bonustype == DMU_OT_ZNODE && lot->lot_num == 0) ||
769 	    (bonustype == DMU_OT_SA && lot->lot_num > 1));
770 
771 	if (bonustype == DMU_OT_SA) {
772 		SA_SET_HDR(sahdr, lot->lot_num,
773 		    buftype == SA_BONUS ? hdrsize : spillhdrsize);
774 	}
775 
776 	kmem_free(attrs, sizeof (sa_attr_type_t) * attr_count);
777 	if (hdl->sa_bonus_tab) {
778 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
779 		hdl->sa_bonus_tab = NULL;
780 	}
781 	if (!sa->sa_force_spill)
782 		VERIFY(0 == sa_build_index(hdl, SA_BONUS));
783 	if (hdl->sa_spill) {
784 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
785 		if (!spilling) {
786 			/*
787 			 * remove spill block that is no longer needed.
788 			 */
789 			dmu_buf_rele(hdl->sa_spill, NULL);
790 			hdl->sa_spill = NULL;
791 			hdl->sa_spill_tab = NULL;
792 			VERIFY(0 == dmu_rm_spill(hdl->sa_os,
793 			    sa_handle_object(hdl), tx));
794 		} else {
795 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
796 		}
797 	}
798 
799 	return (0);
800 }
801 
802 static void
803 sa_free_attr_table(sa_os_t *sa)
804 {
805 	int i;
806 
807 	if (sa->sa_attr_table == NULL)
808 		return;
809 
810 	for (i = 0; i != sa->sa_num_attrs; i++) {
811 		if (sa->sa_attr_table[i].sa_name)
812 			kmem_free(sa->sa_attr_table[i].sa_name,
813 			    strlen(sa->sa_attr_table[i].sa_name) + 1);
814 	}
815 
816 	kmem_free(sa->sa_attr_table,
817 	    sizeof (sa_attr_table_t) * sa->sa_num_attrs);
818 
819 	sa->sa_attr_table = NULL;
820 }
821 
822 static int
823 sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count)
824 {
825 	sa_os_t *sa = os->os_sa;
826 	uint64_t sa_attr_count = 0;
827 	uint64_t sa_reg_count = 0;
828 	int error = 0;
829 	uint64_t attr_value;
830 	sa_attr_table_t *tb;
831 	zap_cursor_t zc;
832 	zap_attribute_t za;
833 	int registered_count = 0;
834 	int i;
835 	dmu_objset_type_t ostype = dmu_objset_type(os);
836 
837 	sa->sa_user_table =
838 	    kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP);
839 	sa->sa_user_table_sz = count * sizeof (sa_attr_type_t);
840 
841 	if (sa->sa_reg_attr_obj != 0) {
842 		error = zap_count(os, sa->sa_reg_attr_obj,
843 		    &sa_attr_count);
844 
845 		/*
846 		 * Make sure we retrieved a count and that it isn't zero
847 		 */
848 		if (error || (error == 0 && sa_attr_count == 0)) {
849 			if (error == 0)
850 				error = SET_ERROR(EINVAL);
851 			goto bail;
852 		}
853 		sa_reg_count = sa_attr_count;
854 	}
855 
856 	if (ostype == DMU_OST_ZFS && sa_attr_count == 0)
857 		sa_attr_count += sa_legacy_attr_count;
858 
859 	/* Allocate attribute numbers for attributes that aren't registered */
860 	for (i = 0; i != count; i++) {
861 		boolean_t found = B_FALSE;
862 		int j;
863 
864 		if (ostype == DMU_OST_ZFS) {
865 			for (j = 0; j != sa_legacy_attr_count; j++) {
866 				if (strcmp(reg_attrs[i].sa_name,
867 				    sa_legacy_attrs[j].sa_name) == 0) {
868 					sa->sa_user_table[i] =
869 					    sa_legacy_attrs[j].sa_attr;
870 					found = B_TRUE;
871 				}
872 			}
873 		}
874 		if (found)
875 			continue;
876 
877 		if (sa->sa_reg_attr_obj)
878 			error = zap_lookup(os, sa->sa_reg_attr_obj,
879 			    reg_attrs[i].sa_name, 8, 1, &attr_value);
880 		else
881 			error = SET_ERROR(ENOENT);
882 		switch (error) {
883 		case ENOENT:
884 			sa->sa_user_table[i] = (sa_attr_type_t)sa_attr_count;
885 			sa_attr_count++;
886 			break;
887 		case 0:
888 			sa->sa_user_table[i] = ATTR_NUM(attr_value);
889 			break;
890 		default:
891 			goto bail;
892 		}
893 	}
894 
895 	sa->sa_num_attrs = sa_attr_count;
896 	tb = sa->sa_attr_table =
897 	    kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP);
898 
899 	/*
900 	 * Attribute table is constructed from requested attribute list,
901 	 * previously foreign registered attributes, and also the legacy
902 	 * ZPL set of attributes.
903 	 */
904 
905 	if (sa->sa_reg_attr_obj) {
906 		for (zap_cursor_init(&zc, os, sa->sa_reg_attr_obj);
907 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
908 		    zap_cursor_advance(&zc)) {
909 			uint64_t value;
910 			value  = za.za_first_integer;
911 
912 			registered_count++;
913 			tb[ATTR_NUM(value)].sa_attr = ATTR_NUM(value);
914 			tb[ATTR_NUM(value)].sa_length = ATTR_LENGTH(value);
915 			tb[ATTR_NUM(value)].sa_byteswap = ATTR_BSWAP(value);
916 			tb[ATTR_NUM(value)].sa_registered = B_TRUE;
917 
918 			if (tb[ATTR_NUM(value)].sa_name) {
919 				continue;
920 			}
921 			tb[ATTR_NUM(value)].sa_name =
922 			    kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP);
923 			(void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name,
924 			    strlen(za.za_name) +1);
925 		}
926 		zap_cursor_fini(&zc);
927 		/*
928 		 * Make sure we processed the correct number of registered
929 		 * attributes
930 		 */
931 		if (registered_count != sa_reg_count) {
932 			ASSERT(error != 0);
933 			goto bail;
934 		}
935 
936 	}
937 
938 	if (ostype == DMU_OST_ZFS) {
939 		for (i = 0; i != sa_legacy_attr_count; i++) {
940 			if (tb[i].sa_name)
941 				continue;
942 			tb[i].sa_attr = sa_legacy_attrs[i].sa_attr;
943 			tb[i].sa_length = sa_legacy_attrs[i].sa_length;
944 			tb[i].sa_byteswap = sa_legacy_attrs[i].sa_byteswap;
945 			tb[i].sa_registered = B_FALSE;
946 			tb[i].sa_name =
947 			    kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1,
948 			    KM_SLEEP);
949 			(void) strlcpy(tb[i].sa_name,
950 			    sa_legacy_attrs[i].sa_name,
951 			    strlen(sa_legacy_attrs[i].sa_name) + 1);
952 		}
953 	}
954 
955 	for (i = 0; i != count; i++) {
956 		sa_attr_type_t attr_id;
957 
958 		attr_id = sa->sa_user_table[i];
959 		if (tb[attr_id].sa_name)
960 			continue;
961 
962 		tb[attr_id].sa_length = reg_attrs[i].sa_length;
963 		tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap;
964 		tb[attr_id].sa_attr = attr_id;
965 		tb[attr_id].sa_name =
966 		    kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP);
967 		(void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name,
968 		    strlen(reg_attrs[i].sa_name) + 1);
969 	}
970 
971 	sa->sa_need_attr_registration =
972 	    (sa_attr_count != registered_count);
973 
974 	return (0);
975 bail:
976 	kmem_free(sa->sa_user_table, count * sizeof (sa_attr_type_t));
977 	sa->sa_user_table = NULL;
978 	sa_free_attr_table(sa);
979 	return ((error != 0) ? error : EINVAL);
980 }
981 
982 int
983 sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count,
984     sa_attr_type_t **user_table)
985 {
986 	zap_cursor_t zc;
987 	zap_attribute_t za;
988 	sa_os_t *sa;
989 	dmu_objset_type_t ostype = dmu_objset_type(os);
990 	sa_attr_type_t *tb;
991 	int error;
992 
993 	mutex_enter(&os->os_user_ptr_lock);
994 	if (os->os_sa) {
995 		mutex_enter(&os->os_sa->sa_lock);
996 		mutex_exit(&os->os_user_ptr_lock);
997 		tb = os->os_sa->sa_user_table;
998 		mutex_exit(&os->os_sa->sa_lock);
999 		*user_table = tb;
1000 		return (0);
1001 	}
1002 
1003 	sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP);
1004 	mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL);
1005 	sa->sa_master_obj = sa_obj;
1006 
1007 	os->os_sa = sa;
1008 	mutex_enter(&sa->sa_lock);
1009 	mutex_exit(&os->os_user_ptr_lock);
1010 	avl_create(&sa->sa_layout_num_tree, layout_num_compare,
1011 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_num_node));
1012 	avl_create(&sa->sa_layout_hash_tree, layout_hash_compare,
1013 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_hash_node));
1014 
1015 	if (sa_obj) {
1016 		error = zap_lookup(os, sa_obj, SA_LAYOUTS,
1017 		    8, 1, &sa->sa_layout_attr_obj);
1018 		if (error != 0 && error != ENOENT)
1019 			goto fail;
1020 		error = zap_lookup(os, sa_obj, SA_REGISTRY,
1021 		    8, 1, &sa->sa_reg_attr_obj);
1022 		if (error != 0 && error != ENOENT)
1023 			goto fail;
1024 	}
1025 
1026 	if ((error = sa_attr_table_setup(os, reg_attrs, count)) != 0)
1027 		goto fail;
1028 
1029 	if (sa->sa_layout_attr_obj != 0) {
1030 		uint64_t layout_count;
1031 
1032 		error = zap_count(os, sa->sa_layout_attr_obj,
1033 		    &layout_count);
1034 
1035 		/*
1036 		 * Layout number count should be > 0
1037 		 */
1038 		if (error || (error == 0 && layout_count == 0)) {
1039 			if (error == 0)
1040 				error = SET_ERROR(EINVAL);
1041 			goto fail;
1042 		}
1043 
1044 		for (zap_cursor_init(&zc, os, sa->sa_layout_attr_obj);
1045 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
1046 		    zap_cursor_advance(&zc)) {
1047 			sa_attr_type_t *lot_attrs;
1048 			uint64_t lot_num;
1049 
1050 			lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) *
1051 			    za.za_num_integers, KM_SLEEP);
1052 
1053 			if ((error = (zap_lookup(os, sa->sa_layout_attr_obj,
1054 			    za.za_name, 2, za.za_num_integers,
1055 			    lot_attrs))) != 0) {
1056 				kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1057 				    za.za_num_integers);
1058 				break;
1059 			}
1060 			VERIFY(ddi_strtoull(za.za_name, NULL, 10,
1061 			    (unsigned long long *)&lot_num) == 0);
1062 
1063 			(void) sa_add_layout_entry(os, lot_attrs,
1064 			    za.za_num_integers, lot_num,
1065 			    sa_layout_info_hash(lot_attrs,
1066 			    za.za_num_integers), B_FALSE, NULL);
1067 			kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1068 			    za.za_num_integers);
1069 		}
1070 		zap_cursor_fini(&zc);
1071 
1072 		/*
1073 		 * Make sure layout count matches number of entries added
1074 		 * to AVL tree
1075 		 */
1076 		if (avl_numnodes(&sa->sa_layout_num_tree) != layout_count) {
1077 			ASSERT(error != 0);
1078 			goto fail;
1079 		}
1080 	}
1081 
1082 	/* Add special layout number for old ZNODES */
1083 	if (ostype == DMU_OST_ZFS) {
1084 		(void) sa_add_layout_entry(os, sa_legacy_zpl_layout,
1085 		    sa_legacy_attr_count, 0,
1086 		    sa_layout_info_hash(sa_legacy_zpl_layout,
1087 		    sa_legacy_attr_count), B_FALSE, NULL);
1088 
1089 		(void) sa_add_layout_entry(os, sa_dummy_zpl_layout, 0, 1,
1090 		    0, B_FALSE, NULL);
1091 	}
1092 	*user_table = os->os_sa->sa_user_table;
1093 	mutex_exit(&sa->sa_lock);
1094 	return (0);
1095 fail:
1096 	os->os_sa = NULL;
1097 	sa_free_attr_table(sa);
1098 	if (sa->sa_user_table)
1099 		kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1100 	mutex_exit(&sa->sa_lock);
1101 	avl_destroy(&sa->sa_layout_hash_tree);
1102 	avl_destroy(&sa->sa_layout_num_tree);
1103 	mutex_destroy(&sa->sa_lock);
1104 	kmem_free(sa, sizeof (sa_os_t));
1105 	return ((error == ECKSUM) ? EIO : error);
1106 }
1107 
1108 void
1109 sa_tear_down(objset_t *os)
1110 {
1111 	sa_os_t *sa = os->os_sa;
1112 	sa_lot_t *layout;
1113 	void *cookie;
1114 
1115 	kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1116 
1117 	/* Free up attr table */
1118 
1119 	sa_free_attr_table(sa);
1120 
1121 	cookie = NULL;
1122 	while (layout = avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie)) {
1123 		sa_idx_tab_t *tab;
1124 		while (tab = list_head(&layout->lot_idx_tab)) {
1125 			ASSERT(zfs_refcount_count(&tab->sa_refcount));
1126 			sa_idx_tab_rele(os, tab);
1127 		}
1128 	}
1129 
1130 	cookie = NULL;
1131 	while (layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie)) {
1132 		kmem_free(layout->lot_attrs,
1133 		    sizeof (sa_attr_type_t) * layout->lot_attr_count);
1134 		kmem_free(layout, sizeof (sa_lot_t));
1135 	}
1136 
1137 	avl_destroy(&sa->sa_layout_hash_tree);
1138 	avl_destroy(&sa->sa_layout_num_tree);
1139 	mutex_destroy(&sa->sa_lock);
1140 
1141 	kmem_free(sa, sizeof (sa_os_t));
1142 	os->os_sa = NULL;
1143 }
1144 
1145 void
1146 sa_build_idx_tab(void *hdr, void *attr_addr, sa_attr_type_t attr,
1147     uint16_t length, int length_idx, boolean_t var_length, void *userp)
1148 {
1149 	sa_idx_tab_t *idx_tab = userp;
1150 
1151 	if (var_length) {
1152 		ASSERT(idx_tab->sa_variable_lengths);
1153 		idx_tab->sa_variable_lengths[length_idx] = length;
1154 	}
1155 	TOC_ATTR_ENCODE(idx_tab->sa_idx_tab[attr], length_idx,
1156 	    (uint32_t)((uintptr_t)attr_addr - (uintptr_t)hdr));
1157 }
1158 
1159 static void
1160 sa_attr_iter(objset_t *os, sa_hdr_phys_t *hdr, dmu_object_type_t type,
1161     sa_iterfunc_t func, sa_lot_t *tab, void *userp)
1162 {
1163 	void *data_start;
1164 	sa_lot_t *tb = tab;
1165 	sa_lot_t search;
1166 	avl_index_t loc;
1167 	sa_os_t *sa = os->os_sa;
1168 	int i;
1169 	uint16_t *length_start = NULL;
1170 	uint8_t length_idx = 0;
1171 
1172 	if (tab == NULL) {
1173 		search.lot_num = SA_LAYOUT_NUM(hdr, type);
1174 		tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1175 		ASSERT(tb);
1176 	}
1177 
1178 	if (IS_SA_BONUSTYPE(type)) {
1179 		data_start = (void *)P2ROUNDUP(((uintptr_t)hdr +
1180 		    offsetof(sa_hdr_phys_t, sa_lengths) +
1181 		    (sizeof (uint16_t) * tb->lot_var_sizes)), 8);
1182 		length_start = hdr->sa_lengths;
1183 	} else {
1184 		data_start = hdr;
1185 	}
1186 
1187 	for (i = 0; i != tb->lot_attr_count; i++) {
1188 		int attr_length, reg_length;
1189 		uint8_t idx_len;
1190 
1191 		reg_length = sa->sa_attr_table[tb->lot_attrs[i]].sa_length;
1192 		if (reg_length) {
1193 			attr_length = reg_length;
1194 			idx_len = 0;
1195 		} else {
1196 			attr_length = length_start[length_idx];
1197 			idx_len = length_idx++;
1198 		}
1199 
1200 		func(hdr, data_start, tb->lot_attrs[i], attr_length,
1201 		    idx_len, reg_length == 0 ? B_TRUE : B_FALSE, userp);
1202 
1203 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
1204 		    attr_length), 8);
1205 	}
1206 }
1207 
1208 /*ARGSUSED*/
1209 void
1210 sa_byteswap_cb(void *hdr, void *attr_addr, sa_attr_type_t attr,
1211     uint16_t length, int length_idx, boolean_t variable_length, void *userp)
1212 {
1213 	sa_handle_t *hdl = userp;
1214 	sa_os_t *sa = hdl->sa_os->os_sa;
1215 
1216 	sa_bswap_table[sa->sa_attr_table[attr].sa_byteswap](attr_addr, length);
1217 }
1218 
1219 void
1220 sa_byteswap(sa_handle_t *hdl, sa_buf_type_t buftype)
1221 {
1222 	sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1223 	dmu_buf_impl_t *db;
1224 	sa_os_t *sa = hdl->sa_os->os_sa;
1225 	int num_lengths = 1;
1226 	int i;
1227 
1228 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1229 	if (sa_hdr_phys->sa_magic == SA_MAGIC)
1230 		return;
1231 
1232 	db = SA_GET_DB(hdl, buftype);
1233 
1234 	if (buftype == SA_SPILL) {
1235 		arc_release(db->db_buf, NULL);
1236 		arc_buf_thaw(db->db_buf);
1237 	}
1238 
1239 	sa_hdr_phys->sa_magic = BSWAP_32(sa_hdr_phys->sa_magic);
1240 	sa_hdr_phys->sa_layout_info = BSWAP_16(sa_hdr_phys->sa_layout_info);
1241 
1242 	/*
1243 	 * Determine number of variable lenghts in header
1244 	 * The standard 8 byte header has one for free and a
1245 	 * 16 byte header would have 4 + 1;
1246 	 */
1247 	if (SA_HDR_SIZE(sa_hdr_phys) > 8)
1248 		num_lengths += (SA_HDR_SIZE(sa_hdr_phys) - 8) >> 1;
1249 	for (i = 0; i != num_lengths; i++)
1250 		sa_hdr_phys->sa_lengths[i] =
1251 		    BSWAP_16(sa_hdr_phys->sa_lengths[i]);
1252 
1253 	sa_attr_iter(hdl->sa_os, sa_hdr_phys, DMU_OT_SA,
1254 	    sa_byteswap_cb, NULL, hdl);
1255 
1256 	if (buftype == SA_SPILL)
1257 		arc_buf_freeze(((dmu_buf_impl_t *)hdl->sa_spill)->db_buf);
1258 }
1259 
1260 static int
1261 sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)
1262 {
1263 	sa_hdr_phys_t *sa_hdr_phys;
1264 	dmu_buf_impl_t *db = SA_GET_DB(hdl, buftype);
1265 	dmu_object_type_t bonustype = SA_BONUSTYPE_FROM_DB(db);
1266 	sa_os_t *sa = hdl->sa_os->os_sa;
1267 	sa_idx_tab_t *idx_tab;
1268 
1269 	sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1270 
1271 	mutex_enter(&sa->sa_lock);
1272 
1273 	/* Do we need to byteswap? */
1274 
1275 	/* only check if not old znode */
1276 	if (IS_SA_BONUSTYPE(bonustype) && sa_hdr_phys->sa_magic != SA_MAGIC &&
1277 	    sa_hdr_phys->sa_magic != 0) {
1278 		VERIFY(BSWAP_32(sa_hdr_phys->sa_magic) == SA_MAGIC);
1279 		sa_byteswap(hdl, buftype);
1280 	}
1281 
1282 	idx_tab = sa_find_idx_tab(hdl->sa_os, bonustype, sa_hdr_phys);
1283 
1284 	if (buftype == SA_BONUS)
1285 		hdl->sa_bonus_tab = idx_tab;
1286 	else
1287 		hdl->sa_spill_tab = idx_tab;
1288 
1289 	mutex_exit(&sa->sa_lock);
1290 	return (0);
1291 }
1292 
1293 /*ARGSUSED*/
1294 static void
1295 sa_evict_sync(void *dbu)
1296 {
1297 	panic("evicting sa dbuf\n");
1298 }
1299 
1300 static void
1301 sa_idx_tab_rele(objset_t *os, void *arg)
1302 {
1303 	sa_os_t *sa = os->os_sa;
1304 	sa_idx_tab_t *idx_tab = arg;
1305 
1306 	if (idx_tab == NULL)
1307 		return;
1308 
1309 	mutex_enter(&sa->sa_lock);
1310 	if (zfs_refcount_remove(&idx_tab->sa_refcount, NULL) == 0) {
1311 		list_remove(&idx_tab->sa_layout->lot_idx_tab, idx_tab);
1312 		if (idx_tab->sa_variable_lengths)
1313 			kmem_free(idx_tab->sa_variable_lengths,
1314 			    sizeof (uint16_t) *
1315 			    idx_tab->sa_layout->lot_var_sizes);
1316 		zfs_refcount_destroy(&idx_tab->sa_refcount);
1317 		kmem_free(idx_tab->sa_idx_tab,
1318 		    sizeof (uint32_t) * sa->sa_num_attrs);
1319 		kmem_free(idx_tab, sizeof (sa_idx_tab_t));
1320 	}
1321 	mutex_exit(&sa->sa_lock);
1322 }
1323 
1324 static void
1325 sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
1326 {
1327 	sa_os_t *sa = os->os_sa;
1328 
1329 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1330 	(void) zfs_refcount_add(&idx_tab->sa_refcount, NULL);
1331 }
1332 
1333 void
1334 sa_handle_destroy(sa_handle_t *hdl)
1335 {
1336 	dmu_buf_t *db = hdl->sa_bonus;
1337 
1338 	mutex_enter(&hdl->sa_lock);
1339 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
1340 
1341 	if (hdl->sa_bonus_tab)
1342 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
1343 
1344 	if (hdl->sa_spill_tab)
1345 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
1346 
1347 	dmu_buf_rele(hdl->sa_bonus, NULL);
1348 
1349 	if (hdl->sa_spill)
1350 		dmu_buf_rele((dmu_buf_t *)hdl->sa_spill, NULL);
1351 	mutex_exit(&hdl->sa_lock);
1352 
1353 	kmem_cache_free(sa_cache, hdl);
1354 }
1355 
1356 int
1357 sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
1358     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1359 {
1360 	int error = 0;
1361 	dmu_object_info_t doi;
1362 	sa_handle_t *handle = NULL;
1363 
1364 #ifdef ZFS_DEBUG
1365 	dmu_object_info_from_db(db, &doi);
1366 	ASSERT(doi.doi_bonus_type == DMU_OT_SA ||
1367 	    doi.doi_bonus_type == DMU_OT_ZNODE);
1368 #endif
1369 	/* find handle, if it exists */
1370 	/* if one doesn't exist then create a new one, and initialize it */
1371 
1372 	if (hdl_type == SA_HDL_SHARED)
1373 		handle = dmu_buf_get_user(db);
1374 
1375 	if (handle == NULL) {
1376 		sa_handle_t *winner = NULL;
1377 
1378 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
1379 		handle->sa_dbu.dbu_evict_func_sync = NULL;
1380 		handle->sa_dbu.dbu_evict_func_async = NULL;
1381 		handle->sa_userp = userp;
1382 		handle->sa_bonus = db;
1383 		handle->sa_os = os;
1384 		handle->sa_spill = NULL;
1385 		handle->sa_bonus_tab = NULL;
1386 		handle->sa_spill_tab = NULL;
1387 
1388 		error = sa_build_index(handle, SA_BONUS);
1389 
1390 		if (hdl_type == SA_HDL_SHARED) {
1391 			dmu_buf_init_user(&handle->sa_dbu, sa_evict_sync, NULL,
1392 			    NULL);
1393 			winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
1394 		}
1395 
1396 		if (winner != NULL) {
1397 			kmem_cache_free(sa_cache, handle);
1398 			handle = winner;
1399 		}
1400 	}
1401 	*handlepp = handle;
1402 
1403 	return (error);
1404 }
1405 
1406 int
1407 sa_handle_get(objset_t *objset, uint64_t objid, void *userp,
1408     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1409 {
1410 	dmu_buf_t *db;
1411 	int error;
1412 
1413 	if (error = dmu_bonus_hold(objset, objid, NULL, &db))
1414 		return (error);
1415 
1416 	return (sa_handle_get_from_db(objset, db, userp, hdl_type,
1417 	    handlepp));
1418 }
1419 
1420 int
1421 sa_buf_hold(objset_t *objset, uint64_t obj_num, void *tag, dmu_buf_t **db)
1422 {
1423 	return (dmu_bonus_hold(objset, obj_num, tag, db));
1424 }
1425 
1426 void
1427 sa_buf_rele(dmu_buf_t *db, void *tag)
1428 {
1429 	dmu_buf_rele(db, tag);
1430 }
1431 
1432 int
1433 sa_lookup_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count)
1434 {
1435 	ASSERT(hdl);
1436 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1437 	return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));
1438 }
1439 
1440 int
1441 sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)
1442 {
1443 	int error;
1444 	sa_bulk_attr_t bulk;
1445 
1446 	bulk.sa_attr = attr;
1447 	bulk.sa_data = buf;
1448 	bulk.sa_length = buflen;
1449 	bulk.sa_data_func = NULL;
1450 
1451 	ASSERT(hdl);
1452 	mutex_enter(&hdl->sa_lock);
1453 	error = sa_lookup_impl(hdl, &bulk, 1);
1454 	mutex_exit(&hdl->sa_lock);
1455 	return (error);
1456 }
1457 
1458 #ifdef _KERNEL
1459 int
1460 sa_lookup_uio(sa_handle_t *hdl, sa_attr_type_t attr, uio_t *uio)
1461 {
1462 	int error;
1463 	sa_bulk_attr_t bulk;
1464 
1465 	bulk.sa_data = NULL;
1466 	bulk.sa_attr = attr;
1467 	bulk.sa_data_func = NULL;
1468 
1469 	ASSERT(hdl);
1470 
1471 	mutex_enter(&hdl->sa_lock);
1472 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) == 0) {
1473 		error = uiomove((void *)bulk.sa_addr, MIN(bulk.sa_size,
1474 		    uio->uio_resid), UIO_READ, uio);
1475 	}
1476 	mutex_exit(&hdl->sa_lock);
1477 	return (error);
1478 
1479 }
1480 #endif
1481 
1482 static sa_idx_tab_t *
1483 sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, sa_hdr_phys_t *hdr)
1484 {
1485 	sa_idx_tab_t *idx_tab;
1486 	sa_os_t *sa = os->os_sa;
1487 	sa_lot_t *tb, search;
1488 	avl_index_t loc;
1489 
1490 	/*
1491 	 * Deterimine layout number.  If SA node and header == 0 then
1492 	 * force the index table to the dummy "1" empty layout.
1493 	 *
1494 	 * The layout number would only be zero for a newly created file
1495 	 * that has not added any attributes yet, or with crypto enabled which
1496 	 * doesn't write any attributes to the bonus buffer.
1497 	 */
1498 
1499 	search.lot_num = SA_LAYOUT_NUM(hdr, bonustype);
1500 
1501 	tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1502 
1503 	/* Verify header size is consistent with layout information */
1504 	ASSERT(tb);
1505 	ASSERT(IS_SA_BONUSTYPE(bonustype) &&
1506 	    SA_HDR_SIZE_MATCH_LAYOUT(hdr, tb) || !IS_SA_BONUSTYPE(bonustype) ||
1507 	    (IS_SA_BONUSTYPE(bonustype) && hdr->sa_layout_info == 0));
1508 
1509 	/*
1510 	 * See if any of the already existing TOC entries can be reused?
1511 	 */
1512 
1513 	for (idx_tab = list_head(&tb->lot_idx_tab); idx_tab;
1514 	    idx_tab = list_next(&tb->lot_idx_tab, idx_tab)) {
1515 		boolean_t valid_idx = B_TRUE;
1516 		int i;
1517 
1518 		if (tb->lot_var_sizes != 0 &&
1519 		    idx_tab->sa_variable_lengths != NULL) {
1520 			for (i = 0; i != tb->lot_var_sizes; i++) {
1521 				if (hdr->sa_lengths[i] !=
1522 				    idx_tab->sa_variable_lengths[i]) {
1523 					valid_idx = B_FALSE;
1524 					break;
1525 				}
1526 			}
1527 		}
1528 		if (valid_idx) {
1529 			sa_idx_tab_hold(os, idx_tab);
1530 			return (idx_tab);
1531 		}
1532 	}
1533 
1534 	/* No such luck, create a new entry */
1535 	idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP);
1536 	idx_tab->sa_idx_tab =
1537 	    kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP);
1538 	idx_tab->sa_layout = tb;
1539 	zfs_refcount_create(&idx_tab->sa_refcount);
1540 	if (tb->lot_var_sizes)
1541 		idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) *
1542 		    tb->lot_var_sizes, KM_SLEEP);
1543 
1544 	sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab,
1545 	    tb, idx_tab);
1546 	sa_idx_tab_hold(os, idx_tab);   /* one hold for consumer */
1547 	sa_idx_tab_hold(os, idx_tab);	/* one for layout */
1548 	list_insert_tail(&tb->lot_idx_tab, idx_tab);
1549 	return (idx_tab);
1550 }
1551 
1552 void
1553 sa_default_locator(void **dataptr, uint32_t *len, uint32_t total_len,
1554     boolean_t start, void *userdata)
1555 {
1556 	ASSERT(start);
1557 
1558 	*dataptr = userdata;
1559 	*len = total_len;
1560 }
1561 
1562 static void
1563 sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)
1564 {
1565 	uint64_t attr_value = 0;
1566 	sa_os_t *sa = hdl->sa_os->os_sa;
1567 	sa_attr_table_t *tb = sa->sa_attr_table;
1568 	int i;
1569 
1570 	mutex_enter(&sa->sa_lock);
1571 
1572 	if (!sa->sa_need_attr_registration || sa->sa_master_obj == 0) {
1573 		mutex_exit(&sa->sa_lock);
1574 		return;
1575 	}
1576 
1577 	if (sa->sa_reg_attr_obj == 0) {
1578 		sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,
1579 		    DMU_OT_SA_ATTR_REGISTRATION,
1580 		    sa->sa_master_obj, SA_REGISTRY, tx);
1581 	}
1582 	for (i = 0; i != sa->sa_num_attrs; i++) {
1583 		if (sa->sa_attr_table[i].sa_registered)
1584 			continue;
1585 		ATTR_ENCODE(attr_value, tb[i].sa_attr, tb[i].sa_length,
1586 		    tb[i].sa_byteswap);
1587 		VERIFY(0 == zap_update(hdl->sa_os, sa->sa_reg_attr_obj,
1588 		    tb[i].sa_name, 8, 1, &attr_value, tx));
1589 		tb[i].sa_registered = B_TRUE;
1590 	}
1591 	sa->sa_need_attr_registration = B_FALSE;
1592 	mutex_exit(&sa->sa_lock);
1593 }
1594 
1595 /*
1596  * Replace all attributes with attributes specified in template.
1597  * If dnode had a spill buffer then those attributes will be
1598  * also be replaced, possibly with just an empty spill block
1599  *
1600  * This interface is intended to only be used for bulk adding of
1601  * attributes for a new file.  It will also be used by the ZPL
1602  * when converting and old formatted znode to native SA support.
1603  */
1604 int
1605 sa_replace_all_by_template_locked(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1606     int attr_count, dmu_tx_t *tx)
1607 {
1608 	sa_os_t *sa = hdl->sa_os->os_sa;
1609 
1610 	if (sa->sa_need_attr_registration)
1611 		sa_attr_register_sync(hdl, tx);
1612 	return (sa_build_layouts(hdl, attr_desc, attr_count, tx));
1613 }
1614 
1615 int
1616 sa_replace_all_by_template(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1617     int attr_count, dmu_tx_t *tx)
1618 {
1619 	int error;
1620 
1621 	mutex_enter(&hdl->sa_lock);
1622 	error = sa_replace_all_by_template_locked(hdl, attr_desc,
1623 	    attr_count, tx);
1624 	mutex_exit(&hdl->sa_lock);
1625 	return (error);
1626 }
1627 
1628 /*
1629  * Add/remove a single attribute or replace a variable-sized attribute value
1630  * with a value of a different size, and then rewrite the entire set
1631  * of attributes.
1632  * Same-length attribute value replacement (including fixed-length attributes)
1633  * is handled more efficiently by the upper layers.
1634  */
1635 static int
1636 sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
1637     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
1638     uint16_t buflen, dmu_tx_t *tx)
1639 {
1640 	sa_os_t *sa = hdl->sa_os->os_sa;
1641 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus;
1642 	dnode_t *dn;
1643 	sa_bulk_attr_t *attr_desc;
1644 	void *old_data[2];
1645 	int bonus_attr_count = 0;
1646 	int bonus_data_size = 0;
1647 	int spill_data_size = 0;
1648 	int spill_attr_count = 0;
1649 	int error;
1650 	uint16_t length, reg_length;
1651 	int i, j, k, length_idx;
1652 	sa_hdr_phys_t *hdr;
1653 	sa_idx_tab_t *idx_tab;
1654 	int attr_count;
1655 	int count;
1656 
1657 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1658 
1659 	/* First make of copy of the old data */
1660 
1661 	DB_DNODE_ENTER(db);
1662 	dn = DB_DNODE(db);
1663 	if (dn->dn_bonuslen != 0) {
1664 		bonus_data_size = hdl->sa_bonus->db_size;
1665 		old_data[0] = kmem_alloc(bonus_data_size, KM_SLEEP);
1666 		bcopy(hdl->sa_bonus->db_data, old_data[0],
1667 		    hdl->sa_bonus->db_size);
1668 		bonus_attr_count = hdl->sa_bonus_tab->sa_layout->lot_attr_count;
1669 	} else {
1670 		old_data[0] = NULL;
1671 	}
1672 	DB_DNODE_EXIT(db);
1673 
1674 	/* Bring spill buffer online if it isn't currently */
1675 
1676 	if ((error = sa_get_spill(hdl)) == 0) {
1677 		spill_data_size = hdl->sa_spill->db_size;
1678 		old_data[1] = kmem_alloc(spill_data_size, KM_SLEEP);
1679 		bcopy(hdl->sa_spill->db_data, old_data[1],
1680 		    hdl->sa_spill->db_size);
1681 		spill_attr_count =
1682 		    hdl->sa_spill_tab->sa_layout->lot_attr_count;
1683 	} else if (error && error != ENOENT) {
1684 		if (old_data[0])
1685 			kmem_free(old_data[0], bonus_data_size);
1686 		return (error);
1687 	} else {
1688 		old_data[1] = NULL;
1689 	}
1690 
1691 	/* build descriptor of all attributes */
1692 
1693 	attr_count = bonus_attr_count + spill_attr_count;
1694 	if (action == SA_ADD)
1695 		attr_count++;
1696 	else if (action == SA_REMOVE)
1697 		attr_count--;
1698 
1699 	attr_desc = kmem_zalloc(sizeof (sa_bulk_attr_t) * attr_count, KM_SLEEP);
1700 
1701 	/*
1702 	 * loop through bonus and spill buffer if it exists, and
1703 	 * build up new attr_descriptor to reset the attributes
1704 	 */
1705 	k = j = 0;
1706 	count = bonus_attr_count;
1707 	hdr = SA_GET_HDR(hdl, SA_BONUS);
1708 	idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);
1709 	for (; k != 2; k++) {
1710 		/*
1711 		 * Iterate over each attribute in layout.  Fetch the
1712 		 * size of variable-length attributes needing rewrite
1713 		 * from sa_lengths[].
1714 		 */
1715 		for (i = 0, length_idx = 0; i != count; i++) {
1716 			sa_attr_type_t attr;
1717 
1718 			attr = idx_tab->sa_layout->lot_attrs[i];
1719 			reg_length = SA_REGISTERED_LEN(sa, attr);
1720 			if (reg_length == 0) {
1721 				length = hdr->sa_lengths[length_idx];
1722 				length_idx++;
1723 			} else {
1724 				length = reg_length;
1725 			}
1726 			if (attr == newattr) {
1727 				/*
1728 				 * There is nothing to do for SA_REMOVE,
1729 				 * so it is just skipped.
1730 				 */
1731 				if (action == SA_REMOVE)
1732 					continue;
1733 
1734 				/*
1735 				 * Duplicate attributes are not allowed, so the
1736 				 * action can not be SA_ADD here.
1737 				 */
1738 				ASSERT3S(action, ==, SA_REPLACE);
1739 
1740 				/*
1741 				 * Only a variable-sized attribute can be
1742 				 * replaced here, and its size must be changing.
1743 				 */
1744 				ASSERT3U(reg_length, ==, 0);
1745 				ASSERT3U(length, !=, buflen);
1746 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1747 				    locator, datastart, buflen);
1748 			} else {
1749 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1750 				    NULL, (void *)
1751 				    (TOC_OFF(idx_tab->sa_idx_tab[attr]) +
1752 				    (uintptr_t)old_data[k]), length);
1753 			}
1754 		}
1755 		if (k == 0 && hdl->sa_spill) {
1756 			hdr = SA_GET_HDR(hdl, SA_SPILL);
1757 			idx_tab = SA_IDX_TAB_GET(hdl, SA_SPILL);
1758 			count = spill_attr_count;
1759 		} else {
1760 			break;
1761 		}
1762 	}
1763 	if (action == SA_ADD) {
1764 		reg_length = SA_REGISTERED_LEN(sa, newattr);
1765 		IMPLY(reg_length != 0, reg_length == buflen);
1766 		SA_ADD_BULK_ATTR(attr_desc, j, newattr, locator,
1767 		    datastart, buflen);
1768 	}
1769 	ASSERT3U(j, ==, attr_count);
1770 
1771 	error = sa_build_layouts(hdl, attr_desc, attr_count, tx);
1772 
1773 	if (old_data[0])
1774 		kmem_free(old_data[0], bonus_data_size);
1775 	if (old_data[1])
1776 		kmem_free(old_data[1], spill_data_size);
1777 	kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
1778 
1779 	return (error);
1780 }
1781 
1782 static int
1783 sa_bulk_update_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
1784     dmu_tx_t *tx)
1785 {
1786 	int error;
1787 	sa_os_t *sa = hdl->sa_os->os_sa;
1788 	dmu_object_type_t bonustype;
1789 
1790 	bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));
1791 
1792 	ASSERT(hdl);
1793 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1794 
1795 	/* sync out registration table if necessary */
1796 	if (sa->sa_need_attr_registration)
1797 		sa_attr_register_sync(hdl, tx);
1798 
1799 	error = sa_attr_op(hdl, bulk, count, SA_UPDATE, tx);
1800 	if (error == 0 && !IS_SA_BONUSTYPE(bonustype) && sa->sa_update_cb)
1801 		sa->sa_update_cb(hdl, tx);
1802 
1803 	return (error);
1804 }
1805 
1806 /*
1807  * update or add new attribute
1808  */
1809 int
1810 sa_update(sa_handle_t *hdl, sa_attr_type_t type,
1811     void *buf, uint32_t buflen, dmu_tx_t *tx)
1812 {
1813 	int error;
1814 	sa_bulk_attr_t bulk;
1815 
1816 	bulk.sa_attr = type;
1817 	bulk.sa_data_func = NULL;
1818 	bulk.sa_length = buflen;
1819 	bulk.sa_data = buf;
1820 
1821 	mutex_enter(&hdl->sa_lock);
1822 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
1823 	mutex_exit(&hdl->sa_lock);
1824 	return (error);
1825 }
1826 
1827 int
1828 sa_update_from_cb(sa_handle_t *hdl, sa_attr_type_t attr,
1829     uint32_t buflen, sa_data_locator_t *locator, void *userdata, dmu_tx_t *tx)
1830 {
1831 	int error;
1832 	sa_bulk_attr_t bulk;
1833 
1834 	bulk.sa_attr = attr;
1835 	bulk.sa_data = userdata;
1836 	bulk.sa_data_func = locator;
1837 	bulk.sa_length = buflen;
1838 
1839 	mutex_enter(&hdl->sa_lock);
1840 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
1841 	mutex_exit(&hdl->sa_lock);
1842 	return (error);
1843 }
1844 
1845 /*
1846  * Return size of an attribute
1847  */
1848 
1849 int
1850 sa_size(sa_handle_t *hdl, sa_attr_type_t attr, int *size)
1851 {
1852 	sa_bulk_attr_t bulk;
1853 	int error;
1854 
1855 	bulk.sa_data = NULL;
1856 	bulk.sa_attr = attr;
1857 	bulk.sa_data_func = NULL;
1858 
1859 	ASSERT(hdl);
1860 	mutex_enter(&hdl->sa_lock);
1861 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) != 0) {
1862 		mutex_exit(&hdl->sa_lock);
1863 		return (error);
1864 	}
1865 	*size = bulk.sa_size;
1866 
1867 	mutex_exit(&hdl->sa_lock);
1868 	return (0);
1869 }
1870 
1871 int
1872 sa_bulk_lookup_locked(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
1873 {
1874 	ASSERT(hdl);
1875 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1876 	return (sa_lookup_impl(hdl, attrs, count));
1877 }
1878 
1879 int
1880 sa_bulk_lookup(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
1881 {
1882 	int error;
1883 
1884 	ASSERT(hdl);
1885 	mutex_enter(&hdl->sa_lock);
1886 	error = sa_bulk_lookup_locked(hdl, attrs, count);
1887 	mutex_exit(&hdl->sa_lock);
1888 	return (error);
1889 }
1890 
1891 int
1892 sa_bulk_update(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count, dmu_tx_t *tx)
1893 {
1894 	int error;
1895 
1896 	ASSERT(hdl);
1897 	mutex_enter(&hdl->sa_lock);
1898 	error = sa_bulk_update_impl(hdl, attrs, count, tx);
1899 	mutex_exit(&hdl->sa_lock);
1900 	return (error);
1901 }
1902 
1903 int
1904 sa_remove(sa_handle_t *hdl, sa_attr_type_t attr, dmu_tx_t *tx)
1905 {
1906 	int error;
1907 
1908 	mutex_enter(&hdl->sa_lock);
1909 	error = sa_modify_attrs(hdl, attr, SA_REMOVE, NULL,
1910 	    NULL, 0, tx);
1911 	mutex_exit(&hdl->sa_lock);
1912 	return (error);
1913 }
1914 
1915 void
1916 sa_object_info(sa_handle_t *hdl, dmu_object_info_t *doi)
1917 {
1918 	dmu_object_info_from_db((dmu_buf_t *)hdl->sa_bonus, doi);
1919 }
1920 
1921 void
1922 sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)
1923 {
1924 	dmu_object_size_from_db((dmu_buf_t *)hdl->sa_bonus,
1925 	    blksize, nblocks);
1926 }
1927 
1928 void
1929 sa_set_userp(sa_handle_t *hdl, void *ptr)
1930 {
1931 	hdl->sa_userp = ptr;
1932 }
1933 
1934 dmu_buf_t *
1935 sa_get_db(sa_handle_t *hdl)
1936 {
1937 	return ((dmu_buf_t *)hdl->sa_bonus);
1938 }
1939 
1940 void *
1941 sa_get_userdata(sa_handle_t *hdl)
1942 {
1943 	return (hdl->sa_userp);
1944 }
1945 
1946 void
1947 sa_register_update_callback_locked(objset_t *os, sa_update_cb_t *func)
1948 {
1949 	ASSERT(MUTEX_HELD(&os->os_sa->sa_lock));
1950 	os->os_sa->sa_update_cb = func;
1951 }
1952 
1953 void
1954 sa_register_update_callback(objset_t *os, sa_update_cb_t *func)
1955 {
1956 
1957 	mutex_enter(&os->os_sa->sa_lock);
1958 	sa_register_update_callback_locked(os, func);
1959 	mutex_exit(&os->os_sa->sa_lock);
1960 }
1961 
1962 uint64_t
1963 sa_handle_object(sa_handle_t *hdl)
1964 {
1965 	return (hdl->sa_bonus->db_object);
1966 }
1967 
1968 boolean_t
1969 sa_enabled(objset_t *os)
1970 {
1971 	return (os->os_sa == NULL);
1972 }
1973 
1974 int
1975 sa_set_sa_object(objset_t *os, uint64_t sa_object)
1976 {
1977 	sa_os_t *sa = os->os_sa;
1978 
1979 	if (sa->sa_master_obj)
1980 		return (1);
1981 
1982 	sa->sa_master_obj = sa_object;
1983 
1984 	return (0);
1985 }
1986 
1987 int
1988 sa_hdrsize(void *arg)
1989 {
1990 	sa_hdr_phys_t *hdr = arg;
1991 
1992 	return (SA_HDR_SIZE(hdr));
1993 }
1994 
1995 void
1996 sa_handle_lock(sa_handle_t *hdl)
1997 {
1998 	ASSERT(hdl);
1999 	mutex_enter(&hdl->sa_lock);
2000 }
2001 
2002 void
2003 sa_handle_unlock(sa_handle_t *hdl)
2004 {
2005 	ASSERT(hdl);
2006 	mutex_exit(&hdl->sa_lock);
2007 }
2008