xref: /illumos-gate/usr/src/uts/common/fs/zfs/sa.c (revision 54811da5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Portions Copyright 2011 iXsystems, Inc
25  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
26  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
27  * Copyright (c) 2014 Integros [integros.com]
28  */
29 
30 #include <sys/zfs_context.h>
31 #include <sys/types.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/sysmacros.h>
35 #include <sys/dmu.h>
36 #include <sys/dmu_impl.h>
37 #include <sys/dmu_objset.h>
38 #include <sys/dmu_tx.h>
39 #include <sys/dbuf.h>
40 #include <sys/dnode.h>
41 #include <sys/zap.h>
42 #include <sys/sa.h>
43 #include <sys/sunddi.h>
44 #include <sys/sa_impl.h>
45 #include <sys/dnode.h>
46 #include <sys/errno.h>
47 #include <sys/zfs_context.h>
48 
49 /*
50  * ZFS System attributes:
51  *
52  * A generic mechanism to allow for arbitrary attributes
53  * to be stored in a dnode.  The data will be stored in the bonus buffer of
54  * the dnode and if necessary a special "spill" block will be used to handle
55  * overflow situations.  The spill block will be sized to fit the data
56  * from 512 - 128K.  When a spill block is used the BP (blkptr_t) for the
57  * spill block is stored at the end of the current bonus buffer.  Any
58  * attributes that would be in the way of the blkptr_t will be relocated
59  * into the spill block.
60  *
61  * Attribute registration:
62  *
63  * Stored persistently on a per dataset basis
64  * a mapping between attribute "string" names and their actual attribute
65  * numeric values, length, and byteswap function.  The names are only used
66  * during registration.  All  attributes are known by their unique attribute
67  * id value.  If an attribute can have a variable size then the value
68  * 0 will be used to indicate this.
69  *
70  * Attribute Layout:
71  *
72  * Attribute layouts are a way to compactly store multiple attributes, but
73  * without taking the overhead associated with managing each attribute
74  * individually.  Since you will typically have the same set of attributes
75  * stored in the same order a single table will be used to represent that
76  * layout.  The ZPL for example will usually have only about 10 different
77  * layouts (regular files, device files, symlinks,
78  * regular files + scanstamp, files/dir with extended attributes, and then
79  * you have the possibility of all of those minus ACL, because it would
80  * be kicked out into the spill block)
81  *
82  * Layouts are simply an array of the attributes and their
83  * ordering i.e. [0, 1, 4, 5, 2]
84  *
85  * Each distinct layout is given a unique layout number and that is whats
86  * stored in the header at the beginning of the SA data buffer.
87  *
88  * A layout only covers a single dbuf (bonus or spill).  If a set of
89  * attributes is split up between the bonus buffer and a spill buffer then
90  * two different layouts will be used.  This allows us to byteswap the
91  * spill without looking at the bonus buffer and keeps the on disk format of
92  * the bonus and spill buffer the same.
93  *
94  * Adding a single attribute will cause the entire set of attributes to
95  * be rewritten and could result in a new layout number being constructed
96  * as part of the rewrite if no such layout exists for the new set of
97  * attribues.  The new attribute will be appended to the end of the already
98  * existing attributes.
99  *
100  * Both the attribute registration and attribute layout information are
101  * stored in normal ZAP attributes.  Their should be a small number of
102  * known layouts and the set of attributes is assumed to typically be quite
103  * small.
104  *
105  * The registered attributes and layout "table" information is maintained
106  * in core and a special "sa_os_t" is attached to the objset_t.
107  *
108  * A special interface is provided to allow for quickly applying
109  * a large set of attributes at once.  sa_replace_all_by_template() is
110  * used to set an array of attributes.  This is used by the ZPL when
111  * creating a brand new file.  The template that is passed into the function
112  * specifies the attribute, size for variable length attributes, location of
113  * data and special "data locator" function if the data isn't in a contiguous
114  * location.
115  *
116  * Byteswap implications:
117  *
118  * Since the SA attributes are not entirely self describing we can't do
119  * the normal byteswap processing.  The special ZAP layout attribute and
120  * attribute registration attributes define the byteswap function and the
121  * size of the attributes, unless it is variable sized.
122  * The normal ZFS byteswapping infrastructure assumes you don't need
123  * to read any objects in order to do the necessary byteswapping.  Whereas
124  * SA attributes can only be properly byteswapped if the dataset is opened
125  * and the layout/attribute ZAP attributes are available.  Because of this
126  * the SA attributes will be byteswapped when they are first accessed by
127  * the SA code that will read the SA data.
128  */
129 
130 typedef void (sa_iterfunc_t)(void *hdr, void *addr, sa_attr_type_t,
131     uint16_t length, int length_idx, boolean_t, void *userp);
132 
133 static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
134 static void sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab);
135 static sa_idx_tab_t *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype,
136     sa_hdr_phys_t *hdr);
137 static void sa_idx_tab_rele(objset_t *os, void *arg);
138 static void sa_copy_data(sa_data_locator_t *func, void *start, void *target,
139     int buflen);
140 static int sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
141     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
142     uint16_t buflen, dmu_tx_t *tx);
143 
144 arc_byteswap_func_t *sa_bswap_table[] = {
145 	byteswap_uint64_array,
146 	byteswap_uint32_array,
147 	byteswap_uint16_array,
148 	byteswap_uint8_array,
149 	zfs_acl_byteswap,
150 };
151 
152 #define	SA_COPY_DATA(f, s, t, l) \
153 	{ \
154 		if (f == NULL) { \
155 			if (l == 8) { \
156 				*(uint64_t *)t = *(uint64_t *)s; \
157 			} else if (l == 16) { \
158 				*(uint64_t *)t = *(uint64_t *)s; \
159 				*(uint64_t *)((uintptr_t)t + 8) = \
160 				    *(uint64_t *)((uintptr_t)s + 8); \
161 			} else { \
162 				bcopy(s, t, l); \
163 			} \
164 		} else \
165 			sa_copy_data(f, s, t, l); \
166 	}
167 
168 /*
169  * This table is fixed and cannot be changed.  Its purpose is to
170  * allow the SA code to work with both old/new ZPL file systems.
171  * It contains the list of legacy attributes.  These attributes aren't
172  * stored in the "attribute" registry zap objects, since older ZPL file systems
173  * won't have the registry.  Only objsets of type ZFS_TYPE_FILESYSTEM will
174  * use this static table.
175  */
176 sa_attr_reg_t sa_legacy_attrs[] = {
177 	{"ZPL_ATIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 0},
178 	{"ZPL_MTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 1},
179 	{"ZPL_CTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 2},
180 	{"ZPL_CRTIME", sizeof (uint64_t) * 2, SA_UINT64_ARRAY, 3},
181 	{"ZPL_GEN", sizeof (uint64_t), SA_UINT64_ARRAY, 4},
182 	{"ZPL_MODE", sizeof (uint64_t), SA_UINT64_ARRAY, 5},
183 	{"ZPL_SIZE", sizeof (uint64_t), SA_UINT64_ARRAY, 6},
184 	{"ZPL_PARENT", sizeof (uint64_t), SA_UINT64_ARRAY, 7},
185 	{"ZPL_LINKS", sizeof (uint64_t), SA_UINT64_ARRAY, 8},
186 	{"ZPL_XATTR", sizeof (uint64_t), SA_UINT64_ARRAY, 9},
187 	{"ZPL_RDEV", sizeof (uint64_t), SA_UINT64_ARRAY, 10},
188 	{"ZPL_FLAGS", sizeof (uint64_t), SA_UINT64_ARRAY, 11},
189 	{"ZPL_UID", sizeof (uint64_t), SA_UINT64_ARRAY, 12},
190 	{"ZPL_GID", sizeof (uint64_t), SA_UINT64_ARRAY, 13},
191 	{"ZPL_PAD", sizeof (uint64_t) * 4, SA_UINT64_ARRAY, 14},
192 	{"ZPL_ZNODE_ACL", 88, SA_UINT8_ARRAY, 15},
193 };
194 
195 /*
196  * This is only used for objects of type DMU_OT_ZNODE
197  */
198 sa_attr_type_t sa_legacy_zpl_layout[] = {
199     0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
200 };
201 
202 /*
203  * Special dummy layout used for buffers with no attributes.
204  */
205 sa_attr_type_t sa_dummy_zpl_layout[] = { 0 };
206 
207 static int sa_legacy_attr_count = 16;
208 static kmem_cache_t *sa_cache = NULL;
209 
210 /*ARGSUSED*/
211 static int
212 sa_cache_constructor(void *buf, void *unused, int kmflag)
213 {
214 	sa_handle_t *hdl = buf;
215 
216 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);
217 	return (0);
218 }
219 
220 /*ARGSUSED*/
221 static void
222 sa_cache_destructor(void *buf, void *unused)
223 {
224 	sa_handle_t *hdl = buf;
225 	mutex_destroy(&hdl->sa_lock);
226 }
227 
228 void
229 sa_cache_init(void)
230 {
231 	sa_cache = kmem_cache_create("sa_cache",
232 	    sizeof (sa_handle_t), 0, sa_cache_constructor,
233 	    sa_cache_destructor, NULL, NULL, NULL, 0);
234 }
235 
236 void
237 sa_cache_fini(void)
238 {
239 	if (sa_cache)
240 		kmem_cache_destroy(sa_cache);
241 }
242 
243 static int
244 layout_num_compare(const void *arg1, const void *arg2)
245 {
246 	const sa_lot_t *node1 = arg1;
247 	const sa_lot_t *node2 = arg2;
248 
249 	if (node1->lot_num > node2->lot_num)
250 		return (1);
251 	else if (node1->lot_num < node2->lot_num)
252 		return (-1);
253 	return (0);
254 }
255 
256 static int
257 layout_hash_compare(const void *arg1, const void *arg2)
258 {
259 	const sa_lot_t *node1 = arg1;
260 	const sa_lot_t *node2 = arg2;
261 
262 	if (node1->lot_hash > node2->lot_hash)
263 		return (1);
264 	if (node1->lot_hash < node2->lot_hash)
265 		return (-1);
266 	if (node1->lot_instance > node2->lot_instance)
267 		return (1);
268 	if (node1->lot_instance < node2->lot_instance)
269 		return (-1);
270 	return (0);
271 }
272 
273 boolean_t
274 sa_layout_equal(sa_lot_t *tbf, sa_attr_type_t *attrs, int count)
275 {
276 	int i;
277 
278 	if (count != tbf->lot_attr_count)
279 		return (1);
280 
281 	for (i = 0; i != count; i++) {
282 		if (attrs[i] != tbf->lot_attrs[i])
283 			return (1);
284 	}
285 	return (0);
286 }
287 
288 #define	SA_ATTR_HASH(attr) (zfs_crc64_table[(-1ULL ^ attr) & 0xFF])
289 
290 static uint64_t
291 sa_layout_info_hash(sa_attr_type_t *attrs, int attr_count)
292 {
293 	int i;
294 	uint64_t crc = -1ULL;
295 
296 	for (i = 0; i != attr_count; i++)
297 		crc ^= SA_ATTR_HASH(attrs[i]);
298 
299 	return (crc);
300 }
301 
302 static int
303 sa_get_spill(sa_handle_t *hdl)
304 {
305 	int rc;
306 	if (hdl->sa_spill == NULL) {
307 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,
308 		    &hdl->sa_spill)) == 0)
309 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
310 	} else {
311 		rc = 0;
312 	}
313 
314 	return (rc);
315 }
316 
317 /*
318  * Main attribute lookup/update function
319  * returns 0 for success or non zero for failures
320  *
321  * Operates on bulk array, first failure will abort further processing
322  */
323 int
324 sa_attr_op(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
325     sa_data_op_t data_op, dmu_tx_t *tx)
326 {
327 	sa_os_t *sa = hdl->sa_os->os_sa;
328 	int i;
329 	int error = 0;
330 	sa_buf_type_t buftypes;
331 
332 	buftypes = 0;
333 
334 	ASSERT(count > 0);
335 	for (i = 0; i != count; i++) {
336 		ASSERT(bulk[i].sa_attr <= hdl->sa_os->os_sa->sa_num_attrs);
337 
338 		bulk[i].sa_addr = NULL;
339 		/* First check the bonus buffer */
340 
341 		if (hdl->sa_bonus_tab && TOC_ATTR_PRESENT(
342 		    hdl->sa_bonus_tab->sa_idx_tab[bulk[i].sa_attr])) {
343 			SA_ATTR_INFO(sa, hdl->sa_bonus_tab,
344 			    SA_GET_HDR(hdl, SA_BONUS),
345 			    bulk[i].sa_attr, bulk[i], SA_BONUS, hdl);
346 			if (tx && !(buftypes & SA_BONUS)) {
347 				dmu_buf_will_dirty(hdl->sa_bonus, tx);
348 				buftypes |= SA_BONUS;
349 			}
350 		}
351 		if (bulk[i].sa_addr == NULL &&
352 		    ((error = sa_get_spill(hdl)) == 0)) {
353 			if (TOC_ATTR_PRESENT(
354 			    hdl->sa_spill_tab->sa_idx_tab[bulk[i].sa_attr])) {
355 				SA_ATTR_INFO(sa, hdl->sa_spill_tab,
356 				    SA_GET_HDR(hdl, SA_SPILL),
357 				    bulk[i].sa_attr, bulk[i], SA_SPILL, hdl);
358 				if (tx && !(buftypes & SA_SPILL) &&
359 				    bulk[i].sa_size == bulk[i].sa_length) {
360 					dmu_buf_will_dirty(hdl->sa_spill, tx);
361 					buftypes |= SA_SPILL;
362 				}
363 			}
364 		}
365 		if (error && error != ENOENT) {
366 			return ((error == ECKSUM) ? EIO : error);
367 		}
368 
369 		switch (data_op) {
370 		case SA_LOOKUP:
371 			if (bulk[i].sa_addr == NULL)
372 				return (SET_ERROR(ENOENT));
373 			if (bulk[i].sa_data) {
374 				SA_COPY_DATA(bulk[i].sa_data_func,
375 				    bulk[i].sa_addr, bulk[i].sa_data,
376 				    bulk[i].sa_size);
377 			}
378 			continue;
379 
380 		case SA_UPDATE:
381 			/* existing rewrite of attr */
382 			if (bulk[i].sa_addr &&
383 			    bulk[i].sa_size == bulk[i].sa_length) {
384 				SA_COPY_DATA(bulk[i].sa_data_func,
385 				    bulk[i].sa_data, bulk[i].sa_addr,
386 				    bulk[i].sa_length);
387 				continue;
388 			} else if (bulk[i].sa_addr) { /* attr size change */
389 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
390 				    SA_REPLACE, bulk[i].sa_data_func,
391 				    bulk[i].sa_data, bulk[i].sa_length, tx);
392 			} else { /* adding new attribute */
393 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,
394 				    SA_ADD, bulk[i].sa_data_func,
395 				    bulk[i].sa_data, bulk[i].sa_length, tx);
396 			}
397 			if (error)
398 				return (error);
399 			break;
400 		}
401 	}
402 	return (error);
403 }
404 
405 static sa_lot_t *
406 sa_add_layout_entry(objset_t *os, sa_attr_type_t *attrs, int attr_count,
407     uint64_t lot_num, uint64_t hash, boolean_t zapadd, dmu_tx_t *tx)
408 {
409 	sa_os_t *sa = os->os_sa;
410 	sa_lot_t *tb, *findtb;
411 	int i;
412 	avl_index_t loc;
413 
414 	ASSERT(MUTEX_HELD(&sa->sa_lock));
415 	tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP);
416 	tb->lot_attr_count = attr_count;
417 	tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
418 	    KM_SLEEP);
419 	bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count);
420 	tb->lot_num = lot_num;
421 	tb->lot_hash = hash;
422 	tb->lot_instance = 0;
423 
424 	if (zapadd) {
425 		char attr_name[8];
426 
427 		if (sa->sa_layout_attr_obj == 0) {
428 			sa->sa_layout_attr_obj = zap_create_link(os,
429 			    DMU_OT_SA_ATTR_LAYOUTS,
430 			    sa->sa_master_obj, SA_LAYOUTS, tx);
431 		}
432 
433 		(void) snprintf(attr_name, sizeof (attr_name),
434 		    "%d", (int)lot_num);
435 		VERIFY(0 == zap_update(os, os->os_sa->sa_layout_attr_obj,
436 		    attr_name, 2, attr_count, attrs, tx));
437 	}
438 
439 	list_create(&tb->lot_idx_tab, sizeof (sa_idx_tab_t),
440 	    offsetof(sa_idx_tab_t, sa_next));
441 
442 	for (i = 0; i != attr_count; i++) {
443 		if (sa->sa_attr_table[tb->lot_attrs[i]].sa_length == 0)
444 			tb->lot_var_sizes++;
445 	}
446 
447 	avl_add(&sa->sa_layout_num_tree, tb);
448 
449 	/* verify we don't have a hash collision */
450 	if ((findtb = avl_find(&sa->sa_layout_hash_tree, tb, &loc)) != NULL) {
451 		for (; findtb && findtb->lot_hash == hash;
452 		    findtb = AVL_NEXT(&sa->sa_layout_hash_tree, findtb)) {
453 			if (findtb->lot_instance != tb->lot_instance)
454 				break;
455 			tb->lot_instance++;
456 		}
457 	}
458 	avl_add(&sa->sa_layout_hash_tree, tb);
459 	return (tb);
460 }
461 
462 static void
463 sa_find_layout(objset_t *os, uint64_t hash, sa_attr_type_t *attrs,
464     int count, dmu_tx_t *tx, sa_lot_t **lot)
465 {
466 	sa_lot_t *tb, tbsearch;
467 	avl_index_t loc;
468 	sa_os_t *sa = os->os_sa;
469 	boolean_t found = B_FALSE;
470 
471 	mutex_enter(&sa->sa_lock);
472 	tbsearch.lot_hash = hash;
473 	tbsearch.lot_instance = 0;
474 	tb = avl_find(&sa->sa_layout_hash_tree, &tbsearch, &loc);
475 	if (tb) {
476 		for (; tb && tb->lot_hash == hash;
477 		    tb = AVL_NEXT(&sa->sa_layout_hash_tree, tb)) {
478 			if (sa_layout_equal(tb, attrs, count) == 0) {
479 				found = B_TRUE;
480 				break;
481 			}
482 		}
483 	}
484 	if (!found) {
485 		tb = sa_add_layout_entry(os, attrs, count,
486 		    avl_numnodes(&sa->sa_layout_num_tree), hash, B_TRUE, tx);
487 	}
488 	mutex_exit(&sa->sa_lock);
489 	*lot = tb;
490 }
491 
492 static int
493 sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)
494 {
495 	int error;
496 	uint32_t blocksize;
497 
498 	if (size == 0) {
499 		blocksize = SPA_MINBLOCKSIZE;
500 	} else if (size > SPA_OLD_MAXBLOCKSIZE) {
501 		ASSERT(0);
502 		return (SET_ERROR(EFBIG));
503 	} else {
504 		blocksize = P2ROUNDUP_TYPED(size, SPA_MINBLOCKSIZE, uint32_t);
505 	}
506 
507 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);
508 	ASSERT(error == 0);
509 	return (error);
510 }
511 
512 static void
513 sa_copy_data(sa_data_locator_t *func, void *datastart, void *target, int buflen)
514 {
515 	if (func == NULL) {
516 		bcopy(datastart, target, buflen);
517 	} else {
518 		boolean_t start;
519 		int bytes;
520 		void *dataptr;
521 		void *saptr = target;
522 		uint32_t length;
523 
524 		start = B_TRUE;
525 		bytes = 0;
526 		while (bytes < buflen) {
527 			func(&dataptr, &length, buflen, start, datastart);
528 			bcopy(dataptr, saptr, length);
529 			saptr = (void *)((caddr_t)saptr + length);
530 			bytes += length;
531 			start = B_FALSE;
532 		}
533 	}
534 }
535 
536 /*
537  * Determine several different sizes
538  * first the sa header size
539  * the number of bytes to be stored
540  * if spill would occur the index in the attribute array is returned
541  *
542  * the boolean will_spill will be set when spilling is necessary.  It
543  * is only set when the buftype is SA_BONUS
544  */
545 static int
546 sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count,
547     dmu_buf_t *db, sa_buf_type_t buftype, int full_space, int *index,
548     int *total, boolean_t *will_spill)
549 {
550 	int var_size = 0;
551 	int i;
552 	int hdrsize;
553 	int extra_hdrsize;
554 
555 	if (buftype == SA_BONUS && sa->sa_force_spill) {
556 		*total = 0;
557 		*index = 0;
558 		*will_spill = B_TRUE;
559 		return (0);
560 	}
561 
562 	*index = -1;
563 	*total = 0;
564 	*will_spill = B_FALSE;
565 
566 	extra_hdrsize = 0;
567 	hdrsize = (SA_BONUSTYPE_FROM_DB(db) == DMU_OT_ZNODE) ? 0 :
568 	    sizeof (sa_hdr_phys_t);
569 
570 	ASSERT(IS_P2ALIGNED(full_space, 8));
571 
572 	for (i = 0; i != attr_count; i++) {
573 		boolean_t is_var_sz;
574 
575 		*total = P2ROUNDUP(*total, 8);
576 		*total += attr_desc[i].sa_length;
577 		if (*will_spill)
578 			continue;
579 
580 		is_var_sz = (SA_REGISTERED_LEN(sa, attr_desc[i].sa_attr) == 0);
581 		if (is_var_sz) {
582 			var_size++;
583 		}
584 
585 		if (is_var_sz && var_size > 1) {
586 			/*
587 			 * Don't worry that the spill block might overflow.
588 			 * It will be resized if needed in sa_build_layouts().
589 			 */
590 			if (buftype == SA_SPILL ||
591 			    P2ROUNDUP(hdrsize + sizeof (uint16_t), 8) +
592 			    *total < full_space) {
593 				/*
594 				 * Account for header space used by array of
595 				 * optional sizes of variable-length attributes.
596 				 * Record the extra header size in case this
597 				 * increase needs to be reversed due to
598 				 * spill-over.
599 				 */
600 				hdrsize += sizeof (uint16_t);
601 				if (*index != -1)
602 					extra_hdrsize += sizeof (uint16_t);
603 			} else {
604 				ASSERT(buftype == SA_BONUS);
605 				if (*index == -1)
606 					*index = i;
607 				*will_spill = B_TRUE;
608 				continue;
609 			}
610 		}
611 
612 		/*
613 		 * find index of where spill *could* occur.
614 		 * Then continue to count of remainder attribute
615 		 * space.  The sum is used later for sizing bonus
616 		 * and spill buffer.
617 		 */
618 		if (buftype == SA_BONUS && *index == -1 &&
619 		    *total + P2ROUNDUP(hdrsize, 8) >
620 		    (full_space - sizeof (blkptr_t))) {
621 			*index = i;
622 		}
623 
624 		if (*total + P2ROUNDUP(hdrsize, 8) > full_space &&
625 		    buftype == SA_BONUS)
626 			*will_spill = B_TRUE;
627 	}
628 
629 	if (*will_spill)
630 		hdrsize -= extra_hdrsize;
631 
632 	hdrsize = P2ROUNDUP(hdrsize, 8);
633 	return (hdrsize);
634 }
635 
636 #define	BUF_SPACE_NEEDED(total, header) (total + header)
637 
638 /*
639  * Find layout that corresponds to ordering of attributes
640  * If not found a new layout number is created and added to
641  * persistent layout tables.
642  */
643 static int
644 sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,
645     dmu_tx_t *tx)
646 {
647 	sa_os_t *sa = hdl->sa_os->os_sa;
648 	uint64_t hash;
649 	sa_buf_type_t buftype;
650 	sa_hdr_phys_t *sahdr;
651 	void *data_start;
652 	int buf_space;
653 	sa_attr_type_t *attrs, *attrs_start;
654 	int i, lot_count;
655 	int dnodesize;
656 	int hdrsize;
657 	int spillhdrsize = 0;
658 	int used;
659 	dmu_object_type_t bonustype;
660 	sa_lot_t *lot;
661 	int len_idx;
662 	int spill_used;
663 	int bonuslen;
664 	boolean_t spilling;
665 
666 	dmu_buf_will_dirty(hdl->sa_bonus, tx);
667 	bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);
668 
669 	dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize);
670 	bonuslen = DN_BONUS_SIZE(dnodesize);
671 
672 	/* first determine bonus header size and sum of all attributes */
673 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,
674 	    SA_BONUS, bonuslen, &i, &used, &spilling);
675 
676 	if (used > SPA_OLD_MAXBLOCKSIZE)
677 		return (SET_ERROR(EFBIG));
678 
679 	VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ?
680 	    MIN(bonuslen - sizeof (blkptr_t), used + hdrsize) :
681 	    used + hdrsize, tx));
682 
683 	ASSERT((bonustype == DMU_OT_ZNODE && spilling == 0) ||
684 	    bonustype == DMU_OT_SA);
685 
686 	/* setup and size spill buffer when needed */
687 	if (spilling) {
688 		boolean_t dummy;
689 
690 		if (hdl->sa_spill == NULL) {
691 			VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, NULL,
692 			    &hdl->sa_spill) == 0);
693 		}
694 		dmu_buf_will_dirty(hdl->sa_spill, tx);
695 
696 		spillhdrsize = sa_find_sizes(sa, &attr_desc[i],
697 		    attr_count - i, hdl->sa_spill, SA_SPILL,
698 		    hdl->sa_spill->db_size, &i, &spill_used, &dummy);
699 
700 		if (spill_used > SPA_OLD_MAXBLOCKSIZE)
701 			return (SET_ERROR(EFBIG));
702 
703 		buf_space = hdl->sa_spill->db_size - spillhdrsize;
704 		if (BUF_SPACE_NEEDED(spill_used, spillhdrsize) >
705 		    hdl->sa_spill->db_size)
706 			VERIFY(0 == sa_resize_spill(hdl,
707 			    BUF_SPACE_NEEDED(spill_used, spillhdrsize), tx));
708 	}
709 
710 	/* setup starting pointers to lay down data */
711 	data_start = (void *)((uintptr_t)hdl->sa_bonus->db_data + hdrsize);
712 	sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;
713 	buftype = SA_BONUS;
714 
715 	if (spilling)
716 		buf_space = (sa->sa_force_spill) ?
717 		    0 : SA_BLKPTR_SPACE - hdrsize;
718 	else
719 		buf_space = hdl->sa_bonus->db_size - hdrsize;
720 
721 	attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
722 	    KM_SLEEP);
723 	lot_count = 0;
724 
725 	for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) {
726 		uint16_t length;
727 
728 		ASSERT(IS_P2ALIGNED(data_start, 8));
729 		ASSERT(IS_P2ALIGNED(buf_space, 8));
730 		attrs[i] = attr_desc[i].sa_attr;
731 		length = SA_REGISTERED_LEN(sa, attrs[i]);
732 		if (length == 0)
733 			length = attr_desc[i].sa_length;
734 
735 		if (buf_space < length) {  /* switch to spill buffer */
736 			VERIFY(spilling);
737 			VERIFY(bonustype == DMU_OT_SA);
738 			if (buftype == SA_BONUS && !sa->sa_force_spill) {
739 				sa_find_layout(hdl->sa_os, hash, attrs_start,
740 				    lot_count, tx, &lot);
741 				SA_SET_HDR(sahdr, lot->lot_num, hdrsize);
742 			}
743 
744 			buftype = SA_SPILL;
745 			hash = -1ULL;
746 			len_idx = 0;
747 
748 			sahdr = (sa_hdr_phys_t *)hdl->sa_spill->db_data;
749 			sahdr->sa_magic = SA_MAGIC;
750 			data_start = (void *)((uintptr_t)sahdr +
751 			    spillhdrsize);
752 			attrs_start = &attrs[i];
753 			buf_space = hdl->sa_spill->db_size - spillhdrsize;
754 			lot_count = 0;
755 		}
756 		hash ^= SA_ATTR_HASH(attrs[i]);
757 		attr_desc[i].sa_addr = data_start;
758 		attr_desc[i].sa_size = length;
759 		SA_COPY_DATA(attr_desc[i].sa_data_func, attr_desc[i].sa_data,
760 		    data_start, length);
761 		if (sa->sa_attr_table[attrs[i]].sa_length == 0) {
762 			sahdr->sa_lengths[len_idx++] = length;
763 		}
764 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
765 		    length), 8);
766 		buf_space -= P2ROUNDUP(length, 8);
767 		lot_count++;
768 	}
769 
770 	sa_find_layout(hdl->sa_os, hash, attrs_start, lot_count, tx, &lot);
771 
772 	/*
773 	 * Verify that old znodes always have layout number 0.
774 	 * Must be DMU_OT_SA for arbitrary layouts
775 	 */
776 	VERIFY((bonustype == DMU_OT_ZNODE && lot->lot_num == 0) ||
777 	    (bonustype == DMU_OT_SA && lot->lot_num > 1));
778 
779 	if (bonustype == DMU_OT_SA) {
780 		SA_SET_HDR(sahdr, lot->lot_num,
781 		    buftype == SA_BONUS ? hdrsize : spillhdrsize);
782 	}
783 
784 	kmem_free(attrs, sizeof (sa_attr_type_t) * attr_count);
785 	if (hdl->sa_bonus_tab) {
786 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
787 		hdl->sa_bonus_tab = NULL;
788 	}
789 	if (!sa->sa_force_spill)
790 		VERIFY(0 == sa_build_index(hdl, SA_BONUS));
791 	if (hdl->sa_spill) {
792 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
793 		if (!spilling) {
794 			/*
795 			 * remove spill block that is no longer needed.
796 			 */
797 			dmu_buf_rele(hdl->sa_spill, NULL);
798 			hdl->sa_spill = NULL;
799 			hdl->sa_spill_tab = NULL;
800 			VERIFY(0 == dmu_rm_spill(hdl->sa_os,
801 			    sa_handle_object(hdl), tx));
802 		} else {
803 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));
804 		}
805 	}
806 
807 	return (0);
808 }
809 
810 static void
811 sa_free_attr_table(sa_os_t *sa)
812 {
813 	int i;
814 
815 	if (sa->sa_attr_table == NULL)
816 		return;
817 
818 	for (i = 0; i != sa->sa_num_attrs; i++) {
819 		if (sa->sa_attr_table[i].sa_name)
820 			kmem_free(sa->sa_attr_table[i].sa_name,
821 			    strlen(sa->sa_attr_table[i].sa_name) + 1);
822 	}
823 
824 	kmem_free(sa->sa_attr_table,
825 	    sizeof (sa_attr_table_t) * sa->sa_num_attrs);
826 
827 	sa->sa_attr_table = NULL;
828 }
829 
830 static int
831 sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count)
832 {
833 	sa_os_t *sa = os->os_sa;
834 	uint64_t sa_attr_count = 0;
835 	uint64_t sa_reg_count = 0;
836 	int error = 0;
837 	uint64_t attr_value;
838 	sa_attr_table_t *tb;
839 	zap_cursor_t zc;
840 	zap_attribute_t za;
841 	int registered_count = 0;
842 	int i;
843 	dmu_objset_type_t ostype = dmu_objset_type(os);
844 
845 	sa->sa_user_table =
846 	    kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP);
847 	sa->sa_user_table_sz = count * sizeof (sa_attr_type_t);
848 
849 	if (sa->sa_reg_attr_obj != 0) {
850 		error = zap_count(os, sa->sa_reg_attr_obj,
851 		    &sa_attr_count);
852 
853 		/*
854 		 * Make sure we retrieved a count and that it isn't zero
855 		 */
856 		if (error || (error == 0 && sa_attr_count == 0)) {
857 			if (error == 0)
858 				error = SET_ERROR(EINVAL);
859 			goto bail;
860 		}
861 		sa_reg_count = sa_attr_count;
862 	}
863 
864 	if (ostype == DMU_OST_ZFS && sa_attr_count == 0)
865 		sa_attr_count += sa_legacy_attr_count;
866 
867 	/* Allocate attribute numbers for attributes that aren't registered */
868 	for (i = 0; i != count; i++) {
869 		boolean_t found = B_FALSE;
870 		int j;
871 
872 		if (ostype == DMU_OST_ZFS) {
873 			for (j = 0; j != sa_legacy_attr_count; j++) {
874 				if (strcmp(reg_attrs[i].sa_name,
875 				    sa_legacy_attrs[j].sa_name) == 0) {
876 					sa->sa_user_table[i] =
877 					    sa_legacy_attrs[j].sa_attr;
878 					found = B_TRUE;
879 				}
880 			}
881 		}
882 		if (found)
883 			continue;
884 
885 		if (sa->sa_reg_attr_obj)
886 			error = zap_lookup(os, sa->sa_reg_attr_obj,
887 			    reg_attrs[i].sa_name, 8, 1, &attr_value);
888 		else
889 			error = SET_ERROR(ENOENT);
890 		switch (error) {
891 		case ENOENT:
892 			sa->sa_user_table[i] = (sa_attr_type_t)sa_attr_count;
893 			sa_attr_count++;
894 			break;
895 		case 0:
896 			sa->sa_user_table[i] = ATTR_NUM(attr_value);
897 			break;
898 		default:
899 			goto bail;
900 		}
901 	}
902 
903 	sa->sa_num_attrs = sa_attr_count;
904 	tb = sa->sa_attr_table =
905 	    kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP);
906 
907 	/*
908 	 * Attribute table is constructed from requested attribute list,
909 	 * previously foreign registered attributes, and also the legacy
910 	 * ZPL set of attributes.
911 	 */
912 
913 	if (sa->sa_reg_attr_obj) {
914 		for (zap_cursor_init(&zc, os, sa->sa_reg_attr_obj);
915 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
916 		    zap_cursor_advance(&zc)) {
917 			uint64_t value;
918 			value  = za.za_first_integer;
919 
920 			registered_count++;
921 			tb[ATTR_NUM(value)].sa_attr = ATTR_NUM(value);
922 			tb[ATTR_NUM(value)].sa_length = ATTR_LENGTH(value);
923 			tb[ATTR_NUM(value)].sa_byteswap = ATTR_BSWAP(value);
924 			tb[ATTR_NUM(value)].sa_registered = B_TRUE;
925 
926 			if (tb[ATTR_NUM(value)].sa_name) {
927 				continue;
928 			}
929 			tb[ATTR_NUM(value)].sa_name =
930 			    kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP);
931 			(void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name,
932 			    strlen(za.za_name) +1);
933 		}
934 		zap_cursor_fini(&zc);
935 		/*
936 		 * Make sure we processed the correct number of registered
937 		 * attributes
938 		 */
939 		if (registered_count != sa_reg_count) {
940 			ASSERT(error != 0);
941 			goto bail;
942 		}
943 
944 	}
945 
946 	if (ostype == DMU_OST_ZFS) {
947 		for (i = 0; i != sa_legacy_attr_count; i++) {
948 			if (tb[i].sa_name)
949 				continue;
950 			tb[i].sa_attr = sa_legacy_attrs[i].sa_attr;
951 			tb[i].sa_length = sa_legacy_attrs[i].sa_length;
952 			tb[i].sa_byteswap = sa_legacy_attrs[i].sa_byteswap;
953 			tb[i].sa_registered = B_FALSE;
954 			tb[i].sa_name =
955 			    kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1,
956 			    KM_SLEEP);
957 			(void) strlcpy(tb[i].sa_name,
958 			    sa_legacy_attrs[i].sa_name,
959 			    strlen(sa_legacy_attrs[i].sa_name) + 1);
960 		}
961 	}
962 
963 	for (i = 0; i != count; i++) {
964 		sa_attr_type_t attr_id;
965 
966 		attr_id = sa->sa_user_table[i];
967 		if (tb[attr_id].sa_name)
968 			continue;
969 
970 		tb[attr_id].sa_length = reg_attrs[i].sa_length;
971 		tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap;
972 		tb[attr_id].sa_attr = attr_id;
973 		tb[attr_id].sa_name =
974 		    kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP);
975 		(void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name,
976 		    strlen(reg_attrs[i].sa_name) + 1);
977 	}
978 
979 	sa->sa_need_attr_registration =
980 	    (sa_attr_count != registered_count);
981 
982 	return (0);
983 bail:
984 	kmem_free(sa->sa_user_table, count * sizeof (sa_attr_type_t));
985 	sa->sa_user_table = NULL;
986 	sa_free_attr_table(sa);
987 	return ((error != 0) ? error : EINVAL);
988 }
989 
990 int
991 sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count,
992     sa_attr_type_t **user_table)
993 {
994 	zap_cursor_t zc;
995 	zap_attribute_t za;
996 	sa_os_t *sa;
997 	dmu_objset_type_t ostype = dmu_objset_type(os);
998 	sa_attr_type_t *tb;
999 	int error;
1000 
1001 	mutex_enter(&os->os_user_ptr_lock);
1002 	if (os->os_sa) {
1003 		mutex_enter(&os->os_sa->sa_lock);
1004 		mutex_exit(&os->os_user_ptr_lock);
1005 		tb = os->os_sa->sa_user_table;
1006 		mutex_exit(&os->os_sa->sa_lock);
1007 		*user_table = tb;
1008 		return (0);
1009 	}
1010 
1011 	sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP);
1012 	mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL);
1013 	sa->sa_master_obj = sa_obj;
1014 
1015 	os->os_sa = sa;
1016 	mutex_enter(&sa->sa_lock);
1017 	mutex_exit(&os->os_user_ptr_lock);
1018 	avl_create(&sa->sa_layout_num_tree, layout_num_compare,
1019 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_num_node));
1020 	avl_create(&sa->sa_layout_hash_tree, layout_hash_compare,
1021 	    sizeof (sa_lot_t), offsetof(sa_lot_t, lot_hash_node));
1022 
1023 	if (sa_obj) {
1024 		error = zap_lookup(os, sa_obj, SA_LAYOUTS,
1025 		    8, 1, &sa->sa_layout_attr_obj);
1026 		if (error != 0 && error != ENOENT)
1027 			goto fail;
1028 		error = zap_lookup(os, sa_obj, SA_REGISTRY,
1029 		    8, 1, &sa->sa_reg_attr_obj);
1030 		if (error != 0 && error != ENOENT)
1031 			goto fail;
1032 	}
1033 
1034 	if ((error = sa_attr_table_setup(os, reg_attrs, count)) != 0)
1035 		goto fail;
1036 
1037 	if (sa->sa_layout_attr_obj != 0) {
1038 		uint64_t layout_count;
1039 
1040 		error = zap_count(os, sa->sa_layout_attr_obj,
1041 		    &layout_count);
1042 
1043 		/*
1044 		 * Layout number count should be > 0
1045 		 */
1046 		if (error || (error == 0 && layout_count == 0)) {
1047 			if (error == 0)
1048 				error = SET_ERROR(EINVAL);
1049 			goto fail;
1050 		}
1051 
1052 		for (zap_cursor_init(&zc, os, sa->sa_layout_attr_obj);
1053 		    (error = zap_cursor_retrieve(&zc, &za)) == 0;
1054 		    zap_cursor_advance(&zc)) {
1055 			sa_attr_type_t *lot_attrs;
1056 			uint64_t lot_num;
1057 
1058 			lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) *
1059 			    za.za_num_integers, KM_SLEEP);
1060 
1061 			if ((error = (zap_lookup(os, sa->sa_layout_attr_obj,
1062 			    za.za_name, 2, za.za_num_integers,
1063 			    lot_attrs))) != 0) {
1064 				kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1065 				    za.za_num_integers);
1066 				break;
1067 			}
1068 			VERIFY(ddi_strtoull(za.za_name, NULL, 10,
1069 			    (unsigned long long *)&lot_num) == 0);
1070 
1071 			(void) sa_add_layout_entry(os, lot_attrs,
1072 			    za.za_num_integers, lot_num,
1073 			    sa_layout_info_hash(lot_attrs,
1074 			    za.za_num_integers), B_FALSE, NULL);
1075 			kmem_free(lot_attrs, sizeof (sa_attr_type_t) *
1076 			    za.za_num_integers);
1077 		}
1078 		zap_cursor_fini(&zc);
1079 
1080 		/*
1081 		 * Make sure layout count matches number of entries added
1082 		 * to AVL tree
1083 		 */
1084 		if (avl_numnodes(&sa->sa_layout_num_tree) != layout_count) {
1085 			ASSERT(error != 0);
1086 			goto fail;
1087 		}
1088 	}
1089 
1090 	/* Add special layout number for old ZNODES */
1091 	if (ostype == DMU_OST_ZFS) {
1092 		(void) sa_add_layout_entry(os, sa_legacy_zpl_layout,
1093 		    sa_legacy_attr_count, 0,
1094 		    sa_layout_info_hash(sa_legacy_zpl_layout,
1095 		    sa_legacy_attr_count), B_FALSE, NULL);
1096 
1097 		(void) sa_add_layout_entry(os, sa_dummy_zpl_layout, 0, 1,
1098 		    0, B_FALSE, NULL);
1099 	}
1100 	*user_table = os->os_sa->sa_user_table;
1101 	mutex_exit(&sa->sa_lock);
1102 	return (0);
1103 fail:
1104 	os->os_sa = NULL;
1105 	sa_free_attr_table(sa);
1106 	if (sa->sa_user_table)
1107 		kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1108 	mutex_exit(&sa->sa_lock);
1109 	avl_destroy(&sa->sa_layout_hash_tree);
1110 	avl_destroy(&sa->sa_layout_num_tree);
1111 	mutex_destroy(&sa->sa_lock);
1112 	kmem_free(sa, sizeof (sa_os_t));
1113 	return ((error == ECKSUM) ? EIO : error);
1114 }
1115 
1116 void
1117 sa_tear_down(objset_t *os)
1118 {
1119 	sa_os_t *sa = os->os_sa;
1120 	sa_lot_t *layout;
1121 	void *cookie;
1122 
1123 	kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
1124 
1125 	/* Free up attr table */
1126 
1127 	sa_free_attr_table(sa);
1128 
1129 	cookie = NULL;
1130 	while (layout = avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie)) {
1131 		sa_idx_tab_t *tab;
1132 		while (tab = list_head(&layout->lot_idx_tab)) {
1133 			ASSERT(refcount_count(&tab->sa_refcount));
1134 			sa_idx_tab_rele(os, tab);
1135 		}
1136 	}
1137 
1138 	cookie = NULL;
1139 	while (layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie)) {
1140 		kmem_free(layout->lot_attrs,
1141 		    sizeof (sa_attr_type_t) * layout->lot_attr_count);
1142 		kmem_free(layout, sizeof (sa_lot_t));
1143 	}
1144 
1145 	avl_destroy(&sa->sa_layout_hash_tree);
1146 	avl_destroy(&sa->sa_layout_num_tree);
1147 	mutex_destroy(&sa->sa_lock);
1148 
1149 	kmem_free(sa, sizeof (sa_os_t));
1150 	os->os_sa = NULL;
1151 }
1152 
1153 void
1154 sa_build_idx_tab(void *hdr, void *attr_addr, sa_attr_type_t attr,
1155     uint16_t length, int length_idx, boolean_t var_length, void *userp)
1156 {
1157 	sa_idx_tab_t *idx_tab = userp;
1158 
1159 	if (var_length) {
1160 		ASSERT(idx_tab->sa_variable_lengths);
1161 		idx_tab->sa_variable_lengths[length_idx] = length;
1162 	}
1163 	TOC_ATTR_ENCODE(idx_tab->sa_idx_tab[attr], length_idx,
1164 	    (uint32_t)((uintptr_t)attr_addr - (uintptr_t)hdr));
1165 }
1166 
1167 static void
1168 sa_attr_iter(objset_t *os, sa_hdr_phys_t *hdr, dmu_object_type_t type,
1169     sa_iterfunc_t func, sa_lot_t *tab, void *userp)
1170 {
1171 	void *data_start;
1172 	sa_lot_t *tb = tab;
1173 	sa_lot_t search;
1174 	avl_index_t loc;
1175 	sa_os_t *sa = os->os_sa;
1176 	int i;
1177 	uint16_t *length_start = NULL;
1178 	uint8_t length_idx = 0;
1179 
1180 	if (tab == NULL) {
1181 		search.lot_num = SA_LAYOUT_NUM(hdr, type);
1182 		tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1183 		ASSERT(tb);
1184 	}
1185 
1186 	if (IS_SA_BONUSTYPE(type)) {
1187 		data_start = (void *)P2ROUNDUP(((uintptr_t)hdr +
1188 		    offsetof(sa_hdr_phys_t, sa_lengths) +
1189 		    (sizeof (uint16_t) * tb->lot_var_sizes)), 8);
1190 		length_start = hdr->sa_lengths;
1191 	} else {
1192 		data_start = hdr;
1193 	}
1194 
1195 	for (i = 0; i != tb->lot_attr_count; i++) {
1196 		int attr_length, reg_length;
1197 		uint8_t idx_len;
1198 
1199 		reg_length = sa->sa_attr_table[tb->lot_attrs[i]].sa_length;
1200 		if (reg_length) {
1201 			attr_length = reg_length;
1202 			idx_len = 0;
1203 		} else {
1204 			attr_length = length_start[length_idx];
1205 			idx_len = length_idx++;
1206 		}
1207 
1208 		func(hdr, data_start, tb->lot_attrs[i], attr_length,
1209 		    idx_len, reg_length == 0 ? B_TRUE : B_FALSE, userp);
1210 
1211 		data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
1212 		    attr_length), 8);
1213 	}
1214 }
1215 
1216 /*ARGSUSED*/
1217 void
1218 sa_byteswap_cb(void *hdr, void *attr_addr, sa_attr_type_t attr,
1219     uint16_t length, int length_idx, boolean_t variable_length, void *userp)
1220 {
1221 	sa_handle_t *hdl = userp;
1222 	sa_os_t *sa = hdl->sa_os->os_sa;
1223 
1224 	sa_bswap_table[sa->sa_attr_table[attr].sa_byteswap](attr_addr, length);
1225 }
1226 
1227 void
1228 sa_byteswap(sa_handle_t *hdl, sa_buf_type_t buftype)
1229 {
1230 	sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1231 	dmu_buf_impl_t *db;
1232 	sa_os_t *sa = hdl->sa_os->os_sa;
1233 	int num_lengths = 1;
1234 	int i;
1235 
1236 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1237 	if (sa_hdr_phys->sa_magic == SA_MAGIC)
1238 		return;
1239 
1240 	db = SA_GET_DB(hdl, buftype);
1241 
1242 	if (buftype == SA_SPILL) {
1243 		arc_release(db->db_buf, NULL);
1244 		arc_buf_thaw(db->db_buf);
1245 	}
1246 
1247 	sa_hdr_phys->sa_magic = BSWAP_32(sa_hdr_phys->sa_magic);
1248 	sa_hdr_phys->sa_layout_info = BSWAP_16(sa_hdr_phys->sa_layout_info);
1249 
1250 	/*
1251 	 * Determine number of variable lenghts in header
1252 	 * The standard 8 byte header has one for free and a
1253 	 * 16 byte header would have 4 + 1;
1254 	 */
1255 	if (SA_HDR_SIZE(sa_hdr_phys) > 8)
1256 		num_lengths += (SA_HDR_SIZE(sa_hdr_phys) - 8) >> 1;
1257 	for (i = 0; i != num_lengths; i++)
1258 		sa_hdr_phys->sa_lengths[i] =
1259 		    BSWAP_16(sa_hdr_phys->sa_lengths[i]);
1260 
1261 	sa_attr_iter(hdl->sa_os, sa_hdr_phys, DMU_OT_SA,
1262 	    sa_byteswap_cb, NULL, hdl);
1263 
1264 	if (buftype == SA_SPILL)
1265 		arc_buf_freeze(((dmu_buf_impl_t *)hdl->sa_spill)->db_buf);
1266 }
1267 
1268 static int
1269 sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)
1270 {
1271 	sa_hdr_phys_t *sa_hdr_phys;
1272 	dmu_buf_impl_t *db = SA_GET_DB(hdl, buftype);
1273 	dmu_object_type_t bonustype = SA_BONUSTYPE_FROM_DB(db);
1274 	sa_os_t *sa = hdl->sa_os->os_sa;
1275 	sa_idx_tab_t *idx_tab;
1276 
1277 	sa_hdr_phys = SA_GET_HDR(hdl, buftype);
1278 
1279 	mutex_enter(&sa->sa_lock);
1280 
1281 	/* Do we need to byteswap? */
1282 
1283 	/* only check if not old znode */
1284 	if (IS_SA_BONUSTYPE(bonustype) && sa_hdr_phys->sa_magic != SA_MAGIC &&
1285 	    sa_hdr_phys->sa_magic != 0) {
1286 		VERIFY(BSWAP_32(sa_hdr_phys->sa_magic) == SA_MAGIC);
1287 		sa_byteswap(hdl, buftype);
1288 	}
1289 
1290 	idx_tab = sa_find_idx_tab(hdl->sa_os, bonustype, sa_hdr_phys);
1291 
1292 	if (buftype == SA_BONUS)
1293 		hdl->sa_bonus_tab = idx_tab;
1294 	else
1295 		hdl->sa_spill_tab = idx_tab;
1296 
1297 	mutex_exit(&sa->sa_lock);
1298 	return (0);
1299 }
1300 
1301 /*ARGSUSED*/
1302 static void
1303 sa_evict_sync(void *dbu)
1304 {
1305 	panic("evicting sa dbuf\n");
1306 }
1307 
1308 static void
1309 sa_idx_tab_rele(objset_t *os, void *arg)
1310 {
1311 	sa_os_t *sa = os->os_sa;
1312 	sa_idx_tab_t *idx_tab = arg;
1313 
1314 	if (idx_tab == NULL)
1315 		return;
1316 
1317 	mutex_enter(&sa->sa_lock);
1318 	if (refcount_remove(&idx_tab->sa_refcount, NULL) == 0) {
1319 		list_remove(&idx_tab->sa_layout->lot_idx_tab, idx_tab);
1320 		if (idx_tab->sa_variable_lengths)
1321 			kmem_free(idx_tab->sa_variable_lengths,
1322 			    sizeof (uint16_t) *
1323 			    idx_tab->sa_layout->lot_var_sizes);
1324 		refcount_destroy(&idx_tab->sa_refcount);
1325 		kmem_free(idx_tab->sa_idx_tab,
1326 		    sizeof (uint32_t) * sa->sa_num_attrs);
1327 		kmem_free(idx_tab, sizeof (sa_idx_tab_t));
1328 	}
1329 	mutex_exit(&sa->sa_lock);
1330 }
1331 
1332 static void
1333 sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
1334 {
1335 	sa_os_t *sa = os->os_sa;
1336 
1337 	ASSERT(MUTEX_HELD(&sa->sa_lock));
1338 	(void) refcount_add(&idx_tab->sa_refcount, NULL);
1339 }
1340 
1341 void
1342 sa_handle_destroy(sa_handle_t *hdl)
1343 {
1344 	dmu_buf_t *db = hdl->sa_bonus;
1345 
1346 	mutex_enter(&hdl->sa_lock);
1347 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
1348 
1349 	if (hdl->sa_bonus_tab)
1350 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
1351 
1352 	if (hdl->sa_spill_tab)
1353 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
1354 
1355 	dmu_buf_rele(hdl->sa_bonus, NULL);
1356 
1357 	if (hdl->sa_spill)
1358 		dmu_buf_rele((dmu_buf_t *)hdl->sa_spill, NULL);
1359 	mutex_exit(&hdl->sa_lock);
1360 
1361 	kmem_cache_free(sa_cache, hdl);
1362 }
1363 
1364 int
1365 sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
1366     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1367 {
1368 	int error = 0;
1369 	dmu_object_info_t doi;
1370 	sa_handle_t *handle = NULL;
1371 
1372 #ifdef ZFS_DEBUG
1373 	dmu_object_info_from_db(db, &doi);
1374 	ASSERT(doi.doi_bonus_type == DMU_OT_SA ||
1375 	    doi.doi_bonus_type == DMU_OT_ZNODE);
1376 #endif
1377 	/* find handle, if it exists */
1378 	/* if one doesn't exist then create a new one, and initialize it */
1379 
1380 	if (hdl_type == SA_HDL_SHARED)
1381 		handle = dmu_buf_get_user(db);
1382 
1383 	if (handle == NULL) {
1384 		sa_handle_t *winner = NULL;
1385 
1386 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
1387 		handle->sa_dbu.dbu_evict_func_sync = NULL;
1388 		handle->sa_dbu.dbu_evict_func_async = NULL;
1389 		handle->sa_userp = userp;
1390 		handle->sa_bonus = db;
1391 		handle->sa_os = os;
1392 		handle->sa_spill = NULL;
1393 		handle->sa_bonus_tab = NULL;
1394 		handle->sa_spill_tab = NULL;
1395 
1396 		error = sa_build_index(handle, SA_BONUS);
1397 
1398 		if (hdl_type == SA_HDL_SHARED) {
1399 			dmu_buf_init_user(&handle->sa_dbu, sa_evict_sync, NULL,
1400 			    NULL);
1401 			winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
1402 		}
1403 
1404 		if (winner != NULL) {
1405 			kmem_cache_free(sa_cache, handle);
1406 			handle = winner;
1407 		}
1408 	}
1409 	*handlepp = handle;
1410 
1411 	return (error);
1412 }
1413 
1414 int
1415 sa_handle_get(objset_t *objset, uint64_t objid, void *userp,
1416     sa_handle_type_t hdl_type, sa_handle_t **handlepp)
1417 {
1418 	dmu_buf_t *db;
1419 	int error;
1420 
1421 	if (error = dmu_bonus_hold(objset, objid, NULL, &db))
1422 		return (error);
1423 
1424 	return (sa_handle_get_from_db(objset, db, userp, hdl_type,
1425 	    handlepp));
1426 }
1427 
1428 int
1429 sa_buf_hold(objset_t *objset, uint64_t obj_num, void *tag, dmu_buf_t **db)
1430 {
1431 	return (dmu_bonus_hold(objset, obj_num, tag, db));
1432 }
1433 
1434 void
1435 sa_buf_rele(dmu_buf_t *db, void *tag)
1436 {
1437 	dmu_buf_rele(db, tag);
1438 }
1439 
1440 int
1441 sa_lookup_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count)
1442 {
1443 	ASSERT(hdl);
1444 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1445 	return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));
1446 }
1447 
1448 int
1449 sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)
1450 {
1451 	int error;
1452 	sa_bulk_attr_t bulk;
1453 
1454 	bulk.sa_attr = attr;
1455 	bulk.sa_data = buf;
1456 	bulk.sa_length = buflen;
1457 	bulk.sa_data_func = NULL;
1458 
1459 	ASSERT(hdl);
1460 	mutex_enter(&hdl->sa_lock);
1461 	error = sa_lookup_impl(hdl, &bulk, 1);
1462 	mutex_exit(&hdl->sa_lock);
1463 	return (error);
1464 }
1465 
1466 #ifdef _KERNEL
1467 int
1468 sa_lookup_uio(sa_handle_t *hdl, sa_attr_type_t attr, uio_t *uio)
1469 {
1470 	int error;
1471 	sa_bulk_attr_t bulk;
1472 
1473 	bulk.sa_data = NULL;
1474 	bulk.sa_attr = attr;
1475 	bulk.sa_data_func = NULL;
1476 
1477 	ASSERT(hdl);
1478 
1479 	mutex_enter(&hdl->sa_lock);
1480 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) == 0) {
1481 		error = uiomove((void *)bulk.sa_addr, MIN(bulk.sa_size,
1482 		    uio->uio_resid), UIO_READ, uio);
1483 	}
1484 	mutex_exit(&hdl->sa_lock);
1485 	return (error);
1486 
1487 }
1488 #endif
1489 
1490 static sa_idx_tab_t *
1491 sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, sa_hdr_phys_t *hdr)
1492 {
1493 	sa_idx_tab_t *idx_tab;
1494 	sa_os_t *sa = os->os_sa;
1495 	sa_lot_t *tb, search;
1496 	avl_index_t loc;
1497 
1498 	/*
1499 	 * Deterimine layout number.  If SA node and header == 0 then
1500 	 * force the index table to the dummy "1" empty layout.
1501 	 *
1502 	 * The layout number would only be zero for a newly created file
1503 	 * that has not added any attributes yet, or with crypto enabled which
1504 	 * doesn't write any attributes to the bonus buffer.
1505 	 */
1506 
1507 	search.lot_num = SA_LAYOUT_NUM(hdr, bonustype);
1508 
1509 	tb = avl_find(&sa->sa_layout_num_tree, &search, &loc);
1510 
1511 	/* Verify header size is consistent with layout information */
1512 	ASSERT(tb);
1513 	ASSERT(IS_SA_BONUSTYPE(bonustype) &&
1514 	    SA_HDR_SIZE_MATCH_LAYOUT(hdr, tb) || !IS_SA_BONUSTYPE(bonustype) ||
1515 	    (IS_SA_BONUSTYPE(bonustype) && hdr->sa_layout_info == 0));
1516 
1517 	/*
1518 	 * See if any of the already existing TOC entries can be reused?
1519 	 */
1520 
1521 	for (idx_tab = list_head(&tb->lot_idx_tab); idx_tab;
1522 	    idx_tab = list_next(&tb->lot_idx_tab, idx_tab)) {
1523 		boolean_t valid_idx = B_TRUE;
1524 		int i;
1525 
1526 		if (tb->lot_var_sizes != 0 &&
1527 		    idx_tab->sa_variable_lengths != NULL) {
1528 			for (i = 0; i != tb->lot_var_sizes; i++) {
1529 				if (hdr->sa_lengths[i] !=
1530 				    idx_tab->sa_variable_lengths[i]) {
1531 					valid_idx = B_FALSE;
1532 					break;
1533 				}
1534 			}
1535 		}
1536 		if (valid_idx) {
1537 			sa_idx_tab_hold(os, idx_tab);
1538 			return (idx_tab);
1539 		}
1540 	}
1541 
1542 	/* No such luck, create a new entry */
1543 	idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP);
1544 	idx_tab->sa_idx_tab =
1545 	    kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP);
1546 	idx_tab->sa_layout = tb;
1547 	refcount_create(&idx_tab->sa_refcount);
1548 	if (tb->lot_var_sizes)
1549 		idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) *
1550 		    tb->lot_var_sizes, KM_SLEEP);
1551 
1552 	sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab,
1553 	    tb, idx_tab);
1554 	sa_idx_tab_hold(os, idx_tab);   /* one hold for consumer */
1555 	sa_idx_tab_hold(os, idx_tab);	/* one for layout */
1556 	list_insert_tail(&tb->lot_idx_tab, idx_tab);
1557 	return (idx_tab);
1558 }
1559 
1560 void
1561 sa_default_locator(void **dataptr, uint32_t *len, uint32_t total_len,
1562     boolean_t start, void *userdata)
1563 {
1564 	ASSERT(start);
1565 
1566 	*dataptr = userdata;
1567 	*len = total_len;
1568 }
1569 
1570 static void
1571 sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)
1572 {
1573 	uint64_t attr_value = 0;
1574 	sa_os_t *sa = hdl->sa_os->os_sa;
1575 	sa_attr_table_t *tb = sa->sa_attr_table;
1576 	int i;
1577 
1578 	mutex_enter(&sa->sa_lock);
1579 
1580 	if (!sa->sa_need_attr_registration || sa->sa_master_obj == NULL) {
1581 		mutex_exit(&sa->sa_lock);
1582 		return;
1583 	}
1584 
1585 	if (sa->sa_reg_attr_obj == NULL) {
1586 		sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,
1587 		    DMU_OT_SA_ATTR_REGISTRATION,
1588 		    sa->sa_master_obj, SA_REGISTRY, tx);
1589 	}
1590 	for (i = 0; i != sa->sa_num_attrs; i++) {
1591 		if (sa->sa_attr_table[i].sa_registered)
1592 			continue;
1593 		ATTR_ENCODE(attr_value, tb[i].sa_attr, tb[i].sa_length,
1594 		    tb[i].sa_byteswap);
1595 		VERIFY(0 == zap_update(hdl->sa_os, sa->sa_reg_attr_obj,
1596 		    tb[i].sa_name, 8, 1, &attr_value, tx));
1597 		tb[i].sa_registered = B_TRUE;
1598 	}
1599 	sa->sa_need_attr_registration = B_FALSE;
1600 	mutex_exit(&sa->sa_lock);
1601 }
1602 
1603 /*
1604  * Replace all attributes with attributes specified in template.
1605  * If dnode had a spill buffer then those attributes will be
1606  * also be replaced, possibly with just an empty spill block
1607  *
1608  * This interface is intended to only be used for bulk adding of
1609  * attributes for a new file.  It will also be used by the ZPL
1610  * when converting and old formatted znode to native SA support.
1611  */
1612 int
1613 sa_replace_all_by_template_locked(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1614     int attr_count, dmu_tx_t *tx)
1615 {
1616 	sa_os_t *sa = hdl->sa_os->os_sa;
1617 
1618 	if (sa->sa_need_attr_registration)
1619 		sa_attr_register_sync(hdl, tx);
1620 	return (sa_build_layouts(hdl, attr_desc, attr_count, tx));
1621 }
1622 
1623 int
1624 sa_replace_all_by_template(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,
1625     int attr_count, dmu_tx_t *tx)
1626 {
1627 	int error;
1628 
1629 	mutex_enter(&hdl->sa_lock);
1630 	error = sa_replace_all_by_template_locked(hdl, attr_desc,
1631 	    attr_count, tx);
1632 	mutex_exit(&hdl->sa_lock);
1633 	return (error);
1634 }
1635 
1636 /*
1637  * Add/remove a single attribute or replace a variable-sized attribute value
1638  * with a value of a different size, and then rewrite the entire set
1639  * of attributes.
1640  * Same-length attribute value replacement (including fixed-length attributes)
1641  * is handled more efficiently by the upper layers.
1642  */
1643 static int
1644 sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
1645     sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
1646     uint16_t buflen, dmu_tx_t *tx)
1647 {
1648 	sa_os_t *sa = hdl->sa_os->os_sa;
1649 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus;
1650 	dnode_t *dn;
1651 	sa_bulk_attr_t *attr_desc;
1652 	void *old_data[2];
1653 	int bonus_attr_count = 0;
1654 	int bonus_data_size = 0;
1655 	int spill_data_size = 0;
1656 	int spill_attr_count = 0;
1657 	int error;
1658 	uint16_t length, reg_length;
1659 	int i, j, k, length_idx;
1660 	sa_hdr_phys_t *hdr;
1661 	sa_idx_tab_t *idx_tab;
1662 	int attr_count;
1663 	int count;
1664 
1665 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1666 
1667 	/* First make of copy of the old data */
1668 
1669 	DB_DNODE_ENTER(db);
1670 	dn = DB_DNODE(db);
1671 	if (dn->dn_bonuslen != 0) {
1672 		bonus_data_size = hdl->sa_bonus->db_size;
1673 		old_data[0] = kmem_alloc(bonus_data_size, KM_SLEEP);
1674 		bcopy(hdl->sa_bonus->db_data, old_data[0],
1675 		    hdl->sa_bonus->db_size);
1676 		bonus_attr_count = hdl->sa_bonus_tab->sa_layout->lot_attr_count;
1677 	} else {
1678 		old_data[0] = NULL;
1679 	}
1680 	DB_DNODE_EXIT(db);
1681 
1682 	/* Bring spill buffer online if it isn't currently */
1683 
1684 	if ((error = sa_get_spill(hdl)) == 0) {
1685 		spill_data_size = hdl->sa_spill->db_size;
1686 		old_data[1] = kmem_alloc(spill_data_size, KM_SLEEP);
1687 		bcopy(hdl->sa_spill->db_data, old_data[1],
1688 		    hdl->sa_spill->db_size);
1689 		spill_attr_count =
1690 		    hdl->sa_spill_tab->sa_layout->lot_attr_count;
1691 	} else if (error && error != ENOENT) {
1692 		if (old_data[0])
1693 			kmem_free(old_data[0], bonus_data_size);
1694 		return (error);
1695 	} else {
1696 		old_data[1] = NULL;
1697 	}
1698 
1699 	/* build descriptor of all attributes */
1700 
1701 	attr_count = bonus_attr_count + spill_attr_count;
1702 	if (action == SA_ADD)
1703 		attr_count++;
1704 	else if (action == SA_REMOVE)
1705 		attr_count--;
1706 
1707 	attr_desc = kmem_zalloc(sizeof (sa_bulk_attr_t) * attr_count, KM_SLEEP);
1708 
1709 	/*
1710 	 * loop through bonus and spill buffer if it exists, and
1711 	 * build up new attr_descriptor to reset the attributes
1712 	 */
1713 	k = j = 0;
1714 	count = bonus_attr_count;
1715 	hdr = SA_GET_HDR(hdl, SA_BONUS);
1716 	idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);
1717 	for (; k != 2; k++) {
1718 		/*
1719 		 * Iterate over each attribute in layout.  Fetch the
1720 		 * size of variable-length attributes needing rewrite
1721 		 * from sa_lengths[].
1722 		 */
1723 		for (i = 0, length_idx = 0; i != count; i++) {
1724 			sa_attr_type_t attr;
1725 
1726 			attr = idx_tab->sa_layout->lot_attrs[i];
1727 			reg_length = SA_REGISTERED_LEN(sa, attr);
1728 			if (reg_length == 0) {
1729 				length = hdr->sa_lengths[length_idx];
1730 				length_idx++;
1731 			} else {
1732 				length = reg_length;
1733 			}
1734 			if (attr == newattr) {
1735 				/*
1736 				 * There is nothing to do for SA_REMOVE,
1737 				 * so it is just skipped.
1738 				 */
1739 				if (action == SA_REMOVE)
1740 					continue;
1741 
1742 				/*
1743 				 * Duplicate attributes are not allowed, so the
1744 				 * action can not be SA_ADD here.
1745 				 */
1746 				ASSERT3S(action, ==, SA_REPLACE);
1747 
1748 				/*
1749 				 * Only a variable-sized attribute can be
1750 				 * replaced here, and its size must be changing.
1751 				 */
1752 				ASSERT3U(reg_length, ==, 0);
1753 				ASSERT3U(length, !=, buflen);
1754 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1755 				    locator, datastart, buflen);
1756 			} else {
1757 				SA_ADD_BULK_ATTR(attr_desc, j, attr,
1758 				    NULL, (void *)
1759 				    (TOC_OFF(idx_tab->sa_idx_tab[attr]) +
1760 				    (uintptr_t)old_data[k]), length);
1761 			}
1762 		}
1763 		if (k == 0 && hdl->sa_spill) {
1764 			hdr = SA_GET_HDR(hdl, SA_SPILL);
1765 			idx_tab = SA_IDX_TAB_GET(hdl, SA_SPILL);
1766 			count = spill_attr_count;
1767 		} else {
1768 			break;
1769 		}
1770 	}
1771 	if (action == SA_ADD) {
1772 		reg_length = SA_REGISTERED_LEN(sa, newattr);
1773 		IMPLY(reg_length != 0, reg_length == buflen);
1774 		SA_ADD_BULK_ATTR(attr_desc, j, newattr, locator,
1775 		    datastart, buflen);
1776 	}
1777 	ASSERT3U(j, ==, attr_count);
1778 
1779 	error = sa_build_layouts(hdl, attr_desc, attr_count, tx);
1780 
1781 	if (old_data[0])
1782 		kmem_free(old_data[0], bonus_data_size);
1783 	if (old_data[1])
1784 		kmem_free(old_data[1], spill_data_size);
1785 	kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
1786 
1787 	return (error);
1788 }
1789 
1790 static int
1791 sa_bulk_update_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,
1792     dmu_tx_t *tx)
1793 {
1794 	int error;
1795 	sa_os_t *sa = hdl->sa_os->os_sa;
1796 	dmu_object_type_t bonustype;
1797 
1798 	bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));
1799 
1800 	ASSERT(hdl);
1801 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1802 
1803 	/* sync out registration table if necessary */
1804 	if (sa->sa_need_attr_registration)
1805 		sa_attr_register_sync(hdl, tx);
1806 
1807 	error = sa_attr_op(hdl, bulk, count, SA_UPDATE, tx);
1808 	if (error == 0 && !IS_SA_BONUSTYPE(bonustype) && sa->sa_update_cb)
1809 		sa->sa_update_cb(hdl, tx);
1810 
1811 	return (error);
1812 }
1813 
1814 /*
1815  * update or add new attribute
1816  */
1817 int
1818 sa_update(sa_handle_t *hdl, sa_attr_type_t type,
1819     void *buf, uint32_t buflen, dmu_tx_t *tx)
1820 {
1821 	int error;
1822 	sa_bulk_attr_t bulk;
1823 
1824 	bulk.sa_attr = type;
1825 	bulk.sa_data_func = NULL;
1826 	bulk.sa_length = buflen;
1827 	bulk.sa_data = buf;
1828 
1829 	mutex_enter(&hdl->sa_lock);
1830 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
1831 	mutex_exit(&hdl->sa_lock);
1832 	return (error);
1833 }
1834 
1835 int
1836 sa_update_from_cb(sa_handle_t *hdl, sa_attr_type_t attr,
1837     uint32_t buflen, sa_data_locator_t *locator, void *userdata, dmu_tx_t *tx)
1838 {
1839 	int error;
1840 	sa_bulk_attr_t bulk;
1841 
1842 	bulk.sa_attr = attr;
1843 	bulk.sa_data = userdata;
1844 	bulk.sa_data_func = locator;
1845 	bulk.sa_length = buflen;
1846 
1847 	mutex_enter(&hdl->sa_lock);
1848 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
1849 	mutex_exit(&hdl->sa_lock);
1850 	return (error);
1851 }
1852 
1853 /*
1854  * Return size of an attribute
1855  */
1856 
1857 int
1858 sa_size(sa_handle_t *hdl, sa_attr_type_t attr, int *size)
1859 {
1860 	sa_bulk_attr_t bulk;
1861 	int error;
1862 
1863 	bulk.sa_data = NULL;
1864 	bulk.sa_attr = attr;
1865 	bulk.sa_data_func = NULL;
1866 
1867 	ASSERT(hdl);
1868 	mutex_enter(&hdl->sa_lock);
1869 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) != 0) {
1870 		mutex_exit(&hdl->sa_lock);
1871 		return (error);
1872 	}
1873 	*size = bulk.sa_size;
1874 
1875 	mutex_exit(&hdl->sa_lock);
1876 	return (0);
1877 }
1878 
1879 int
1880 sa_bulk_lookup_locked(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
1881 {
1882 	ASSERT(hdl);
1883 	ASSERT(MUTEX_HELD(&hdl->sa_lock));
1884 	return (sa_lookup_impl(hdl, attrs, count));
1885 }
1886 
1887 int
1888 sa_bulk_lookup(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)
1889 {
1890 	int error;
1891 
1892 	ASSERT(hdl);
1893 	mutex_enter(&hdl->sa_lock);
1894 	error = sa_bulk_lookup_locked(hdl, attrs, count);
1895 	mutex_exit(&hdl->sa_lock);
1896 	return (error);
1897 }
1898 
1899 int
1900 sa_bulk_update(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count, dmu_tx_t *tx)
1901 {
1902 	int error;
1903 
1904 	ASSERT(hdl);
1905 	mutex_enter(&hdl->sa_lock);
1906 	error = sa_bulk_update_impl(hdl, attrs, count, tx);
1907 	mutex_exit(&hdl->sa_lock);
1908 	return (error);
1909 }
1910 
1911 int
1912 sa_remove(sa_handle_t *hdl, sa_attr_type_t attr, dmu_tx_t *tx)
1913 {
1914 	int error;
1915 
1916 	mutex_enter(&hdl->sa_lock);
1917 	error = sa_modify_attrs(hdl, attr, SA_REMOVE, NULL,
1918 	    NULL, 0, tx);
1919 	mutex_exit(&hdl->sa_lock);
1920 	return (error);
1921 }
1922 
1923 void
1924 sa_object_info(sa_handle_t *hdl, dmu_object_info_t *doi)
1925 {
1926 	dmu_object_info_from_db((dmu_buf_t *)hdl->sa_bonus, doi);
1927 }
1928 
1929 void
1930 sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)
1931 {
1932 	dmu_object_size_from_db((dmu_buf_t *)hdl->sa_bonus,
1933 	    blksize, nblocks);
1934 }
1935 
1936 void
1937 sa_set_userp(sa_handle_t *hdl, void *ptr)
1938 {
1939 	hdl->sa_userp = ptr;
1940 }
1941 
1942 dmu_buf_t *
1943 sa_get_db(sa_handle_t *hdl)
1944 {
1945 	return ((dmu_buf_t *)hdl->sa_bonus);
1946 }
1947 
1948 void *
1949 sa_get_userdata(sa_handle_t *hdl)
1950 {
1951 	return (hdl->sa_userp);
1952 }
1953 
1954 void
1955 sa_register_update_callback_locked(objset_t *os, sa_update_cb_t *func)
1956 {
1957 	ASSERT(MUTEX_HELD(&os->os_sa->sa_lock));
1958 	os->os_sa->sa_update_cb = func;
1959 }
1960 
1961 void
1962 sa_register_update_callback(objset_t *os, sa_update_cb_t *func)
1963 {
1964 
1965 	mutex_enter(&os->os_sa->sa_lock);
1966 	sa_register_update_callback_locked(os, func);
1967 	mutex_exit(&os->os_sa->sa_lock);
1968 }
1969 
1970 uint64_t
1971 sa_handle_object(sa_handle_t *hdl)
1972 {
1973 	return (hdl->sa_bonus->db_object);
1974 }
1975 
1976 boolean_t
1977 sa_enabled(objset_t *os)
1978 {
1979 	return (os->os_sa == NULL);
1980 }
1981 
1982 int
1983 sa_set_sa_object(objset_t *os, uint64_t sa_object)
1984 {
1985 	sa_os_t *sa = os->os_sa;
1986 
1987 	if (sa->sa_master_obj)
1988 		return (1);
1989 
1990 	sa->sa_master_obj = sa_object;
1991 
1992 	return (0);
1993 }
1994 
1995 int
1996 sa_hdrsize(void *arg)
1997 {
1998 	sa_hdr_phys_t *hdr = arg;
1999 
2000 	return (SA_HDR_SIZE(hdr));
2001 }
2002 
2003 void
2004 sa_handle_lock(sa_handle_t *hdl)
2005 {
2006 	ASSERT(hdl);
2007 	mutex_enter(&hdl->sa_lock);
2008 }
2009 
2010 void
2011 sa_handle_unlock(sa_handle_t *hdl)
2012 {
2013 	ASSERT(hdl);
2014 	mutex_exit(&hdl->sa_lock);
2015 }
2016