xref: /illumos-gate/usr/src/common/acl/acl_common.c (revision da6c28aa)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/avl.h>
31 #if defined(_KERNEL)
32 #include <sys/systm.h>
33 #include <sys/sysmacros.h>
34 #include <acl/acl_common.h>
35 #else
36 #include <errno.h>
37 #include <stdlib.h>
38 #include <stddef.h>
39 #include <strings.h>
40 #include <unistd.h>
41 #include <assert.h>
42 #include <grp.h>
43 #include <pwd.h>
44 #include <acl_common.h>
45 #define	ASSERT	assert
46 #endif
47 
48 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
49     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
50     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
51 
52 
53 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
54 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
55 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
56 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
57 
58 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
59 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
60 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
61 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
62 
63 #define	ACL_DELETE_SET_DENY			0x0000100
64 #define	ACL_DELETE_SET_ALLOW			0x0000200
65 #define	ACL_DELETE_ERR_DENY			0x0000400
66 #define	ACL_DELETE_ERR_ALLOW			0x0000800
67 
68 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
69 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
70 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
71 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
72 
73 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
74 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
75 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
76 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
77 
78 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
79 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
80 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
81 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
82 
83 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
84 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
85 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
86 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
87 
88 
89 #define	ACE_VALID_MASK_BITS (\
90     ACE_READ_DATA | \
91     ACE_LIST_DIRECTORY | \
92     ACE_WRITE_DATA | \
93     ACE_ADD_FILE | \
94     ACE_APPEND_DATA | \
95     ACE_ADD_SUBDIRECTORY | \
96     ACE_READ_NAMED_ATTRS | \
97     ACE_WRITE_NAMED_ATTRS | \
98     ACE_EXECUTE | \
99     ACE_DELETE_CHILD | \
100     ACE_READ_ATTRIBUTES | \
101     ACE_WRITE_ATTRIBUTES | \
102     ACE_DELETE | \
103     ACE_READ_ACL | \
104     ACE_WRITE_ACL | \
105     ACE_WRITE_OWNER | \
106     ACE_SYNCHRONIZE)
107 
108 #define	ACE_MASK_UNDEFINED			0x80000000
109 
110 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
111     ACE_DIRECTORY_INHERIT_ACE | \
112     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
113     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
114     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
115 
116 /*
117  * ACL conversion helpers
118  */
119 
120 typedef enum {
121 	ace_unused,
122 	ace_user_obj,
123 	ace_user,
124 	ace_group, /* includes GROUP and GROUP_OBJ */
125 	ace_other_obj
126 } ace_to_aent_state_t;
127 
128 typedef struct acevals {
129 	uid_t key;
130 	avl_node_t avl;
131 	uint32_t mask;
132 	uint32_t allowed;
133 	uint32_t denied;
134 	int aent_type;
135 } acevals_t;
136 
137 typedef struct ace_list {
138 	acevals_t user_obj;
139 	avl_tree_t user;
140 	int numusers;
141 	acevals_t group_obj;
142 	avl_tree_t group;
143 	int numgroups;
144 	acevals_t other_obj;
145 	uint32_t acl_mask;
146 	int hasmask;
147 	int dfacl_flag;
148 	ace_to_aent_state_t state;
149 	int seen; /* bitmask of all aclent_t a_type values seen */
150 } ace_list_t;
151 
152 ace_t trivial_acl[] = {
153 	{(uid_t)-1, 0, ACE_OWNER, ACE_ACCESS_DENIED_ACE_TYPE},
154 	{(uid_t)-1, ACE_WRITE_ACL|ACE_WRITE_OWNER|ACE_WRITE_ATTRIBUTES|
155 	    ACE_WRITE_NAMED_ATTRS, ACE_OWNER, ACE_ACCESS_ALLOWED_ACE_TYPE},
156 	{(uid_t)-1, 0, ACE_GROUP|ACE_IDENTIFIER_GROUP,
157 	    ACE_ACCESS_DENIED_ACE_TYPE},
158 	{(uid_t)-1, 0, ACE_GROUP|ACE_IDENTIFIER_GROUP,
159 	    ACE_ACCESS_ALLOWED_ACE_TYPE},
160 	{(uid_t)-1, ACE_WRITE_ACL|ACE_WRITE_OWNER| ACE_WRITE_ATTRIBUTES|
161 	    ACE_WRITE_NAMED_ATTRS, ACE_EVERYONE, ACE_ACCESS_DENIED_ACE_TYPE},
162 	{(uid_t)-1, ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
163 	    ACE_SYNCHRONIZE, ACE_EVERYONE, ACE_ACCESS_ALLOWED_ACE_TYPE}
164 };
165 
166 
167 void
168 adjust_ace_pair_common(void *pair, size_t access_off,
169     size_t pairsize, mode_t mode)
170 {
171 	char *datap = (char *)pair;
172 	uint32_t *amask0 = (uint32_t *)(uintptr_t)(datap + access_off);
173 	uint32_t *amask1 = (uint32_t *)(uintptr_t)(datap + pairsize +
174 	    access_off);
175 	if (mode & S_IROTH)
176 		*amask1 |= ACE_READ_DATA;
177 	else
178 		*amask0 |= ACE_READ_DATA;
179 	if (mode & S_IWOTH)
180 		*amask1 |= ACE_WRITE_DATA|ACE_APPEND_DATA;
181 	else
182 		*amask0 |= ACE_WRITE_DATA|ACE_APPEND_DATA;
183 	if (mode & S_IXOTH)
184 		*amask1 |= ACE_EXECUTE;
185 	else
186 		*amask0 |= ACE_EXECUTE;
187 }
188 
189 void
190 adjust_ace_pair(ace_t *pair, mode_t mode)
191 {
192 	adjust_ace_pair_common(pair, offsetof(ace_t, a_access_mask),
193 	    sizeof (ace_t), mode);
194 }
195 
196 /*
197  * ace_trivial:
198  * determine whether an ace_t acl is trivial
199  *
200  * Trivialness implies that the acl is composed of only
201  * owner, group, everyone entries.  ACL can't
202  * have read_acl denied, and write_owner/write_acl/write_attributes
203  * can only be owner@ entry.
204  */
205 int
206 ace_trivial_common(void *acep, int aclcnt,
207     uint64_t (*walk)(void *, uint64_t, int aclcnt,
208     uint16_t *, uint16_t *, uint32_t *))
209 {
210 	int owner_seen = 0;
211 	int group_seen = 0;
212 	int everyone_seen = 0;
213 	uint16_t flags;
214 	uint32_t mask;
215 	uint16_t type;
216 	uint64_t cookie = 0;
217 
218 	while (cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask)) {
219 		switch (flags & ACE_TYPE_FLAGS) {
220 		case ACE_OWNER:
221 			if (group_seen || everyone_seen)
222 				return (1);
223 			owner_seen++;
224 			break;
225 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
226 			if (everyone_seen || owner_seen == 0)
227 				return (1);
228 			group_seen++;
229 			break;
230 
231 		case ACE_EVERYONE:
232 			if (owner_seen == 0 || group_seen == 0)
233 				return (1);
234 			everyone_seen++;
235 			break;
236 		default:
237 			return (1);
238 
239 		}
240 
241 		if (flags & (ACE_FILE_INHERIT_ACE|
242 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
243 		    ACE_INHERIT_ONLY_ACE))
244 			return (1);
245 
246 		/*
247 		 * Special check for some special bits
248 		 *
249 		 * Don't allow anybody to deny reading basic
250 		 * attributes or a files ACL.
251 		 */
252 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
253 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
254 			return (1);
255 
256 		/*
257 		 * Allow on owner@ to allow
258 		 * write_acl/write_owner/write_attributes
259 		 */
260 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
261 		    (!(flags & ACE_OWNER) && (mask &
262 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL|ACE_WRITE_ATTRIBUTES))))
263 			return (1);
264 
265 	}
266 
267 	if ((owner_seen == 0) || (group_seen == 0) || (everyone_seen == 0))
268 		return (1);
269 
270 	return (0);
271 }
272 
273 uint64_t
274 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
275     uint16_t *type, uint32_t *mask)
276 {
277 	ace_t *acep = datap;
278 
279 	*flags = acep[cookie].a_flags;
280 	*type = acep[cookie].a_type;
281 	*mask = acep[cookie++].a_access_mask;
282 
283 	if (cookie > aclcnt)
284 		return (0);
285 	else
286 		return (cookie);
287 }
288 
289 int
290 ace_trivial(ace_t *acep, int aclcnt)
291 {
292 	return (ace_trivial_common(acep, aclcnt, ace_walk));
293 }
294 
295 /*
296  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
297  * v = Ptr to array/vector of objs
298  * n = # objs in the array
299  * s = size of each obj (must be multiples of a word size)
300  * f = ptr to function to compare two objs
301  *	returns (-1 = less than, 0 = equal, 1 = greater than
302  */
303 void
304 ksort(caddr_t v, int n, int s, int (*f)())
305 {
306 	int g, i, j, ii;
307 	unsigned int *p1, *p2;
308 	unsigned int tmp;
309 
310 	/* No work to do */
311 	if (v == NULL || n <= 1)
312 		return;
313 
314 	/* Sanity check on arguments */
315 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
316 	ASSERT(s > 0);
317 	for (g = n / 2; g > 0; g /= 2) {
318 		for (i = g; i < n; i++) {
319 			for (j = i - g; j >= 0 &&
320 			    (*f)(v + j * s, v + (j + g) * s) == 1;
321 			    j -= g) {
322 				p1 = (void *)(v + j * s);
323 				p2 = (void *)(v + (j + g) * s);
324 				for (ii = 0; ii < s / 4; ii++) {
325 					tmp = *p1;
326 					*p1++ = *p2;
327 					*p2++ = tmp;
328 				}
329 			}
330 		}
331 	}
332 }
333 
334 /*
335  * Compare two acls, all fields.  Returns:
336  * -1 (less than)
337  *  0 (equal)
338  * +1 (greater than)
339  */
340 int
341 cmp2acls(void *a, void *b)
342 {
343 	aclent_t *x = (aclent_t *)a;
344 	aclent_t *y = (aclent_t *)b;
345 
346 	/* Compare types */
347 	if (x->a_type < y->a_type)
348 		return (-1);
349 	if (x->a_type > y->a_type)
350 		return (1);
351 	/* Equal types; compare id's */
352 	if (x->a_id < y->a_id)
353 		return (-1);
354 	if (x->a_id > y->a_id)
355 		return (1);
356 	/* Equal ids; compare perms */
357 	if (x->a_perm < y->a_perm)
358 		return (-1);
359 	if (x->a_perm > y->a_perm)
360 		return (1);
361 	/* Totally equal */
362 	return (0);
363 }
364 
365 /*ARGSUSED*/
366 static void *
367 cacl_realloc(void *ptr, size_t size, size_t new_size)
368 {
369 #if defined(_KERNEL)
370 	void *tmp;
371 
372 	tmp = kmem_alloc(new_size, KM_SLEEP);
373 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
374 	kmem_free(ptr, size);
375 	return (tmp);
376 #else
377 	return (realloc(ptr, new_size));
378 #endif
379 }
380 
381 static int
382 cacl_malloc(void **ptr, size_t size)
383 {
384 #if defined(_KERNEL)
385 	*ptr = kmem_zalloc(size, KM_SLEEP);
386 	return (0);
387 #else
388 	*ptr = calloc(1, size);
389 	if (*ptr == NULL)
390 		return (errno);
391 
392 	return (0);
393 #endif
394 }
395 
396 /*ARGSUSED*/
397 static void
398 cacl_free(void *ptr, size_t size)
399 {
400 #if defined(_KERNEL)
401 	kmem_free(ptr, size);
402 #else
403 	free(ptr);
404 #endif
405 }
406 
407 acl_t *
408 acl_alloc(enum acl_type type)
409 {
410 	acl_t *aclp;
411 
412 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
413 		return (NULL);
414 
415 	aclp->acl_aclp = NULL;
416 	aclp->acl_cnt = 0;
417 
418 	switch (type) {
419 	case ACE_T:
420 		aclp->acl_type = ACE_T;
421 		aclp->acl_entry_size = sizeof (ace_t);
422 		break;
423 	case ACLENT_T:
424 		aclp->acl_type = ACLENT_T;
425 		aclp->acl_entry_size = sizeof (aclent_t);
426 		break;
427 	default:
428 		acl_free(aclp);
429 		aclp = NULL;
430 	}
431 	return (aclp);
432 }
433 
434 /*
435  * Free acl_t structure
436  */
437 void
438 acl_free(acl_t *aclp)
439 {
440 	int acl_size;
441 
442 	if (aclp == NULL)
443 		return;
444 
445 	if (aclp->acl_aclp) {
446 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
447 		cacl_free(aclp->acl_aclp, acl_size);
448 	}
449 
450 	cacl_free(aclp, sizeof (acl_t));
451 }
452 
453 static uint32_t
454 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
455 {
456 	uint32_t access_mask = 0;
457 	int acl_produce;
458 	int synchronize_set = 0, write_owner_set = 0;
459 	int delete_set = 0, write_attrs_set = 0;
460 	int read_named_set = 0, write_named_set = 0;
461 
462 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
463 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
464 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
465 
466 	if (isallow) {
467 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
468 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
469 		delete_set = ACL_DELETE_SET_ALLOW;
470 		if (hasreadperm)
471 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
472 		if (haswriteperm)
473 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
474 		if (isowner)
475 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
476 		else if (haswriteperm)
477 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
478 	} else {
479 
480 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
481 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
482 		delete_set = ACL_DELETE_SET_DENY;
483 		if (hasreadperm)
484 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
485 		if (haswriteperm)
486 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
487 		if (isowner)
488 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
489 		else if (haswriteperm)
490 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
491 		else
492 			/*
493 			 * If the entity is not the owner and does not
494 			 * have write permissions ACE_WRITE_ATTRIBUTES will
495 			 * always go in the DENY ACE.
496 			 */
497 			access_mask |= ACE_WRITE_ATTRIBUTES;
498 	}
499 
500 	if (acl_produce & synchronize_set)
501 		access_mask |= ACE_SYNCHRONIZE;
502 	if (acl_produce & write_owner_set)
503 		access_mask |= ACE_WRITE_OWNER;
504 	if (acl_produce & delete_set)
505 		access_mask |= ACE_DELETE;
506 	if (acl_produce & write_attrs_set)
507 		access_mask |= ACE_WRITE_ATTRIBUTES;
508 	if (acl_produce & read_named_set)
509 		access_mask |= ACE_READ_NAMED_ATTRS;
510 	if (acl_produce & write_named_set)
511 		access_mask |= ACE_WRITE_NAMED_ATTRS;
512 
513 	return (access_mask);
514 }
515 
516 /*
517  * Given an mode_t, convert it into an access_mask as used
518  * by nfsace, assuming aclent_t -> nfsace semantics.
519  */
520 static uint32_t
521 mode_to_ace_access(mode_t mode, int isdir, int isowner, int isallow)
522 {
523 	uint32_t access = 0;
524 	int haswriteperm = 0;
525 	int hasreadperm = 0;
526 
527 	if (isallow) {
528 		haswriteperm = (mode & S_IWOTH);
529 		hasreadperm = (mode & S_IROTH);
530 	} else {
531 		haswriteperm = !(mode & S_IWOTH);
532 		hasreadperm = !(mode & S_IROTH);
533 	}
534 
535 	/*
536 	 * The following call takes care of correctly setting the following
537 	 * mask bits in the access_mask:
538 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
539 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
540 	 */
541 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
542 
543 	if (isallow) {
544 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
545 		if (isowner)
546 			access |= ACE_WRITE_ACL;
547 	} else {
548 		if (! isowner)
549 			access |= ACE_WRITE_ACL;
550 	}
551 
552 	/* read */
553 	if (mode & S_IROTH) {
554 		access |= ACE_READ_DATA;
555 	}
556 	/* write */
557 	if (mode & S_IWOTH) {
558 		access |= ACE_WRITE_DATA |
559 		    ACE_APPEND_DATA;
560 		if (isdir)
561 			access |= ACE_DELETE_CHILD;
562 	}
563 	/* exec */
564 	if (mode & 01) {
565 		access |= ACE_EXECUTE;
566 	}
567 
568 	return (access);
569 }
570 
571 /*
572  * Given an nfsace (presumably an ALLOW entry), make a
573  * corresponding DENY entry at the address given.
574  */
575 static void
576 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
577 {
578 	(void) memcpy(deny, allow, sizeof (ace_t));
579 
580 	deny->a_who = allow->a_who;
581 
582 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
583 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
584 	if (isdir)
585 		deny->a_access_mask ^= ACE_DELETE_CHILD;
586 
587 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
588 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
589 	    ACE_WRITE_NAMED_ATTRS);
590 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
591 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
592 	    B_FALSE);
593 }
594 /*
595  * Make an initial pass over an array of aclent_t's.  Gather
596  * information such as an ACL_MASK (if any), number of users,
597  * number of groups, and whether the array needs to be sorted.
598  */
599 static int
600 ln_aent_preprocess(aclent_t *aclent, int n,
601     int *hasmask, mode_t *mask,
602     int *numuser, int *numgroup, int *needsort)
603 {
604 	int error = 0;
605 	int i;
606 	int curtype = 0;
607 
608 	*hasmask = 0;
609 	*mask = 07;
610 	*needsort = 0;
611 	*numuser = 0;
612 	*numgroup = 0;
613 
614 	for (i = 0; i < n; i++) {
615 		if (aclent[i].a_type < curtype)
616 			*needsort = 1;
617 		else if (aclent[i].a_type > curtype)
618 			curtype = aclent[i].a_type;
619 		if (aclent[i].a_type & USER)
620 			(*numuser)++;
621 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
622 			(*numgroup)++;
623 		if (aclent[i].a_type & CLASS_OBJ) {
624 			if (*hasmask) {
625 				error = EINVAL;
626 				goto out;
627 			} else {
628 				*hasmask = 1;
629 				*mask = aclent[i].a_perm;
630 			}
631 		}
632 	}
633 
634 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
635 		error = EINVAL;
636 		goto out;
637 	}
638 
639 out:
640 	return (error);
641 }
642 
643 /*
644  * Convert an array of aclent_t into an array of nfsace entries,
645  * following POSIX draft -> nfsv4 conversion semantics as outlined in
646  * the IETF draft.
647  */
648 static int
649 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
650 {
651 	int error = 0;
652 	mode_t mask;
653 	int numuser, numgroup, needsort;
654 	int resultsize = 0;
655 	int i, groupi = 0, skip;
656 	ace_t *acep, *result = NULL;
657 	int hasmask;
658 
659 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
660 	    &numuser, &numgroup, &needsort);
661 	if (error != 0)
662 		goto out;
663 
664 	/* allow + deny for each aclent */
665 	resultsize = n * 2;
666 	if (hasmask) {
667 		/*
668 		 * stick extra deny on the group_obj and on each
669 		 * user|group for the mask (the group_obj was added
670 		 * into the count for numgroup)
671 		 */
672 		resultsize += numuser + numgroup;
673 		/* ... and don't count the mask itself */
674 		resultsize -= 2;
675 	}
676 
677 	/* sort the source if necessary */
678 	if (needsort)
679 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
680 
681 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
682 		goto out;
683 
684 	acep = result;
685 
686 	for (i = 0; i < n; i++) {
687 		/*
688 		 * don't process CLASS_OBJ (mask); mask was grabbed in
689 		 * ln_aent_preprocess()
690 		 */
691 		if (aclent[i].a_type & CLASS_OBJ)
692 			continue;
693 
694 		/* If we need an ACL_MASK emulator, prepend it now */
695 		if ((hasmask) &&
696 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
697 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
698 			acep->a_flags = 0;
699 			if (aclent[i].a_type & GROUP_OBJ) {
700 				acep->a_who = (uid_t)-1;
701 				acep->a_flags |=
702 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
703 			} else if (aclent[i].a_type & USER) {
704 				acep->a_who = aclent[i].a_id;
705 			} else {
706 				acep->a_who = aclent[i].a_id;
707 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
708 			}
709 			if (aclent[i].a_type & ACL_DEFAULT) {
710 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
711 				    ACE_FILE_INHERIT_ACE |
712 				    ACE_DIRECTORY_INHERIT_ACE;
713 			}
714 			/*
715 			 * Set the access mask for the prepended deny
716 			 * ace.  To do this, we invert the mask (found
717 			 * in ln_aent_preprocess()) then convert it to an
718 			 * DENY ace access_mask.
719 			 */
720 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
721 			    isdir, 0, 0);
722 			acep += 1;
723 		}
724 
725 		/* handle a_perm -> access_mask */
726 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
727 		    isdir, aclent[i].a_type & USER_OBJ, 1);
728 
729 		/* emulate a default aclent */
730 		if (aclent[i].a_type & ACL_DEFAULT) {
731 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
732 			    ACE_FILE_INHERIT_ACE |
733 			    ACE_DIRECTORY_INHERIT_ACE;
734 		}
735 
736 		/*
737 		 * handle a_perm and a_id
738 		 *
739 		 * this must be done last, since it involves the
740 		 * corresponding deny aces, which are handled
741 		 * differently for each different a_type.
742 		 */
743 		if (aclent[i].a_type & USER_OBJ) {
744 			acep->a_who = (uid_t)-1;
745 			acep->a_flags |= ACE_OWNER;
746 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
747 			acep += 2;
748 		} else if (aclent[i].a_type & USER) {
749 			acep->a_who = aclent[i].a_id;
750 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
751 			acep += 2;
752 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
753 			if (aclent[i].a_type & GROUP_OBJ) {
754 				acep->a_who = (uid_t)-1;
755 				acep->a_flags |= ACE_GROUP;
756 			} else {
757 				acep->a_who = aclent[i].a_id;
758 			}
759 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
760 			/*
761 			 * Set the corresponding deny for the group ace.
762 			 *
763 			 * The deny aces go after all of the groups, unlike
764 			 * everything else, where they immediately follow
765 			 * the allow ace.
766 			 *
767 			 * We calculate "skip", the number of slots to
768 			 * skip ahead for the deny ace, here.
769 			 *
770 			 * The pattern is:
771 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
772 			 * thus, skip is
773 			 * (2 * numgroup) - 1 - groupi
774 			 * (2 * numgroup) to account for MD + A
775 			 * - 1 to account for the fact that we're on the
776 			 * access (A), not the mask (MD)
777 			 * - groupi to account for the fact that we have
778 			 * passed up groupi number of MD's.
779 			 */
780 			skip = (2 * numgroup) - 1 - groupi;
781 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
782 			/*
783 			 * If we just did the last group, skip acep past
784 			 * all of the denies; else, just move ahead one.
785 			 */
786 			if (++groupi >= numgroup)
787 				acep += numgroup + 1;
788 			else
789 				acep += 1;
790 		} else if (aclent[i].a_type & OTHER_OBJ) {
791 			acep->a_who = (uid_t)-1;
792 			acep->a_flags |= ACE_EVERYONE;
793 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
794 			acep += 2;
795 		} else {
796 			error = EINVAL;
797 			goto out;
798 		}
799 	}
800 
801 	*acepp = result;
802 	*rescount = resultsize;
803 
804 out:
805 	if (error != 0) {
806 		if ((result != NULL) && (resultsize > 0)) {
807 			cacl_free(result, resultsize * sizeof (ace_t));
808 		}
809 	}
810 
811 	return (error);
812 }
813 
814 static int
815 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, int isdir,
816     ace_t **retacep, int *retacecnt)
817 {
818 	ace_t *acep;
819 	ace_t *dfacep;
820 	int acecnt = 0;
821 	int dfacecnt = 0;
822 	int dfaclstart = 0;
823 	int dfaclcnt = 0;
824 	aclent_t *aclp;
825 	int i;
826 	int error;
827 	int acesz, dfacesz;
828 
829 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
830 
831 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
832 		if (aclp->a_type & ACL_DEFAULT)
833 			break;
834 	}
835 
836 	if (i < aclcnt) {
837 		dfaclstart = i;
838 		dfaclcnt = aclcnt - i;
839 	}
840 
841 	if (dfaclcnt && isdir == 0) {
842 		return (EINVAL);
843 	}
844 
845 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
846 	if (error)
847 		return (error);
848 
849 	if (dfaclcnt) {
850 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
851 		    &dfacep, &dfacecnt, isdir);
852 		if (error) {
853 			if (acep) {
854 				cacl_free(acep, acecnt * sizeof (ace_t));
855 			}
856 			return (error);
857 		}
858 	}
859 
860 	if (dfacecnt != 0) {
861 		acesz = sizeof (ace_t) * acecnt;
862 		dfacesz = sizeof (ace_t) * dfacecnt;
863 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
864 		if (acep == NULL)
865 			return (ENOMEM);
866 		if (dfaclcnt) {
867 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
868 		}
869 	}
870 	if (dfaclcnt)
871 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
872 
873 	*retacecnt = acecnt + dfacecnt;
874 	*retacep = acep;
875 	return (0);
876 }
877 
878 static int
879 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, int isdir)
880 {
881 	int error = 0;
882 	o_mode_t mode = 0;
883 	uint32_t bits, wantbits;
884 
885 	/* read */
886 	if (mask & ACE_READ_DATA)
887 		mode |= S_IROTH;
888 
889 	/* write */
890 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
891 	if (isdir)
892 		wantbits |= ACE_DELETE_CHILD;
893 	bits = mask & wantbits;
894 	if (bits != 0) {
895 		if (bits != wantbits) {
896 			error = ENOTSUP;
897 			goto out;
898 		}
899 		mode |= S_IWOTH;
900 	}
901 
902 	/* exec */
903 	if (mask & ACE_EXECUTE) {
904 		mode |= S_IXOTH;
905 	}
906 
907 	*modep = mode;
908 
909 out:
910 	return (error);
911 }
912 
913 static void
914 acevals_init(acevals_t *vals, uid_t key)
915 {
916 	bzero(vals, sizeof (*vals));
917 	vals->allowed = ACE_MASK_UNDEFINED;
918 	vals->denied = ACE_MASK_UNDEFINED;
919 	vals->mask = ACE_MASK_UNDEFINED;
920 	vals->key = key;
921 }
922 
923 static void
924 ace_list_init(ace_list_t *al, int dfacl_flag)
925 {
926 	acevals_init(&al->user_obj, NULL);
927 	acevals_init(&al->group_obj, NULL);
928 	acevals_init(&al->other_obj, NULL);
929 	al->numusers = 0;
930 	al->numgroups = 0;
931 	al->acl_mask = 0;
932 	al->hasmask = 0;
933 	al->state = ace_unused;
934 	al->seen = 0;
935 	al->dfacl_flag = dfacl_flag;
936 }
937 
938 /*
939  * Find or create an acevals holder for a given id and avl tree.
940  *
941  * Note that only one thread will ever touch these avl trees, so
942  * there is no need for locking.
943  */
944 static acevals_t *
945 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
946 {
947 	acevals_t key, *rc;
948 	avl_index_t where;
949 
950 	key.key = ace->a_who;
951 	rc = avl_find(avl, &key, &where);
952 	if (rc != NULL)
953 		return (rc);
954 
955 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
956 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
957 		return (NULL);
958 
959 	acevals_init(rc, ace->a_who);
960 	avl_insert(avl, rc, where);
961 	(*num)++;
962 
963 	return (rc);
964 }
965 
966 static int
967 access_mask_check(ace_t *acep, int mask_bit, int isowner)
968 {
969 	int set_deny, err_deny;
970 	int set_allow, err_allow;
971 	int acl_consume;
972 	int haswriteperm, hasreadperm;
973 
974 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
975 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
976 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
977 	} else {
978 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
979 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
980 	}
981 
982 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
983 	    ACL_DELETE_ERR_DENY |
984 	    ACL_WRITE_OWNER_ERR_DENY |
985 	    ACL_WRITE_OWNER_ERR_ALLOW |
986 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
987 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
988 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
989 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
990 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
991 	    ACL_READ_NAMED_READER_ERR_DENY);
992 
993 	if (mask_bit == ACE_SYNCHRONIZE) {
994 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
995 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
996 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
997 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
998 	} else if (mask_bit == ACE_WRITE_OWNER) {
999 		set_deny = ACL_WRITE_OWNER_SET_DENY;
1000 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
1001 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
1002 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
1003 	} else if (mask_bit == ACE_DELETE) {
1004 		set_deny = ACL_DELETE_SET_DENY;
1005 		err_deny =  ACL_DELETE_ERR_DENY;
1006 		set_allow = ACL_DELETE_SET_ALLOW;
1007 		err_allow = ACL_DELETE_ERR_ALLOW;
1008 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
1009 		if (isowner) {
1010 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
1011 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
1012 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
1013 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
1014 		} else if (haswriteperm) {
1015 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
1016 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
1017 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
1018 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
1019 		} else {
1020 			if ((acep->a_access_mask & mask_bit) &&
1021 			    (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1022 				return (ENOTSUP);
1023 			}
1024 			return (0);
1025 		}
1026 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
1027 		if (!hasreadperm)
1028 			return (0);
1029 
1030 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
1031 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
1032 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
1033 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
1034 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
1035 		if (!haswriteperm)
1036 			return (0);
1037 
1038 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
1039 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
1040 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
1041 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
1042 	} else {
1043 		return (EINVAL);
1044 	}
1045 
1046 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
1047 		if (acl_consume & set_deny) {
1048 			if (!(acep->a_access_mask & mask_bit)) {
1049 				return (ENOTSUP);
1050 			}
1051 		} else if (acl_consume & err_deny) {
1052 			if (acep->a_access_mask & mask_bit) {
1053 				return (ENOTSUP);
1054 			}
1055 		}
1056 	} else {
1057 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
1058 		if (acl_consume & set_allow) {
1059 			if (!(acep->a_access_mask & mask_bit)) {
1060 				return (ENOTSUP);
1061 			}
1062 		} else if (acl_consume & err_allow) {
1063 			if (acep->a_access_mask & mask_bit) {
1064 				return (ENOTSUP);
1065 			}
1066 		}
1067 	}
1068 	return (0);
1069 }
1070 
1071 static int
1072 ace_to_aent_legal(ace_t *acep)
1073 {
1074 	int error = 0;
1075 	int isowner;
1076 
1077 	/* only ALLOW or DENY */
1078 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1079 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
1080 		error = ENOTSUP;
1081 		goto out;
1082 	}
1083 
1084 	/* check for invalid flags */
1085 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
1086 		error = EINVAL;
1087 		goto out;
1088 	}
1089 
1090 	/* some flags are illegal */
1091 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
1092 	    ACE_FAILED_ACCESS_ACE_FLAG |
1093 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
1094 		error = ENOTSUP;
1095 		goto out;
1096 	}
1097 
1098 	/* check for invalid masks */
1099 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
1100 		error = EINVAL;
1101 		goto out;
1102 	}
1103 
1104 	if ((acep->a_flags & ACE_OWNER)) {
1105 		isowner = 1;
1106 	} else {
1107 		isowner = 0;
1108 	}
1109 
1110 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
1111 	if (error)
1112 		goto out;
1113 
1114 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
1115 	if (error)
1116 		goto out;
1117 
1118 	error = access_mask_check(acep, ACE_DELETE, isowner);
1119 	if (error)
1120 		goto out;
1121 
1122 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
1123 	if (error)
1124 		goto out;
1125 
1126 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
1127 	if (error)
1128 		goto out;
1129 
1130 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
1131 	if (error)
1132 		goto out;
1133 
1134 	/* more detailed checking of masks */
1135 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1136 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
1137 			error = ENOTSUP;
1138 			goto out;
1139 		}
1140 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
1141 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
1142 			error = ENOTSUP;
1143 			goto out;
1144 		}
1145 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1146 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1147 			error = ENOTSUP;
1148 			goto out;
1149 		}
1150 	}
1151 
1152 	/* ACL enforcement */
1153 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1154 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1155 		error = ENOTSUP;
1156 		goto out;
1157 	}
1158 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1159 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1160 		    (isowner)) {
1161 			error = ENOTSUP;
1162 			goto out;
1163 		}
1164 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1165 		    (! isowner)) {
1166 			error = ENOTSUP;
1167 			goto out;
1168 		}
1169 	}
1170 
1171 out:
1172 	return (error);
1173 }
1174 
1175 static int
1176 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, int isdir)
1177 {
1178 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1179 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1180 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1181 		return (ENOTSUP);
1182 	}
1183 
1184 	return (ace_mask_to_mode(mask, modep, isdir));
1185 }
1186 
1187 static int
1188 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1189     uid_t owner, gid_t group, int isdir)
1190 {
1191 	int error;
1192 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1193 
1194 	if (isdir)
1195 		flips |= ACE_DELETE_CHILD;
1196 	if (vals->allowed != (vals->denied ^ flips)) {
1197 		error = ENOTSUP;
1198 		goto out;
1199 	}
1200 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1201 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1202 		error = ENOTSUP;
1203 		goto out;
1204 	}
1205 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1206 	if (error != 0)
1207 		goto out;
1208 	dest->a_type = vals->aent_type;
1209 	if (dest->a_type & (USER | GROUP)) {
1210 		dest->a_id = vals->key;
1211 	} else if (dest->a_type & USER_OBJ) {
1212 		dest->a_id = owner;
1213 	} else if (dest->a_type & GROUP_OBJ) {
1214 		dest->a_id = group;
1215 	} else if (dest->a_type & OTHER_OBJ) {
1216 		dest->a_id = 0;
1217 	} else {
1218 		error = EINVAL;
1219 		goto out;
1220 	}
1221 
1222 out:
1223 	return (error);
1224 }
1225 
1226 
1227 static int
1228 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1229     uid_t owner, gid_t group, int isdir)
1230 {
1231 	int error = 0;
1232 	aclent_t *aent, *result = NULL;
1233 	acevals_t *vals;
1234 	int resultcount;
1235 
1236 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1237 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1238 		error = ENOTSUP;
1239 		goto out;
1240 	}
1241 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1242 		error = ENOTSUP;
1243 		goto out;
1244 	}
1245 
1246 	resultcount = 3 + list->numusers + list->numgroups;
1247 	/*
1248 	 * This must be the same condition as below, when we add the CLASS_OBJ
1249 	 * (aka ACL mask)
1250 	 */
1251 	if ((list->hasmask) || (! list->dfacl_flag))
1252 		resultcount += 1;
1253 
1254 	if (cacl_malloc((void **)&result,
1255 	    resultcount * sizeof (aclent_t)) != 0) {
1256 		error = ENOMEM;
1257 		goto out;
1258 	}
1259 	aent = result;
1260 
1261 	/* USER_OBJ */
1262 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1263 		error = EINVAL;
1264 		goto out;
1265 	}
1266 
1267 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1268 	    isdir);
1269 
1270 	if (error != 0)
1271 		goto out;
1272 	++aent;
1273 	/* USER */
1274 	vals = NULL;
1275 	for (vals = avl_first(&list->user); vals != NULL;
1276 	    vals = AVL_NEXT(&list->user, vals)) {
1277 		if (!(vals->aent_type & USER)) {
1278 			error = EINVAL;
1279 			goto out;
1280 		}
1281 		error = acevals_to_aent(vals, aent, list, owner, group,
1282 		    isdir);
1283 		if (error != 0)
1284 			goto out;
1285 		++aent;
1286 	}
1287 	/* GROUP_OBJ */
1288 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1289 		error = EINVAL;
1290 		goto out;
1291 	}
1292 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1293 	    isdir);
1294 	if (error != 0)
1295 		goto out;
1296 	++aent;
1297 	/* GROUP */
1298 	vals = NULL;
1299 	for (vals = avl_first(&list->group); vals != NULL;
1300 	    vals = AVL_NEXT(&list->group, vals)) {
1301 		if (!(vals->aent_type & GROUP)) {
1302 			error = EINVAL;
1303 			goto out;
1304 		}
1305 		error = acevals_to_aent(vals, aent, list, owner, group,
1306 		    isdir);
1307 		if (error != 0)
1308 			goto out;
1309 		++aent;
1310 	}
1311 	/*
1312 	 * CLASS_OBJ (aka ACL_MASK)
1313 	 *
1314 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1315 	 * This is to follow UFS's behavior.
1316 	 */
1317 	if ((list->hasmask) || (! list->dfacl_flag)) {
1318 		if (list->hasmask) {
1319 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1320 			if (isdir)
1321 				flips |= ACE_DELETE_CHILD;
1322 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1323 			    &aent->a_perm, isdir);
1324 			if (error != 0)
1325 				goto out;
1326 		} else {
1327 			/* fabricate the ACL_MASK from the group permissions */
1328 			error = ace_mask_to_mode(list->group_obj.allowed,
1329 			    &aent->a_perm, isdir);
1330 			if (error != 0)
1331 				goto out;
1332 		}
1333 		aent->a_id = 0;
1334 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1335 		++aent;
1336 	}
1337 	/* OTHER_OBJ */
1338 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1339 		error = EINVAL;
1340 		goto out;
1341 	}
1342 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1343 	    isdir);
1344 	if (error != 0)
1345 		goto out;
1346 	++aent;
1347 
1348 	*aclentp = result;
1349 	*aclcnt = resultcount;
1350 
1351 out:
1352 	if (error != 0) {
1353 		if (result != NULL)
1354 			cacl_free(result, resultcount * sizeof (aclent_t));
1355 	}
1356 
1357 	return (error);
1358 }
1359 
1360 
1361 /*
1362  * free all data associated with an ace_list
1363  */
1364 static void
1365 ace_list_free(ace_list_t *al)
1366 {
1367 	acevals_t *node;
1368 	void *cookie;
1369 
1370 	if (al == NULL)
1371 		return;
1372 
1373 	cookie = NULL;
1374 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1375 		cacl_free(node, sizeof (acevals_t));
1376 	cookie = NULL;
1377 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1378 		cacl_free(node, sizeof (acevals_t));
1379 
1380 	avl_destroy(&al->user);
1381 	avl_destroy(&al->group);
1382 
1383 	/* free the container itself */
1384 	cacl_free(al, sizeof (ace_list_t));
1385 }
1386 
1387 static int
1388 acevals_compare(const void *va, const void *vb)
1389 {
1390 	const acevals_t *a = va, *b = vb;
1391 
1392 	if (a->key == b->key)
1393 		return (0);
1394 
1395 	if (a->key > b->key)
1396 		return (1);
1397 
1398 	else
1399 		return (-1);
1400 }
1401 
1402 /*
1403  * Convert a list of ace_t entries to equivalent regular and default
1404  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1405  */
1406 static int
1407 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1408     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1409     int isdir)
1410 {
1411 	int error = 0;
1412 	ace_t *acep;
1413 	uint32_t bits;
1414 	int i;
1415 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1416 	acevals_t *vals;
1417 
1418 	*aclentp = NULL;
1419 	*aclcnt = 0;
1420 	*dfaclentp = NULL;
1421 	*dfaclcnt = 0;
1422 
1423 	/* we need at least user_obj, group_obj, and other_obj */
1424 	if (n < 6) {
1425 		error = ENOTSUP;
1426 		goto out;
1427 	}
1428 	if (ace == NULL) {
1429 		error = EINVAL;
1430 		goto out;
1431 	}
1432 
1433 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1434 	if (error != 0)
1435 		goto out;
1436 
1437 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1438 	    offsetof(acevals_t, avl));
1439 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1440 	    offsetof(acevals_t, avl));
1441 
1442 	ace_list_init(normacl, 0);
1443 
1444 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1445 	if (error != 0)
1446 		goto out;
1447 
1448 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1449 	    offsetof(acevals_t, avl));
1450 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1451 	    offsetof(acevals_t, avl));
1452 	ace_list_init(dfacl, ACL_DEFAULT);
1453 
1454 	/* process every ace_t... */
1455 	for (i = 0; i < n; i++) {
1456 		acep = &ace[i];
1457 
1458 		/* rule out certain cases quickly */
1459 		error = ace_to_aent_legal(acep);
1460 		if (error != 0)
1461 			goto out;
1462 
1463 		/*
1464 		 * Turn off these bits in order to not have to worry about
1465 		 * them when doing the checks for compliments.
1466 		 */
1467 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1468 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1469 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1470 
1471 		/* see if this should be a regular or default acl */
1472 		bits = acep->a_flags &
1473 		    (ACE_INHERIT_ONLY_ACE |
1474 		    ACE_FILE_INHERIT_ACE |
1475 		    ACE_DIRECTORY_INHERIT_ACE);
1476 		if (bits != 0) {
1477 			/* all or nothing on these inherit bits */
1478 			if (bits != (ACE_INHERIT_ONLY_ACE |
1479 			    ACE_FILE_INHERIT_ACE |
1480 			    ACE_DIRECTORY_INHERIT_ACE)) {
1481 				error = ENOTSUP;
1482 				goto out;
1483 			}
1484 			acl = dfacl;
1485 		} else {
1486 			acl = normacl;
1487 		}
1488 
1489 		if ((acep->a_flags & ACE_OWNER)) {
1490 			if (acl->state > ace_user_obj) {
1491 				error = ENOTSUP;
1492 				goto out;
1493 			}
1494 			acl->state = ace_user_obj;
1495 			acl->seen |= USER_OBJ;
1496 			vals = &acl->user_obj;
1497 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1498 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1499 			acl->state = ace_other_obj;
1500 			acl->seen |= OTHER_OBJ;
1501 			vals = &acl->other_obj;
1502 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1503 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1504 			if (acl->state > ace_group) {
1505 				error = ENOTSUP;
1506 				goto out;
1507 			}
1508 			if ((acep->a_flags & ACE_GROUP)) {
1509 				acl->seen |= GROUP_OBJ;
1510 				vals = &acl->group_obj;
1511 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1512 			} else {
1513 				acl->seen |= GROUP;
1514 				vals = acevals_find(acep, &acl->group,
1515 				    &acl->numgroups);
1516 				if (vals == NULL) {
1517 					error = ENOMEM;
1518 					goto out;
1519 				}
1520 				vals->aent_type = GROUP | acl->dfacl_flag;
1521 			}
1522 			acl->state = ace_group;
1523 		} else {
1524 			if (acl->state > ace_user) {
1525 				error = ENOTSUP;
1526 				goto out;
1527 			}
1528 			acl->state = ace_user;
1529 			acl->seen |= USER;
1530 			vals = acevals_find(acep, &acl->user,
1531 			    &acl->numusers);
1532 			if (vals == NULL) {
1533 				error = ENOMEM;
1534 				goto out;
1535 			}
1536 			vals->aent_type = USER | acl->dfacl_flag;
1537 		}
1538 
1539 		if (!(acl->state > ace_unused)) {
1540 			error = EINVAL;
1541 			goto out;
1542 		}
1543 
1544 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1545 			/* no more than one allowed per aclent_t */
1546 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1547 				error = ENOTSUP;
1548 				goto out;
1549 			}
1550 			vals->allowed = acep->a_access_mask;
1551 		} else {
1552 			/*
1553 			 * it's a DENY; if there was a previous DENY, it
1554 			 * must have been an ACL_MASK.
1555 			 */
1556 			if (vals->denied != ACE_MASK_UNDEFINED) {
1557 				/* ACL_MASK is for USER and GROUP only */
1558 				if ((acl->state != ace_user) &&
1559 				    (acl->state != ace_group)) {
1560 					error = ENOTSUP;
1561 					goto out;
1562 				}
1563 
1564 				if (! acl->hasmask) {
1565 					acl->hasmask = 1;
1566 					acl->acl_mask = vals->denied;
1567 				/* check for mismatched ACL_MASK emulations */
1568 				} else if (acl->acl_mask != vals->denied) {
1569 					error = ENOTSUP;
1570 					goto out;
1571 				}
1572 				vals->mask = vals->denied;
1573 			}
1574 			vals->denied = acep->a_access_mask;
1575 		}
1576 	}
1577 
1578 	/* done collating; produce the aclent_t lists */
1579 	if (normacl->state != ace_unused) {
1580 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1581 		    owner, group, isdir);
1582 		if (error != 0) {
1583 			goto out;
1584 		}
1585 	}
1586 	if (dfacl->state != ace_unused) {
1587 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1588 		    owner, group, isdir);
1589 		if (error != 0) {
1590 			goto out;
1591 		}
1592 	}
1593 
1594 out:
1595 	if (normacl != NULL)
1596 		ace_list_free(normacl);
1597 	if (dfacl != NULL)
1598 		ace_list_free(dfacl);
1599 
1600 	return (error);
1601 }
1602 
1603 static int
1604 convert_ace_to_aent(ace_t *acebufp, int acecnt, int isdir,
1605     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1606 {
1607 	int error = 0;
1608 	aclent_t *aclentp, *dfaclentp;
1609 	int aclcnt, dfaclcnt;
1610 	int aclsz, dfaclsz;
1611 
1612 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1613 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1614 
1615 	if (error)
1616 		return (error);
1617 
1618 
1619 	if (dfaclcnt != 0) {
1620 		/*
1621 		 * Slap aclentp and dfaclentp into a single array.
1622 		 */
1623 		aclsz = sizeof (aclent_t) * aclcnt;
1624 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1625 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1626 		if (aclentp != NULL) {
1627 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1628 		} else {
1629 			error = ENOMEM;
1630 		}
1631 	}
1632 
1633 	if (aclentp) {
1634 		*retaclentp = aclentp;
1635 		*retaclcnt = aclcnt + dfaclcnt;
1636 	}
1637 
1638 	if (dfaclentp)
1639 		cacl_free(dfaclentp, dfaclsz);
1640 
1641 	return (error);
1642 }
1643 
1644 
1645 int
1646 acl_translate(acl_t *aclp, int target_flavor, int isdir, uid_t owner,
1647     gid_t group)
1648 {
1649 	int aclcnt;
1650 	void *acldata;
1651 	int error;
1652 
1653 	/*
1654 	 * See if we need to translate
1655 	 */
1656 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1657 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1658 	    aclp->acl_type == ACLENT_T))
1659 		return (0);
1660 
1661 	if (target_flavor == -1) {
1662 		error = EINVAL;
1663 		goto out;
1664 	}
1665 
1666 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1667 	    aclp->acl_type == ACLENT_T) {
1668 		error = convert_aent_to_ace(aclp->acl_aclp,
1669 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1670 		if (error)
1671 			goto out;
1672 
1673 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1674 	    aclp->acl_type == ACE_T) {
1675 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1676 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1677 		if (error)
1678 			goto out;
1679 	} else {
1680 		error = ENOTSUP;
1681 		goto out;
1682 	}
1683 
1684 	/*
1685 	 * replace old acl with newly translated acl
1686 	 */
1687 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1688 	aclp->acl_aclp = acldata;
1689 	aclp->acl_cnt = aclcnt;
1690 	if (target_flavor == _ACL_ACE_ENABLED) {
1691 		aclp->acl_type = ACE_T;
1692 		aclp->acl_entry_size = sizeof (ace_t);
1693 	} else {
1694 		aclp->acl_type = ACLENT_T;
1695 		aclp->acl_entry_size = sizeof (aclent_t);
1696 	}
1697 	return (0);
1698 
1699 out:
1700 
1701 #if !defined(_KERNEL)
1702 	errno = error;
1703 	return (-1);
1704 #else
1705 	return (error);
1706 #endif
1707 }
1708