xref: /illumos-gate/usr/src/common/acl/acl_common.c (revision 1eb4e906)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <sys/errno.h>
29 #include <sys/avl.h>
30 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
31 #include <sys/debug.h>
32 #include <sys/kmem.h>
33 #include <sys/systm.h>
34 #include <sys/sysmacros.h>
35 #include <acl/acl_common.h>
36 #else
37 #include <errno.h>
38 #include <stdlib.h>
39 #include <stddef.h>
40 #include <strings.h>
41 #include <unistd.h>
42 #include <assert.h>
43 #include <grp.h>
44 #include <pwd.h>
45 #include <acl_common.h>
46 #define	ASSERT	assert
47 #endif
48 
49 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
50     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
51     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
52 
53 
54 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
55 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
56 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
57 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
58 
59 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
60 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
61 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
62 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
63 
64 #define	ACL_DELETE_SET_DENY			0x0000100
65 #define	ACL_DELETE_SET_ALLOW			0x0000200
66 #define	ACL_DELETE_ERR_DENY			0x0000400
67 #define	ACL_DELETE_ERR_ALLOW			0x0000800
68 
69 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
70 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
71 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
72 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
73 
74 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
75 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
76 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
77 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
78 
79 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
80 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
81 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
82 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
83 
84 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
85 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
86 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
87 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
88 
89 
90 #define	ACE_VALID_MASK_BITS (\
91     ACE_READ_DATA | \
92     ACE_LIST_DIRECTORY | \
93     ACE_WRITE_DATA | \
94     ACE_ADD_FILE | \
95     ACE_APPEND_DATA | \
96     ACE_ADD_SUBDIRECTORY | \
97     ACE_READ_NAMED_ATTRS | \
98     ACE_WRITE_NAMED_ATTRS | \
99     ACE_EXECUTE | \
100     ACE_DELETE_CHILD | \
101     ACE_READ_ATTRIBUTES | \
102     ACE_WRITE_ATTRIBUTES | \
103     ACE_DELETE | \
104     ACE_READ_ACL | \
105     ACE_WRITE_ACL | \
106     ACE_WRITE_OWNER | \
107     ACE_SYNCHRONIZE)
108 
109 #define	ACE_MASK_UNDEFINED			0x80000000
110 
111 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
112     ACE_DIRECTORY_INHERIT_ACE | \
113     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
114     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
115     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
116 
117 /*
118  * ACL conversion helpers
119  */
120 
121 typedef enum {
122 	ace_unused,
123 	ace_user_obj,
124 	ace_user,
125 	ace_group, /* includes GROUP and GROUP_OBJ */
126 	ace_other_obj
127 } ace_to_aent_state_t;
128 
129 typedef struct acevals {
130 	uid_t key;
131 	avl_node_t avl;
132 	uint32_t mask;
133 	uint32_t allowed;
134 	uint32_t denied;
135 	int aent_type;
136 } acevals_t;
137 
138 typedef struct ace_list {
139 	acevals_t user_obj;
140 	avl_tree_t user;
141 	int numusers;
142 	acevals_t group_obj;
143 	avl_tree_t group;
144 	int numgroups;
145 	acevals_t other_obj;
146 	uint32_t acl_mask;
147 	int hasmask;
148 	int dfacl_flag;
149 	ace_to_aent_state_t state;
150 	int seen; /* bitmask of all aclent_t a_type values seen */
151 } ace_list_t;
152 
153 /*
154  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
155  * v = Ptr to array/vector of objs
156  * n = # objs in the array
157  * s = size of each obj (must be multiples of a word size)
158  * f = ptr to function to compare two objs
159  *	returns (-1 = less than, 0 = equal, 1 = greater than
160  */
161 void
162 ksort(caddr_t v, int n, int s, int (*f)())
163 {
164 	int g, i, j, ii;
165 	unsigned int *p1, *p2;
166 	unsigned int tmp;
167 
168 	/* No work to do */
169 	if (v == NULL || n <= 1)
170 		return;
171 
172 	/* Sanity check on arguments */
173 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
174 	ASSERT(s > 0);
175 	for (g = n / 2; g > 0; g /= 2) {
176 		for (i = g; i < n; i++) {
177 			for (j = i - g; j >= 0 &&
178 			    (*f)(v + j * s, v + (j + g) * s) == 1;
179 			    j -= g) {
180 				p1 = (void *)(v + j * s);
181 				p2 = (void *)(v + (j + g) * s);
182 				for (ii = 0; ii < s / 4; ii++) {
183 					tmp = *p1;
184 					*p1++ = *p2;
185 					*p2++ = tmp;
186 				}
187 			}
188 		}
189 	}
190 }
191 
192 /*
193  * Compare two acls, all fields.  Returns:
194  * -1 (less than)
195  *  0 (equal)
196  * +1 (greater than)
197  */
198 int
199 cmp2acls(void *a, void *b)
200 {
201 	aclent_t *x = (aclent_t *)a;
202 	aclent_t *y = (aclent_t *)b;
203 
204 	/* Compare types */
205 	if (x->a_type < y->a_type)
206 		return (-1);
207 	if (x->a_type > y->a_type)
208 		return (1);
209 	/* Equal types; compare id's */
210 	if (x->a_id < y->a_id)
211 		return (-1);
212 	if (x->a_id > y->a_id)
213 		return (1);
214 	/* Equal ids; compare perms */
215 	if (x->a_perm < y->a_perm)
216 		return (-1);
217 	if (x->a_perm > y->a_perm)
218 		return (1);
219 	/* Totally equal */
220 	return (0);
221 }
222 
223 /*ARGSUSED*/
224 static void *
225 cacl_realloc(void *ptr, size_t size, size_t new_size)
226 {
227 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
228 	void *tmp;
229 
230 	tmp = kmem_alloc(new_size, KM_SLEEP);
231 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
232 	kmem_free(ptr, size);
233 	return (tmp);
234 #else
235 	return (realloc(ptr, new_size));
236 #endif
237 }
238 
239 static int
240 cacl_malloc(void **ptr, size_t size)
241 {
242 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
243 	*ptr = kmem_zalloc(size, KM_SLEEP);
244 	return (0);
245 #else
246 	*ptr = calloc(1, size);
247 	if (*ptr == NULL)
248 		return (errno);
249 
250 	return (0);
251 #endif
252 }
253 
254 /*ARGSUSED*/
255 static void
256 cacl_free(void *ptr, size_t size)
257 {
258 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
259 	kmem_free(ptr, size);
260 #else
261 	free(ptr);
262 #endif
263 }
264 
265 acl_t *
266 acl_alloc(enum acl_type type)
267 {
268 	acl_t *aclp;
269 
270 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
271 		return (NULL);
272 
273 	aclp->acl_aclp = NULL;
274 	aclp->acl_cnt = 0;
275 
276 	switch (type) {
277 	case ACE_T:
278 		aclp->acl_type = ACE_T;
279 		aclp->acl_entry_size = sizeof (ace_t);
280 		break;
281 	case ACLENT_T:
282 		aclp->acl_type = ACLENT_T;
283 		aclp->acl_entry_size = sizeof (aclent_t);
284 		break;
285 	default:
286 		acl_free(aclp);
287 		aclp = NULL;
288 	}
289 	return (aclp);
290 }
291 
292 /*
293  * Free acl_t structure
294  */
295 void
296 acl_free(acl_t *aclp)
297 {
298 	int acl_size;
299 
300 	if (aclp == NULL)
301 		return;
302 
303 	if (aclp->acl_aclp) {
304 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
305 		cacl_free(aclp->acl_aclp, acl_size);
306 	}
307 
308 	cacl_free(aclp, sizeof (acl_t));
309 }
310 
311 static uint32_t
312 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
313 {
314 	uint32_t access_mask = 0;
315 	int acl_produce;
316 	int synchronize_set = 0, write_owner_set = 0;
317 	int delete_set = 0, write_attrs_set = 0;
318 	int read_named_set = 0, write_named_set = 0;
319 
320 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
321 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
322 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
323 
324 	if (isallow) {
325 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
326 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
327 		delete_set = ACL_DELETE_SET_ALLOW;
328 		if (hasreadperm)
329 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
330 		if (haswriteperm)
331 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
332 		if (isowner)
333 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
334 		else if (haswriteperm)
335 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
336 	} else {
337 
338 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
339 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
340 		delete_set = ACL_DELETE_SET_DENY;
341 		if (hasreadperm)
342 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
343 		if (haswriteperm)
344 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
345 		if (isowner)
346 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
347 		else if (haswriteperm)
348 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
349 		else
350 			/*
351 			 * If the entity is not the owner and does not
352 			 * have write permissions ACE_WRITE_ATTRIBUTES will
353 			 * always go in the DENY ACE.
354 			 */
355 			access_mask |= ACE_WRITE_ATTRIBUTES;
356 	}
357 
358 	if (acl_produce & synchronize_set)
359 		access_mask |= ACE_SYNCHRONIZE;
360 	if (acl_produce & write_owner_set)
361 		access_mask |= ACE_WRITE_OWNER;
362 	if (acl_produce & delete_set)
363 		access_mask |= ACE_DELETE;
364 	if (acl_produce & write_attrs_set)
365 		access_mask |= ACE_WRITE_ATTRIBUTES;
366 	if (acl_produce & read_named_set)
367 		access_mask |= ACE_READ_NAMED_ATTRS;
368 	if (acl_produce & write_named_set)
369 		access_mask |= ACE_WRITE_NAMED_ATTRS;
370 
371 	return (access_mask);
372 }
373 
374 /*
375  * Given an mode_t, convert it into an access_mask as used
376  * by nfsace, assuming aclent_t -> nfsace semantics.
377  */
378 static uint32_t
379 mode_to_ace_access(mode_t mode, boolean_t isdir, int isowner, int isallow)
380 {
381 	uint32_t access = 0;
382 	int haswriteperm = 0;
383 	int hasreadperm = 0;
384 
385 	if (isallow) {
386 		haswriteperm = (mode & S_IWOTH);
387 		hasreadperm = (mode & S_IROTH);
388 	} else {
389 		haswriteperm = !(mode & S_IWOTH);
390 		hasreadperm = !(mode & S_IROTH);
391 	}
392 
393 	/*
394 	 * The following call takes care of correctly setting the following
395 	 * mask bits in the access_mask:
396 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
397 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
398 	 */
399 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
400 
401 	if (isallow) {
402 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
403 		if (isowner)
404 			access |= ACE_WRITE_ACL;
405 	} else {
406 		if (! isowner)
407 			access |= ACE_WRITE_ACL;
408 	}
409 
410 	/* read */
411 	if (mode & S_IROTH) {
412 		access |= ACE_READ_DATA;
413 	}
414 	/* write */
415 	if (mode & S_IWOTH) {
416 		access |= ACE_WRITE_DATA |
417 		    ACE_APPEND_DATA;
418 		if (isdir)
419 			access |= ACE_DELETE_CHILD;
420 	}
421 	/* exec */
422 	if (mode & S_IXOTH) {
423 		access |= ACE_EXECUTE;
424 	}
425 
426 	return (access);
427 }
428 
429 /*
430  * Given an nfsace (presumably an ALLOW entry), make a
431  * corresponding DENY entry at the address given.
432  */
433 static void
434 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
435 {
436 	(void) memcpy(deny, allow, sizeof (ace_t));
437 
438 	deny->a_who = allow->a_who;
439 
440 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
441 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
442 	if (isdir)
443 		deny->a_access_mask ^= ACE_DELETE_CHILD;
444 
445 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
446 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
447 	    ACE_WRITE_NAMED_ATTRS);
448 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
449 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
450 	    B_FALSE);
451 }
452 /*
453  * Make an initial pass over an array of aclent_t's.  Gather
454  * information such as an ACL_MASK (if any), number of users,
455  * number of groups, and whether the array needs to be sorted.
456  */
457 static int
458 ln_aent_preprocess(aclent_t *aclent, int n,
459     int *hasmask, mode_t *mask,
460     int *numuser, int *numgroup, int *needsort)
461 {
462 	int error = 0;
463 	int i;
464 	int curtype = 0;
465 
466 	*hasmask = 0;
467 	*mask = 07;
468 	*needsort = 0;
469 	*numuser = 0;
470 	*numgroup = 0;
471 
472 	for (i = 0; i < n; i++) {
473 		if (aclent[i].a_type < curtype)
474 			*needsort = 1;
475 		else if (aclent[i].a_type > curtype)
476 			curtype = aclent[i].a_type;
477 		if (aclent[i].a_type & USER)
478 			(*numuser)++;
479 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
480 			(*numgroup)++;
481 		if (aclent[i].a_type & CLASS_OBJ) {
482 			if (*hasmask) {
483 				error = EINVAL;
484 				goto out;
485 			} else {
486 				*hasmask = 1;
487 				*mask = aclent[i].a_perm;
488 			}
489 		}
490 	}
491 
492 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
493 		error = EINVAL;
494 		goto out;
495 	}
496 
497 out:
498 	return (error);
499 }
500 
501 /*
502  * Convert an array of aclent_t into an array of nfsace entries,
503  * following POSIX draft -> nfsv4 conversion semantics as outlined in
504  * the IETF draft.
505  */
506 static int
507 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
508 {
509 	int error = 0;
510 	mode_t mask;
511 	int numuser, numgroup, needsort;
512 	int resultsize = 0;
513 	int i, groupi = 0, skip;
514 	ace_t *acep, *result = NULL;
515 	int hasmask;
516 
517 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
518 	    &numuser, &numgroup, &needsort);
519 	if (error != 0)
520 		goto out;
521 
522 	/* allow + deny for each aclent */
523 	resultsize = n * 2;
524 	if (hasmask) {
525 		/*
526 		 * stick extra deny on the group_obj and on each
527 		 * user|group for the mask (the group_obj was added
528 		 * into the count for numgroup)
529 		 */
530 		resultsize += numuser + numgroup;
531 		/* ... and don't count the mask itself */
532 		resultsize -= 2;
533 	}
534 
535 	/* sort the source if necessary */
536 	if (needsort)
537 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
538 
539 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
540 		goto out;
541 
542 	acep = result;
543 
544 	for (i = 0; i < n; i++) {
545 		/*
546 		 * don't process CLASS_OBJ (mask); mask was grabbed in
547 		 * ln_aent_preprocess()
548 		 */
549 		if (aclent[i].a_type & CLASS_OBJ)
550 			continue;
551 
552 		/* If we need an ACL_MASK emulator, prepend it now */
553 		if ((hasmask) &&
554 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
555 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
556 			acep->a_flags = 0;
557 			if (aclent[i].a_type & GROUP_OBJ) {
558 				acep->a_who = (uid_t)-1;
559 				acep->a_flags |=
560 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
561 			} else if (aclent[i].a_type & USER) {
562 				acep->a_who = aclent[i].a_id;
563 			} else {
564 				acep->a_who = aclent[i].a_id;
565 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
566 			}
567 			if (aclent[i].a_type & ACL_DEFAULT) {
568 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
569 				    ACE_FILE_INHERIT_ACE |
570 				    ACE_DIRECTORY_INHERIT_ACE;
571 			}
572 			/*
573 			 * Set the access mask for the prepended deny
574 			 * ace.  To do this, we invert the mask (found
575 			 * in ln_aent_preprocess()) then convert it to an
576 			 * DENY ace access_mask.
577 			 */
578 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
579 			    isdir, 0, 0);
580 			acep += 1;
581 		}
582 
583 		/* handle a_perm -> access_mask */
584 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
585 		    isdir, aclent[i].a_type & USER_OBJ, 1);
586 
587 		/* emulate a default aclent */
588 		if (aclent[i].a_type & ACL_DEFAULT) {
589 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
590 			    ACE_FILE_INHERIT_ACE |
591 			    ACE_DIRECTORY_INHERIT_ACE;
592 		}
593 
594 		/*
595 		 * handle a_perm and a_id
596 		 *
597 		 * this must be done last, since it involves the
598 		 * corresponding deny aces, which are handled
599 		 * differently for each different a_type.
600 		 */
601 		if (aclent[i].a_type & USER_OBJ) {
602 			acep->a_who = (uid_t)-1;
603 			acep->a_flags |= ACE_OWNER;
604 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
605 			acep += 2;
606 		} else if (aclent[i].a_type & USER) {
607 			acep->a_who = aclent[i].a_id;
608 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
609 			acep += 2;
610 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
611 			if (aclent[i].a_type & GROUP_OBJ) {
612 				acep->a_who = (uid_t)-1;
613 				acep->a_flags |= ACE_GROUP;
614 			} else {
615 				acep->a_who = aclent[i].a_id;
616 			}
617 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
618 			/*
619 			 * Set the corresponding deny for the group ace.
620 			 *
621 			 * The deny aces go after all of the groups, unlike
622 			 * everything else, where they immediately follow
623 			 * the allow ace.
624 			 *
625 			 * We calculate "skip", the number of slots to
626 			 * skip ahead for the deny ace, here.
627 			 *
628 			 * The pattern is:
629 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
630 			 * thus, skip is
631 			 * (2 * numgroup) - 1 - groupi
632 			 * (2 * numgroup) to account for MD + A
633 			 * - 1 to account for the fact that we're on the
634 			 * access (A), not the mask (MD)
635 			 * - groupi to account for the fact that we have
636 			 * passed up groupi number of MD's.
637 			 */
638 			skip = (2 * numgroup) - 1 - groupi;
639 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
640 			/*
641 			 * If we just did the last group, skip acep past
642 			 * all of the denies; else, just move ahead one.
643 			 */
644 			if (++groupi >= numgroup)
645 				acep += numgroup + 1;
646 			else
647 				acep += 1;
648 		} else if (aclent[i].a_type & OTHER_OBJ) {
649 			acep->a_who = (uid_t)-1;
650 			acep->a_flags |= ACE_EVERYONE;
651 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
652 			acep += 2;
653 		} else {
654 			error = EINVAL;
655 			goto out;
656 		}
657 	}
658 
659 	*acepp = result;
660 	*rescount = resultsize;
661 
662 out:
663 	if (error != 0) {
664 		if ((result != NULL) && (resultsize > 0)) {
665 			cacl_free(result, resultsize * sizeof (ace_t));
666 		}
667 	}
668 
669 	return (error);
670 }
671 
672 static int
673 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, boolean_t isdir,
674     ace_t **retacep, int *retacecnt)
675 {
676 	ace_t *acep;
677 	ace_t *dfacep;
678 	int acecnt = 0;
679 	int dfacecnt = 0;
680 	int dfaclstart = 0;
681 	int dfaclcnt = 0;
682 	aclent_t *aclp;
683 	int i;
684 	int error;
685 	int acesz, dfacesz;
686 
687 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
688 
689 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
690 		if (aclp->a_type & ACL_DEFAULT)
691 			break;
692 	}
693 
694 	if (i < aclcnt) {
695 		dfaclstart = i;
696 		dfaclcnt = aclcnt - i;
697 	}
698 
699 	if (dfaclcnt && !isdir) {
700 		return (EINVAL);
701 	}
702 
703 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
704 	if (error)
705 		return (error);
706 
707 	if (dfaclcnt) {
708 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
709 		    &dfacep, &dfacecnt, isdir);
710 		if (error) {
711 			if (acep) {
712 				cacl_free(acep, acecnt * sizeof (ace_t));
713 			}
714 			return (error);
715 		}
716 	}
717 
718 	if (dfacecnt != 0) {
719 		acesz = sizeof (ace_t) * acecnt;
720 		dfacesz = sizeof (ace_t) * dfacecnt;
721 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
722 		if (acep == NULL)
723 			return (ENOMEM);
724 		if (dfaclcnt) {
725 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
726 		}
727 	}
728 	if (dfaclcnt)
729 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
730 
731 	*retacecnt = acecnt + dfacecnt;
732 	*retacep = acep;
733 	return (0);
734 }
735 
736 static int
737 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, boolean_t isdir)
738 {
739 	int error = 0;
740 	o_mode_t mode = 0;
741 	uint32_t bits, wantbits;
742 
743 	/* read */
744 	if (mask & ACE_READ_DATA)
745 		mode |= S_IROTH;
746 
747 	/* write */
748 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
749 	if (isdir)
750 		wantbits |= ACE_DELETE_CHILD;
751 	bits = mask & wantbits;
752 	if (bits != 0) {
753 		if (bits != wantbits) {
754 			error = ENOTSUP;
755 			goto out;
756 		}
757 		mode |= S_IWOTH;
758 	}
759 
760 	/* exec */
761 	if (mask & ACE_EXECUTE) {
762 		mode |= S_IXOTH;
763 	}
764 
765 	*modep = mode;
766 
767 out:
768 	return (error);
769 }
770 
771 static void
772 acevals_init(acevals_t *vals, uid_t key)
773 {
774 	bzero(vals, sizeof (*vals));
775 	vals->allowed = ACE_MASK_UNDEFINED;
776 	vals->denied = ACE_MASK_UNDEFINED;
777 	vals->mask = ACE_MASK_UNDEFINED;
778 	vals->key = key;
779 }
780 
781 static void
782 ace_list_init(ace_list_t *al, int dfacl_flag)
783 {
784 	acevals_init(&al->user_obj, NULL);
785 	acevals_init(&al->group_obj, NULL);
786 	acevals_init(&al->other_obj, NULL);
787 	al->numusers = 0;
788 	al->numgroups = 0;
789 	al->acl_mask = 0;
790 	al->hasmask = 0;
791 	al->state = ace_unused;
792 	al->seen = 0;
793 	al->dfacl_flag = dfacl_flag;
794 }
795 
796 /*
797  * Find or create an acevals holder for a given id and avl tree.
798  *
799  * Note that only one thread will ever touch these avl trees, so
800  * there is no need for locking.
801  */
802 static acevals_t *
803 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
804 {
805 	acevals_t key, *rc;
806 	avl_index_t where;
807 
808 	key.key = ace->a_who;
809 	rc = avl_find(avl, &key, &where);
810 	if (rc != NULL)
811 		return (rc);
812 
813 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
814 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
815 		return (NULL);
816 
817 	acevals_init(rc, ace->a_who);
818 	avl_insert(avl, rc, where);
819 	(*num)++;
820 
821 	return (rc);
822 }
823 
824 static int
825 access_mask_check(ace_t *acep, int mask_bit, int isowner)
826 {
827 	int set_deny, err_deny;
828 	int set_allow, err_allow;
829 	int acl_consume;
830 	int haswriteperm, hasreadperm;
831 
832 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
833 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
834 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
835 	} else {
836 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
837 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
838 	}
839 
840 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
841 	    ACL_DELETE_ERR_DENY |
842 	    ACL_WRITE_OWNER_ERR_DENY |
843 	    ACL_WRITE_OWNER_ERR_ALLOW |
844 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
845 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
846 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
847 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
848 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
849 	    ACL_READ_NAMED_READER_ERR_DENY);
850 
851 	if (mask_bit == ACE_SYNCHRONIZE) {
852 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
853 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
854 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
855 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
856 	} else if (mask_bit == ACE_WRITE_OWNER) {
857 		set_deny = ACL_WRITE_OWNER_SET_DENY;
858 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
859 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
860 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
861 	} else if (mask_bit == ACE_DELETE) {
862 		set_deny = ACL_DELETE_SET_DENY;
863 		err_deny =  ACL_DELETE_ERR_DENY;
864 		set_allow = ACL_DELETE_SET_ALLOW;
865 		err_allow = ACL_DELETE_ERR_ALLOW;
866 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
867 		if (isowner) {
868 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
869 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
870 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
871 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
872 		} else if (haswriteperm) {
873 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
874 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
875 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
876 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
877 		} else {
878 			if ((acep->a_access_mask & mask_bit) &&
879 			    (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) {
880 				return (ENOTSUP);
881 			}
882 			return (0);
883 		}
884 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
885 		if (!hasreadperm)
886 			return (0);
887 
888 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
889 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
890 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
891 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
892 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
893 		if (!haswriteperm)
894 			return (0);
895 
896 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
897 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
898 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
899 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
900 	} else {
901 		return (EINVAL);
902 	}
903 
904 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
905 		if (acl_consume & set_deny) {
906 			if (!(acep->a_access_mask & mask_bit)) {
907 				return (ENOTSUP);
908 			}
909 		} else if (acl_consume & err_deny) {
910 			if (acep->a_access_mask & mask_bit) {
911 				return (ENOTSUP);
912 			}
913 		}
914 	} else {
915 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
916 		if (acl_consume & set_allow) {
917 			if (!(acep->a_access_mask & mask_bit)) {
918 				return (ENOTSUP);
919 			}
920 		} else if (acl_consume & err_allow) {
921 			if (acep->a_access_mask & mask_bit) {
922 				return (ENOTSUP);
923 			}
924 		}
925 	}
926 	return (0);
927 }
928 
929 static int
930 ace_to_aent_legal(ace_t *acep)
931 {
932 	int error = 0;
933 	int isowner;
934 
935 	/* only ALLOW or DENY */
936 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
937 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
938 		error = ENOTSUP;
939 		goto out;
940 	}
941 
942 	/* check for invalid flags */
943 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
944 		error = EINVAL;
945 		goto out;
946 	}
947 
948 	/* some flags are illegal */
949 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
950 	    ACE_FAILED_ACCESS_ACE_FLAG |
951 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
952 		error = ENOTSUP;
953 		goto out;
954 	}
955 
956 	/* check for invalid masks */
957 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
958 		error = EINVAL;
959 		goto out;
960 	}
961 
962 	if ((acep->a_flags & ACE_OWNER)) {
963 		isowner = 1;
964 	} else {
965 		isowner = 0;
966 	}
967 
968 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
969 	if (error)
970 		goto out;
971 
972 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
973 	if (error)
974 		goto out;
975 
976 	error = access_mask_check(acep, ACE_DELETE, isowner);
977 	if (error)
978 		goto out;
979 
980 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
981 	if (error)
982 		goto out;
983 
984 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
985 	if (error)
986 		goto out;
987 
988 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
989 	if (error)
990 		goto out;
991 
992 	/* more detailed checking of masks */
993 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
994 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
995 			error = ENOTSUP;
996 			goto out;
997 		}
998 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
999 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
1000 			error = ENOTSUP;
1001 			goto out;
1002 		}
1003 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1004 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1005 			error = ENOTSUP;
1006 			goto out;
1007 		}
1008 	}
1009 
1010 	/* ACL enforcement */
1011 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1012 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1013 		error = ENOTSUP;
1014 		goto out;
1015 	}
1016 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1017 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1018 		    (isowner)) {
1019 			error = ENOTSUP;
1020 			goto out;
1021 		}
1022 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1023 		    (! isowner)) {
1024 			error = ENOTSUP;
1025 			goto out;
1026 		}
1027 	}
1028 
1029 out:
1030 	return (error);
1031 }
1032 
1033 static int
1034 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, boolean_t isdir)
1035 {
1036 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1037 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1038 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1039 		return (ENOTSUP);
1040 	}
1041 
1042 	return (ace_mask_to_mode(mask, modep, isdir));
1043 }
1044 
1045 static int
1046 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1047     uid_t owner, gid_t group, boolean_t isdir)
1048 {
1049 	int error;
1050 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1051 
1052 	if (isdir)
1053 		flips |= ACE_DELETE_CHILD;
1054 	if (vals->allowed != (vals->denied ^ flips)) {
1055 		error = ENOTSUP;
1056 		goto out;
1057 	}
1058 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1059 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1060 		error = ENOTSUP;
1061 		goto out;
1062 	}
1063 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1064 	if (error != 0)
1065 		goto out;
1066 	dest->a_type = vals->aent_type;
1067 	if (dest->a_type & (USER | GROUP)) {
1068 		dest->a_id = vals->key;
1069 	} else if (dest->a_type & USER_OBJ) {
1070 		dest->a_id = owner;
1071 	} else if (dest->a_type & GROUP_OBJ) {
1072 		dest->a_id = group;
1073 	} else if (dest->a_type & OTHER_OBJ) {
1074 		dest->a_id = 0;
1075 	} else {
1076 		error = EINVAL;
1077 		goto out;
1078 	}
1079 
1080 out:
1081 	return (error);
1082 }
1083 
1084 
1085 static int
1086 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1087     uid_t owner, gid_t group, boolean_t isdir)
1088 {
1089 	int error = 0;
1090 	aclent_t *aent, *result = NULL;
1091 	acevals_t *vals;
1092 	int resultcount;
1093 
1094 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1095 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1096 		error = ENOTSUP;
1097 		goto out;
1098 	}
1099 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1100 		error = ENOTSUP;
1101 		goto out;
1102 	}
1103 
1104 	resultcount = 3 + list->numusers + list->numgroups;
1105 	/*
1106 	 * This must be the same condition as below, when we add the CLASS_OBJ
1107 	 * (aka ACL mask)
1108 	 */
1109 	if ((list->hasmask) || (! list->dfacl_flag))
1110 		resultcount += 1;
1111 
1112 	if (cacl_malloc((void **)&result,
1113 	    resultcount * sizeof (aclent_t)) != 0) {
1114 		error = ENOMEM;
1115 		goto out;
1116 	}
1117 	aent = result;
1118 
1119 	/* USER_OBJ */
1120 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1121 		error = EINVAL;
1122 		goto out;
1123 	}
1124 
1125 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1126 	    isdir);
1127 
1128 	if (error != 0)
1129 		goto out;
1130 	++aent;
1131 	/* USER */
1132 	vals = NULL;
1133 	for (vals = avl_first(&list->user); vals != NULL;
1134 	    vals = AVL_NEXT(&list->user, vals)) {
1135 		if (!(vals->aent_type & USER)) {
1136 			error = EINVAL;
1137 			goto out;
1138 		}
1139 		error = acevals_to_aent(vals, aent, list, owner, group,
1140 		    isdir);
1141 		if (error != 0)
1142 			goto out;
1143 		++aent;
1144 	}
1145 	/* GROUP_OBJ */
1146 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1147 		error = EINVAL;
1148 		goto out;
1149 	}
1150 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1151 	    isdir);
1152 	if (error != 0)
1153 		goto out;
1154 	++aent;
1155 	/* GROUP */
1156 	vals = NULL;
1157 	for (vals = avl_first(&list->group); vals != NULL;
1158 	    vals = AVL_NEXT(&list->group, vals)) {
1159 		if (!(vals->aent_type & GROUP)) {
1160 			error = EINVAL;
1161 			goto out;
1162 		}
1163 		error = acevals_to_aent(vals, aent, list, owner, group,
1164 		    isdir);
1165 		if (error != 0)
1166 			goto out;
1167 		++aent;
1168 	}
1169 	/*
1170 	 * CLASS_OBJ (aka ACL_MASK)
1171 	 *
1172 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1173 	 * This is to follow UFS's behavior.
1174 	 */
1175 	if ((list->hasmask) || (! list->dfacl_flag)) {
1176 		if (list->hasmask) {
1177 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1178 			if (isdir)
1179 				flips |= ACE_DELETE_CHILD;
1180 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1181 			    &aent->a_perm, isdir);
1182 			if (error != 0)
1183 				goto out;
1184 		} else {
1185 			/* fabricate the ACL_MASK from the group permissions */
1186 			error = ace_mask_to_mode(list->group_obj.allowed,
1187 			    &aent->a_perm, isdir);
1188 			if (error != 0)
1189 				goto out;
1190 		}
1191 		aent->a_id = 0;
1192 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1193 		++aent;
1194 	}
1195 	/* OTHER_OBJ */
1196 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1197 		error = EINVAL;
1198 		goto out;
1199 	}
1200 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1201 	    isdir);
1202 	if (error != 0)
1203 		goto out;
1204 	++aent;
1205 
1206 	*aclentp = result;
1207 	*aclcnt = resultcount;
1208 
1209 out:
1210 	if (error != 0) {
1211 		if (result != NULL)
1212 			cacl_free(result, resultcount * sizeof (aclent_t));
1213 	}
1214 
1215 	return (error);
1216 }
1217 
1218 
1219 /*
1220  * free all data associated with an ace_list
1221  */
1222 static void
1223 ace_list_free(ace_list_t *al)
1224 {
1225 	acevals_t *node;
1226 	void *cookie;
1227 
1228 	if (al == NULL)
1229 		return;
1230 
1231 	cookie = NULL;
1232 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1233 		cacl_free(node, sizeof (acevals_t));
1234 	cookie = NULL;
1235 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1236 		cacl_free(node, sizeof (acevals_t));
1237 
1238 	avl_destroy(&al->user);
1239 	avl_destroy(&al->group);
1240 
1241 	/* free the container itself */
1242 	cacl_free(al, sizeof (ace_list_t));
1243 }
1244 
1245 static int
1246 acevals_compare(const void *va, const void *vb)
1247 {
1248 	const acevals_t *a = va, *b = vb;
1249 
1250 	if (a->key == b->key)
1251 		return (0);
1252 
1253 	if (a->key > b->key)
1254 		return (1);
1255 
1256 	else
1257 		return (-1);
1258 }
1259 
1260 /*
1261  * Convert a list of ace_t entries to equivalent regular and default
1262  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1263  */
1264 static int
1265 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1266     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1267     boolean_t isdir)
1268 {
1269 	int error = 0;
1270 	ace_t *acep;
1271 	uint32_t bits;
1272 	int i;
1273 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1274 	acevals_t *vals;
1275 
1276 	*aclentp = NULL;
1277 	*aclcnt = 0;
1278 	*dfaclentp = NULL;
1279 	*dfaclcnt = 0;
1280 
1281 	/* we need at least user_obj, group_obj, and other_obj */
1282 	if (n < 6) {
1283 		error = ENOTSUP;
1284 		goto out;
1285 	}
1286 	if (ace == NULL) {
1287 		error = EINVAL;
1288 		goto out;
1289 	}
1290 
1291 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1292 	if (error != 0)
1293 		goto out;
1294 
1295 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1296 	    offsetof(acevals_t, avl));
1297 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1298 	    offsetof(acevals_t, avl));
1299 
1300 	ace_list_init(normacl, 0);
1301 
1302 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1303 	if (error != 0)
1304 		goto out;
1305 
1306 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1307 	    offsetof(acevals_t, avl));
1308 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1309 	    offsetof(acevals_t, avl));
1310 	ace_list_init(dfacl, ACL_DEFAULT);
1311 
1312 	/* process every ace_t... */
1313 	for (i = 0; i < n; i++) {
1314 		acep = &ace[i];
1315 
1316 		/* rule out certain cases quickly */
1317 		error = ace_to_aent_legal(acep);
1318 		if (error != 0)
1319 			goto out;
1320 
1321 		/*
1322 		 * Turn off these bits in order to not have to worry about
1323 		 * them when doing the checks for compliments.
1324 		 */
1325 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1326 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1327 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1328 
1329 		/* see if this should be a regular or default acl */
1330 		bits = acep->a_flags &
1331 		    (ACE_INHERIT_ONLY_ACE |
1332 		    ACE_FILE_INHERIT_ACE |
1333 		    ACE_DIRECTORY_INHERIT_ACE);
1334 		if (bits != 0) {
1335 			/* all or nothing on these inherit bits */
1336 			if (bits != (ACE_INHERIT_ONLY_ACE |
1337 			    ACE_FILE_INHERIT_ACE |
1338 			    ACE_DIRECTORY_INHERIT_ACE)) {
1339 				error = ENOTSUP;
1340 				goto out;
1341 			}
1342 			acl = dfacl;
1343 		} else {
1344 			acl = normacl;
1345 		}
1346 
1347 		if ((acep->a_flags & ACE_OWNER)) {
1348 			if (acl->state > ace_user_obj) {
1349 				error = ENOTSUP;
1350 				goto out;
1351 			}
1352 			acl->state = ace_user_obj;
1353 			acl->seen |= USER_OBJ;
1354 			vals = &acl->user_obj;
1355 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1356 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1357 			acl->state = ace_other_obj;
1358 			acl->seen |= OTHER_OBJ;
1359 			vals = &acl->other_obj;
1360 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1361 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1362 			if (acl->state > ace_group) {
1363 				error = ENOTSUP;
1364 				goto out;
1365 			}
1366 			if ((acep->a_flags & ACE_GROUP)) {
1367 				acl->seen |= GROUP_OBJ;
1368 				vals = &acl->group_obj;
1369 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1370 			} else {
1371 				acl->seen |= GROUP;
1372 				vals = acevals_find(acep, &acl->group,
1373 				    &acl->numgroups);
1374 				if (vals == NULL) {
1375 					error = ENOMEM;
1376 					goto out;
1377 				}
1378 				vals->aent_type = GROUP | acl->dfacl_flag;
1379 			}
1380 			acl->state = ace_group;
1381 		} else {
1382 			if (acl->state > ace_user) {
1383 				error = ENOTSUP;
1384 				goto out;
1385 			}
1386 			acl->state = ace_user;
1387 			acl->seen |= USER;
1388 			vals = acevals_find(acep, &acl->user,
1389 			    &acl->numusers);
1390 			if (vals == NULL) {
1391 				error = ENOMEM;
1392 				goto out;
1393 			}
1394 			vals->aent_type = USER | acl->dfacl_flag;
1395 		}
1396 
1397 		if (!(acl->state > ace_unused)) {
1398 			error = EINVAL;
1399 			goto out;
1400 		}
1401 
1402 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1403 			/* no more than one allowed per aclent_t */
1404 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1405 				error = ENOTSUP;
1406 				goto out;
1407 			}
1408 			vals->allowed = acep->a_access_mask;
1409 		} else {
1410 			/*
1411 			 * it's a DENY; if there was a previous DENY, it
1412 			 * must have been an ACL_MASK.
1413 			 */
1414 			if (vals->denied != ACE_MASK_UNDEFINED) {
1415 				/* ACL_MASK is for USER and GROUP only */
1416 				if ((acl->state != ace_user) &&
1417 				    (acl->state != ace_group)) {
1418 					error = ENOTSUP;
1419 					goto out;
1420 				}
1421 
1422 				if (! acl->hasmask) {
1423 					acl->hasmask = 1;
1424 					acl->acl_mask = vals->denied;
1425 				/* check for mismatched ACL_MASK emulations */
1426 				} else if (acl->acl_mask != vals->denied) {
1427 					error = ENOTSUP;
1428 					goto out;
1429 				}
1430 				vals->mask = vals->denied;
1431 			}
1432 			vals->denied = acep->a_access_mask;
1433 		}
1434 	}
1435 
1436 	/* done collating; produce the aclent_t lists */
1437 	if (normacl->state != ace_unused) {
1438 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1439 		    owner, group, isdir);
1440 		if (error != 0) {
1441 			goto out;
1442 		}
1443 	}
1444 	if (dfacl->state != ace_unused) {
1445 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1446 		    owner, group, isdir);
1447 		if (error != 0) {
1448 			goto out;
1449 		}
1450 	}
1451 
1452 out:
1453 	if (normacl != NULL)
1454 		ace_list_free(normacl);
1455 	if (dfacl != NULL)
1456 		ace_list_free(dfacl);
1457 
1458 	return (error);
1459 }
1460 
1461 static int
1462 convert_ace_to_aent(ace_t *acebufp, int acecnt, boolean_t isdir,
1463     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1464 {
1465 	int error = 0;
1466 	aclent_t *aclentp, *dfaclentp;
1467 	int aclcnt, dfaclcnt;
1468 	int aclsz, dfaclsz = 0;
1469 
1470 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1471 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1472 
1473 	if (error)
1474 		return (error);
1475 
1476 
1477 	if (dfaclcnt != 0) {
1478 		/*
1479 		 * Slap aclentp and dfaclentp into a single array.
1480 		 */
1481 		aclsz = sizeof (aclent_t) * aclcnt;
1482 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1483 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1484 		if (aclentp != NULL) {
1485 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1486 		} else {
1487 			error = ENOMEM;
1488 		}
1489 	}
1490 
1491 	if (aclentp) {
1492 		*retaclentp = aclentp;
1493 		*retaclcnt = aclcnt + dfaclcnt;
1494 	}
1495 
1496 	if (dfaclentp)
1497 		cacl_free(dfaclentp, dfaclsz);
1498 
1499 	return (error);
1500 }
1501 
1502 
1503 int
1504 acl_translate(acl_t *aclp, int target_flavor, boolean_t isdir, uid_t owner,
1505     gid_t group)
1506 {
1507 	int aclcnt;
1508 	void *acldata;
1509 	int error;
1510 
1511 	/*
1512 	 * See if we need to translate
1513 	 */
1514 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1515 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1516 	    aclp->acl_type == ACLENT_T))
1517 		return (0);
1518 
1519 	if (target_flavor == -1) {
1520 		error = EINVAL;
1521 		goto out;
1522 	}
1523 
1524 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1525 	    aclp->acl_type == ACLENT_T) {
1526 		error = convert_aent_to_ace(aclp->acl_aclp,
1527 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1528 		if (error)
1529 			goto out;
1530 
1531 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1532 	    aclp->acl_type == ACE_T) {
1533 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1534 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1535 		if (error)
1536 			goto out;
1537 	} else {
1538 		error = ENOTSUP;
1539 		goto out;
1540 	}
1541 
1542 	/*
1543 	 * replace old acl with newly translated acl
1544 	 */
1545 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1546 	aclp->acl_aclp = acldata;
1547 	aclp->acl_cnt = aclcnt;
1548 	if (target_flavor == _ACL_ACE_ENABLED) {
1549 		aclp->acl_type = ACE_T;
1550 		aclp->acl_entry_size = sizeof (ace_t);
1551 	} else {
1552 		aclp->acl_type = ACLENT_T;
1553 		aclp->acl_entry_size = sizeof (aclent_t);
1554 	}
1555 	return (0);
1556 
1557 out:
1558 
1559 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
1560 	return (error);
1561 #else
1562 	errno = error;
1563 	return (-1);
1564 #endif
1565 }
1566 
1567 #define	SET_ACE(acl, index, who, mask, type, flags) { \
1568 	acl[0][index].a_who = (uint32_t)who; \
1569 	acl[0][index].a_type = type; \
1570 	acl[0][index].a_flags = flags; \
1571 	acl[0][index++].a_access_mask = mask; \
1572 }
1573 
1574 void
1575 acl_trivial_access_masks(mode_t mode, boolean_t isdir, trivial_acl_t *masks)
1576 {
1577 	uint32_t read_mask = ACE_READ_DATA;
1578 	uint32_t write_mask = ACE_WRITE_DATA|ACE_APPEND_DATA;
1579 	uint32_t execute_mask = ACE_EXECUTE;
1580 
1581 	if (isdir)
1582 		write_mask |= ACE_DELETE_CHILD;
1583 
1584 	masks->deny1 = 0;
1585 	if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH)))
1586 		masks->deny1 |= read_mask;
1587 	if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH)))
1588 		masks->deny1 |= write_mask;
1589 	if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH)))
1590 		masks->deny1 |= execute_mask;
1591 
1592 	masks->deny2 = 0;
1593 	if (!(mode & S_IRGRP) && (mode & S_IROTH))
1594 		masks->deny2 |= read_mask;
1595 	if (!(mode & S_IWGRP) && (mode & S_IWOTH))
1596 		masks->deny2 |= write_mask;
1597 	if (!(mode & S_IXGRP) && (mode & S_IXOTH))
1598 		masks->deny2 |= execute_mask;
1599 
1600 	masks->allow0 = 0;
1601 	if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH)))
1602 		masks->allow0 |= read_mask;
1603 	if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH)))
1604 		masks->allow0 |= write_mask;
1605 	if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH)))
1606 		masks->allow0 |= execute_mask;
1607 
1608 	masks->owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL|
1609 	    ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES|
1610 	    ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE;
1611 	if (mode & S_IRUSR)
1612 		masks->owner |= read_mask;
1613 	if (mode & S_IWUSR)
1614 		masks->owner |= write_mask;
1615 	if (mode & S_IXUSR)
1616 		masks->owner |= execute_mask;
1617 
1618 	masks->group = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1619 	    ACE_SYNCHRONIZE;
1620 	if (mode & S_IRGRP)
1621 		masks->group |= read_mask;
1622 	if (mode & S_IWGRP)
1623 		masks->group |= write_mask;
1624 	if (mode & S_IXGRP)
1625 		masks->group |= execute_mask;
1626 
1627 	masks->everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1628 	    ACE_SYNCHRONIZE;
1629 	if (mode & S_IROTH)
1630 		masks->everyone |= read_mask;
1631 	if (mode & S_IWOTH)
1632 		masks->everyone |= write_mask;
1633 	if (mode & S_IXOTH)
1634 		masks->everyone |= execute_mask;
1635 }
1636 
1637 int
1638 acl_trivial_create(mode_t mode, boolean_t isdir, ace_t **acl, int *count)
1639 {
1640 	int 		index = 0;
1641 	int		error;
1642 	trivial_acl_t	masks;
1643 
1644 	*count = 3;
1645 	acl_trivial_access_masks(mode, isdir, &masks);
1646 
1647 	if (masks.allow0)
1648 		(*count)++;
1649 	if (masks.deny1)
1650 		(*count)++;
1651 	if (masks.deny2)
1652 		(*count)++;
1653 
1654 	if ((error = cacl_malloc((void **)acl, *count * sizeof (ace_t))) != 0)
1655 		return (error);
1656 
1657 	if (masks.allow0) {
1658 		SET_ACE(acl, index, -1, masks.allow0,
1659 		    ACE_ACCESS_ALLOWED_ACE_TYPE, ACE_OWNER);
1660 	}
1661 	if (masks.deny1) {
1662 		SET_ACE(acl, index, -1, masks.deny1,
1663 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_OWNER);
1664 	}
1665 	if (masks.deny2) {
1666 		SET_ACE(acl, index, -1, masks.deny2,
1667 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_GROUP|ACE_IDENTIFIER_GROUP);
1668 	}
1669 
1670 	SET_ACE(acl, index, -1, masks.owner, ACE_ACCESS_ALLOWED_ACE_TYPE,
1671 	    ACE_OWNER);
1672 	SET_ACE(acl, index, -1, masks.group, ACE_ACCESS_ALLOWED_ACE_TYPE,
1673 	    ACE_IDENTIFIER_GROUP|ACE_GROUP);
1674 	SET_ACE(acl, index, -1, masks.everyone, ACE_ACCESS_ALLOWED_ACE_TYPE,
1675 	    ACE_EVERYONE);
1676 
1677 	return (0);
1678 }
1679 
1680 /*
1681  * ace_trivial:
1682  * determine whether an ace_t acl is trivial
1683  *
1684  * Trivialness implies that the acl is composed of only
1685  * owner, group, everyone entries.  ACL can't
1686  * have read_acl denied, and write_owner/write_acl/write_attributes
1687  * can only be owner@ entry.
1688  */
1689 int
1690 ace_trivial_common(void *acep, int aclcnt,
1691     uint64_t (*walk)(void *, uint64_t, int aclcnt,
1692     uint16_t *, uint16_t *, uint32_t *))
1693 {
1694 	uint16_t flags;
1695 	uint32_t mask;
1696 	uint16_t type;
1697 	uint64_t cookie = 0;
1698 
1699 	while (cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask)) {
1700 		switch (flags & ACE_TYPE_FLAGS) {
1701 		case ACE_OWNER:
1702 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
1703 		case ACE_EVERYONE:
1704 			break;
1705 		default:
1706 			return (1);
1707 
1708 		}
1709 
1710 		if (flags & (ACE_FILE_INHERIT_ACE|
1711 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
1712 		    ACE_INHERIT_ONLY_ACE))
1713 			return (1);
1714 
1715 		/*
1716 		 * Special check for some special bits
1717 		 *
1718 		 * Don't allow anybody to deny reading basic
1719 		 * attributes or a files ACL.
1720 		 */
1721 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
1722 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
1723 			return (1);
1724 
1725 		/*
1726 		 * Delete permission is never set by default
1727 		 */
1728 		if (mask & ACE_DELETE)
1729 			return (1);
1730 
1731 		/*
1732 		 * Child delete permission should be accompanied by write
1733 		 */
1734 		if ((mask & ACE_DELETE_CHILD) && !(mask & ACE_WRITE_DATA))
1735 			return (1);
1736 
1737 		/*
1738 		 * only allow owner@ to have
1739 		 * write_acl/write_owner/write_attributes/write_xattr/
1740 		 */
1741 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
1742 		    (!(flags & ACE_OWNER) && (mask &
1743 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES|
1744 		    ACE_WRITE_NAMED_ATTRS))))
1745 			return (1);
1746 
1747 	}
1748 	return (0);
1749 }
1750 
1751 uint64_t
1752 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
1753     uint16_t *type, uint32_t *mask)
1754 {
1755 	ace_t *acep = datap;
1756 
1757 	if (cookie >= aclcnt)
1758 		return (0);
1759 
1760 	*flags = acep[cookie].a_flags;
1761 	*type = acep[cookie].a_type;
1762 	*mask = acep[cookie++].a_access_mask;
1763 
1764 	return (cookie);
1765 }
1766 
1767 int
1768 ace_trivial(ace_t *acep, int aclcnt)
1769 {
1770 	return (ace_trivial_common(acep, aclcnt, ace_walk));
1771 }
1772