xref: /illumos-gate/usr/src/common/acl/acl_common.c (revision ef9416a8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <sys/errno.h>
29 #include <sys/avl.h>
30 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
31 #include <sys/debug.h>
32 #include <sys/kmem.h>
33 #include <sys/systm.h>
34 #include <sys/sysmacros.h>
35 #include <acl/acl_common.h>
36 #else
37 #include <errno.h>
38 #include <stdlib.h>
39 #include <stddef.h>
40 #include <strings.h>
41 #include <unistd.h>
42 #include <assert.h>
43 #include <grp.h>
44 #include <pwd.h>
45 #include <acl_common.h>
46 #define	ASSERT	assert
47 #endif
48 
49 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
50     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
51     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
52 
53 
54 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
55 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
56 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
57 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
58 
59 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
60 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
61 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
62 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
63 
64 #define	ACL_DELETE_SET_DENY			0x0000100
65 #define	ACL_DELETE_SET_ALLOW			0x0000200
66 #define	ACL_DELETE_ERR_DENY			0x0000400
67 #define	ACL_DELETE_ERR_ALLOW			0x0000800
68 
69 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
70 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
71 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
72 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
73 
74 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
75 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
76 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
77 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
78 
79 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
80 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
81 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
82 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
83 
84 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
85 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
86 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
87 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
88 
89 
90 #define	ACE_VALID_MASK_BITS (\
91     ACE_READ_DATA | \
92     ACE_LIST_DIRECTORY | \
93     ACE_WRITE_DATA | \
94     ACE_ADD_FILE | \
95     ACE_APPEND_DATA | \
96     ACE_ADD_SUBDIRECTORY | \
97     ACE_READ_NAMED_ATTRS | \
98     ACE_WRITE_NAMED_ATTRS | \
99     ACE_EXECUTE | \
100     ACE_DELETE_CHILD | \
101     ACE_READ_ATTRIBUTES | \
102     ACE_WRITE_ATTRIBUTES | \
103     ACE_DELETE | \
104     ACE_READ_ACL | \
105     ACE_WRITE_ACL | \
106     ACE_WRITE_OWNER | \
107     ACE_SYNCHRONIZE)
108 
109 #define	ACE_MASK_UNDEFINED			0x80000000
110 
111 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
112     ACE_DIRECTORY_INHERIT_ACE | \
113     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
114     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
115     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
116 
117 /*
118  * ACL conversion helpers
119  */
120 
121 typedef enum {
122 	ace_unused,
123 	ace_user_obj,
124 	ace_user,
125 	ace_group, /* includes GROUP and GROUP_OBJ */
126 	ace_other_obj
127 } ace_to_aent_state_t;
128 
129 typedef struct acevals {
130 	uid_t key;
131 	avl_node_t avl;
132 	uint32_t mask;
133 	uint32_t allowed;
134 	uint32_t denied;
135 	int aent_type;
136 } acevals_t;
137 
138 typedef struct ace_list {
139 	acevals_t user_obj;
140 	avl_tree_t user;
141 	int numusers;
142 	acevals_t group_obj;
143 	avl_tree_t group;
144 	int numgroups;
145 	acevals_t other_obj;
146 	uint32_t acl_mask;
147 	int hasmask;
148 	int dfacl_flag;
149 	ace_to_aent_state_t state;
150 	int seen; /* bitmask of all aclent_t a_type values seen */
151 } ace_list_t;
152 
153 /*
154  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
155  * v = Ptr to array/vector of objs
156  * n = # objs in the array
157  * s = size of each obj (must be multiples of a word size)
158  * f = ptr to function to compare two objs
159  *	returns (-1 = less than, 0 = equal, 1 = greater than
160  */
161 void
162 ksort(caddr_t v, int n, int s, int (*f)())
163 {
164 	int g, i, j, ii;
165 	unsigned int *p1, *p2;
166 	unsigned int tmp;
167 
168 	/* No work to do */
169 	if (v == NULL || n <= 1)
170 		return;
171 
172 	/* Sanity check on arguments */
173 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
174 	ASSERT(s > 0);
175 	for (g = n / 2; g > 0; g /= 2) {
176 		for (i = g; i < n; i++) {
177 			for (j = i - g; j >= 0 &&
178 			    (*f)(v + j * s, v + (j + g) * s) == 1;
179 			    j -= g) {
180 				p1 = (void *)(v + j * s);
181 				p2 = (void *)(v + (j + g) * s);
182 				for (ii = 0; ii < s / 4; ii++) {
183 					tmp = *p1;
184 					*p1++ = *p2;
185 					*p2++ = tmp;
186 				}
187 			}
188 		}
189 	}
190 }
191 
192 /*
193  * Compare two acls, all fields.  Returns:
194  * -1 (less than)
195  *  0 (equal)
196  * +1 (greater than)
197  */
198 int
199 cmp2acls(void *a, void *b)
200 {
201 	aclent_t *x = (aclent_t *)a;
202 	aclent_t *y = (aclent_t *)b;
203 
204 	/* Compare types */
205 	if (x->a_type < y->a_type)
206 		return (-1);
207 	if (x->a_type > y->a_type)
208 		return (1);
209 	/* Equal types; compare id's */
210 	if (x->a_id < y->a_id)
211 		return (-1);
212 	if (x->a_id > y->a_id)
213 		return (1);
214 	/* Equal ids; compare perms */
215 	if (x->a_perm < y->a_perm)
216 		return (-1);
217 	if (x->a_perm > y->a_perm)
218 		return (1);
219 	/* Totally equal */
220 	return (0);
221 }
222 
223 /*ARGSUSED*/
224 static void *
225 cacl_realloc(void *ptr, size_t size, size_t new_size)
226 {
227 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
228 	void *tmp;
229 
230 	tmp = kmem_alloc(new_size, KM_SLEEP);
231 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
232 	kmem_free(ptr, size);
233 	return (tmp);
234 #else
235 	return (realloc(ptr, new_size));
236 #endif
237 }
238 
239 static int
240 cacl_malloc(void **ptr, size_t size)
241 {
242 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
243 	*ptr = kmem_zalloc(size, KM_SLEEP);
244 	return (0);
245 #else
246 	*ptr = calloc(1, size);
247 	if (*ptr == NULL)
248 		return (errno);
249 
250 	return (0);
251 #endif
252 }
253 
254 /*ARGSUSED*/
255 static void
256 cacl_free(void *ptr, size_t size)
257 {
258 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
259 	kmem_free(ptr, size);
260 #else
261 	free(ptr);
262 #endif
263 }
264 
265 acl_t *
266 acl_alloc(enum acl_type type)
267 {
268 	acl_t *aclp;
269 
270 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
271 		return (NULL);
272 
273 	aclp->acl_aclp = NULL;
274 	aclp->acl_cnt = 0;
275 
276 	switch (type) {
277 	case ACE_T:
278 		aclp->acl_type = ACE_T;
279 		aclp->acl_entry_size = sizeof (ace_t);
280 		break;
281 	case ACLENT_T:
282 		aclp->acl_type = ACLENT_T;
283 		aclp->acl_entry_size = sizeof (aclent_t);
284 		break;
285 	default:
286 		acl_free(aclp);
287 		aclp = NULL;
288 	}
289 	return (aclp);
290 }
291 
292 /*
293  * Free acl_t structure
294  */
295 void
296 acl_free(acl_t *aclp)
297 {
298 	int acl_size;
299 
300 	if (aclp == NULL)
301 		return;
302 
303 	if (aclp->acl_aclp) {
304 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
305 		cacl_free(aclp->acl_aclp, acl_size);
306 	}
307 
308 	cacl_free(aclp, sizeof (acl_t));
309 }
310 
311 static uint32_t
312 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
313 {
314 	uint32_t access_mask = 0;
315 	int acl_produce;
316 	int synchronize_set = 0, write_owner_set = 0;
317 	int delete_set = 0, write_attrs_set = 0;
318 	int read_named_set = 0, write_named_set = 0;
319 
320 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
321 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
322 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
323 
324 	if (isallow) {
325 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
326 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
327 		delete_set = ACL_DELETE_SET_ALLOW;
328 		if (hasreadperm)
329 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
330 		if (haswriteperm)
331 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
332 		if (isowner)
333 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
334 		else if (haswriteperm)
335 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
336 	} else {
337 
338 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
339 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
340 		delete_set = ACL_DELETE_SET_DENY;
341 		if (hasreadperm)
342 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
343 		if (haswriteperm)
344 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
345 		if (isowner)
346 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
347 		else if (haswriteperm)
348 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
349 		else
350 			/*
351 			 * If the entity is not the owner and does not
352 			 * have write permissions ACE_WRITE_ATTRIBUTES will
353 			 * always go in the DENY ACE.
354 			 */
355 			access_mask |= ACE_WRITE_ATTRIBUTES;
356 	}
357 
358 	if (acl_produce & synchronize_set)
359 		access_mask |= ACE_SYNCHRONIZE;
360 	if (acl_produce & write_owner_set)
361 		access_mask |= ACE_WRITE_OWNER;
362 	if (acl_produce & delete_set)
363 		access_mask |= ACE_DELETE;
364 	if (acl_produce & write_attrs_set)
365 		access_mask |= ACE_WRITE_ATTRIBUTES;
366 	if (acl_produce & read_named_set)
367 		access_mask |= ACE_READ_NAMED_ATTRS;
368 	if (acl_produce & write_named_set)
369 		access_mask |= ACE_WRITE_NAMED_ATTRS;
370 
371 	return (access_mask);
372 }
373 
374 /*
375  * Given an mode_t, convert it into an access_mask as used
376  * by nfsace, assuming aclent_t -> nfsace semantics.
377  */
378 static uint32_t
379 mode_to_ace_access(mode_t mode, boolean_t isdir, int isowner, int isallow)
380 {
381 	uint32_t access = 0;
382 	int haswriteperm = 0;
383 	int hasreadperm = 0;
384 
385 	if (isallow) {
386 		haswriteperm = (mode & S_IWOTH);
387 		hasreadperm = (mode & S_IROTH);
388 	} else {
389 		haswriteperm = !(mode & S_IWOTH);
390 		hasreadperm = !(mode & S_IROTH);
391 	}
392 
393 	/*
394 	 * The following call takes care of correctly setting the following
395 	 * mask bits in the access_mask:
396 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
397 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
398 	 */
399 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
400 
401 	if (isallow) {
402 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
403 		if (isowner)
404 			access |= ACE_WRITE_ACL;
405 	} else {
406 		if (! isowner)
407 			access |= ACE_WRITE_ACL;
408 	}
409 
410 	/* read */
411 	if (mode & S_IROTH) {
412 		access |= ACE_READ_DATA;
413 	}
414 	/* write */
415 	if (mode & S_IWOTH) {
416 		access |= ACE_WRITE_DATA |
417 		    ACE_APPEND_DATA;
418 		if (isdir)
419 			access |= ACE_DELETE_CHILD;
420 	}
421 	/* exec */
422 	if (mode & S_IXOTH) {
423 		access |= ACE_EXECUTE;
424 	}
425 
426 	return (access);
427 }
428 
429 /*
430  * Given an nfsace (presumably an ALLOW entry), make a
431  * corresponding DENY entry at the address given.
432  */
433 static void
434 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
435 {
436 	(void) memcpy(deny, allow, sizeof (ace_t));
437 
438 	deny->a_who = allow->a_who;
439 
440 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
441 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
442 	if (isdir)
443 		deny->a_access_mask ^= ACE_DELETE_CHILD;
444 
445 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
446 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
447 	    ACE_WRITE_NAMED_ATTRS);
448 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
449 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
450 	    B_FALSE);
451 }
452 /*
453  * Make an initial pass over an array of aclent_t's.  Gather
454  * information such as an ACL_MASK (if any), number of users,
455  * number of groups, and whether the array needs to be sorted.
456  */
457 static int
458 ln_aent_preprocess(aclent_t *aclent, int n,
459     int *hasmask, mode_t *mask,
460     int *numuser, int *numgroup, int *needsort)
461 {
462 	int error = 0;
463 	int i;
464 	int curtype = 0;
465 
466 	*hasmask = 0;
467 	*mask = 07;
468 	*needsort = 0;
469 	*numuser = 0;
470 	*numgroup = 0;
471 
472 	for (i = 0; i < n; i++) {
473 		if (aclent[i].a_type < curtype)
474 			*needsort = 1;
475 		else if (aclent[i].a_type > curtype)
476 			curtype = aclent[i].a_type;
477 		if (aclent[i].a_type & USER)
478 			(*numuser)++;
479 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
480 			(*numgroup)++;
481 		if (aclent[i].a_type & CLASS_OBJ) {
482 			if (*hasmask) {
483 				error = EINVAL;
484 				goto out;
485 			} else {
486 				*hasmask = 1;
487 				*mask = aclent[i].a_perm;
488 			}
489 		}
490 	}
491 
492 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
493 		error = EINVAL;
494 		goto out;
495 	}
496 
497 out:
498 	return (error);
499 }
500 
501 /*
502  * Convert an array of aclent_t into an array of nfsace entries,
503  * following POSIX draft -> nfsv4 conversion semantics as outlined in
504  * the IETF draft.
505  */
506 static int
507 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
508 {
509 	int error = 0;
510 	mode_t mask;
511 	int numuser, numgroup, needsort;
512 	int resultsize = 0;
513 	int i, groupi = 0, skip;
514 	ace_t *acep, *result = NULL;
515 	int hasmask;
516 
517 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
518 	    &numuser, &numgroup, &needsort);
519 	if (error != 0)
520 		goto out;
521 
522 	/* allow + deny for each aclent */
523 	resultsize = n * 2;
524 	if (hasmask) {
525 		/*
526 		 * stick extra deny on the group_obj and on each
527 		 * user|group for the mask (the group_obj was added
528 		 * into the count for numgroup)
529 		 */
530 		resultsize += numuser + numgroup;
531 		/* ... and don't count the mask itself */
532 		resultsize -= 2;
533 	}
534 
535 	/* sort the source if necessary */
536 	if (needsort)
537 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
538 
539 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
540 		goto out;
541 
542 	acep = result;
543 
544 	for (i = 0; i < n; i++) {
545 		/*
546 		 * don't process CLASS_OBJ (mask); mask was grabbed in
547 		 * ln_aent_preprocess()
548 		 */
549 		if (aclent[i].a_type & CLASS_OBJ)
550 			continue;
551 
552 		/* If we need an ACL_MASK emulator, prepend it now */
553 		if ((hasmask) &&
554 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
555 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
556 			acep->a_flags = 0;
557 			if (aclent[i].a_type & GROUP_OBJ) {
558 				acep->a_who = (uid_t)-1;
559 				acep->a_flags |=
560 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
561 			} else if (aclent[i].a_type & USER) {
562 				acep->a_who = aclent[i].a_id;
563 			} else {
564 				acep->a_who = aclent[i].a_id;
565 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
566 			}
567 			if (aclent[i].a_type & ACL_DEFAULT) {
568 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
569 				    ACE_FILE_INHERIT_ACE |
570 				    ACE_DIRECTORY_INHERIT_ACE;
571 			}
572 			/*
573 			 * Set the access mask for the prepended deny
574 			 * ace.  To do this, we invert the mask (found
575 			 * in ln_aent_preprocess()) then convert it to an
576 			 * DENY ace access_mask.
577 			 */
578 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
579 			    isdir, 0, 0);
580 			acep += 1;
581 		}
582 
583 		/* handle a_perm -> access_mask */
584 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
585 		    isdir, aclent[i].a_type & USER_OBJ, 1);
586 
587 		/* emulate a default aclent */
588 		if (aclent[i].a_type & ACL_DEFAULT) {
589 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
590 			    ACE_FILE_INHERIT_ACE |
591 			    ACE_DIRECTORY_INHERIT_ACE;
592 		}
593 
594 		/*
595 		 * handle a_perm and a_id
596 		 *
597 		 * this must be done last, since it involves the
598 		 * corresponding deny aces, which are handled
599 		 * differently for each different a_type.
600 		 */
601 		if (aclent[i].a_type & USER_OBJ) {
602 			acep->a_who = (uid_t)-1;
603 			acep->a_flags |= ACE_OWNER;
604 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
605 			acep += 2;
606 		} else if (aclent[i].a_type & USER) {
607 			acep->a_who = aclent[i].a_id;
608 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
609 			acep += 2;
610 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
611 			if (aclent[i].a_type & GROUP_OBJ) {
612 				acep->a_who = (uid_t)-1;
613 				acep->a_flags |= ACE_GROUP;
614 			} else {
615 				acep->a_who = aclent[i].a_id;
616 			}
617 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
618 			/*
619 			 * Set the corresponding deny for the group ace.
620 			 *
621 			 * The deny aces go after all of the groups, unlike
622 			 * everything else, where they immediately follow
623 			 * the allow ace.
624 			 *
625 			 * We calculate "skip", the number of slots to
626 			 * skip ahead for the deny ace, here.
627 			 *
628 			 * The pattern is:
629 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
630 			 * thus, skip is
631 			 * (2 * numgroup) - 1 - groupi
632 			 * (2 * numgroup) to account for MD + A
633 			 * - 1 to account for the fact that we're on the
634 			 * access (A), not the mask (MD)
635 			 * - groupi to account for the fact that we have
636 			 * passed up groupi number of MD's.
637 			 */
638 			skip = (2 * numgroup) - 1 - groupi;
639 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
640 			/*
641 			 * If we just did the last group, skip acep past
642 			 * all of the denies; else, just move ahead one.
643 			 */
644 			if (++groupi >= numgroup)
645 				acep += numgroup + 1;
646 			else
647 				acep += 1;
648 		} else if (aclent[i].a_type & OTHER_OBJ) {
649 			acep->a_who = (uid_t)-1;
650 			acep->a_flags |= ACE_EVERYONE;
651 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
652 			acep += 2;
653 		} else {
654 			error = EINVAL;
655 			goto out;
656 		}
657 	}
658 
659 	*acepp = result;
660 	*rescount = resultsize;
661 
662 out:
663 	if (error != 0) {
664 		if ((result != NULL) && (resultsize > 0)) {
665 			cacl_free(result, resultsize * sizeof (ace_t));
666 		}
667 	}
668 
669 	return (error);
670 }
671 
672 static int
673 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, boolean_t isdir,
674     ace_t **retacep, int *retacecnt)
675 {
676 	ace_t *acep;
677 	ace_t *dfacep;
678 	int acecnt = 0;
679 	int dfacecnt = 0;
680 	int dfaclstart = 0;
681 	int dfaclcnt = 0;
682 	aclent_t *aclp;
683 	int i;
684 	int error;
685 	int acesz, dfacesz;
686 
687 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
688 
689 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
690 		if (aclp->a_type & ACL_DEFAULT)
691 			break;
692 	}
693 
694 	if (i < aclcnt) {
695 		dfaclstart = i;
696 		dfaclcnt = aclcnt - i;
697 	}
698 
699 	if (dfaclcnt && !isdir) {
700 		return (EINVAL);
701 	}
702 
703 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
704 	if (error)
705 		return (error);
706 
707 	if (dfaclcnt) {
708 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
709 		    &dfacep, &dfacecnt, isdir);
710 		if (error) {
711 			if (acep) {
712 				cacl_free(acep, acecnt * sizeof (ace_t));
713 			}
714 			return (error);
715 		}
716 	}
717 
718 	if (dfacecnt != 0) {
719 		acesz = sizeof (ace_t) * acecnt;
720 		dfacesz = sizeof (ace_t) * dfacecnt;
721 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
722 		if (acep == NULL)
723 			return (ENOMEM);
724 		if (dfaclcnt) {
725 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
726 		}
727 	}
728 	if (dfaclcnt)
729 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
730 
731 	*retacecnt = acecnt + dfacecnt;
732 	*retacep = acep;
733 	return (0);
734 }
735 
736 static int
737 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, boolean_t isdir)
738 {
739 	int error = 0;
740 	o_mode_t mode = 0;
741 	uint32_t bits, wantbits;
742 
743 	/* read */
744 	if (mask & ACE_READ_DATA)
745 		mode |= S_IROTH;
746 
747 	/* write */
748 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
749 	if (isdir)
750 		wantbits |= ACE_DELETE_CHILD;
751 	bits = mask & wantbits;
752 	if (bits != 0) {
753 		if (bits != wantbits) {
754 			error = ENOTSUP;
755 			goto out;
756 		}
757 		mode |= S_IWOTH;
758 	}
759 
760 	/* exec */
761 	if (mask & ACE_EXECUTE) {
762 		mode |= S_IXOTH;
763 	}
764 
765 	*modep = mode;
766 
767 out:
768 	return (error);
769 }
770 
771 static void
772 acevals_init(acevals_t *vals, uid_t key)
773 {
774 	bzero(vals, sizeof (*vals));
775 	vals->allowed = ACE_MASK_UNDEFINED;
776 	vals->denied = ACE_MASK_UNDEFINED;
777 	vals->mask = ACE_MASK_UNDEFINED;
778 	vals->key = key;
779 }
780 
781 static void
782 ace_list_init(ace_list_t *al, int dfacl_flag)
783 {
784 	acevals_init(&al->user_obj, 0);
785 	acevals_init(&al->group_obj, 0);
786 	acevals_init(&al->other_obj, 0);
787 	al->numusers = 0;
788 	al->numgroups = 0;
789 	al->acl_mask = 0;
790 	al->hasmask = 0;
791 	al->state = ace_unused;
792 	al->seen = 0;
793 	al->dfacl_flag = dfacl_flag;
794 }
795 
796 /*
797  * Find or create an acevals holder for a given id and avl tree.
798  *
799  * Note that only one thread will ever touch these avl trees, so
800  * there is no need for locking.
801  */
802 static acevals_t *
803 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
804 {
805 	acevals_t key, *rc;
806 	avl_index_t where;
807 
808 	key.key = ace->a_who;
809 	rc = avl_find(avl, &key, &where);
810 	if (rc != NULL)
811 		return (rc);
812 
813 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
814 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
815 		return (NULL);
816 
817 	acevals_init(rc, ace->a_who);
818 	avl_insert(avl, rc, where);
819 	(*num)++;
820 
821 	return (rc);
822 }
823 
824 static int
825 access_mask_check(ace_t *acep, int mask_bit, int isowner)
826 {
827 	int set_deny, err_deny;
828 	int set_allow, err_allow;
829 	int acl_consume;
830 	int haswriteperm, hasreadperm;
831 
832 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
833 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
834 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
835 	} else {
836 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
837 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
838 	}
839 
840 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
841 	    ACL_DELETE_ERR_DENY |
842 	    ACL_WRITE_OWNER_ERR_DENY |
843 	    ACL_WRITE_OWNER_ERR_ALLOW |
844 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
845 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
846 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
847 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
848 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
849 	    ACL_READ_NAMED_READER_ERR_DENY);
850 
851 	if (mask_bit == ACE_SYNCHRONIZE) {
852 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
853 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
854 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
855 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
856 	} else if (mask_bit == ACE_WRITE_OWNER) {
857 		set_deny = ACL_WRITE_OWNER_SET_DENY;
858 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
859 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
860 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
861 	} else if (mask_bit == ACE_DELETE) {
862 		set_deny = ACL_DELETE_SET_DENY;
863 		err_deny =  ACL_DELETE_ERR_DENY;
864 		set_allow = ACL_DELETE_SET_ALLOW;
865 		err_allow = ACL_DELETE_ERR_ALLOW;
866 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
867 		if (isowner) {
868 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
869 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
870 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
871 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
872 		} else if (haswriteperm) {
873 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
874 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
875 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
876 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
877 		} else {
878 			if (((acep->a_access_mask & mask_bit) != 0) &&
879 			    (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE)) {
880 				return (ENOTSUP);
881 			}
882 			return (0);
883 		}
884 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
885 		if (!hasreadperm)
886 			return (0);
887 
888 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
889 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
890 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
891 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
892 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
893 		if (!haswriteperm)
894 			return (0);
895 
896 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
897 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
898 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
899 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
900 	} else {
901 		return (EINVAL);
902 	}
903 
904 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
905 		if (acl_consume & set_deny) {
906 			if (!(acep->a_access_mask & mask_bit)) {
907 				return (ENOTSUP);
908 			}
909 		} else if (acl_consume & err_deny) {
910 			if (acep->a_access_mask & mask_bit) {
911 				return (ENOTSUP);
912 			}
913 		}
914 	} else {
915 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
916 		if (acl_consume & set_allow) {
917 			if (!(acep->a_access_mask & mask_bit)) {
918 				return (ENOTSUP);
919 			}
920 		} else if (acl_consume & err_allow) {
921 			if (acep->a_access_mask & mask_bit) {
922 				return (ENOTSUP);
923 			}
924 		}
925 	}
926 	return (0);
927 }
928 
929 static int
930 ace_to_aent_legal(ace_t *acep)
931 {
932 	int error = 0;
933 	int isowner;
934 
935 	/* only ALLOW or DENY */
936 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
937 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
938 		error = ENOTSUP;
939 		goto out;
940 	}
941 
942 	/* check for invalid flags */
943 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
944 		error = EINVAL;
945 		goto out;
946 	}
947 
948 	/* some flags are illegal */
949 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
950 	    ACE_FAILED_ACCESS_ACE_FLAG |
951 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
952 		error = ENOTSUP;
953 		goto out;
954 	}
955 
956 	/* check for invalid masks */
957 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
958 		error = EINVAL;
959 		goto out;
960 	}
961 
962 	if ((acep->a_flags & ACE_OWNER)) {
963 		isowner = 1;
964 	} else {
965 		isowner = 0;
966 	}
967 
968 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
969 	if (error)
970 		goto out;
971 
972 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
973 	if (error)
974 		goto out;
975 
976 	error = access_mask_check(acep, ACE_DELETE, isowner);
977 	if (error)
978 		goto out;
979 
980 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
981 	if (error)
982 		goto out;
983 
984 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
985 	if (error)
986 		goto out;
987 
988 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
989 	if (error)
990 		goto out;
991 
992 	/* more detailed checking of masks */
993 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
994 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
995 			error = ENOTSUP;
996 			goto out;
997 		}
998 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
999 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
1000 			error = ENOTSUP;
1001 			goto out;
1002 		}
1003 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1004 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1005 			error = ENOTSUP;
1006 			goto out;
1007 		}
1008 	}
1009 
1010 	/* ACL enforcement */
1011 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1012 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1013 		error = ENOTSUP;
1014 		goto out;
1015 	}
1016 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1017 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1018 		    (isowner)) {
1019 			error = ENOTSUP;
1020 			goto out;
1021 		}
1022 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1023 		    (! isowner)) {
1024 			error = ENOTSUP;
1025 			goto out;
1026 		}
1027 	}
1028 
1029 out:
1030 	return (error);
1031 }
1032 
1033 static int
1034 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, boolean_t isdir)
1035 {
1036 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1037 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1038 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1039 		return (ENOTSUP);
1040 	}
1041 
1042 	return (ace_mask_to_mode(mask, modep, isdir));
1043 }
1044 
1045 static int
1046 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1047     uid_t owner, gid_t group, boolean_t isdir)
1048 {
1049 	int error;
1050 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1051 
1052 	if (isdir)
1053 		flips |= ACE_DELETE_CHILD;
1054 	if (vals->allowed != (vals->denied ^ flips)) {
1055 		error = ENOTSUP;
1056 		goto out;
1057 	}
1058 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1059 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1060 		error = ENOTSUP;
1061 		goto out;
1062 	}
1063 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1064 	if (error != 0)
1065 		goto out;
1066 	dest->a_type = vals->aent_type;
1067 	if (dest->a_type & (USER | GROUP)) {
1068 		dest->a_id = vals->key;
1069 	} else if (dest->a_type & USER_OBJ) {
1070 		dest->a_id = owner;
1071 	} else if (dest->a_type & GROUP_OBJ) {
1072 		dest->a_id = group;
1073 	} else if (dest->a_type & OTHER_OBJ) {
1074 		dest->a_id = 0;
1075 	} else {
1076 		error = EINVAL;
1077 		goto out;
1078 	}
1079 
1080 out:
1081 	return (error);
1082 }
1083 
1084 
1085 static int
1086 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1087     uid_t owner, gid_t group, boolean_t isdir)
1088 {
1089 	int error = 0;
1090 	aclent_t *aent, *result = NULL;
1091 	acevals_t *vals;
1092 	int resultcount;
1093 
1094 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1095 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1096 		return (ENOTSUP);
1097 	}
1098 
1099 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1100 		return (ENOTSUP);
1101 	}
1102 
1103 	resultcount = 3 + list->numusers + list->numgroups;
1104 	/*
1105 	 * This must be the same condition as below, when we add the CLASS_OBJ
1106 	 * (aka ACL mask)
1107 	 */
1108 	if ((list->hasmask) || (! list->dfacl_flag))
1109 		resultcount += 1;
1110 
1111 	if (cacl_malloc((void **)&result,
1112 	    resultcount * sizeof (aclent_t)) != 0) {
1113 		error = ENOMEM;
1114 		goto out;
1115 	}
1116 	aent = result;
1117 
1118 	/* USER_OBJ */
1119 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1120 		error = EINVAL;
1121 		goto out;
1122 	}
1123 
1124 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1125 	    isdir);
1126 
1127 	if (error != 0)
1128 		goto out;
1129 	++aent;
1130 	/* USER */
1131 	vals = NULL;
1132 	for (vals = avl_first(&list->user); vals != NULL;
1133 	    vals = AVL_NEXT(&list->user, vals)) {
1134 		if (!(vals->aent_type & USER)) {
1135 			error = EINVAL;
1136 			goto out;
1137 		}
1138 		error = acevals_to_aent(vals, aent, list, owner, group,
1139 		    isdir);
1140 		if (error != 0)
1141 			goto out;
1142 		++aent;
1143 	}
1144 	/* GROUP_OBJ */
1145 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1146 		error = EINVAL;
1147 		goto out;
1148 	}
1149 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1150 	    isdir);
1151 	if (error != 0)
1152 		goto out;
1153 	++aent;
1154 	/* GROUP */
1155 	vals = NULL;
1156 	for (vals = avl_first(&list->group); vals != NULL;
1157 	    vals = AVL_NEXT(&list->group, vals)) {
1158 		if (!(vals->aent_type & GROUP)) {
1159 			error = EINVAL;
1160 			goto out;
1161 		}
1162 		error = acevals_to_aent(vals, aent, list, owner, group,
1163 		    isdir);
1164 		if (error != 0)
1165 			goto out;
1166 		++aent;
1167 	}
1168 	/*
1169 	 * CLASS_OBJ (aka ACL_MASK)
1170 	 *
1171 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1172 	 * This is to follow UFS's behavior.
1173 	 */
1174 	if ((list->hasmask) || (! list->dfacl_flag)) {
1175 		if (list->hasmask) {
1176 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1177 			if (isdir)
1178 				flips |= ACE_DELETE_CHILD;
1179 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1180 			    &aent->a_perm, isdir);
1181 			if (error != 0)
1182 				goto out;
1183 		} else {
1184 			/* fabricate the ACL_MASK from the group permissions */
1185 			error = ace_mask_to_mode(list->group_obj.allowed,
1186 			    &aent->a_perm, isdir);
1187 			if (error != 0)
1188 				goto out;
1189 		}
1190 		aent->a_id = 0;
1191 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1192 		++aent;
1193 	}
1194 	/* OTHER_OBJ */
1195 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1196 		error = EINVAL;
1197 		goto out;
1198 	}
1199 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1200 	    isdir);
1201 	if (error != 0)
1202 		goto out;
1203 	++aent;
1204 
1205 	*aclentp = result;
1206 	*aclcnt = resultcount;
1207 
1208 out:
1209 	if (error != 0) {
1210 		if (result != NULL)
1211 			cacl_free(result, resultcount * sizeof (aclent_t));
1212 	}
1213 
1214 	return (error);
1215 }
1216 
1217 
1218 /*
1219  * free all data associated with an ace_list
1220  */
1221 static void
1222 ace_list_free(ace_list_t *al)
1223 {
1224 	acevals_t *node;
1225 	void *cookie;
1226 
1227 	if (al == NULL)
1228 		return;
1229 
1230 	cookie = NULL;
1231 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1232 		cacl_free(node, sizeof (acevals_t));
1233 	cookie = NULL;
1234 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1235 		cacl_free(node, sizeof (acevals_t));
1236 
1237 	avl_destroy(&al->user);
1238 	avl_destroy(&al->group);
1239 
1240 	/* free the container itself */
1241 	cacl_free(al, sizeof (ace_list_t));
1242 }
1243 
1244 static int
1245 acevals_compare(const void *va, const void *vb)
1246 {
1247 	const acevals_t *a = va, *b = vb;
1248 
1249 	if (a->key == b->key)
1250 		return (0);
1251 
1252 	if (a->key > b->key)
1253 		return (1);
1254 
1255 	else
1256 		return (-1);
1257 }
1258 
1259 /*
1260  * Convert a list of ace_t entries to equivalent regular and default
1261  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1262  */
1263 static int
1264 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1265     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1266     boolean_t isdir)
1267 {
1268 	int error = 0;
1269 	ace_t *acep;
1270 	uint32_t bits;
1271 	int i;
1272 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1273 	acevals_t *vals;
1274 
1275 	*aclentp = NULL;
1276 	*aclcnt = 0;
1277 	*dfaclentp = NULL;
1278 	*dfaclcnt = 0;
1279 
1280 	/* we need at least user_obj, group_obj, and other_obj */
1281 	if (n < 6) {
1282 		error = ENOTSUP;
1283 		goto out;
1284 	}
1285 	if (ace == NULL) {
1286 		error = EINVAL;
1287 		goto out;
1288 	}
1289 
1290 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1291 	if (error != 0)
1292 		goto out;
1293 
1294 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1295 	    offsetof(acevals_t, avl));
1296 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1297 	    offsetof(acevals_t, avl));
1298 
1299 	ace_list_init(normacl, 0);
1300 
1301 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1302 	if (error != 0)
1303 		goto out;
1304 
1305 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1306 	    offsetof(acevals_t, avl));
1307 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1308 	    offsetof(acevals_t, avl));
1309 	ace_list_init(dfacl, ACL_DEFAULT);
1310 
1311 	/* process every ace_t... */
1312 	for (i = 0; i < n; i++) {
1313 		acep = &ace[i];
1314 
1315 		/* rule out certain cases quickly */
1316 		error = ace_to_aent_legal(acep);
1317 		if (error != 0)
1318 			goto out;
1319 
1320 		/*
1321 		 * Turn off these bits in order to not have to worry about
1322 		 * them when doing the checks for compliments.
1323 		 */
1324 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1325 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1326 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1327 
1328 		/* see if this should be a regular or default acl */
1329 		bits = acep->a_flags &
1330 		    (ACE_INHERIT_ONLY_ACE |
1331 		    ACE_FILE_INHERIT_ACE |
1332 		    ACE_DIRECTORY_INHERIT_ACE);
1333 		if (bits != 0) {
1334 			/* all or nothing on these inherit bits */
1335 			if (bits != (ACE_INHERIT_ONLY_ACE |
1336 			    ACE_FILE_INHERIT_ACE |
1337 			    ACE_DIRECTORY_INHERIT_ACE)) {
1338 				error = ENOTSUP;
1339 				goto out;
1340 			}
1341 			acl = dfacl;
1342 		} else {
1343 			acl = normacl;
1344 		}
1345 
1346 		if ((acep->a_flags & ACE_OWNER)) {
1347 			if (acl->state > ace_user_obj) {
1348 				error = ENOTSUP;
1349 				goto out;
1350 			}
1351 			acl->state = ace_user_obj;
1352 			acl->seen |= USER_OBJ;
1353 			vals = &acl->user_obj;
1354 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1355 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1356 			acl->state = ace_other_obj;
1357 			acl->seen |= OTHER_OBJ;
1358 			vals = &acl->other_obj;
1359 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1360 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1361 			if (acl->state > ace_group) {
1362 				error = ENOTSUP;
1363 				goto out;
1364 			}
1365 			if ((acep->a_flags & ACE_GROUP)) {
1366 				acl->seen |= GROUP_OBJ;
1367 				vals = &acl->group_obj;
1368 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1369 			} else {
1370 				acl->seen |= GROUP;
1371 				vals = acevals_find(acep, &acl->group,
1372 				    &acl->numgroups);
1373 				if (vals == NULL) {
1374 					error = ENOMEM;
1375 					goto out;
1376 				}
1377 				vals->aent_type = GROUP | acl->dfacl_flag;
1378 			}
1379 			acl->state = ace_group;
1380 		} else {
1381 			if (acl->state > ace_user) {
1382 				error = ENOTSUP;
1383 				goto out;
1384 			}
1385 			acl->state = ace_user;
1386 			acl->seen |= USER;
1387 			vals = acevals_find(acep, &acl->user,
1388 			    &acl->numusers);
1389 			if (vals == NULL) {
1390 				error = ENOMEM;
1391 				goto out;
1392 			}
1393 			vals->aent_type = USER | acl->dfacl_flag;
1394 		}
1395 
1396 		if (!(acl->state > ace_unused)) {
1397 			error = EINVAL;
1398 			goto out;
1399 		}
1400 
1401 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1402 			/* no more than one allowed per aclent_t */
1403 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1404 				error = ENOTSUP;
1405 				goto out;
1406 			}
1407 			vals->allowed = acep->a_access_mask;
1408 		} else {
1409 			/*
1410 			 * it's a DENY; if there was a previous DENY, it
1411 			 * must have been an ACL_MASK.
1412 			 */
1413 			if (vals->denied != ACE_MASK_UNDEFINED) {
1414 				/* ACL_MASK is for USER and GROUP only */
1415 				if ((acl->state != ace_user) &&
1416 				    (acl->state != ace_group)) {
1417 					error = ENOTSUP;
1418 					goto out;
1419 				}
1420 
1421 				if (! acl->hasmask) {
1422 					acl->hasmask = 1;
1423 					acl->acl_mask = vals->denied;
1424 				/* check for mismatched ACL_MASK emulations */
1425 				} else if (acl->acl_mask != vals->denied) {
1426 					error = ENOTSUP;
1427 					goto out;
1428 				}
1429 				vals->mask = vals->denied;
1430 			}
1431 			vals->denied = acep->a_access_mask;
1432 		}
1433 	}
1434 
1435 	/* done collating; produce the aclent_t lists */
1436 	if (normacl->state != ace_unused) {
1437 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1438 		    owner, group, isdir);
1439 		if (error != 0) {
1440 			goto out;
1441 		}
1442 	}
1443 	if (dfacl->state != ace_unused) {
1444 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1445 		    owner, group, isdir);
1446 		if (error != 0) {
1447 			goto out;
1448 		}
1449 	}
1450 
1451 out:
1452 	if (normacl != NULL)
1453 		ace_list_free(normacl);
1454 	if (dfacl != NULL)
1455 		ace_list_free(dfacl);
1456 
1457 	return (error);
1458 }
1459 
1460 static int
1461 convert_ace_to_aent(ace_t *acebufp, int acecnt, boolean_t isdir,
1462     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1463 {
1464 	int error = 0;
1465 	aclent_t *aclentp, *dfaclentp;
1466 	int aclcnt, dfaclcnt;
1467 	int aclsz, dfaclsz = 0;
1468 
1469 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1470 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1471 
1472 	if (error)
1473 		return (error);
1474 
1475 
1476 	if (dfaclcnt != 0) {
1477 		/*
1478 		 * Slap aclentp and dfaclentp into a single array.
1479 		 */
1480 		aclsz = sizeof (aclent_t) * aclcnt;
1481 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1482 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1483 		if (aclentp != NULL) {
1484 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1485 		} else {
1486 			error = ENOMEM;
1487 		}
1488 	}
1489 
1490 	if (aclentp) {
1491 		*retaclentp = aclentp;
1492 		*retaclcnt = aclcnt + dfaclcnt;
1493 	}
1494 
1495 	if (dfaclentp)
1496 		cacl_free(dfaclentp, dfaclsz);
1497 
1498 	return (error);
1499 }
1500 
1501 
1502 int
1503 acl_translate(acl_t *aclp, int target_flavor, boolean_t isdir, uid_t owner,
1504     gid_t group)
1505 {
1506 	int aclcnt;
1507 	void *acldata;
1508 	int error;
1509 
1510 	/*
1511 	 * See if we need to translate
1512 	 */
1513 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1514 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1515 	    aclp->acl_type == ACLENT_T))
1516 		return (0);
1517 
1518 	if (target_flavor == -1) {
1519 		error = EINVAL;
1520 		goto out;
1521 	}
1522 
1523 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1524 	    aclp->acl_type == ACLENT_T) {
1525 		error = convert_aent_to_ace(aclp->acl_aclp,
1526 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1527 		if (error)
1528 			goto out;
1529 
1530 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1531 	    aclp->acl_type == ACE_T) {
1532 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1533 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1534 		if (error)
1535 			goto out;
1536 	} else {
1537 		error = ENOTSUP;
1538 		goto out;
1539 	}
1540 
1541 	/*
1542 	 * replace old acl with newly translated acl
1543 	 */
1544 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1545 	aclp->acl_aclp = acldata;
1546 	aclp->acl_cnt = aclcnt;
1547 	if (target_flavor == _ACL_ACE_ENABLED) {
1548 		aclp->acl_type = ACE_T;
1549 		aclp->acl_entry_size = sizeof (ace_t);
1550 	} else {
1551 		aclp->acl_type = ACLENT_T;
1552 		aclp->acl_entry_size = sizeof (aclent_t);
1553 	}
1554 	return (0);
1555 
1556 out:
1557 
1558 #if defined(_KERNEL) || defined(_FAKE_KERNEL)
1559 	return (error);
1560 #else
1561 	errno = error;
1562 	return (-1);
1563 #endif
1564 }
1565 
1566 #define	SET_ACE(acl, index, who, mask, type, flags) { \
1567 	acl[0][index].a_who = (uint32_t)who; \
1568 	acl[0][index].a_type = type; \
1569 	acl[0][index].a_flags = flags; \
1570 	acl[0][index++].a_access_mask = mask; \
1571 }
1572 
1573 void
1574 acl_trivial_access_masks(mode_t mode, boolean_t isdir, trivial_acl_t *masks)
1575 {
1576 	uint32_t read_mask = ACE_READ_DATA;
1577 	uint32_t write_mask = ACE_WRITE_DATA|ACE_APPEND_DATA;
1578 	uint32_t execute_mask = ACE_EXECUTE;
1579 
1580 	if (isdir)
1581 		write_mask |= ACE_DELETE_CHILD;
1582 
1583 	masks->deny1 = 0;
1584 	if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH)))
1585 		masks->deny1 |= read_mask;
1586 	if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH)))
1587 		masks->deny1 |= write_mask;
1588 	if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH)))
1589 		masks->deny1 |= execute_mask;
1590 
1591 	masks->deny2 = 0;
1592 	if (!(mode & S_IRGRP) && (mode & S_IROTH))
1593 		masks->deny2 |= read_mask;
1594 	if (!(mode & S_IWGRP) && (mode & S_IWOTH))
1595 		masks->deny2 |= write_mask;
1596 	if (!(mode & S_IXGRP) && (mode & S_IXOTH))
1597 		masks->deny2 |= execute_mask;
1598 
1599 	masks->allow0 = 0;
1600 	if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH)))
1601 		masks->allow0 |= read_mask;
1602 	if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH)))
1603 		masks->allow0 |= write_mask;
1604 	if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH)))
1605 		masks->allow0 |= execute_mask;
1606 
1607 	masks->owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL|
1608 	    ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES|
1609 	    ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE;
1610 	if (mode & S_IRUSR)
1611 		masks->owner |= read_mask;
1612 	if (mode & S_IWUSR)
1613 		masks->owner |= write_mask;
1614 	if (mode & S_IXUSR)
1615 		masks->owner |= execute_mask;
1616 
1617 	masks->group = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1618 	    ACE_SYNCHRONIZE;
1619 	if (mode & S_IRGRP)
1620 		masks->group |= read_mask;
1621 	if (mode & S_IWGRP)
1622 		masks->group |= write_mask;
1623 	if (mode & S_IXGRP)
1624 		masks->group |= execute_mask;
1625 
1626 	masks->everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1627 	    ACE_SYNCHRONIZE;
1628 	if (mode & S_IROTH)
1629 		masks->everyone |= read_mask;
1630 	if (mode & S_IWOTH)
1631 		masks->everyone |= write_mask;
1632 	if (mode & S_IXOTH)
1633 		masks->everyone |= execute_mask;
1634 }
1635 
1636 int
1637 acl_trivial_create(mode_t mode, boolean_t isdir, ace_t **acl, int *count)
1638 {
1639 	int		index = 0;
1640 	int		error;
1641 	trivial_acl_t	masks;
1642 
1643 	*count = 3;
1644 	acl_trivial_access_masks(mode, isdir, &masks);
1645 
1646 	if (masks.allow0)
1647 		(*count)++;
1648 	if (masks.deny1)
1649 		(*count)++;
1650 	if (masks.deny2)
1651 		(*count)++;
1652 
1653 	if ((error = cacl_malloc((void **)acl, *count * sizeof (ace_t))) != 0)
1654 		return (error);
1655 
1656 	if (masks.allow0) {
1657 		SET_ACE(acl, index, -1, masks.allow0,
1658 		    ACE_ACCESS_ALLOWED_ACE_TYPE, ACE_OWNER);
1659 	}
1660 	if (masks.deny1) {
1661 		SET_ACE(acl, index, -1, masks.deny1,
1662 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_OWNER);
1663 	}
1664 	if (masks.deny2) {
1665 		SET_ACE(acl, index, -1, masks.deny2,
1666 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_GROUP|ACE_IDENTIFIER_GROUP);
1667 	}
1668 
1669 	SET_ACE(acl, index, -1, masks.owner, ACE_ACCESS_ALLOWED_ACE_TYPE,
1670 	    ACE_OWNER);
1671 	SET_ACE(acl, index, -1, masks.group, ACE_ACCESS_ALLOWED_ACE_TYPE,
1672 	    ACE_IDENTIFIER_GROUP|ACE_GROUP);
1673 	SET_ACE(acl, index, -1, masks.everyone, ACE_ACCESS_ALLOWED_ACE_TYPE,
1674 	    ACE_EVERYONE);
1675 
1676 	return (0);
1677 }
1678 
1679 /*
1680  * ace_trivial:
1681  * determine whether an ace_t acl is trivial
1682  *
1683  * Trivialness implies that the acl is composed of only
1684  * owner, group, everyone entries.  ACL can't
1685  * have read_acl denied, and write_owner/write_acl/write_attributes
1686  * can only be owner@ entry.
1687  */
1688 int
1689 ace_trivial_common(void *acep, int aclcnt,
1690     uint64_t (*walk)(void *, uint64_t, int aclcnt,
1691     uint16_t *, uint16_t *, uint32_t *))
1692 {
1693 	uint16_t flags;
1694 	uint32_t mask;
1695 	uint16_t type;
1696 	uint64_t cookie = 0;
1697 
1698 	while ((cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask))
1699 	    != 0) {
1700 		switch (flags & ACE_TYPE_FLAGS) {
1701 		case ACE_OWNER:
1702 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
1703 		case ACE_EVERYONE:
1704 			break;
1705 		default:
1706 			return (1);
1707 
1708 		}
1709 
1710 		if (flags & (ACE_FILE_INHERIT_ACE|
1711 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
1712 		    ACE_INHERIT_ONLY_ACE))
1713 			return (1);
1714 
1715 		/*
1716 		 * Special check for some special bits
1717 		 *
1718 		 * Don't allow anybody to deny reading basic
1719 		 * attributes or a files ACL.
1720 		 */
1721 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
1722 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
1723 			return (1);
1724 
1725 		/*
1726 		 * Delete permission is never set by default
1727 		 */
1728 		if (mask & ACE_DELETE)
1729 			return (1);
1730 
1731 		/*
1732 		 * Child delete permission should be accompanied by write
1733 		 */
1734 		if ((mask & ACE_DELETE_CHILD) && !(mask & ACE_WRITE_DATA))
1735 			return (1);
1736 
1737 		/*
1738 		 * only allow owner@ to have
1739 		 * write_acl/write_owner/write_attributes/write_xattr/
1740 		 */
1741 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
1742 		    (!(flags & ACE_OWNER) && (mask &
1743 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES|
1744 		    ACE_WRITE_NAMED_ATTRS))))
1745 			return (1);
1746 
1747 	}
1748 	return (0);
1749 }
1750 
1751 uint64_t
1752 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
1753     uint16_t *type, uint32_t *mask)
1754 {
1755 	ace_t *acep = datap;
1756 
1757 	if (cookie >= aclcnt)
1758 		return (0);
1759 
1760 	*flags = acep[cookie].a_flags;
1761 	*type = acep[cookie].a_type;
1762 	*mask = acep[cookie++].a_access_mask;
1763 
1764 	return (cookie);
1765 }
1766 
1767 int
1768 ace_trivial(ace_t *acep, int aclcnt)
1769 {
1770 	return (ace_trivial_common(acep, aclcnt, ace_walk));
1771 }
1772