xref: /illumos-gate/usr/src/uts/common/os/fio.c (revision cf96e8bf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2015, Joyent Inc.
25  */
26 
27 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	All Rights Reserved */
29 
30 #include <sys/types.h>
31 #include <sys/sysmacros.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/errno.h>
35 #include <sys/signal.h>
36 #include <sys/cred.h>
37 #include <sys/user.h>
38 #include <sys/conf.h>
39 #include <sys/vfs.h>
40 #include <sys/vnode.h>
41 #include <sys/pathname.h>
42 #include <sys/file.h>
43 #include <sys/flock.h>
44 #include <sys/proc.h>
45 #include <sys/var.h>
46 #include <sys/cpuvar.h>
47 #include <sys/open.h>
48 #include <sys/cmn_err.h>
49 #include <sys/priocntl.h>
50 #include <sys/procset.h>
51 #include <sys/prsystm.h>
52 #include <sys/debug.h>
53 #include <sys/kmem.h>
54 #include <sys/atomic.h>
55 #include <sys/fcntl.h>
56 #include <sys/poll.h>
57 #include <sys/rctl.h>
58 #include <sys/port_impl.h>
59 #include <sys/dtrace.h>
60 
61 #include <c2/audit.h>
62 #include <sys/nbmlock.h>
63 
64 #ifdef DEBUG
65 
66 static uint32_t afd_maxfd;	/* # of entries in maximum allocated array */
67 static uint32_t afd_alloc;	/* count of kmem_alloc()s */
68 static uint32_t afd_free;	/* count of kmem_free()s */
69 static uint32_t afd_wait;	/* count of waits on non-zero ref count */
70 #define	MAXFD(x)	(afd_maxfd = ((afd_maxfd >= (x))? afd_maxfd : (x)))
71 #define	COUNT(x)	atomic_inc_32(&x)
72 
73 #else	/* DEBUG */
74 
75 #define	MAXFD(x)
76 #define	COUNT(x)
77 
78 #endif	/* DEBUG */
79 
80 kmem_cache_t *file_cache;
81 
82 static void port_close_fd(portfd_t *);
83 
84 /*
85  * File descriptor allocation.
86  *
87  * fd_find(fip, minfd) finds the first available descriptor >= minfd.
88  * The most common case is open(2), in which minfd = 0, but we must also
89  * support fcntl(fd, F_DUPFD, minfd).
90  *
91  * The algorithm is as follows: we keep all file descriptors in an infix
92  * binary tree in which each node records the number of descriptors
93  * allocated in its right subtree, including itself.  Starting at minfd,
94  * we ascend the tree until we find a non-fully allocated right subtree.
95  * We then descend that subtree in a binary search for the smallest fd.
96  * Finally, we ascend the tree again to increment the allocation count
97  * of every subtree containing the newly-allocated fd.  Freeing an fd
98  * requires only the last step: we ascend the tree to decrement allocation
99  * counts.  Each of these three steps (ascent to find non-full subtree,
100  * descent to find lowest fd, ascent to update allocation counts) is
101  * O(log n), thus the algorithm as a whole is O(log n).
102  *
103  * We don't implement the fd tree using the customary left/right/parent
104  * pointers, but instead take advantage of the glorious mathematics of
105  * full infix binary trees.  For reference, here's an illustration of the
106  * logical structure of such a tree, rooted at 4 (binary 100), covering
107  * the range 1-7 (binary 001-111).  Our canonical trees do not include
108  * fd 0; we'll deal with that later.
109  *
110  *	      100
111  *	     /	 \
112  *	    /	  \
113  *	  010	  110
114  *	  / \	  / \
115  *	001 011 101 111
116  *
117  * We make the following observations, all of which are easily proven by
118  * induction on the depth of the tree:
119  *
120  * (T1) The least-significant bit (LSB) of any node is equal to its level
121  *      in the tree.  In our example, nodes 001, 011, 101 and 111 are at
122  *      level 0; nodes 010 and 110 are at level 1; and node 100 is at level 2.
123  *
124  * (T2) The child size (CSIZE) of node N -- that is, the total number of
125  *	right-branch descendants in a child of node N, including itself -- is
126  *	given by clearing all but the least significant bit of N.  This
127  *	follows immediately from (T1).  Applying this rule to our example, we
128  *	see that CSIZE(100) = 100, CSIZE(x10) = 10, and CSIZE(xx1) = 1.
129  *
130  * (T3) The nearest left ancestor (LPARENT) of node N -- that is, the nearest
131  *	ancestor containing node N in its right child -- is given by clearing
132  *	the LSB of N.  For example, LPARENT(111) = 110 and LPARENT(110) = 100.
133  *	Clearing the LSB of nodes 001, 010 or 100 yields zero, reflecting
134  *	the fact that these are leftmost nodes.  Note that this algorithm
135  *	automatically skips generations as necessary.  For example, the parent
136  *      of node 101 is 110, which is a *right* ancestor (not what we want);
137  *      but its grandparent is 100, which is a left ancestor. Clearing the LSB
138  *      of 101 gets us to 100 directly, skipping right past the uninteresting
139  *      generation (110).
140  *
141  *      Note that since LPARENT clears the LSB, whereas CSIZE clears all *but*
142  *	the LSB, we can express LPARENT() nicely in terms of CSIZE():
143  *
144  *	LPARENT(N) = N - CSIZE(N)
145  *
146  * (T4) The nearest right ancestor (RPARENT) of node N is given by:
147  *
148  *	RPARENT(N) = N + CSIZE(N)
149  *
150  * (T5) For every interior node, the children differ from their parent by
151  *	CSIZE(parent) / 2.  In our example, CSIZE(100) / 2 = 2 = 10 binary,
152  *      and indeed, the children of 100 are 100 +/- 10 = 010 and 110.
153  *
154  * Next, we'll need a few two's-complement math tricks.  Suppose a number,
155  * N, has the following form:
156  *
157  *		N = xxxx10...0
158  *
159  * That is, the binary representation of N consists of some string of bits,
160  * then a 1, then all zeroes.  This amounts to nothing more than saying that
161  * N has a least-significant bit, which is true for any N != 0.  If we look
162  * at N and N - 1 together, we see that we can combine them in useful ways:
163  *
164  *		  N = xxxx10...0
165  *	      N - 1 = xxxx01...1
166  *	------------------------
167  *	N & (N - 1) = xxxx000000
168  *	N | (N - 1) = xxxx111111
169  *	N ^ (N - 1) =     111111
170  *
171  * In particular, this suggests several easy ways to clear all but the LSB,
172  * which by (T2) is exactly what we need to determine CSIZE(N) = 10...0.
173  * We'll opt for this formulation:
174  *
175  *	(C1) CSIZE(N) = (N - 1) ^ (N | (N - 1))
176  *
177  * Similarly, we have an easy way to determine LPARENT(N), which requires
178  * that we clear the LSB of N:
179  *
180  *	(L1) LPARENT(N) = N & (N - 1)
181  *
182  * We note in the above relations that (N | (N - 1)) - N = CSIZE(N) - 1.
183  * When combined with (T4), this yields an easy way to compute RPARENT(N):
184  *
185  *	(R1) RPARENT(N) = (N | (N - 1)) + 1
186  *
187  * Finally, to accommodate fd 0 we must adjust all of our results by +/-1 to
188  * move the fd range from [1, 2^n) to [0, 2^n - 1).  This is straightforward,
189  * so there's no need to belabor the algebra; the revised relations become:
190  *
191  *	(C1a) CSIZE(N) = N ^ (N | (N + 1))
192  *
193  *	(L1a) LPARENT(N) = (N & (N + 1)) - 1
194  *
195  *	(R1a) RPARENT(N) = N | (N + 1)
196  *
197  * This completes the mathematical framework.  We now have all the tools
198  * we need to implement fd_find() and fd_reserve().
199  *
200  * fd_find(fip, minfd) finds the smallest available file descriptor >= minfd.
201  * It does not actually allocate the descriptor; that's done by fd_reserve().
202  * fd_find() proceeds in two steps:
203  *
204  * (1) Find the leftmost subtree that contains a descriptor >= minfd.
205  *     We start at the right subtree rooted at minfd.  If this subtree is
206  *     not full -- if fip->fi_list[minfd].uf_alloc != CSIZE(minfd) -- then
207  *     step 1 is done.  Otherwise, we know that all fds in this subtree
208  *     are taken, so we ascend to RPARENT(minfd) using (R1a).  We repeat
209  *     this process until we either find a candidate subtree or exceed
210  *     fip->fi_nfiles.  We use (C1a) to compute CSIZE().
211  *
212  * (2) Find the smallest fd in the subtree discovered by step 1.
213  *     Starting at the root of this subtree, we descend to find the
214  *     smallest available fd.  Since the left children have the smaller
215  *     fds, we will descend rightward only when the left child is full.
216  *
217  *     We begin by comparing the number of allocated fds in the root
218  *     to the number of allocated fds in its right child; if they differ
219  *     by exactly CSIZE(child), we know the left subtree is full, so we
220  *     descend right; that is, the right child becomes the search root.
221  *     Otherwise we leave the root alone and start following the right
222  *     child's left children.  As fortune would have it, this is very
223  *     simple computationally: by (T5), the right child of fd is just
224  *     fd + size, where size = CSIZE(fd) / 2.  Applying (T5) again,
225  *     we find that the right child's left child is fd + size - (size / 2) =
226  *     fd + (size / 2); *its* left child is fd + (size / 2) - (size / 4) =
227  *     fd + (size / 4), and so on.  In general, fd's right child's
228  *     leftmost nth descendant is fd + (size >> n).  Thus, to follow
229  *     the right child's left descendants, we just halve the size in
230  *     each iteration of the search.
231  *
232  *     When we descend leftward, we must keep track of the number of fds
233  *     that were allocated in all the right subtrees we rejected, so we
234  *     know how many of the root fd's allocations are in the remaining
235  *     (as yet unexplored) leftmost part of its right subtree.  When we
236  *     encounter a fully-allocated left child -- that is, when we find
237  *     that fip->fi_list[fd].uf_alloc == ralloc + size -- we descend right
238  *     (as described earlier), resetting ralloc to zero.
239  *
240  * fd_reserve(fip, fd, incr) either allocates or frees fd, depending
241  * on whether incr is 1 or -1.  Starting at fd, fd_reserve() ascends
242  * the leftmost ancestors (see (T3)) and updates the allocation counts.
243  * At each step we use (L1a) to compute LPARENT(), the next left ancestor.
244  *
245  * flist_minsize() finds the minimal tree that still covers all
246  * used fds; as long as the allocation count of a root node is zero, we
247  * don't need that node or its right subtree.
248  *
249  * flist_nalloc() counts the number of allocated fds in the tree, by starting
250  * at the top of the tree and summing the right-subtree allocation counts as
251  * it descends leftwards.
252  *
253  * Note: we assume that flist_grow() will keep fip->fi_nfiles of the form
254  * 2^n - 1.  This ensures that the fd trees are always full, which saves
255  * quite a bit of boundary checking.
256  */
257 static int
fd_find(uf_info_t * fip,int minfd)258 fd_find(uf_info_t *fip, int minfd)
259 {
260 	int size, ralloc, fd;
261 
262 	ASSERT(MUTEX_HELD(&fip->fi_lock));
263 	ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
264 
265 	for (fd = minfd; (uint_t)fd < fip->fi_nfiles; fd |= fd + 1) {
266 		size = fd ^ (fd | (fd + 1));
267 		if (fip->fi_list[fd].uf_alloc == size)
268 			continue;
269 		for (ralloc = 0, size >>= 1; size != 0; size >>= 1) {
270 			ralloc += fip->fi_list[fd + size].uf_alloc;
271 			if (fip->fi_list[fd].uf_alloc == ralloc + size) {
272 				fd += size;
273 				ralloc = 0;
274 			}
275 		}
276 		return (fd);
277 	}
278 	return (-1);
279 }
280 
281 static void
fd_reserve(uf_info_t * fip,int fd,int incr)282 fd_reserve(uf_info_t *fip, int fd, int incr)
283 {
284 	int pfd;
285 	uf_entry_t *ufp = &fip->fi_list[fd];
286 
287 	ASSERT((uint_t)fd < fip->fi_nfiles);
288 	ASSERT((ufp->uf_busy == 0 && incr == 1) ||
289 	    (ufp->uf_busy == 1 && incr == -1));
290 	ASSERT(MUTEX_HELD(&ufp->uf_lock));
291 	ASSERT(MUTEX_HELD(&fip->fi_lock));
292 
293 	for (pfd = fd; pfd >= 0; pfd = (pfd & (pfd + 1)) - 1)
294 		fip->fi_list[pfd].uf_alloc += incr;
295 
296 	ufp->uf_busy += incr;
297 }
298 
299 static int
flist_minsize(uf_info_t * fip)300 flist_minsize(uf_info_t *fip)
301 {
302 	int fd;
303 
304 	/*
305 	 * We'd like to ASSERT(MUTEX_HELD(&fip->fi_lock)), but we're called
306 	 * by flist_fork(), which relies on other mechanisms for mutual
307 	 * exclusion.
308 	 */
309 	ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
310 
311 	for (fd = fip->fi_nfiles; fd != 0; fd >>= 1)
312 		if (fip->fi_list[fd >> 1].uf_alloc != 0)
313 			break;
314 
315 	return (fd);
316 }
317 
318 static int
flist_nalloc(uf_info_t * fip)319 flist_nalloc(uf_info_t *fip)
320 {
321 	int fd;
322 	int nalloc = 0;
323 
324 	ASSERT(MUTEX_HELD(&fip->fi_lock));
325 	ASSERT((fip->fi_nfiles & (fip->fi_nfiles + 1)) == 0);
326 
327 	for (fd = fip->fi_nfiles; fd != 0; fd >>= 1)
328 		nalloc += fip->fi_list[fd >> 1].uf_alloc;
329 
330 	return (nalloc);
331 }
332 
333 /*
334  * Increase size of the fi_list array to accommodate at least maxfd.
335  * We keep the size of the form 2^n - 1 for benefit of fd_find().
336  */
337 static void
flist_grow(int maxfd)338 flist_grow(int maxfd)
339 {
340 	uf_info_t *fip = P_FINFO(curproc);
341 	int newcnt, oldcnt;
342 	uf_entry_t *src, *dst, *newlist, *oldlist, *newend, *oldend;
343 	uf_rlist_t *urp;
344 
345 	for (newcnt = 1; newcnt <= maxfd; newcnt = (newcnt << 1) | 1)
346 		continue;
347 
348 	newlist = kmem_zalloc(newcnt * sizeof (uf_entry_t), KM_SLEEP);
349 
350 	mutex_enter(&fip->fi_lock);
351 	oldcnt = fip->fi_nfiles;
352 	if (newcnt <= oldcnt) {
353 		mutex_exit(&fip->fi_lock);
354 		kmem_free(newlist, newcnt * sizeof (uf_entry_t));
355 		return;
356 	}
357 	ASSERT((newcnt & (newcnt + 1)) == 0);
358 	oldlist = fip->fi_list;
359 	oldend = oldlist + oldcnt;
360 	newend = newlist + oldcnt;	/* no need to lock beyond old end */
361 
362 	/*
363 	 * fi_list and fi_nfiles cannot change while any uf_lock is held,
364 	 * so we must grab all the old locks *and* the new locks up to oldcnt.
365 	 * (Locks beyond the end of oldcnt aren't visible until we store
366 	 * the new fi_nfiles, which is the last thing we do before dropping
367 	 * all the locks, so there's no need to acquire these locks).
368 	 * Holding the new locks is necessary because when fi_list changes
369 	 * to point to the new list, fi_nfiles won't have been stored yet.
370 	 * If we *didn't* hold the new locks, someone doing a UF_ENTER()
371 	 * could see the new fi_list, grab the new uf_lock, and then see
372 	 * fi_nfiles change while the lock is held -- in violation of
373 	 * UF_ENTER() semantics.
374 	 */
375 	for (src = oldlist; src < oldend; src++)
376 		mutex_enter(&src->uf_lock);
377 
378 	for (dst = newlist; dst < newend; dst++)
379 		mutex_enter(&dst->uf_lock);
380 
381 	for (src = oldlist, dst = newlist; src < oldend; src++, dst++) {
382 		dst->uf_file = src->uf_file;
383 		dst->uf_fpollinfo = src->uf_fpollinfo;
384 		dst->uf_refcnt = src->uf_refcnt;
385 		dst->uf_alloc = src->uf_alloc;
386 		dst->uf_flag = src->uf_flag;
387 		dst->uf_busy = src->uf_busy;
388 		dst->uf_portfd = src->uf_portfd;
389 		dst->uf_gen = src->uf_gen;
390 	}
391 
392 	/*
393 	 * As soon as we store the new flist, future locking operations
394 	 * will use it.  Therefore, we must ensure that all the state
395 	 * we've just established reaches global visibility before the
396 	 * new flist does.
397 	 */
398 	membar_producer();
399 	fip->fi_list = newlist;
400 
401 	/*
402 	 * Routines like getf() make an optimistic check on the validity
403 	 * of the supplied file descriptor: if it's less than the current
404 	 * value of fi_nfiles -- examined without any locks -- then it's
405 	 * safe to attempt a UF_ENTER() on that fd (which is a valid
406 	 * assumption because fi_nfiles only increases).  Therefore, it
407 	 * is critical that the new value of fi_nfiles not reach global
408 	 * visibility until after the new fi_list: if it happened the
409 	 * other way around, getf() could see the new fi_nfiles and attempt
410 	 * a UF_ENTER() on the old fi_list, which would write beyond its
411 	 * end if the fd exceeded the old fi_nfiles.
412 	 */
413 	membar_producer();
414 	fip->fi_nfiles = newcnt;
415 
416 	/*
417 	 * The new state is consistent now, so we can drop all the locks.
418 	 */
419 	for (dst = newlist; dst < newend; dst++)
420 		mutex_exit(&dst->uf_lock);
421 
422 	for (src = oldlist; src < oldend; src++) {
423 		/*
424 		 * If any threads are blocked on the old cvs, wake them.
425 		 * This will force them to wake up, discover that fi_list
426 		 * has changed, and go back to sleep on the new cvs.
427 		 */
428 		cv_broadcast(&src->uf_wanted_cv);
429 		cv_broadcast(&src->uf_closing_cv);
430 		mutex_exit(&src->uf_lock);
431 	}
432 
433 	mutex_exit(&fip->fi_lock);
434 
435 	/*
436 	 * Retire the old flist.  We can't actually kmem_free() it now
437 	 * because someone may still have a pointer to it.  Instead,
438 	 * we link it onto a list of retired flists.  The new flist
439 	 * is at least double the size of the previous flist, so the
440 	 * total size of all retired flists will be less than the size
441 	 * of the current one (to prove, consider the sum of a geometric
442 	 * series in powers of 2).  exit() frees the retired flists.
443 	 */
444 	urp = kmem_zalloc(sizeof (uf_rlist_t), KM_SLEEP);
445 	urp->ur_list = oldlist;
446 	urp->ur_nfiles = oldcnt;
447 
448 	mutex_enter(&fip->fi_lock);
449 	urp->ur_next = fip->fi_rlist;
450 	fip->fi_rlist = urp;
451 	mutex_exit(&fip->fi_lock);
452 }
453 
454 /*
455  * Utility functions for keeping track of the active file descriptors.
456  */
457 void
clear_stale_fd()458 clear_stale_fd()		/* called from post_syscall() */
459 {
460 	afd_t *afd = &curthread->t_activefd;
461 	int i;
462 
463 	/* uninitialized is ok here, a_nfd is then zero */
464 	for (i = 0; i < afd->a_nfd; i++) {
465 		/* assert that this should not be necessary */
466 		ASSERT(afd->a_fd[i] == -1);
467 		afd->a_fd[i] = -1;
468 	}
469 	afd->a_stale = 0;
470 }
471 
472 void
free_afd(afd_t * afd)473 free_afd(afd_t *afd)		/* called below and from thread_free() */
474 {
475 	int i;
476 
477 	/* free the buffer if it was kmem_alloc()ed */
478 	if (afd->a_nfd > sizeof (afd->a_buf) / sizeof (afd->a_buf[0])) {
479 		COUNT(afd_free);
480 		kmem_free(afd->a_fd, afd->a_nfd * sizeof (afd->a_fd[0]));
481 	}
482 
483 	/* (re)initialize the structure */
484 	afd->a_fd = &afd->a_buf[0];
485 	afd->a_nfd = sizeof (afd->a_buf) / sizeof (afd->a_buf[0]);
486 	afd->a_stale = 0;
487 	for (i = 0; i < afd->a_nfd; i++)
488 		afd->a_fd[i] = -1;
489 }
490 
491 static void
set_active_fd(int fd)492 set_active_fd(int fd)
493 {
494 	afd_t *afd = &curthread->t_activefd;
495 	int i;
496 	int *old_fd;
497 	int old_nfd;
498 	int *new_fd;
499 	int new_nfd;
500 
501 	if (afd->a_nfd == 0) {	/* first time initialization */
502 		ASSERT(fd == -1);
503 		mutex_enter(&afd->a_fdlock);
504 		free_afd(afd);
505 		mutex_exit(&afd->a_fdlock);
506 	}
507 
508 	/* insert fd into vacant slot, if any */
509 	for (i = 0; i < afd->a_nfd; i++) {
510 		if (afd->a_fd[i] == -1) {
511 			afd->a_fd[i] = fd;
512 			return;
513 		}
514 	}
515 
516 	/*
517 	 * Reallocate the a_fd[] array to add one more slot.
518 	 */
519 	ASSERT(fd == -1);
520 	old_nfd = afd->a_nfd;
521 	old_fd = afd->a_fd;
522 	new_nfd = old_nfd + 1;
523 	new_fd = kmem_alloc(new_nfd * sizeof (afd->a_fd[0]), KM_SLEEP);
524 	MAXFD(new_nfd);
525 	COUNT(afd_alloc);
526 
527 	mutex_enter(&afd->a_fdlock);
528 	afd->a_fd = new_fd;
529 	afd->a_nfd = new_nfd;
530 	for (i = 0; i < old_nfd; i++)
531 		afd->a_fd[i] = old_fd[i];
532 	afd->a_fd[i] = fd;
533 	mutex_exit(&afd->a_fdlock);
534 
535 	if (old_nfd > sizeof (afd->a_buf) / sizeof (afd->a_buf[0])) {
536 		COUNT(afd_free);
537 		kmem_free(old_fd, old_nfd * sizeof (afd->a_fd[0]));
538 	}
539 }
540 
541 void
clear_active_fd(int fd)542 clear_active_fd(int fd)		/* called below and from aio.c */
543 {
544 	afd_t *afd = &curthread->t_activefd;
545 	int i;
546 
547 	for (i = 0; i < afd->a_nfd; i++) {
548 		if (afd->a_fd[i] == fd) {
549 			afd->a_fd[i] = -1;
550 			break;
551 		}
552 	}
553 	ASSERT(i < afd->a_nfd);		/* not found is not ok */
554 }
555 
556 /*
557  * Does this thread have this fd active?
558  */
559 static int
is_active_fd(kthread_t * t,int fd)560 is_active_fd(kthread_t *t, int fd)
561 {
562 	afd_t *afd = &t->t_activefd;
563 	int i;
564 
565 	ASSERT(t != curthread);
566 	mutex_enter(&afd->a_fdlock);
567 	/* uninitialized is ok here, a_nfd is then zero */
568 	for (i = 0; i < afd->a_nfd; i++) {
569 		if (afd->a_fd[i] == fd) {
570 			mutex_exit(&afd->a_fdlock);
571 			return (1);
572 		}
573 	}
574 	mutex_exit(&afd->a_fdlock);
575 	return (0);
576 }
577 
578 /*
579  * Convert a user supplied file descriptor into a pointer to a file structure.
580  * Only task is to check range of the descriptor (soft resource limit was
581  * enforced at open time and shouldn't be checked here).
582  */
583 file_t *
getf_gen(int fd,uf_entry_gen_t * genp)584 getf_gen(int fd, uf_entry_gen_t *genp)
585 {
586 	uf_info_t *fip = P_FINFO(curproc);
587 	uf_entry_t *ufp;
588 	file_t *fp;
589 
590 	if ((uint_t)fd >= fip->fi_nfiles)
591 		return (NULL);
592 
593 	/*
594 	 * Reserve a slot in the active fd array now so we can call
595 	 * set_active_fd(fd) for real below, while still inside UF_ENTER().
596 	 */
597 	set_active_fd(-1);
598 
599 	UF_ENTER(ufp, fip, fd);
600 
601 	if ((fp = ufp->uf_file) == NULL) {
602 		UF_EXIT(ufp);
603 
604 		if (fd == fip->fi_badfd && fip->fi_action > 0)
605 			tsignal(curthread, fip->fi_action);
606 
607 		return (NULL);
608 	}
609 	ufp->uf_refcnt++;
610 	if (genp != NULL) {
611 		*genp = ufp->uf_gen;
612 	}
613 
614 	set_active_fd(fd);	/* record the active file descriptor */
615 
616 	UF_EXIT(ufp);
617 
618 	return (fp);
619 }
620 
621 file_t *
getf(int fd)622 getf(int fd)
623 {
624 	return (getf_gen(fd, NULL));
625 }
626 
627 /*
628  * Close whatever file currently occupies the file descriptor slot
629  * and install the new file, usually NULL, in the file descriptor slot.
630  * The close must complete before we release the file descriptor slot.
631  * If newfp != NULL we only return an error if we can't allocate the
632  * slot so the caller knows that it needs to free the filep;
633  * in the other cases we return the error number from closef().
634  */
635 int
closeandsetf(int fd,file_t * newfp)636 closeandsetf(int fd, file_t *newfp)
637 {
638 	proc_t *p = curproc;
639 	uf_info_t *fip = P_FINFO(p);
640 	uf_entry_t *ufp;
641 	file_t *fp;
642 	fpollinfo_t *fpip;
643 	portfd_t *pfd;
644 	int error;
645 
646 	if ((uint_t)fd >= fip->fi_nfiles) {
647 		if (newfp == NULL)
648 			return (EBADF);
649 		flist_grow(fd);
650 	}
651 
652 	if (newfp != NULL) {
653 		/*
654 		 * If ufp is reserved but has no file pointer, it's in the
655 		 * transition between ufalloc() and setf().  We must wait
656 		 * for this transition to complete before assigning the
657 		 * new non-NULL file pointer.
658 		 */
659 		mutex_enter(&fip->fi_lock);
660 		if (fd == fip->fi_badfd) {
661 			mutex_exit(&fip->fi_lock);
662 			if (fip->fi_action > 0)
663 				tsignal(curthread, fip->fi_action);
664 			return (EBADF);
665 		}
666 		UF_ENTER(ufp, fip, fd);
667 		while (ufp->uf_busy && ufp->uf_file == NULL) {
668 			mutex_exit(&fip->fi_lock);
669 			cv_wait_stop(&ufp->uf_wanted_cv, &ufp->uf_lock, 250);
670 			UF_EXIT(ufp);
671 			mutex_enter(&fip->fi_lock);
672 			UF_ENTER(ufp, fip, fd);
673 		}
674 		if ((fp = ufp->uf_file) == NULL) {
675 			ASSERT(ufp->uf_fpollinfo == NULL);
676 			ASSERT(ufp->uf_flag == 0);
677 			fd_reserve(fip, fd, 1);
678 			ufp->uf_file = newfp;
679 			ufp->uf_gen++;
680 			UF_EXIT(ufp);
681 			mutex_exit(&fip->fi_lock);
682 			return (0);
683 		}
684 		mutex_exit(&fip->fi_lock);
685 	} else {
686 		UF_ENTER(ufp, fip, fd);
687 		if ((fp = ufp->uf_file) == NULL) {
688 			UF_EXIT(ufp);
689 			return (EBADF);
690 		}
691 	}
692 
693 	ASSERT(ufp->uf_busy);
694 	ufp->uf_file = NULL;
695 	ufp->uf_flag = 0;
696 
697 	/*
698 	 * If the file descriptor reference count is non-zero, then
699 	 * some other lwp in the process is performing system call
700 	 * activity on the file.  To avoid blocking here for a long
701 	 * time (the other lwp might be in a long term sleep in its
702 	 * system call), we scan all other lwps in the process to
703 	 * find the ones with this fd as one of their active fds,
704 	 * set their a_stale flag, and set them running if they
705 	 * are in an interruptible sleep so they will emerge from
706 	 * their system calls immediately.  post_syscall() will
707 	 * test the a_stale flag and set errno to EBADF.
708 	 */
709 	ASSERT(ufp->uf_refcnt == 0 || p->p_lwpcnt > 1);
710 	if (ufp->uf_refcnt > 0) {
711 		kthread_t *t;
712 
713 		/*
714 		 * We call sprlock_proc(p) to ensure that the thread
715 		 * list will not change while we are scanning it.
716 		 * To do this, we must drop ufp->uf_lock and then
717 		 * reacquire it (so we are not holding both p->p_lock
718 		 * and ufp->uf_lock at the same time).  ufp->uf_lock
719 		 * must be held for is_active_fd() to be correct
720 		 * (set_active_fd() is called while holding ufp->uf_lock).
721 		 *
722 		 * This is a convoluted dance, but it is better than
723 		 * the old brute-force method of stopping every thread
724 		 * in the process by calling holdlwps(SHOLDFORK1).
725 		 */
726 
727 		UF_EXIT(ufp);
728 		COUNT(afd_wait);
729 
730 		mutex_enter(&p->p_lock);
731 		sprlock_proc(p);
732 		mutex_exit(&p->p_lock);
733 
734 		UF_ENTER(ufp, fip, fd);
735 		ASSERT(ufp->uf_file == NULL);
736 
737 		if (ufp->uf_refcnt > 0) {
738 			for (t = curthread->t_forw;
739 			    t != curthread;
740 			    t = t->t_forw) {
741 				if (is_active_fd(t, fd)) {
742 					thread_lock(t);
743 					t->t_activefd.a_stale = 1;
744 					t->t_post_sys = 1;
745 					if (ISWAKEABLE(t))
746 						setrun_locked(t);
747 					thread_unlock(t);
748 				}
749 			}
750 		}
751 
752 		UF_EXIT(ufp);
753 
754 		mutex_enter(&p->p_lock);
755 		sprunlock(p);
756 
757 		UF_ENTER(ufp, fip, fd);
758 		ASSERT(ufp->uf_file == NULL);
759 	}
760 
761 	/*
762 	 * Wait for other lwps to stop using this file descriptor.
763 	 */
764 	while (ufp->uf_refcnt > 0) {
765 		cv_wait_stop(&ufp->uf_closing_cv, &ufp->uf_lock, 250);
766 		/*
767 		 * cv_wait_stop() drops ufp->uf_lock, so the file list
768 		 * can change.  Drop the lock on our (possibly) stale
769 		 * ufp and let UF_ENTER() find and lock the current ufp.
770 		 */
771 		UF_EXIT(ufp);
772 		UF_ENTER(ufp, fip, fd);
773 	}
774 
775 #ifdef DEBUG
776 	/*
777 	 * catch a watchfd on device's pollhead list but not on fpollinfo list
778 	 */
779 	if (ufp->uf_fpollinfo != NULL)
780 		checkwfdlist(fp->f_vnode, ufp->uf_fpollinfo);
781 #endif	/* DEBUG */
782 
783 	/*
784 	 * We may need to cleanup some cached poll states in t_pollstate
785 	 * before the fd can be reused. It is important that we don't
786 	 * access a stale thread structure. We will do the cleanup in two
787 	 * phases to avoid deadlock and holding uf_lock for too long.
788 	 * In phase 1, hold the uf_lock and call pollblockexit() to set
789 	 * state in t_pollstate struct so that a thread does not exit on
790 	 * us. In phase 2, we drop the uf_lock and call pollcacheclean().
791 	 */
792 	pfd = ufp->uf_portfd;
793 	ufp->uf_portfd = NULL;
794 	fpip = ufp->uf_fpollinfo;
795 	ufp->uf_fpollinfo = NULL;
796 	if (fpip != NULL)
797 		pollblockexit(fpip);
798 	UF_EXIT(ufp);
799 	if (fpip != NULL)
800 		pollcacheclean(fpip, fd);
801 	if (pfd)
802 		port_close_fd(pfd);
803 
804 	/*
805 	 * Keep the file descriptor entry reserved across the closef().
806 	 */
807 	error = closef(fp);
808 
809 	setf(fd, newfp);
810 
811 	/* Only return closef() error when closing is all we do */
812 	return (newfp == NULL ? error : 0);
813 }
814 
815 /*
816  * Decrement uf_refcnt; wakeup anyone waiting to close the file.
817  */
818 void
releasef(int fd)819 releasef(int fd)
820 {
821 	uf_info_t *fip = P_FINFO(curproc);
822 	uf_entry_t *ufp;
823 
824 	UF_ENTER(ufp, fip, fd);
825 	ASSERT(ufp->uf_refcnt > 0);
826 	clear_active_fd(fd);	/* clear the active file descriptor */
827 	if (--ufp->uf_refcnt == 0)
828 		cv_broadcast(&ufp->uf_closing_cv);
829 	UF_EXIT(ufp);
830 }
831 
832 /*
833  * Identical to releasef() but can be called from another process.
834  */
835 void
areleasef(int fd,uf_info_t * fip)836 areleasef(int fd, uf_info_t *fip)
837 {
838 	uf_entry_t *ufp;
839 
840 	UF_ENTER(ufp, fip, fd);
841 	ASSERT(ufp->uf_refcnt > 0);
842 	if (--ufp->uf_refcnt == 0)
843 		cv_broadcast(&ufp->uf_closing_cv);
844 	UF_EXIT(ufp);
845 }
846 
847 /*
848  * Duplicate all file descriptors across a fork.
849  */
850 void
flist_fork(uf_info_t * pfip,uf_info_t * cfip)851 flist_fork(uf_info_t *pfip, uf_info_t *cfip)
852 {
853 	int fd, nfiles;
854 	uf_entry_t *pufp, *cufp;
855 
856 	mutex_init(&cfip->fi_lock, NULL, MUTEX_DEFAULT, NULL);
857 	cfip->fi_rlist = NULL;
858 
859 	/*
860 	 * We don't need to hold fi_lock because all other lwp's in the
861 	 * parent have been held.
862 	 */
863 	cfip->fi_nfiles = nfiles = flist_minsize(pfip);
864 
865 	cfip->fi_list = nfiles == 0 ? NULL :
866 	    kmem_zalloc(nfiles * sizeof (uf_entry_t), KM_SLEEP);
867 
868 	for (fd = 0, pufp = pfip->fi_list, cufp = cfip->fi_list; fd < nfiles;
869 	    fd++, pufp++, cufp++) {
870 		cufp->uf_file = pufp->uf_file;
871 		cufp->uf_alloc = pufp->uf_alloc;
872 		cufp->uf_flag = pufp->uf_flag;
873 		cufp->uf_busy = pufp->uf_busy;
874 		cufp->uf_gen = pufp->uf_gen;
875 		if (pufp->uf_file == NULL) {
876 			ASSERT(pufp->uf_flag == 0);
877 			if (pufp->uf_busy) {
878 				/*
879 				 * Grab locks to appease ASSERTs in fd_reserve
880 				 */
881 				mutex_enter(&cfip->fi_lock);
882 				mutex_enter(&cufp->uf_lock);
883 				fd_reserve(cfip, fd, -1);
884 				mutex_exit(&cufp->uf_lock);
885 				mutex_exit(&cfip->fi_lock);
886 			}
887 		}
888 	}
889 }
890 
891 /*
892  * Close all open file descriptors for the current process.
893  * This is only called from exit(), which is single-threaded,
894  * so we don't need any locking.
895  */
896 void
closeall(uf_info_t * fip)897 closeall(uf_info_t *fip)
898 {
899 	int fd;
900 	file_t *fp;
901 	uf_entry_t *ufp;
902 
903 	ufp = fip->fi_list;
904 	for (fd = 0; fd < fip->fi_nfiles; fd++, ufp++) {
905 		if ((fp = ufp->uf_file) != NULL) {
906 			ufp->uf_file = NULL;
907 			if (ufp->uf_portfd != NULL) {
908 				portfd_t *pfd;
909 				/* remove event port association */
910 				pfd = ufp->uf_portfd;
911 				ufp->uf_portfd = NULL;
912 				port_close_fd(pfd);
913 			}
914 			ASSERT(ufp->uf_fpollinfo == NULL);
915 			(void) closef(fp);
916 		}
917 	}
918 
919 	kmem_free(fip->fi_list, fip->fi_nfiles * sizeof (uf_entry_t));
920 	fip->fi_list = NULL;
921 	fip->fi_nfiles = 0;
922 	while (fip->fi_rlist != NULL) {
923 		uf_rlist_t *urp = fip->fi_rlist;
924 		fip->fi_rlist = urp->ur_next;
925 		kmem_free(urp->ur_list, urp->ur_nfiles * sizeof (uf_entry_t));
926 		kmem_free(urp, sizeof (uf_rlist_t));
927 	}
928 }
929 
930 /*
931  * Internal form of close.  Decrement reference count on file
932  * structure.  Decrement reference count on the vnode following
933  * removal of the referencing file structure.
934  */
935 int
closef(file_t * fp)936 closef(file_t *fp)
937 {
938 	vnode_t *vp;
939 	int error;
940 	int count;
941 	int flag;
942 	offset_t offset;
943 
944 	/*
945 	 * audit close of file (may be exit)
946 	 */
947 	if (AU_AUDITING())
948 		audit_closef(fp);
949 	ASSERT(MUTEX_NOT_HELD(&P_FINFO(curproc)->fi_lock));
950 
951 	mutex_enter(&fp->f_tlock);
952 
953 	ASSERT(fp->f_count > 0);
954 
955 	count = fp->f_count--;
956 	flag = fp->f_flag;
957 	offset = fp->f_offset;
958 
959 	vp = fp->f_vnode;
960 
961 	error = VOP_CLOSE(vp, flag, count, offset, fp->f_cred, NULL);
962 
963 	if (count > 1) {
964 		mutex_exit(&fp->f_tlock);
965 		return (error);
966 	}
967 	ASSERT(fp->f_count == 0);
968 	/* Last reference, remove any OFD style lock for the file_t */
969 	ofdcleanlock(fp);
970 	mutex_exit(&fp->f_tlock);
971 
972 	/*
973 	 * If DTrace has getf() subroutines active, it will set dtrace_closef
974 	 * to point to code that implements a barrier with respect to probe
975 	 * context.  This must be called before the file_t is freed (and the
976 	 * vnode that it refers to is released) -- but it must be after the
977 	 * file_t has been removed from the uf_entry_t.  That is, there must
978 	 * be no way for a racing getf() in probe context to yield the fp that
979 	 * we're operating upon.
980 	 */
981 	if (dtrace_closef != NULL)
982 		(*dtrace_closef)();
983 
984 	VN_RELE(vp);
985 	/*
986 	 * deallocate resources to audit_data
987 	 */
988 	if (audit_active)
989 		audit_unfalloc(fp);
990 	crfree(fp->f_cred);
991 	kmem_cache_free(file_cache, fp);
992 	return (error);
993 }
994 
995 /*
996  * This is a combination of ufalloc() and setf().
997  */
998 int
ufalloc_file(int start,file_t * fp)999 ufalloc_file(int start, file_t *fp)
1000 {
1001 	proc_t *p = curproc;
1002 	uf_info_t *fip = P_FINFO(p);
1003 	int filelimit;
1004 	uf_entry_t *ufp;
1005 	int nfiles;
1006 	int fd;
1007 
1008 	/*
1009 	 * Assertion is to convince the correctness of the following
1010 	 * assignment for filelimit after casting to int.
1011 	 */
1012 	ASSERT(p->p_fno_ctl <= INT_MAX);
1013 	filelimit = (int)p->p_fno_ctl;
1014 
1015 	for (;;) {
1016 		mutex_enter(&fip->fi_lock);
1017 		fd = fd_find(fip, start);
1018 		if (fd >= 0 && fd == fip->fi_badfd) {
1019 			start = fd + 1;
1020 			mutex_exit(&fip->fi_lock);
1021 			continue;
1022 		}
1023 		if ((uint_t)fd < filelimit)
1024 			break;
1025 		if (fd >= filelimit) {
1026 			mutex_exit(&fip->fi_lock);
1027 			mutex_enter(&p->p_lock);
1028 			(void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
1029 			    p->p_rctls, p, RCA_SAFE);
1030 			mutex_exit(&p->p_lock);
1031 			return (-1);
1032 		}
1033 		/* fd_find() returned -1 */
1034 		nfiles = fip->fi_nfiles;
1035 		mutex_exit(&fip->fi_lock);
1036 		flist_grow(MAX(start, nfiles));
1037 	}
1038 
1039 	UF_ENTER(ufp, fip, fd);
1040 	fd_reserve(fip, fd, 1);
1041 	ASSERT(ufp->uf_file == NULL);
1042 	ufp->uf_file = fp;
1043 	if (fp != NULL) {
1044 		ufp->uf_gen++;
1045 	}
1046 	UF_EXIT(ufp);
1047 	mutex_exit(&fip->fi_lock);
1048 	return (fd);
1049 }
1050 
1051 /*
1052  * Allocate a user file descriptor greater than or equal to "start".
1053  */
1054 int
ufalloc(int start)1055 ufalloc(int start)
1056 {
1057 	return (ufalloc_file(start, NULL));
1058 }
1059 
1060 /*
1061  * Check that a future allocation of count fds on proc p has a good
1062  * chance of succeeding.  If not, do rctl processing as if we'd failed
1063  * the allocation.
1064  *
1065  * Our caller must guarantee that p cannot disappear underneath us.
1066  */
1067 int
ufcanalloc(proc_t * p,uint_t count)1068 ufcanalloc(proc_t *p, uint_t count)
1069 {
1070 	uf_info_t *fip = P_FINFO(p);
1071 	int filelimit;
1072 	int current;
1073 
1074 	if (count == 0)
1075 		return (1);
1076 
1077 	ASSERT(p->p_fno_ctl <= INT_MAX);
1078 	filelimit = (int)p->p_fno_ctl;
1079 
1080 	mutex_enter(&fip->fi_lock);
1081 	current = flist_nalloc(fip);		/* # of in-use descriptors */
1082 	mutex_exit(&fip->fi_lock);
1083 
1084 	/*
1085 	 * If count is a positive integer, the worst that can happen is
1086 	 * an overflow to a negative value, which is caught by the >= 0 check.
1087 	 */
1088 	current += count;
1089 	if (count <= INT_MAX && current >= 0 && current <= filelimit)
1090 		return (1);
1091 
1092 	mutex_enter(&p->p_lock);
1093 	(void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
1094 	    p->p_rctls, p, RCA_SAFE);
1095 	mutex_exit(&p->p_lock);
1096 	return (0);
1097 }
1098 
1099 /*
1100  * Allocate a user file descriptor and a file structure.
1101  * Initialize the descriptor to point at the file structure.
1102  * If fdp is NULL, the user file descriptor will not be allocated.
1103  */
1104 int
falloc(vnode_t * vp,int flag,file_t ** fpp,int * fdp)1105 falloc(vnode_t *vp, int flag, file_t **fpp, int *fdp)
1106 {
1107 	file_t *fp;
1108 	int fd;
1109 
1110 	if (fdp) {
1111 		if ((fd = ufalloc(0)) == -1)
1112 			return (EMFILE);
1113 	}
1114 	fp = kmem_cache_alloc(file_cache, KM_SLEEP);
1115 	/*
1116 	 * Note: falloc returns the fp locked
1117 	 */
1118 	mutex_enter(&fp->f_tlock);
1119 	fp->f_count = 1;
1120 	fp->f_flag = (ushort_t)flag;
1121 	fp->f_flag2 = (flag & (FSEARCH|FEXEC)) >> 16;
1122 	fp->f_vnode = vp;
1123 	fp->f_offset = 0;
1124 	fp->f_audit_data = 0;
1125 	crhold(fp->f_cred = CRED());
1126 	/*
1127 	 * allocate resources to audit_data
1128 	 */
1129 	if (audit_active)
1130 		audit_falloc(fp);
1131 	*fpp = fp;
1132 	if (fdp)
1133 		*fdp = fd;
1134 	return (0);
1135 }
1136 
1137 /*ARGSUSED*/
1138 static int
file_cache_constructor(void * buf,void * cdrarg,int kmflags)1139 file_cache_constructor(void *buf, void *cdrarg, int kmflags)
1140 {
1141 	file_t *fp = buf;
1142 
1143 	mutex_init(&fp->f_tlock, NULL, MUTEX_DEFAULT, NULL);
1144 	return (0);
1145 }
1146 
1147 /*ARGSUSED*/
1148 static void
file_cache_destructor(void * buf,void * cdrarg)1149 file_cache_destructor(void *buf, void *cdrarg)
1150 {
1151 	file_t *fp = buf;
1152 
1153 	mutex_destroy(&fp->f_tlock);
1154 }
1155 
1156 void
finit()1157 finit()
1158 {
1159 	file_cache = kmem_cache_create("file_cache", sizeof (file_t), 0,
1160 	    file_cache_constructor, file_cache_destructor, NULL, NULL, NULL, 0);
1161 }
1162 
1163 void
unfalloc(file_t * fp)1164 unfalloc(file_t *fp)
1165 {
1166 	ASSERT(MUTEX_HELD(&fp->f_tlock));
1167 	if (--fp->f_count <= 0) {
1168 		/*
1169 		 * deallocate resources to audit_data
1170 		 */
1171 		if (audit_active)
1172 			audit_unfalloc(fp);
1173 		crfree(fp->f_cred);
1174 		mutex_exit(&fp->f_tlock);
1175 		kmem_cache_free(file_cache, fp);
1176 	} else
1177 		mutex_exit(&fp->f_tlock);
1178 }
1179 
1180 /*
1181  * Given a file descriptor, set the user's
1182  * file pointer to the given parameter.
1183  */
1184 void
setf(int fd,file_t * fp)1185 setf(int fd, file_t *fp)
1186 {
1187 	uf_info_t *fip = P_FINFO(curproc);
1188 	uf_entry_t *ufp;
1189 
1190 	if (AU_AUDITING())
1191 		audit_setf(fp, fd);
1192 
1193 	if (fp == NULL) {
1194 		mutex_enter(&fip->fi_lock);
1195 		UF_ENTER(ufp, fip, fd);
1196 		fd_reserve(fip, fd, -1);
1197 		mutex_exit(&fip->fi_lock);
1198 	} else {
1199 		UF_ENTER(ufp, fip, fd);
1200 		ASSERT(ufp->uf_busy);
1201 		ufp->uf_gen++;
1202 	}
1203 	ASSERT(ufp->uf_fpollinfo == NULL);
1204 	ASSERT(ufp->uf_flag == 0);
1205 	ufp->uf_file = fp;
1206 	cv_broadcast(&ufp->uf_wanted_cv);
1207 	UF_EXIT(ufp);
1208 }
1209 
1210 /*
1211  * Given a file descriptor, return the file table flags, plus,
1212  * if this is a socket in asynchronous mode, the FASYNC flag.
1213  * getf() may or may not have been called before calling f_getfl().
1214  */
1215 int
f_getfl(int fd,int * flagp)1216 f_getfl(int fd, int *flagp)
1217 {
1218 	uf_info_t *fip = P_FINFO(curproc);
1219 	uf_entry_t *ufp;
1220 	file_t *fp;
1221 	int error;
1222 
1223 	if ((uint_t)fd >= fip->fi_nfiles)
1224 		error = EBADF;
1225 	else {
1226 		UF_ENTER(ufp, fip, fd);
1227 		if ((fp = ufp->uf_file) == NULL)
1228 			error = EBADF;
1229 		else {
1230 			vnode_t *vp = fp->f_vnode;
1231 			int flag = fp->f_flag | (fp->f_flag2 << 16);
1232 
1233 			/*
1234 			 * BSD fcntl() FASYNC compatibility.
1235 			 */
1236 			if (vp->v_type == VSOCK)
1237 				flag |= sock_getfasync(vp);
1238 			*flagp = flag;
1239 			error = 0;
1240 		}
1241 		UF_EXIT(ufp);
1242 	}
1243 
1244 	return (error);
1245 }
1246 
1247 /*
1248  * Given a file descriptor, return the user's file flags.
1249  * Force the FD_CLOEXEC flag for writable self-open /proc files.
1250  * getf() may or may not have been called before calling f_getfd_error().
1251  */
1252 int
f_getfd_error(int fd,int * flagp)1253 f_getfd_error(int fd, int *flagp)
1254 {
1255 	uf_info_t *fip = P_FINFO(curproc);
1256 	uf_entry_t *ufp;
1257 	file_t *fp;
1258 	int flag;
1259 	int error;
1260 
1261 	if ((uint_t)fd >= fip->fi_nfiles)
1262 		error = EBADF;
1263 	else {
1264 		UF_ENTER(ufp, fip, fd);
1265 		if ((fp = ufp->uf_file) == NULL)
1266 			error = EBADF;
1267 		else {
1268 			flag = ufp->uf_flag;
1269 			if ((fp->f_flag & FWRITE) && pr_isself(fp->f_vnode))
1270 				flag |= FD_CLOEXEC;
1271 			*flagp = flag;
1272 			error = 0;
1273 		}
1274 		UF_EXIT(ufp);
1275 	}
1276 
1277 	return (error);
1278 }
1279 
1280 /*
1281  * getf() must have been called before calling f_getfd().
1282  */
1283 char
f_getfd(int fd)1284 f_getfd(int fd)
1285 {
1286 	int flag = 0;
1287 	(void) f_getfd_error(fd, &flag);
1288 	return ((char)flag);
1289 }
1290 
1291 /*
1292  * Given a file descriptor and file flags, set the user's file flags.
1293  * At present, the only valid flag is FD_CLOEXEC.
1294  * getf() may or may not have been called before calling f_setfd_error().
1295  */
1296 int
f_setfd_error(int fd,int flags)1297 f_setfd_error(int fd, int flags)
1298 {
1299 	uf_info_t *fip = P_FINFO(curproc);
1300 	uf_entry_t *ufp;
1301 	int error;
1302 
1303 	if ((uint_t)fd >= fip->fi_nfiles)
1304 		error = EBADF;
1305 	else {
1306 		UF_ENTER(ufp, fip, fd);
1307 		if (ufp->uf_file == NULL)
1308 			error = EBADF;
1309 		else {
1310 			ufp->uf_flag = flags & FD_CLOEXEC;
1311 			error = 0;
1312 		}
1313 		UF_EXIT(ufp);
1314 	}
1315 	return (error);
1316 }
1317 
1318 void
f_setfd(int fd,char flags)1319 f_setfd(int fd, char flags)
1320 {
1321 	(void) f_setfd_error(fd, flags);
1322 }
1323 
1324 #define	BADFD_MIN	3
1325 #define	BADFD_MAX	255
1326 
1327 /*
1328  * Attempt to allocate a file descriptor which is bad and which
1329  * is "poison" to the application.  It cannot be closed (except
1330  * on exec), allocated for a different use, etc.
1331  */
1332 int
f_badfd(int start,int * fdp,int action)1333 f_badfd(int start, int *fdp, int action)
1334 {
1335 	int fdr;
1336 	int badfd;
1337 	uf_info_t *fip = P_FINFO(curproc);
1338 
1339 #ifdef _LP64
1340 	/* No restrictions on 64 bit _file */
1341 	if (get_udatamodel() != DATAMODEL_ILP32)
1342 		return (EINVAL);
1343 #endif
1344 
1345 	if (start > BADFD_MAX || start < BADFD_MIN)
1346 		return (EINVAL);
1347 
1348 	if (action >= NSIG || action < 0)
1349 		return (EINVAL);
1350 
1351 	mutex_enter(&fip->fi_lock);
1352 	badfd = fip->fi_badfd;
1353 	mutex_exit(&fip->fi_lock);
1354 
1355 	if (badfd != -1)
1356 		return (EAGAIN);
1357 
1358 	fdr = ufalloc(start);
1359 
1360 	if (fdr > BADFD_MAX) {
1361 		setf(fdr, NULL);
1362 		return (EMFILE);
1363 	}
1364 	if (fdr < 0)
1365 		return (EMFILE);
1366 
1367 	mutex_enter(&fip->fi_lock);
1368 	if (fip->fi_badfd != -1) {
1369 		/* Lost race */
1370 		mutex_exit(&fip->fi_lock);
1371 		setf(fdr, NULL);
1372 		return (EAGAIN);
1373 	}
1374 	fip->fi_action = action;
1375 	fip->fi_badfd = fdr;
1376 	mutex_exit(&fip->fi_lock);
1377 	setf(fdr, NULL);
1378 
1379 	*fdp = fdr;
1380 
1381 	return (0);
1382 }
1383 
1384 /*
1385  * Allocate a file descriptor and assign it to the vnode "*vpp",
1386  * performing the usual open protocol upon it and returning the
1387  * file descriptor allocated.  It is the responsibility of the
1388  * caller to dispose of "*vpp" if any error occurs.
1389  */
1390 int
fassign(vnode_t ** vpp,int mode,int * fdp)1391 fassign(vnode_t **vpp, int mode, int *fdp)
1392 {
1393 	file_t *fp;
1394 	int error;
1395 	int fd;
1396 
1397 	if (error = falloc((vnode_t *)NULL, mode, &fp, &fd))
1398 		return (error);
1399 	if (error = VOP_OPEN(vpp, mode, fp->f_cred, NULL)) {
1400 		setf(fd, NULL);
1401 		unfalloc(fp);
1402 		return (error);
1403 	}
1404 	fp->f_vnode = *vpp;
1405 	mutex_exit(&fp->f_tlock);
1406 	/*
1407 	 * Fill in the slot falloc reserved.
1408 	 */
1409 	setf(fd, fp);
1410 	*fdp = fd;
1411 	return (0);
1412 }
1413 
1414 /*
1415  * When a process forks it must increment the f_count of all file pointers
1416  * since there is a new process pointing at them.  fcnt_add(fip, 1) does this.
1417  * Since we are called when there is only 1 active lwp we don't need to
1418  * hold fi_lock or any uf_lock.  If the fork fails, fork_fail() calls
1419  * fcnt_add(fip, -1) to restore the counts.
1420  */
1421 void
fcnt_add(uf_info_t * fip,int incr)1422 fcnt_add(uf_info_t *fip, int incr)
1423 {
1424 	int i;
1425 	uf_entry_t *ufp;
1426 	file_t *fp;
1427 
1428 	ufp = fip->fi_list;
1429 	for (i = 0; i < fip->fi_nfiles; i++, ufp++) {
1430 		if ((fp = ufp->uf_file) != NULL) {
1431 			mutex_enter(&fp->f_tlock);
1432 			ASSERT((incr == 1 && fp->f_count >= 1) ||
1433 			    (incr == -1 && fp->f_count >= 2));
1434 			fp->f_count += incr;
1435 			mutex_exit(&fp->f_tlock);
1436 		}
1437 	}
1438 }
1439 
1440 /*
1441  * This is called from exec to close all fd's that have the FD_CLOEXEC flag
1442  * set and also to close all self-open for write /proc file descriptors.
1443  */
1444 void
close_exec(uf_info_t * fip)1445 close_exec(uf_info_t *fip)
1446 {
1447 	int fd;
1448 	file_t *fp;
1449 	fpollinfo_t *fpip;
1450 	uf_entry_t *ufp;
1451 	portfd_t *pfd;
1452 
1453 	ufp = fip->fi_list;
1454 	for (fd = 0; fd < fip->fi_nfiles; fd++, ufp++) {
1455 		if ((fp = ufp->uf_file) != NULL &&
1456 		    ((ufp->uf_flag & FD_CLOEXEC) ||
1457 		    ((fp->f_flag & FWRITE) && pr_isself(fp->f_vnode)))) {
1458 			fpip = ufp->uf_fpollinfo;
1459 			mutex_enter(&fip->fi_lock);
1460 			mutex_enter(&ufp->uf_lock);
1461 			fd_reserve(fip, fd, -1);
1462 			mutex_exit(&fip->fi_lock);
1463 			ufp->uf_file = NULL;
1464 			ufp->uf_fpollinfo = NULL;
1465 			ufp->uf_flag = 0;
1466 			/*
1467 			 * We may need to cleanup some cached poll states
1468 			 * in t_pollstate before the fd can be reused. It
1469 			 * is important that we don't access a stale thread
1470 			 * structure. We will do the cleanup in two
1471 			 * phases to avoid deadlock and holding uf_lock for
1472 			 * too long. In phase 1, hold the uf_lock and call
1473 			 * pollblockexit() to set state in t_pollstate struct
1474 			 * so that a thread does not exit on us. In phase 2,
1475 			 * we drop the uf_lock and call pollcacheclean().
1476 			 */
1477 			pfd = ufp->uf_portfd;
1478 			ufp->uf_portfd = NULL;
1479 			if (fpip != NULL)
1480 				pollblockexit(fpip);
1481 			mutex_exit(&ufp->uf_lock);
1482 			if (fpip != NULL)
1483 				pollcacheclean(fpip, fd);
1484 			if (pfd)
1485 				port_close_fd(pfd);
1486 			(void) closef(fp);
1487 		}
1488 	}
1489 
1490 	/* Reset bad fd */
1491 	fip->fi_badfd = -1;
1492 	fip->fi_action = -1;
1493 }
1494 
1495 /*
1496  * Utility function called by most of the *at() system call interfaces.
1497  *
1498  * Generate a starting vnode pointer for an (fd, path) pair where 'fd'
1499  * is an open file descriptor for a directory to be used as the starting
1500  * point for the lookup of the relative pathname 'path' (or, if path is
1501  * NULL, generate a vnode pointer for the direct target of the operation).
1502  *
1503  * If we successfully return a non-NULL startvp, it has been the target
1504  * of VN_HOLD() and the caller must call VN_RELE() on it.
1505  */
1506 int
fgetstartvp(int fd,char * path,vnode_t ** startvpp)1507 fgetstartvp(int fd, char *path, vnode_t **startvpp)
1508 {
1509 	vnode_t		*startvp;
1510 	file_t		*startfp;
1511 	char		startchar;
1512 
1513 	if (fd == AT_FDCWD && path == NULL)
1514 		return (EFAULT);
1515 
1516 	if (fd == AT_FDCWD) {
1517 		/*
1518 		 * Start from the current working directory.
1519 		 */
1520 		startvp = NULL;
1521 	} else {
1522 		if (path == NULL)
1523 			startchar = '\0';
1524 		else if (copyin(path, &startchar, sizeof (char)))
1525 			return (EFAULT);
1526 
1527 		if (startchar == '/') {
1528 			/*
1529 			 * 'path' is an absolute pathname.
1530 			 */
1531 			startvp = NULL;
1532 		} else {
1533 			/*
1534 			 * 'path' is a relative pathname or we will
1535 			 * be applying the operation to 'fd' itself.
1536 			 */
1537 			if ((startfp = getf(fd)) == NULL)
1538 				return (EBADF);
1539 			startvp = startfp->f_vnode;
1540 			VN_HOLD(startvp);
1541 			releasef(fd);
1542 		}
1543 	}
1544 	*startvpp = startvp;
1545 	return (0);
1546 }
1547 
1548 /*
1549  * Called from fchownat() and fchmodat() to set ownership and mode.
1550  * The contents of *vap must be set before calling here.
1551  */
1552 int
fsetattrat(int fd,char * path,int flags,struct vattr * vap)1553 fsetattrat(int fd, char *path, int flags, struct vattr *vap)
1554 {
1555 	vnode_t		*startvp;
1556 	vnode_t		*vp;
1557 	int		error;
1558 
1559 	/*
1560 	 * Since we are never called to set the size of a file, we don't
1561 	 * need to check for non-blocking locks (via nbl_need_check(vp)).
1562 	 */
1563 	ASSERT(!(vap->va_mask & AT_SIZE));
1564 
1565 	if ((error = fgetstartvp(fd, path, &startvp)) != 0)
1566 		return (error);
1567 	if (AU_AUDITING() && startvp != NULL)
1568 		audit_setfsat_path(1);
1569 
1570 	/*
1571 	 * Do lookup for fchownat/fchmodat when path not NULL
1572 	 */
1573 	if (path != NULL) {
1574 		if (error = lookupnameat(path, UIO_USERSPACE,
1575 		    (flags == AT_SYMLINK_NOFOLLOW) ?
1576 		    NO_FOLLOW : FOLLOW,
1577 		    NULLVPP, &vp, startvp)) {
1578 			if (startvp != NULL)
1579 				VN_RELE(startvp);
1580 			return (error);
1581 		}
1582 	} else {
1583 		vp = startvp;
1584 		ASSERT(vp);
1585 		VN_HOLD(vp);
1586 	}
1587 
1588 	if (vp->v_type == VLNK && (vap->va_mask & AT_MODE) != 0) {
1589 		error = EOPNOTSUPP;
1590 	} else if (vn_is_readonly(vp)) {
1591 		error = EROFS;
1592 	} else {
1593 		error = VOP_SETATTR(vp, vap, 0, CRED(), NULL);
1594 	}
1595 
1596 	if (startvp != NULL)
1597 		VN_RELE(startvp);
1598 	VN_RELE(vp);
1599 
1600 	return (error);
1601 }
1602 
1603 /*
1604  * Return true if the given vnode is referenced by any
1605  * entry in the current process's file descriptor table.
1606  */
1607 int
fisopen(vnode_t * vp)1608 fisopen(vnode_t *vp)
1609 {
1610 	int fd;
1611 	file_t *fp;
1612 	vnode_t *ovp;
1613 	uf_info_t *fip = P_FINFO(curproc);
1614 	uf_entry_t *ufp;
1615 
1616 	mutex_enter(&fip->fi_lock);
1617 	for (fd = 0; fd < fip->fi_nfiles; fd++) {
1618 		UF_ENTER(ufp, fip, fd);
1619 		if ((fp = ufp->uf_file) != NULL &&
1620 		    (ovp = fp->f_vnode) != NULL && VN_CMP(vp, ovp)) {
1621 			UF_EXIT(ufp);
1622 			mutex_exit(&fip->fi_lock);
1623 			return (1);
1624 		}
1625 		UF_EXIT(ufp);
1626 	}
1627 	mutex_exit(&fip->fi_lock);
1628 	return (0);
1629 }
1630 
1631 /*
1632  * Return zero if at least one file currently open (by curproc) shouldn't be
1633  * allowed to change zones.
1634  */
1635 int
files_can_change_zones(void)1636 files_can_change_zones(void)
1637 {
1638 	int fd;
1639 	file_t *fp;
1640 	uf_info_t *fip = P_FINFO(curproc);
1641 	uf_entry_t *ufp;
1642 
1643 	mutex_enter(&fip->fi_lock);
1644 	for (fd = 0; fd < fip->fi_nfiles; fd++) {
1645 		UF_ENTER(ufp, fip, fd);
1646 		if ((fp = ufp->uf_file) != NULL &&
1647 		    !vn_can_change_zones(fp->f_vnode)) {
1648 			UF_EXIT(ufp);
1649 			mutex_exit(&fip->fi_lock);
1650 			return (0);
1651 		}
1652 		UF_EXIT(ufp);
1653 	}
1654 	mutex_exit(&fip->fi_lock);
1655 	return (1);
1656 }
1657 
1658 #ifdef DEBUG
1659 
1660 /*
1661  * The following functions are only used in ASSERT()s elsewhere.
1662  * They do not modify the state of the system.
1663  */
1664 
1665 /*
1666  * Return true (1) if the current thread is in the fpollinfo
1667  * list for this file descriptor, else false (0).
1668  */
1669 static int
curthread_in_plist(uf_entry_t * ufp)1670 curthread_in_plist(uf_entry_t *ufp)
1671 {
1672 	fpollinfo_t *fpip;
1673 
1674 	ASSERT(MUTEX_HELD(&ufp->uf_lock));
1675 	for (fpip = ufp->uf_fpollinfo; fpip; fpip = fpip->fp_next)
1676 		if (fpip->fp_thread == curthread)
1677 			return (1);
1678 	return (0);
1679 }
1680 
1681 /*
1682  * Sanity check to make sure that after lwp_exit(),
1683  * curthread does not appear on any fd's fpollinfo list.
1684  */
1685 void
checkfpollinfo(void)1686 checkfpollinfo(void)
1687 {
1688 	int fd;
1689 	uf_info_t *fip = P_FINFO(curproc);
1690 	uf_entry_t *ufp;
1691 
1692 	mutex_enter(&fip->fi_lock);
1693 	for (fd = 0; fd < fip->fi_nfiles; fd++) {
1694 		UF_ENTER(ufp, fip, fd);
1695 		ASSERT(!curthread_in_plist(ufp));
1696 		UF_EXIT(ufp);
1697 	}
1698 	mutex_exit(&fip->fi_lock);
1699 }
1700 
1701 /*
1702  * Return true (1) if the current thread is in the fpollinfo
1703  * list for this file descriptor, else false (0).
1704  * This is the same as curthread_in_plist(),
1705  * but is called w/o holding uf_lock.
1706  */
1707 int
infpollinfo(int fd)1708 infpollinfo(int fd)
1709 {
1710 	uf_info_t *fip = P_FINFO(curproc);
1711 	uf_entry_t *ufp;
1712 	int rc;
1713 
1714 	UF_ENTER(ufp, fip, fd);
1715 	rc = curthread_in_plist(ufp);
1716 	UF_EXIT(ufp);
1717 	return (rc);
1718 }
1719 
1720 #endif	/* DEBUG */
1721 
1722 /*
1723  * Add the curthread to fpollinfo list, meaning this fd is currently in the
1724  * thread's poll cache. Each lwp polling this file descriptor should call
1725  * this routine once.
1726  */
1727 void
addfpollinfo(int fd)1728 addfpollinfo(int fd)
1729 {
1730 	struct uf_entry *ufp;
1731 	fpollinfo_t *fpip;
1732 	uf_info_t *fip = P_FINFO(curproc);
1733 
1734 	fpip = kmem_zalloc(sizeof (fpollinfo_t), KM_SLEEP);
1735 	fpip->fp_thread = curthread;
1736 	UF_ENTER(ufp, fip, fd);
1737 	/*
1738 	 * Assert we are not already on the list, that is, that
1739 	 * this lwp did not call addfpollinfo twice for the same fd.
1740 	 */
1741 	ASSERT(!curthread_in_plist(ufp));
1742 	/*
1743 	 * addfpollinfo is always done inside the getf/releasef pair.
1744 	 */
1745 	ASSERT(ufp->uf_refcnt >= 1);
1746 	fpip->fp_next = ufp->uf_fpollinfo;
1747 	ufp->uf_fpollinfo = fpip;
1748 	UF_EXIT(ufp);
1749 }
1750 
1751 /*
1752  * Delete curthread from fpollinfo list if it is there.
1753  */
1754 void
delfpollinfo(int fd)1755 delfpollinfo(int fd)
1756 {
1757 	struct uf_entry *ufp;
1758 	struct fpollinfo *fpip;
1759 	struct fpollinfo **fpipp;
1760 	uf_info_t *fip = P_FINFO(curproc);
1761 
1762 	UF_ENTER(ufp, fip, fd);
1763 	for (fpipp = &ufp->uf_fpollinfo;
1764 	    (fpip = *fpipp) != NULL;
1765 	    fpipp = &fpip->fp_next) {
1766 		if (fpip->fp_thread == curthread) {
1767 			*fpipp = fpip->fp_next;
1768 			kmem_free(fpip, sizeof (fpollinfo_t));
1769 			break;
1770 		}
1771 	}
1772 	/*
1773 	 * Assert that we are not still on the list, that is, that
1774 	 * this lwp did not call addfpollinfo twice for the same fd.
1775 	 */
1776 	ASSERT(!curthread_in_plist(ufp));
1777 	UF_EXIT(ufp);
1778 }
1779 
1780 /*
1781  * fd is associated with a port. pfd is a pointer to the fd entry in the
1782  * cache of the port.
1783  */
1784 
1785 void
addfd_port(int fd,portfd_t * pfd)1786 addfd_port(int fd, portfd_t *pfd)
1787 {
1788 	struct uf_entry *ufp;
1789 	uf_info_t *fip = P_FINFO(curproc);
1790 
1791 	UF_ENTER(ufp, fip, fd);
1792 	/*
1793 	 * addfd_port is always done inside the getf/releasef pair.
1794 	 */
1795 	ASSERT(ufp->uf_refcnt >= 1);
1796 	if (ufp->uf_portfd == NULL) {
1797 		/* first entry */
1798 		ufp->uf_portfd = pfd;
1799 		pfd->pfd_next = NULL;
1800 	} else {
1801 		pfd->pfd_next = ufp->uf_portfd;
1802 		ufp->uf_portfd = pfd;
1803 		pfd->pfd_next->pfd_prev = pfd;
1804 	}
1805 	UF_EXIT(ufp);
1806 }
1807 
1808 void
delfd_port(int fd,portfd_t * pfd)1809 delfd_port(int fd, portfd_t *pfd)
1810 {
1811 	struct uf_entry *ufp;
1812 	uf_info_t *fip = P_FINFO(curproc);
1813 
1814 	UF_ENTER(ufp, fip, fd);
1815 	/*
1816 	 * delfd_port is always done inside the getf/releasef pair.
1817 	 */
1818 	ASSERT(ufp->uf_refcnt >= 1);
1819 	if (ufp->uf_portfd == pfd) {
1820 		/* remove first entry */
1821 		ufp->uf_portfd = pfd->pfd_next;
1822 	} else {
1823 		pfd->pfd_prev->pfd_next = pfd->pfd_next;
1824 		if (pfd->pfd_next != NULL)
1825 			pfd->pfd_next->pfd_prev = pfd->pfd_prev;
1826 	}
1827 	UF_EXIT(ufp);
1828 }
1829 
1830 static void
port_close_fd(portfd_t * pfd)1831 port_close_fd(portfd_t *pfd)
1832 {
1833 	portfd_t	*pfdn;
1834 
1835 	/*
1836 	 * At this point, no other thread should access
1837 	 * the portfd_t list for this fd. The uf_file, uf_portfd
1838 	 * pointers in the uf_entry_t struct for this fd would
1839 	 * be set to NULL.
1840 	 */
1841 	for (; pfd != NULL; pfd = pfdn) {
1842 		pfdn = pfd->pfd_next;
1843 		port_close_pfd(pfd);
1844 	}
1845 }
1846