17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
534709573Sraf * Common Development and Distribution License (the "License").
634709573Sraf * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
2134709573Sraf
227c478bd9Sstevel@tonic-gate /*
238fd04b83SRoger A. Faulkner * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
247c478bd9Sstevel@tonic-gate * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate */
267c478bd9Sstevel@tonic-gate
277c478bd9Sstevel@tonic-gate /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
282c76d751SPatrick Mooney /* All Rights Reserved */
297c478bd9Sstevel@tonic-gate
30cd1c8b85SMatthew Ahrens /*
3148bbca81SDaniel Hoffman * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
32f3bb54f3SPatrick Mooney * Copyright 2015, Joyent, Inc.
332c76d751SPatrick Mooney * Copyright 2022 Oxide Computer Company
34cd1c8b85SMatthew Ahrens */
35cd1c8b85SMatthew Ahrens
367c478bd9Sstevel@tonic-gate /*
377c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD
387c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California.
397c478bd9Sstevel@tonic-gate */
407c478bd9Sstevel@tonic-gate
417c478bd9Sstevel@tonic-gate #include <sys/param.h>
427c478bd9Sstevel@tonic-gate #include <sys/isa_defs.h>
437c478bd9Sstevel@tonic-gate #include <sys/types.h>
447c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
457c478bd9Sstevel@tonic-gate #include <sys/user.h>
467c478bd9Sstevel@tonic-gate #include <sys/systm.h>
477c478bd9Sstevel@tonic-gate #include <sys/errno.h>
487c478bd9Sstevel@tonic-gate #include <sys/time.h>
497c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
507c478bd9Sstevel@tonic-gate #include <sys/file.h>
517c478bd9Sstevel@tonic-gate #include <sys/mode.h>
527c478bd9Sstevel@tonic-gate #include <sys/proc.h>
537c478bd9Sstevel@tonic-gate #include <sys/uio.h>
547c478bd9Sstevel@tonic-gate #include <sys/poll_impl.h>
557c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
567c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
577c478bd9Sstevel@tonic-gate #include <sys/debug.h>
587c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
597c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
607c478bd9Sstevel@tonic-gate #include <sys/rctl.h>
6111dc39ddSpraks #include <sys/port_impl.h>
627c478bd9Sstevel@tonic-gate #include <sys/schedctl.h>
63eb4462acSDavid Plauger #include <sys/cpu.h>
647c478bd9Sstevel@tonic-gate
657c478bd9Sstevel@tonic-gate #define NPHLOCKS 64 /* Number of locks; must be power of 2 */
667c478bd9Sstevel@tonic-gate #define PHLOCKADDR(php) &plocks[(((uintptr_t)(php)) >> 8) & (NPHLOCKS - 1)]
677c478bd9Sstevel@tonic-gate #define PHLOCK(php) PHLOCKADDR(php).pp_lock
687c478bd9Sstevel@tonic-gate #define PH_ENTER(php) mutex_enter(PHLOCK(php))
697c478bd9Sstevel@tonic-gate #define PH_EXIT(php) mutex_exit(PHLOCK(php))
707c478bd9Sstevel@tonic-gate #define VALID_POLL_EVENTS (POLLIN | POLLPRI | POLLOUT | POLLRDNORM \
717c478bd9Sstevel@tonic-gate | POLLRDBAND | POLLWRBAND | POLLHUP | POLLERR | POLLNVAL)
727c478bd9Sstevel@tonic-gate
737c478bd9Sstevel@tonic-gate /*
747c478bd9Sstevel@tonic-gate * global counters to collect some stats
757c478bd9Sstevel@tonic-gate */
767c478bd9Sstevel@tonic-gate static struct {
777c478bd9Sstevel@tonic-gate kstat_named_t polllistmiss; /* failed to find a cached poll list */
787c478bd9Sstevel@tonic-gate kstat_named_t pollcachehit; /* list matched 100% w/ cached one */
797c478bd9Sstevel@tonic-gate kstat_named_t pollcachephit; /* list matched < 100% w/ cached one */
807c478bd9Sstevel@tonic-gate kstat_named_t pollcachemiss; /* every list entry is dif from cache */
81f3bb54f3SPatrick Mooney kstat_named_t pollunlockfail; /* failed to perform pollunlock */
827c478bd9Sstevel@tonic-gate } pollstats = {
837c478bd9Sstevel@tonic-gate { "polllistmiss", KSTAT_DATA_UINT64 },
847c478bd9Sstevel@tonic-gate { "pollcachehit", KSTAT_DATA_UINT64 },
857c478bd9Sstevel@tonic-gate { "pollcachephit", KSTAT_DATA_UINT64 },
86f3bb54f3SPatrick Mooney { "pollcachemiss", KSTAT_DATA_UINT64 },
87f3bb54f3SPatrick Mooney { "pollunlockfail", KSTAT_DATA_UINT64 }
887c478bd9Sstevel@tonic-gate };
897c478bd9Sstevel@tonic-gate
907c478bd9Sstevel@tonic-gate kstat_named_t *pollstats_ptr = (kstat_named_t *)&pollstats;
917c478bd9Sstevel@tonic-gate uint_t pollstats_ndata = sizeof (pollstats) / sizeof (kstat_named_t);
927c478bd9Sstevel@tonic-gate
937c478bd9Sstevel@tonic-gate struct pplock {
947c478bd9Sstevel@tonic-gate kmutex_t pp_lock;
957c478bd9Sstevel@tonic-gate short pp_flag;
967c478bd9Sstevel@tonic-gate kcondvar_t pp_wait_cv;
977c478bd9Sstevel@tonic-gate int32_t pp_pad; /* to a nice round 16 bytes */
987c478bd9Sstevel@tonic-gate };
997c478bd9Sstevel@tonic-gate
1007c478bd9Sstevel@tonic-gate static struct pplock plocks[NPHLOCKS]; /* Hash array of pollhead locks */
1017c478bd9Sstevel@tonic-gate
102f3bb54f3SPatrick Mooney /* Contention lock & list for preventing deadlocks in recursive /dev/poll. */
103f3bb54f3SPatrick Mooney static kmutex_t pollstate_contenders_lock;
104f3bb54f3SPatrick Mooney static pollstate_t *pollstate_contenders = NULL;
105f3bb54f3SPatrick Mooney
1067c478bd9Sstevel@tonic-gate #ifdef DEBUG
1077c478bd9Sstevel@tonic-gate static int pollchecksanity(pollstate_t *, nfds_t);
1087c478bd9Sstevel@tonic-gate static int pollcheckxref(pollstate_t *, int);
1097c478bd9Sstevel@tonic-gate static void pollcheckphlist(void);
1107c478bd9Sstevel@tonic-gate static int pollcheckrevents(pollstate_t *, int, int, int);
1117c478bd9Sstevel@tonic-gate static void checkpolldat(pollstate_t *);
1127c478bd9Sstevel@tonic-gate #endif /* DEBUG */
1137c478bd9Sstevel@tonic-gate static int plist_chkdupfd(file_t *, polldat_t *, pollstate_t *, pollfd_t *, int,
1147c478bd9Sstevel@tonic-gate int *);
1157c478bd9Sstevel@tonic-gate
1167c478bd9Sstevel@tonic-gate /*
1177c478bd9Sstevel@tonic-gate * Data structure overview:
1187c478bd9Sstevel@tonic-gate * The per-thread poll state consists of
1197c478bd9Sstevel@tonic-gate * one pollstate_t
1207c478bd9Sstevel@tonic-gate * one pollcache_t
1217c478bd9Sstevel@tonic-gate * one bitmap with one event bit per fd
1227c478bd9Sstevel@tonic-gate * a (two-dimensional) hashed array of polldat_t structures - one entry
1237c478bd9Sstevel@tonic-gate * per fd
1247c478bd9Sstevel@tonic-gate *
1257c478bd9Sstevel@tonic-gate * This conglomerate of data structures interact with
1267c478bd9Sstevel@tonic-gate * the pollhead which is used by VOP_POLL and pollwakeup
1277c478bd9Sstevel@tonic-gate * (protected by the PHLOCK, cached array of plocks), and
1287c478bd9Sstevel@tonic-gate * the fpollinfo list hanging off the fi_list which is used to notify
1297c478bd9Sstevel@tonic-gate * poll when a cached fd is closed. This is protected by uf_lock.
1307c478bd9Sstevel@tonic-gate *
1317c478bd9Sstevel@tonic-gate * Invariants:
1327c478bd9Sstevel@tonic-gate * pd_php (pollhead pointer) is set iff (if and only if) the polldat
1337c478bd9Sstevel@tonic-gate * is on that pollhead. This is modified atomically under pc_lock.
1347c478bd9Sstevel@tonic-gate *
1357c478bd9Sstevel@tonic-gate * pd_fp (file_t pointer) is set iff the thread is on the fpollinfo
1367c478bd9Sstevel@tonic-gate * list for that open file.
1377c478bd9Sstevel@tonic-gate * This is modified atomically under pc_lock.
1387c478bd9Sstevel@tonic-gate *
1397c478bd9Sstevel@tonic-gate * pd_count is the sum (over all values of i) of pd_ref[i].xf_refcnt.
1407c478bd9Sstevel@tonic-gate * Iff pd_ref[i].xf_refcnt >= 1 then
1417c478bd9Sstevel@tonic-gate * ps_pcacheset[i].pcs_pollfd[pd_ref[i].xf_position].fd == pd_fd
1427c478bd9Sstevel@tonic-gate * Iff pd_ref[i].xf_refcnt > 1 then
1437c478bd9Sstevel@tonic-gate * In ps_pcacheset[i].pcs_pollfd between index
1447c478bd9Sstevel@tonic-gate * pd_ref[i].xf_position] and the end of the list
1457c478bd9Sstevel@tonic-gate * there are xf_refcnt entries with .fd == pd_fd
1467c478bd9Sstevel@tonic-gate *
1477c478bd9Sstevel@tonic-gate * Locking design:
1487c478bd9Sstevel@tonic-gate * Whenever possible the design relies on the fact that the poll cache state
1497c478bd9Sstevel@tonic-gate * is per thread thus for both poll and exit it is self-synchronizing.
1507c478bd9Sstevel@tonic-gate * Thus the key interactions where other threads access the state are:
1517c478bd9Sstevel@tonic-gate * pollwakeup (and polltime), and
1527c478bd9Sstevel@tonic-gate * close cleaning up the cached references to an open file
1537c478bd9Sstevel@tonic-gate *
1547c478bd9Sstevel@tonic-gate * The two key locks in poll proper is ps_lock and pc_lock.
1557c478bd9Sstevel@tonic-gate *
1567c478bd9Sstevel@tonic-gate * The ps_lock is used for synchronization between poll, (lwp_)exit and close
1577c478bd9Sstevel@tonic-gate * to ensure that modifications to pollcacheset structure are serialized.
1587c478bd9Sstevel@tonic-gate * This lock is held through most of poll() except where poll sleeps
1597c478bd9Sstevel@tonic-gate * since there is little need to handle closes concurrently with the execution
1607c478bd9Sstevel@tonic-gate * of poll.
1617c478bd9Sstevel@tonic-gate * The pc_lock protects most of the fields in pollcache structure and polldat
1627c478bd9Sstevel@tonic-gate * structures (which are accessed by poll, pollwakeup, and polltime)
1637c478bd9Sstevel@tonic-gate * with the exception of fields that are only modified when only one thread
1647c478bd9Sstevel@tonic-gate * can access this per-thread state.
1657c478bd9Sstevel@tonic-gate * Those exceptions occur in poll when first allocating the per-thread state,
1667c478bd9Sstevel@tonic-gate * when poll grows the number of polldat (never shrinks), and when
1677c478bd9Sstevel@tonic-gate * exit/pollcleanup has ensured that there are no references from either
1687c478bd9Sstevel@tonic-gate * pollheads or fpollinfo to the threads poll state.
1697c478bd9Sstevel@tonic-gate *
1707c478bd9Sstevel@tonic-gate * Poll(2) system call is the only path which ps_lock and pc_lock are both
1717c478bd9Sstevel@tonic-gate * held, in that order. It needs ps_lock to synchronize with close and
1727c478bd9Sstevel@tonic-gate * lwp_exit; and pc_lock with pollwakeup.
1737c478bd9Sstevel@tonic-gate *
1747c478bd9Sstevel@tonic-gate * The locking interaction between pc_lock and PHLOCK take into account
1757c478bd9Sstevel@tonic-gate * that poll acquires these locks in the order of pc_lock and then PHLOCK
1767c478bd9Sstevel@tonic-gate * while pollwakeup does it in the reverse order. Thus pollwakeup implements
1777c478bd9Sstevel@tonic-gate * deadlock avoidance by dropping the locks and reacquiring them in the
1787c478bd9Sstevel@tonic-gate * reverse order. For this to work pollwakeup needs to prevent the thread
1797c478bd9Sstevel@tonic-gate * from exiting and freeing all of the poll related state. Thus is done
1807c478bd9Sstevel@tonic-gate * using
1817c478bd9Sstevel@tonic-gate * the pc_no_exit lock
1827c478bd9Sstevel@tonic-gate * the pc_busy counter
1837c478bd9Sstevel@tonic-gate * the pc_busy_cv condition variable
1847c478bd9Sstevel@tonic-gate *
1857c478bd9Sstevel@tonic-gate * The locking interaction between pc_lock and uf_lock has similar
1867c478bd9Sstevel@tonic-gate * issues. Poll holds ps_lock and/or pc_lock across calls to getf/releasef
1877c478bd9Sstevel@tonic-gate * which acquire uf_lock. The poll cleanup in close needs to hold uf_lock
1887c478bd9Sstevel@tonic-gate * to prevent poll or exit from doing a delfpollinfo after which the thread
1897c478bd9Sstevel@tonic-gate * might exit. But the cleanup needs to acquire pc_lock when modifying
1907c478bd9Sstevel@tonic-gate * the poll cache state. The solution is to use pc_busy and do the close
1917c478bd9Sstevel@tonic-gate * cleanup in two phases:
1927c478bd9Sstevel@tonic-gate * First close calls pollblockexit which increments pc_busy.
1937c478bd9Sstevel@tonic-gate * This prevents the per-thread poll related state from being freed.
1947c478bd9Sstevel@tonic-gate * Then close drops uf_lock and calls pollcacheclean.
1957c478bd9Sstevel@tonic-gate * This routine can then acquire pc_lock and remove any references
1967c478bd9Sstevel@tonic-gate * to the closing fd (as well as recording that it has been closed
1977c478bd9Sstevel@tonic-gate * so that a POLLNVAL can be generated even if the fd is reused before
1987c478bd9Sstevel@tonic-gate * poll has been woken up and checked getf() again).
1997c478bd9Sstevel@tonic-gate *
2007c478bd9Sstevel@tonic-gate * When removing a polled fd from poll cache, the fd is always removed
2017c478bd9Sstevel@tonic-gate * from pollhead list first and then from fpollinfo list, i.e.,
2022c76d751SPatrick Mooney * polldat_disassociate() is called before delfpollinfo().
2037c478bd9Sstevel@tonic-gate *
2047c478bd9Sstevel@tonic-gate *
2057c478bd9Sstevel@tonic-gate * Locking hierarchy:
2067c478bd9Sstevel@tonic-gate * pc_no_exit is a leaf level lock.
2077c478bd9Sstevel@tonic-gate * ps_lock is held when acquiring pc_lock (except when pollwakeup
2087c478bd9Sstevel@tonic-gate * acquires pc_lock).
2092c76d751SPatrick Mooney * pc_lock might be held when acquiring PHLOCK (polldat_associate/
2102c76d751SPatrick Mooney * polldat_disassociate)
2117c478bd9Sstevel@tonic-gate * pc_lock is always held (but this is not required)
2122c76d751SPatrick Mooney * when acquiring PHLOCK (in polladd/polldat_disassociate and pollwakeup
2132c76d751SPatrick Mooney * called from pcache_clean_entry).
2147c478bd9Sstevel@tonic-gate * pc_lock is held across addfpollinfo/delfpollinfo which acquire
2157c478bd9Sstevel@tonic-gate * uf_lock.
2167c478bd9Sstevel@tonic-gate * pc_lock is held across getf/releasef which acquire uf_lock.
2177c478bd9Sstevel@tonic-gate * ps_lock might be held across getf/releasef which acquire uf_lock.
2187c478bd9Sstevel@tonic-gate * pollwakeup tries to acquire pc_lock while holding PHLOCK
2197c478bd9Sstevel@tonic-gate * but drops the locks and reacquire them in reverse order to avoid
2207c478bd9Sstevel@tonic-gate * deadlock.
2217c478bd9Sstevel@tonic-gate *
2227c478bd9Sstevel@tonic-gate * Note also that there is deadlock avoidance support for VOP_POLL routines
2237c478bd9Sstevel@tonic-gate * and pollwakeup involving a file system or driver lock.
2247c478bd9Sstevel@tonic-gate * See below.
2257c478bd9Sstevel@tonic-gate */
2267c478bd9Sstevel@tonic-gate
2277c478bd9Sstevel@tonic-gate /*
2287c478bd9Sstevel@tonic-gate * Deadlock avoidance support for VOP_POLL() routines. This is
2297c478bd9Sstevel@tonic-gate * sometimes necessary to prevent deadlock between polling threads
2307c478bd9Sstevel@tonic-gate * (which hold poll locks on entry to xx_poll(), then acquire foo)
2317c478bd9Sstevel@tonic-gate * and pollwakeup() threads (which hold foo, then acquire poll locks).
2327c478bd9Sstevel@tonic-gate *
233f3bb54f3SPatrick Mooney * pollunlock(*cookie) releases whatever poll locks the current thread holds,
234f3bb54f3SPatrick Mooney * setting a cookie for use by pollrelock();
2357c478bd9Sstevel@tonic-gate *
2367c478bd9Sstevel@tonic-gate * pollrelock(cookie) reacquires previously dropped poll locks;
2377c478bd9Sstevel@tonic-gate *
2387c478bd9Sstevel@tonic-gate * polllock(php, mutex) does the common case: pollunlock(),
2397c478bd9Sstevel@tonic-gate * acquire the problematic mutex, pollrelock().
240f3bb54f3SPatrick Mooney *
241f3bb54f3SPatrick Mooney * If polllock() or pollunlock() return non-zero, it indicates that a recursive
242f3bb54f3SPatrick Mooney * /dev/poll is in progress and pollcache locks cannot be dropped. Callers
243f3bb54f3SPatrick Mooney * must handle this by indicating a POLLNVAL in the revents of the VOP_POLL.
2447c478bd9Sstevel@tonic-gate */
2457c478bd9Sstevel@tonic-gate int
pollunlock(int * lockstate)246f3bb54f3SPatrick Mooney pollunlock(int *lockstate)
2477c478bd9Sstevel@tonic-gate {
248f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
2497c478bd9Sstevel@tonic-gate pollcache_t *pcp;
250f3bb54f3SPatrick Mooney
251f3bb54f3SPatrick Mooney ASSERT(lockstate != NULL);
252f3bb54f3SPatrick Mooney
253f3bb54f3SPatrick Mooney /*
254f3bb54f3SPatrick Mooney * There is no way to safely perform a pollunlock() while in the depths
255f3bb54f3SPatrick Mooney * of a recursive /dev/poll operation.
256f3bb54f3SPatrick Mooney */
257f3bb54f3SPatrick Mooney if (ps != NULL && ps->ps_depth > 1) {
258f3bb54f3SPatrick Mooney ps->ps_flags |= POLLSTATE_ULFAIL;
259f3bb54f3SPatrick Mooney pollstats.pollunlockfail.value.ui64++;
260f3bb54f3SPatrick Mooney return (-1);
261f3bb54f3SPatrick Mooney }
2627c478bd9Sstevel@tonic-gate
2637c478bd9Sstevel@tonic-gate /*
2647c478bd9Sstevel@tonic-gate * t_pollcache is set by /dev/poll and event ports (port_fd.c).
2657c478bd9Sstevel@tonic-gate * If the pollrelock/pollunlock is called as a result of poll(2),
2667c478bd9Sstevel@tonic-gate * the t_pollcache should be NULL.
2677c478bd9Sstevel@tonic-gate */
2687c478bd9Sstevel@tonic-gate if (curthread->t_pollcache == NULL)
269f3bb54f3SPatrick Mooney pcp = ps->ps_pcache;
2707c478bd9Sstevel@tonic-gate else
2717c478bd9Sstevel@tonic-gate pcp = curthread->t_pollcache;
2727c478bd9Sstevel@tonic-gate
273f3bb54f3SPatrick Mooney if (!mutex_owned(&pcp->pc_lock)) {
274f3bb54f3SPatrick Mooney *lockstate = 0;
275f3bb54f3SPatrick Mooney } else {
276f3bb54f3SPatrick Mooney *lockstate = 1;
2777c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
2787c478bd9Sstevel@tonic-gate }
279f3bb54f3SPatrick Mooney return (0);
2807c478bd9Sstevel@tonic-gate }
2817c478bd9Sstevel@tonic-gate
282*f23ed011SPatrick Mooney /*
283*f23ed011SPatrick Mooney * The pc_lock and pc_flag fields of port_fdcache_t must exactly match those of
284*f23ed011SPatrick Mooney * pollcache_t as they are accessed through t_pollcache as if they were part of
285*f23ed011SPatrick Mooney * a "real" pollcache.
286*f23ed011SPatrick Mooney */
287*f23ed011SPatrick Mooney CTASSERT(offsetof(pollcache_t, pc_lock) == offsetof(port_fdcache_t, pc_lock));
288*f23ed011SPatrick Mooney CTASSERT(offsetof(pollcache_t, pc_flag) == offsetof(port_fdcache_t, pc_flag));
289*f23ed011SPatrick Mooney
2907c478bd9Sstevel@tonic-gate void
pollrelock(int lockstate)2917c478bd9Sstevel@tonic-gate pollrelock(int lockstate)
2927c478bd9Sstevel@tonic-gate {
293f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
2947c478bd9Sstevel@tonic-gate pollcache_t *pcp;
2957c478bd9Sstevel@tonic-gate
296f3bb54f3SPatrick Mooney /* Skip this whole ordeal if the pollcache was not locked to begin */
297f3bb54f3SPatrick Mooney if (lockstate == 0)
298f3bb54f3SPatrick Mooney return;
299f3bb54f3SPatrick Mooney
3007c478bd9Sstevel@tonic-gate /*
3017c478bd9Sstevel@tonic-gate * t_pollcache is set by /dev/poll and event ports (port_fd.c).
3027c478bd9Sstevel@tonic-gate * If the pollrelock/pollunlock is called as a result of poll(2),
3037c478bd9Sstevel@tonic-gate * the t_pollcache should be NULL.
3047c478bd9Sstevel@tonic-gate */
3057c478bd9Sstevel@tonic-gate if (curthread->t_pollcache == NULL)
306f3bb54f3SPatrick Mooney pcp = ps->ps_pcache;
3077c478bd9Sstevel@tonic-gate else
3087c478bd9Sstevel@tonic-gate pcp = curthread->t_pollcache;
3097c478bd9Sstevel@tonic-gate
310f3bb54f3SPatrick Mooney mutex_enter(&pcp->pc_lock);
3117c478bd9Sstevel@tonic-gate }
3127c478bd9Sstevel@tonic-gate
3137c478bd9Sstevel@tonic-gate /* ARGSUSED */
314f3bb54f3SPatrick Mooney int
polllock(pollhead_t * php,kmutex_t * lp)3157c478bd9Sstevel@tonic-gate polllock(pollhead_t *php, kmutex_t *lp)
3167c478bd9Sstevel@tonic-gate {
317f3bb54f3SPatrick Mooney if (mutex_tryenter(lp) == 0) {
318f3bb54f3SPatrick Mooney int state;
319f3bb54f3SPatrick Mooney
320f3bb54f3SPatrick Mooney if (pollunlock(&state) != 0) {
321f3bb54f3SPatrick Mooney return (-1);
322f3bb54f3SPatrick Mooney }
3237c478bd9Sstevel@tonic-gate mutex_enter(lp);
324f3bb54f3SPatrick Mooney pollrelock(state);
3257c478bd9Sstevel@tonic-gate }
326f3bb54f3SPatrick Mooney return (0);
3277c478bd9Sstevel@tonic-gate }
3287c478bd9Sstevel@tonic-gate
3297c478bd9Sstevel@tonic-gate static int
poll_common(pollfd_t * fds,nfds_t nfds,timespec_t * tsp,k_sigset_t * ksetp)3307c478bd9Sstevel@tonic-gate poll_common(pollfd_t *fds, nfds_t nfds, timespec_t *tsp, k_sigset_t *ksetp)
3317c478bd9Sstevel@tonic-gate {
3327c478bd9Sstevel@tonic-gate kthread_t *t = curthread;
3337c478bd9Sstevel@tonic-gate klwp_t *lwp = ttolwp(t);
3347c478bd9Sstevel@tonic-gate proc_t *p = ttoproc(t);
3357c478bd9Sstevel@tonic-gate int fdcnt = 0;
3367c478bd9Sstevel@tonic-gate int i;
337cd1c8b85SMatthew Ahrens hrtime_t deadline; /* hrtime value when we want to return */
3387c478bd9Sstevel@tonic-gate pollfd_t *pollfdp;
3397c478bd9Sstevel@tonic-gate pollstate_t *ps;
3407c478bd9Sstevel@tonic-gate pollcache_t *pcp;
3417c478bd9Sstevel@tonic-gate int error = 0;
3427c478bd9Sstevel@tonic-gate nfds_t old_nfds;
3437c478bd9Sstevel@tonic-gate int cacheindex = 0; /* which cache set is used */
3447c478bd9Sstevel@tonic-gate
3457c478bd9Sstevel@tonic-gate /*
3467c478bd9Sstevel@tonic-gate * Determine the precise future time of the requested timeout, if any.
3477c478bd9Sstevel@tonic-gate */
348cd1c8b85SMatthew Ahrens if (tsp == NULL) {
349cd1c8b85SMatthew Ahrens deadline = -1;
350cd1c8b85SMatthew Ahrens } else if (tsp->tv_sec == 0 && tsp->tv_nsec == 0) {
351cd1c8b85SMatthew Ahrens deadline = 0;
352cd1c8b85SMatthew Ahrens } else {
353cd1c8b85SMatthew Ahrens /* They must wait at least a tick. */
3547f0270d8SRichard Lowe deadline = ((hrtime_t)tsp->tv_sec * NANOSEC) + tsp->tv_nsec;
355cd1c8b85SMatthew Ahrens deadline = MAX(deadline, nsec_per_tick);
356cd1c8b85SMatthew Ahrens deadline += gethrtime();
3577c478bd9Sstevel@tonic-gate }
3587c478bd9Sstevel@tonic-gate
3597c478bd9Sstevel@tonic-gate /*
3607c478bd9Sstevel@tonic-gate * Reset our signal mask, if requested.
3617c478bd9Sstevel@tonic-gate */
3627c478bd9Sstevel@tonic-gate if (ksetp != NULL) {
3637c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock);
3647c478bd9Sstevel@tonic-gate schedctl_finish_sigblock(t);
3657c478bd9Sstevel@tonic-gate lwp->lwp_sigoldmask = t->t_hold;
3667c478bd9Sstevel@tonic-gate t->t_hold = *ksetp;
3677c478bd9Sstevel@tonic-gate t->t_flag |= T_TOMASK;
3687c478bd9Sstevel@tonic-gate /*
369d3d50737SRafael Vanoni * Call cv_reltimedwait_sig() just to check for signals.
3707c478bd9Sstevel@tonic-gate * We will return immediately with either 0 or -1.
3717c478bd9Sstevel@tonic-gate */
372d3d50737SRafael Vanoni if (!cv_reltimedwait_sig(&t->t_delay_cv, &p->p_lock, 0,
373d3d50737SRafael Vanoni TR_CLOCK_TICK)) {
3747c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock);
3757c478bd9Sstevel@tonic-gate error = EINTR;
3767c478bd9Sstevel@tonic-gate goto pollout;
3777c478bd9Sstevel@tonic-gate }
3787c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock);
3797c478bd9Sstevel@tonic-gate }
3807c478bd9Sstevel@tonic-gate
3817c478bd9Sstevel@tonic-gate /*
38248bbca81SDaniel Hoffman * Check to see if this one just wants to use poll() as a timeout.
38348bbca81SDaniel Hoffman * If yes then bypass all the other stuff and make it sleep.
3847c478bd9Sstevel@tonic-gate */
3857c478bd9Sstevel@tonic-gate if (nfds == 0) {
3867c478bd9Sstevel@tonic-gate /*
3877c478bd9Sstevel@tonic-gate * Sleep until we have passed the requested future
3887c478bd9Sstevel@tonic-gate * time or until interrupted by a signal.
389cd1c8b85SMatthew Ahrens * Do not check for signals if we do not want to wait.
3907c478bd9Sstevel@tonic-gate */
391cd1c8b85SMatthew Ahrens if (deadline != 0) {
3927c478bd9Sstevel@tonic-gate mutex_enter(&t->t_delay_lock);
393cd1c8b85SMatthew Ahrens while ((error = cv_timedwait_sig_hrtime(&t->t_delay_cv,
394cd1c8b85SMatthew Ahrens &t->t_delay_lock, deadline)) > 0)
3957c478bd9Sstevel@tonic-gate continue;
3967c478bd9Sstevel@tonic-gate mutex_exit(&t->t_delay_lock);
397cd1c8b85SMatthew Ahrens error = (error == 0) ? EINTR : 0;
3987c478bd9Sstevel@tonic-gate }
3997c478bd9Sstevel@tonic-gate goto pollout;
4007c478bd9Sstevel@tonic-gate }
4017c478bd9Sstevel@tonic-gate
4025f684e24Ssp if (nfds > p->p_fno_ctl) {
4037c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock);
4047c478bd9Sstevel@tonic-gate (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
4057c478bd9Sstevel@tonic-gate p->p_rctls, p, RCA_SAFE);
4067c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock);
4077c478bd9Sstevel@tonic-gate error = EINVAL;
4087c478bd9Sstevel@tonic-gate goto pollout;
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate
4117c478bd9Sstevel@tonic-gate /*
4127c478bd9Sstevel@tonic-gate * Need to allocate memory for pollstate before anything because
4137c478bd9Sstevel@tonic-gate * the mutex and cv are created in this space
4147c478bd9Sstevel@tonic-gate */
415f3bb54f3SPatrick Mooney ps = pollstate_create();
4167c478bd9Sstevel@tonic-gate
4177c478bd9Sstevel@tonic-gate if (ps->ps_pcache == NULL)
4187c478bd9Sstevel@tonic-gate ps->ps_pcache = pcache_alloc();
4197c478bd9Sstevel@tonic-gate pcp = ps->ps_pcache;
4207c478bd9Sstevel@tonic-gate
4217c478bd9Sstevel@tonic-gate /*
4227c478bd9Sstevel@tonic-gate * NOTE: for performance, buffers are saved across poll() calls.
4237c478bd9Sstevel@tonic-gate * The theory is that if a process polls heavily, it tends to poll
4247c478bd9Sstevel@tonic-gate * on the same set of descriptors. Therefore, we only reallocate
4257c478bd9Sstevel@tonic-gate * buffers when nfds changes. There is no hysteresis control,
4267c478bd9Sstevel@tonic-gate * because there is no data to suggest that this is necessary;
4277c478bd9Sstevel@tonic-gate * the penalty of reallocating is not *that* great in any event.
4287c478bd9Sstevel@tonic-gate */
4297c478bd9Sstevel@tonic-gate old_nfds = ps->ps_nfds;
4307c478bd9Sstevel@tonic-gate if (nfds != old_nfds) {
4317c478bd9Sstevel@tonic-gate
4327d5e0ac2Sok kmem_free(ps->ps_pollfd, old_nfds * sizeof (pollfd_t));
4337d5e0ac2Sok pollfdp = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
4347c478bd9Sstevel@tonic-gate ps->ps_pollfd = pollfdp;
4357c478bd9Sstevel@tonic-gate ps->ps_nfds = nfds;
4367c478bd9Sstevel@tonic-gate }
4377c478bd9Sstevel@tonic-gate
4387c478bd9Sstevel@tonic-gate pollfdp = ps->ps_pollfd;
4397c478bd9Sstevel@tonic-gate if (copyin(fds, pollfdp, nfds * sizeof (pollfd_t))) {
4407c478bd9Sstevel@tonic-gate error = EFAULT;
4417c478bd9Sstevel@tonic-gate goto pollout;
4427c478bd9Sstevel@tonic-gate }
4437c478bd9Sstevel@tonic-gate
4447c478bd9Sstevel@tonic-gate if (fds == NULL) {
4457c478bd9Sstevel@tonic-gate /*
4467c478bd9Sstevel@tonic-gate * If the process has page 0 mapped, then the copyin() above
4477c478bd9Sstevel@tonic-gate * will succeed even if fds is NULL. However, our cached
4487c478bd9Sstevel@tonic-gate * poll lists are keyed by the address of the passed-in fds
4497c478bd9Sstevel@tonic-gate * structure, and we use the value NULL to indicate an unused
4507c478bd9Sstevel@tonic-gate * poll cache list entry. As such, we elect not to support
4517c478bd9Sstevel@tonic-gate * NULL as a valid (user) memory address and fail the poll()
4527c478bd9Sstevel@tonic-gate * call.
4537c478bd9Sstevel@tonic-gate */
4547c478bd9Sstevel@tonic-gate error = EINVAL;
4557c478bd9Sstevel@tonic-gate goto pollout;
4567c478bd9Sstevel@tonic-gate }
4577c478bd9Sstevel@tonic-gate
4587c478bd9Sstevel@tonic-gate /*
4597c478bd9Sstevel@tonic-gate * If this thread polls for the first time, allocate ALL poll
4607c478bd9Sstevel@tonic-gate * cache data structures and cache the poll fd list. This
4617c478bd9Sstevel@tonic-gate * allocation is delayed till now because lwp's polling 0 fd
4627c478bd9Sstevel@tonic-gate * (i.e. using poll as timeout()) don't need this memory.
4637c478bd9Sstevel@tonic-gate */
4647c478bd9Sstevel@tonic-gate mutex_enter(&ps->ps_lock);
4657c478bd9Sstevel@tonic-gate pcp = ps->ps_pcache;
4667c478bd9Sstevel@tonic-gate ASSERT(pcp != NULL);
4677c478bd9Sstevel@tonic-gate if (pcp->pc_bitmap == NULL) {
4687c478bd9Sstevel@tonic-gate pcache_create(pcp, nfds);
4697c478bd9Sstevel@tonic-gate /*
4707c478bd9Sstevel@tonic-gate * poll and cache this poll fd list in ps_pcacheset[0].
4717c478bd9Sstevel@tonic-gate */
4727c478bd9Sstevel@tonic-gate error = pcacheset_cache_list(ps, fds, &fdcnt, cacheindex);
4737c478bd9Sstevel@tonic-gate if (fdcnt || error) {
4747c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
4757c478bd9Sstevel@tonic-gate goto pollout;
4767c478bd9Sstevel@tonic-gate }
4777c478bd9Sstevel@tonic-gate } else {
4787c478bd9Sstevel@tonic-gate pollcacheset_t *pcset = ps->ps_pcacheset;
4797c478bd9Sstevel@tonic-gate
4807c478bd9Sstevel@tonic-gate /*
4817c478bd9Sstevel@tonic-gate * Not first time polling. Select a cached poll list by
4827c478bd9Sstevel@tonic-gate * matching user pollfd list buffer address.
4837c478bd9Sstevel@tonic-gate */
4847c478bd9Sstevel@tonic-gate for (cacheindex = 0; cacheindex < ps->ps_nsets; cacheindex++) {
4857c478bd9Sstevel@tonic-gate if (pcset[cacheindex].pcs_usradr == (uintptr_t)fds) {
4867c478bd9Sstevel@tonic-gate if ((++pcset[cacheindex].pcs_count) == 0) {
4877c478bd9Sstevel@tonic-gate /*
4887c478bd9Sstevel@tonic-gate * counter is wrapping around.
4897c478bd9Sstevel@tonic-gate */
4907c478bd9Sstevel@tonic-gate pcacheset_reset_count(ps, cacheindex);
4917c478bd9Sstevel@tonic-gate }
4927c478bd9Sstevel@tonic-gate /*
4937c478bd9Sstevel@tonic-gate * examine and resolve possible
4947c478bd9Sstevel@tonic-gate * difference of the current poll
4957c478bd9Sstevel@tonic-gate * list and previously cached one.
4967c478bd9Sstevel@tonic-gate * If there is an error during resolve(),
4977c478bd9Sstevel@tonic-gate * the callee will guarantee the consistency
4987c478bd9Sstevel@tonic-gate * of cached poll list and cache content.
4997c478bd9Sstevel@tonic-gate */
5007c478bd9Sstevel@tonic-gate error = pcacheset_resolve(ps, nfds, &fdcnt,
5017c478bd9Sstevel@tonic-gate cacheindex);
5027c478bd9Sstevel@tonic-gate if (error) {
5037c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
5047c478bd9Sstevel@tonic-gate goto pollout;
5057c478bd9Sstevel@tonic-gate }
5067c478bd9Sstevel@tonic-gate break;
5077c478bd9Sstevel@tonic-gate }
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate /*
5107c478bd9Sstevel@tonic-gate * Note that pcs_usradr field of an used entry won't be
5117e12ceb3SToomas Soome * 0 because it stores the address of passed-in fds,
5127e12ceb3SToomas Soome * and 0 fds will not be cached (Then it is either
5137c478bd9Sstevel@tonic-gate * the special timeout case when nfds is 0 or it returns
5147c478bd9Sstevel@tonic-gate * failure directly).
5157c478bd9Sstevel@tonic-gate */
5167e12ceb3SToomas Soome if (pcset[cacheindex].pcs_usradr == (uintptr_t)NULL) {
5177c478bd9Sstevel@tonic-gate /*
5187c478bd9Sstevel@tonic-gate * found an unused entry. Use it to cache
5197c478bd9Sstevel@tonic-gate * this poll list.
5207c478bd9Sstevel@tonic-gate */
5217c478bd9Sstevel@tonic-gate error = pcacheset_cache_list(ps, fds, &fdcnt,
5227c478bd9Sstevel@tonic-gate cacheindex);
5237c478bd9Sstevel@tonic-gate if (fdcnt || error) {
5247c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
5257c478bd9Sstevel@tonic-gate goto pollout;
5267c478bd9Sstevel@tonic-gate }
5277c478bd9Sstevel@tonic-gate break;
5287c478bd9Sstevel@tonic-gate }
5297c478bd9Sstevel@tonic-gate }
5307c478bd9Sstevel@tonic-gate if (cacheindex == ps->ps_nsets) {
5317c478bd9Sstevel@tonic-gate /*
5327c478bd9Sstevel@tonic-gate * We failed to find a matching cached poll fd list.
5337c478bd9Sstevel@tonic-gate * replace an old list.
5347c478bd9Sstevel@tonic-gate */
5357c478bd9Sstevel@tonic-gate pollstats.polllistmiss.value.ui64++;
5367c478bd9Sstevel@tonic-gate cacheindex = pcacheset_replace(ps);
5377c478bd9Sstevel@tonic-gate ASSERT(cacheindex < ps->ps_nsets);
5387c478bd9Sstevel@tonic-gate pcset[cacheindex].pcs_usradr = (uintptr_t)fds;
5397c478bd9Sstevel@tonic-gate error = pcacheset_resolve(ps, nfds, &fdcnt, cacheindex);
5407c478bd9Sstevel@tonic-gate if (error) {
5417c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
5427c478bd9Sstevel@tonic-gate goto pollout;
5437c478bd9Sstevel@tonic-gate }
5447c478bd9Sstevel@tonic-gate }
5457c478bd9Sstevel@tonic-gate }
5467c478bd9Sstevel@tonic-gate
5477c478bd9Sstevel@tonic-gate /*
5487c478bd9Sstevel@tonic-gate * Always scan the bitmap with the lock on the pollcache held.
5497c478bd9Sstevel@tonic-gate * This is to make sure that a wakeup does not come undetected.
5507c478bd9Sstevel@tonic-gate * If the lock is not held, a pollwakeup could have come for an
5517c478bd9Sstevel@tonic-gate * fd we already checked but before this thread sleeps, in which
5527c478bd9Sstevel@tonic-gate * case the wakeup is missed. Now we hold the pcache lock and
5537c478bd9Sstevel@tonic-gate * check the bitmap again. This will prevent wakeup from happening
5547c478bd9Sstevel@tonic-gate * while we hold pcache lock since pollwakeup() will also lock
5557c478bd9Sstevel@tonic-gate * the pcache before updating poll bitmap.
5567c478bd9Sstevel@tonic-gate */
5577c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
5587c478bd9Sstevel@tonic-gate for (;;) {
5597c478bd9Sstevel@tonic-gate pcp->pc_flag = 0;
5607c478bd9Sstevel@tonic-gate error = pcache_poll(pollfdp, ps, nfds, &fdcnt, cacheindex);
5617c478bd9Sstevel@tonic-gate if (fdcnt || error) {
5627c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
5637c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
5647c478bd9Sstevel@tonic-gate break;
5657c478bd9Sstevel@tonic-gate }
5667c478bd9Sstevel@tonic-gate
5677c478bd9Sstevel@tonic-gate /*
568a5eb7107SBryan Cantrill * If PC_POLLWAKE is set, a pollwakeup() was performed on
5697c478bd9Sstevel@tonic-gate * one of the file descriptors. This can happen only if
5707c478bd9Sstevel@tonic-gate * one of the VOP_POLL() functions dropped pcp->pc_lock.
5717c478bd9Sstevel@tonic-gate * The only current cases of this is in procfs (prpoll())
5727c478bd9Sstevel@tonic-gate * and STREAMS (strpoll()).
5737c478bd9Sstevel@tonic-gate */
574a5eb7107SBryan Cantrill if (pcp->pc_flag & PC_POLLWAKE)
5757c478bd9Sstevel@tonic-gate continue;
5767c478bd9Sstevel@tonic-gate
5777c478bd9Sstevel@tonic-gate /*
5787c478bd9Sstevel@tonic-gate * If you get here, the poll of fds was unsuccessful.
5797c478bd9Sstevel@tonic-gate * Wait until some fd becomes readable, writable, or gets
5807c478bd9Sstevel@tonic-gate * an exception, or until a signal or a timeout occurs.
5817c478bd9Sstevel@tonic-gate * Do not check for signals if we have a zero timeout.
5827c478bd9Sstevel@tonic-gate */
5837c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
584cd1c8b85SMatthew Ahrens if (deadline == 0) {
585cd1c8b85SMatthew Ahrens error = -1;
586fe234e7cSMatt Amdur } else {
587cd1c8b85SMatthew Ahrens error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
588cd1c8b85SMatthew Ahrens &pcp->pc_lock, deadline);
589fe234e7cSMatt Amdur }
5907c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
5917c478bd9Sstevel@tonic-gate /*
5927c478bd9Sstevel@tonic-gate * If we have received a signal or timed out
5937c478bd9Sstevel@tonic-gate * then break out and return.
5947c478bd9Sstevel@tonic-gate */
595cd1c8b85SMatthew Ahrens if (error <= 0) {
596cd1c8b85SMatthew Ahrens error = (error == 0) ? EINTR : 0;
5977c478bd9Sstevel@tonic-gate break;
5987c478bd9Sstevel@tonic-gate }
5997c478bd9Sstevel@tonic-gate /*
6007c478bd9Sstevel@tonic-gate * We have not received a signal or timed out.
6017c478bd9Sstevel@tonic-gate * Continue around and poll fds again.
6027c478bd9Sstevel@tonic-gate */
6037c478bd9Sstevel@tonic-gate mutex_enter(&ps->ps_lock);
6047c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
6057c478bd9Sstevel@tonic-gate }
6067c478bd9Sstevel@tonic-gate
6077c478bd9Sstevel@tonic-gate pollout:
6087c478bd9Sstevel@tonic-gate /*
6097c478bd9Sstevel@tonic-gate * If we changed the signal mask but we received
6107c478bd9Sstevel@tonic-gate * no signal then restore the signal mask.
6117c478bd9Sstevel@tonic-gate * Otherwise psig() will deal with the signal mask.
6127c478bd9Sstevel@tonic-gate */
6137c478bd9Sstevel@tonic-gate if (ksetp != NULL) {
6147c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock);
6157c478bd9Sstevel@tonic-gate if (lwp->lwp_cursig == 0) {
6167c478bd9Sstevel@tonic-gate t->t_hold = lwp->lwp_sigoldmask;
6177c478bd9Sstevel@tonic-gate t->t_flag &= ~T_TOMASK;
6187c478bd9Sstevel@tonic-gate }
6197c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock);
6207c478bd9Sstevel@tonic-gate }
6217c478bd9Sstevel@tonic-gate
6227c478bd9Sstevel@tonic-gate if (error)
6237c478bd9Sstevel@tonic-gate return (set_errno(error));
6247c478bd9Sstevel@tonic-gate
6257c478bd9Sstevel@tonic-gate /*
6267c478bd9Sstevel@tonic-gate * Copy out the events and return the fdcnt to the user.
6277c478bd9Sstevel@tonic-gate */
6287c478bd9Sstevel@tonic-gate if (nfds != 0 &&
6297c478bd9Sstevel@tonic-gate copyout(pollfdp, fds, nfds * sizeof (pollfd_t)))
6307c478bd9Sstevel@tonic-gate return (set_errno(EFAULT));
6317c478bd9Sstevel@tonic-gate
6327c478bd9Sstevel@tonic-gate #ifdef DEBUG
6337c478bd9Sstevel@tonic-gate /*
6347c478bd9Sstevel@tonic-gate * Another sanity check:
6357c478bd9Sstevel@tonic-gate */
6367c478bd9Sstevel@tonic-gate if (fdcnt) {
6377c478bd9Sstevel@tonic-gate int reventcnt = 0;
6387c478bd9Sstevel@tonic-gate
6397c478bd9Sstevel@tonic-gate for (i = 0; i < nfds; i++) {
6407c478bd9Sstevel@tonic-gate if (pollfdp[i].fd < 0) {
6417c478bd9Sstevel@tonic-gate ASSERT(pollfdp[i].revents == 0);
6427c478bd9Sstevel@tonic-gate continue;
6437c478bd9Sstevel@tonic-gate }
6447c478bd9Sstevel@tonic-gate if (pollfdp[i].revents) {
6457c478bd9Sstevel@tonic-gate reventcnt++;
6467c478bd9Sstevel@tonic-gate }
6477c478bd9Sstevel@tonic-gate }
6487c478bd9Sstevel@tonic-gate ASSERT(fdcnt == reventcnt);
6497c478bd9Sstevel@tonic-gate } else {
6507c478bd9Sstevel@tonic-gate for (i = 0; i < nfds; i++) {
6517c478bd9Sstevel@tonic-gate ASSERT(pollfdp[i].revents == 0);
6527c478bd9Sstevel@tonic-gate }
6537c478bd9Sstevel@tonic-gate }
6547c478bd9Sstevel@tonic-gate #endif /* DEBUG */
6557c478bd9Sstevel@tonic-gate
6567c478bd9Sstevel@tonic-gate return (fdcnt);
6577c478bd9Sstevel@tonic-gate }
6587c478bd9Sstevel@tonic-gate
6597c478bd9Sstevel@tonic-gate /*
6607c478bd9Sstevel@tonic-gate * This is the system call trap that poll(),
6617c478bd9Sstevel@tonic-gate * select() and pselect() are built upon.
6627c478bd9Sstevel@tonic-gate * It is a private interface between libc and the kernel.
6637c478bd9Sstevel@tonic-gate */
6647c478bd9Sstevel@tonic-gate int
pollsys(pollfd_t * fds,nfds_t nfds,timespec_t * timeoutp,sigset_t * setp)6657c478bd9Sstevel@tonic-gate pollsys(pollfd_t *fds, nfds_t nfds, timespec_t *timeoutp, sigset_t *setp)
6667c478bd9Sstevel@tonic-gate {
6677c478bd9Sstevel@tonic-gate timespec_t ts;
6687c478bd9Sstevel@tonic-gate timespec_t *tsp;
6697c478bd9Sstevel@tonic-gate sigset_t set;
6707c478bd9Sstevel@tonic-gate k_sigset_t kset;
6717c478bd9Sstevel@tonic-gate k_sigset_t *ksetp;
6727c478bd9Sstevel@tonic-gate model_t datamodel = get_udatamodel();
6737c478bd9Sstevel@tonic-gate
6747c478bd9Sstevel@tonic-gate if (timeoutp == NULL)
6757c478bd9Sstevel@tonic-gate tsp = NULL;
6767c478bd9Sstevel@tonic-gate else {
6777c478bd9Sstevel@tonic-gate if (datamodel == DATAMODEL_NATIVE) {
6787c478bd9Sstevel@tonic-gate if (copyin(timeoutp, &ts, sizeof (ts)))
6797c478bd9Sstevel@tonic-gate return (set_errno(EFAULT));
6807c478bd9Sstevel@tonic-gate } else {
6817c478bd9Sstevel@tonic-gate timespec32_t ts32;
6827c478bd9Sstevel@tonic-gate
6837c478bd9Sstevel@tonic-gate if (copyin(timeoutp, &ts32, sizeof (ts32)))
6847c478bd9Sstevel@tonic-gate return (set_errno(EFAULT));
6857c478bd9Sstevel@tonic-gate TIMESPEC32_TO_TIMESPEC(&ts, &ts32)
6867c478bd9Sstevel@tonic-gate }
6877c478bd9Sstevel@tonic-gate
6887c478bd9Sstevel@tonic-gate if (itimerspecfix(&ts))
6897c478bd9Sstevel@tonic-gate return (set_errno(EINVAL));
6907c478bd9Sstevel@tonic-gate tsp = &ts;
6917c478bd9Sstevel@tonic-gate }
6927c478bd9Sstevel@tonic-gate
6937c478bd9Sstevel@tonic-gate if (setp == NULL)
6947c478bd9Sstevel@tonic-gate ksetp = NULL;
6957c478bd9Sstevel@tonic-gate else {
6967c478bd9Sstevel@tonic-gate if (copyin(setp, &set, sizeof (set)))
6977c478bd9Sstevel@tonic-gate return (set_errno(EFAULT));
6987c478bd9Sstevel@tonic-gate sigutok(&set, &kset);
6997c478bd9Sstevel@tonic-gate ksetp = &kset;
7007c478bd9Sstevel@tonic-gate }
7017c478bd9Sstevel@tonic-gate
7027c478bd9Sstevel@tonic-gate return (poll_common(fds, nfds, tsp, ksetp));
7037c478bd9Sstevel@tonic-gate }
7047c478bd9Sstevel@tonic-gate
7057c478bd9Sstevel@tonic-gate /*
7067c478bd9Sstevel@tonic-gate * Clean up any state left around by poll(2). Called when a thread exits.
7077c478bd9Sstevel@tonic-gate */
7087c478bd9Sstevel@tonic-gate void
pollcleanup()7097c478bd9Sstevel@tonic-gate pollcleanup()
7107c478bd9Sstevel@tonic-gate {
7117c478bd9Sstevel@tonic-gate pollstate_t *ps = curthread->t_pollstate;
7127c478bd9Sstevel@tonic-gate pollcache_t *pcp;
7137c478bd9Sstevel@tonic-gate
7147c478bd9Sstevel@tonic-gate if (ps == NULL)
7157c478bd9Sstevel@tonic-gate return;
7167c478bd9Sstevel@tonic-gate pcp = ps->ps_pcache;
7177c478bd9Sstevel@tonic-gate /*
7187c478bd9Sstevel@tonic-gate * free up all cached poll fds
7197c478bd9Sstevel@tonic-gate */
7207c478bd9Sstevel@tonic-gate if (pcp == NULL) {
7217c478bd9Sstevel@tonic-gate /* this pollstate is used by /dev/poll */
7227c478bd9Sstevel@tonic-gate goto pollcleanout;
7237c478bd9Sstevel@tonic-gate }
7247c478bd9Sstevel@tonic-gate
7257c478bd9Sstevel@tonic-gate if (pcp->pc_bitmap != NULL) {
7267c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(&ps->ps_lock));
7277c478bd9Sstevel@tonic-gate /*
7287c478bd9Sstevel@tonic-gate * a close lwp can race with us when cleaning up a polldat
7297c478bd9Sstevel@tonic-gate * entry. We hold the ps_lock when cleaning hash table.
7307c478bd9Sstevel@tonic-gate * Since this pollcache is going away anyway, there is no
7317c478bd9Sstevel@tonic-gate * need to hold the pc_lock.
7327c478bd9Sstevel@tonic-gate */
7337c478bd9Sstevel@tonic-gate mutex_enter(&ps->ps_lock);
7347c478bd9Sstevel@tonic-gate pcache_clean(pcp);
7357c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
7367c478bd9Sstevel@tonic-gate #ifdef DEBUG
7377c478bd9Sstevel@tonic-gate /*
7387c478bd9Sstevel@tonic-gate * At this point, all fds cached by this lwp should be
7397c478bd9Sstevel@tonic-gate * cleaned up. There should be no fd in fi_list still
7407c478bd9Sstevel@tonic-gate * reference this thread.
7417c478bd9Sstevel@tonic-gate */
7427c478bd9Sstevel@tonic-gate checkfpollinfo(); /* sanity check */
7437c478bd9Sstevel@tonic-gate pollcheckphlist(); /* sanity check */
7447c478bd9Sstevel@tonic-gate #endif /* DEBUG */
7457c478bd9Sstevel@tonic-gate }
7467c478bd9Sstevel@tonic-gate /*
7477c478bd9Sstevel@tonic-gate * Be sure no one is referencing thread before exiting
7487c478bd9Sstevel@tonic-gate */
7497c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
7507c478bd9Sstevel@tonic-gate ASSERT(pcp->pc_busy >= 0);
7517c478bd9Sstevel@tonic-gate while (pcp->pc_busy > 0)
7527c478bd9Sstevel@tonic-gate cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
7537c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
7547c478bd9Sstevel@tonic-gate pollcleanout:
7557c478bd9Sstevel@tonic-gate pollstate_destroy(ps);
7567c478bd9Sstevel@tonic-gate curthread->t_pollstate = NULL;
7577c478bd9Sstevel@tonic-gate }
7587c478bd9Sstevel@tonic-gate
7597c478bd9Sstevel@tonic-gate /*
7607c478bd9Sstevel@tonic-gate * pollwakeup() - poke threads waiting in poll() for some event
7617c478bd9Sstevel@tonic-gate * on a particular object.
7627c478bd9Sstevel@tonic-gate *
7637c478bd9Sstevel@tonic-gate * The threads hanging off of the specified pollhead structure are scanned.
7647c478bd9Sstevel@tonic-gate * If their event mask matches the specified event(s), then pollnotify() is
7657c478bd9Sstevel@tonic-gate * called to poke the thread.
7667c478bd9Sstevel@tonic-gate *
7677c478bd9Sstevel@tonic-gate * Multiple events may be specified. When POLLHUP or POLLERR are specified,
7687c478bd9Sstevel@tonic-gate * all waiting threads are poked.
7697c478bd9Sstevel@tonic-gate *
7707c478bd9Sstevel@tonic-gate * It is important that pollnotify() not drop the lock protecting the list
7717c478bd9Sstevel@tonic-gate * of threads.
7727c478bd9Sstevel@tonic-gate */
7737c478bd9Sstevel@tonic-gate void
pollwakeup(pollhead_t * php,short events_arg)7747c478bd9Sstevel@tonic-gate pollwakeup(pollhead_t *php, short events_arg)
7757c478bd9Sstevel@tonic-gate {
7767c478bd9Sstevel@tonic-gate polldat_t *pdp;
7777c478bd9Sstevel@tonic-gate int events = (ushort_t)events_arg;
77811dc39ddSpraks struct plist {
77911dc39ddSpraks port_t *pp;
78011dc39ddSpraks int pevents;
78111dc39ddSpraks struct plist *next;
78211dc39ddSpraks };
78311dc39ddSpraks struct plist *plhead = NULL, *pltail = NULL;
7847c478bd9Sstevel@tonic-gate
7857c478bd9Sstevel@tonic-gate retry:
7867c478bd9Sstevel@tonic-gate PH_ENTER(php);
7877c478bd9Sstevel@tonic-gate
7887c478bd9Sstevel@tonic-gate for (pdp = php->ph_list; pdp; pdp = pdp->pd_next) {
7897c478bd9Sstevel@tonic-gate if ((pdp->pd_events & events) ||
7907c478bd9Sstevel@tonic-gate (events & (POLLHUP | POLLERR))) {
7917c478bd9Sstevel@tonic-gate
7922c76d751SPatrick Mooney pollcache_t *pcp;
7937c478bd9Sstevel@tonic-gate
7947c478bd9Sstevel@tonic-gate if (pdp->pd_portev != NULL) {
7957c478bd9Sstevel@tonic-gate port_kevent_t *pkevp = pdp->pd_portev;
7967c478bd9Sstevel@tonic-gate /*
7977c478bd9Sstevel@tonic-gate * Object (fd) is associated with an event port,
7987c478bd9Sstevel@tonic-gate * => send event notification to the port.
7997c478bd9Sstevel@tonic-gate */
80011dc39ddSpraks ASSERT(pkevp->portkev_source == PORT_SOURCE_FD);
80161b4b1efSpraks mutex_enter(&pkevp->portkev_lock);
8027c478bd9Sstevel@tonic-gate if (pkevp->portkev_flags & PORT_KEV_VALID) {
80311dc39ddSpraks int pevents;
80411dc39ddSpraks
8057c478bd9Sstevel@tonic-gate pkevp->portkev_flags &= ~PORT_KEV_VALID;
80661b4b1efSpraks pkevp->portkev_events |= events &
80761b4b1efSpraks (pdp->pd_events | POLLHUP |
80861b4b1efSpraks POLLERR);
80961b4b1efSpraks /*
81061b4b1efSpraks * portkev_lock mutex will be released
81111dc39ddSpraks * by port_send_event().
81211dc39ddSpraks */
81311dc39ddSpraks port_send_event(pkevp);
81411dc39ddSpraks
81511dc39ddSpraks /*
81611dc39ddSpraks * If we have some thread polling the
81711dc39ddSpraks * port's fd, add it to the list. They
81811dc39ddSpraks * will be notified later.
81911dc39ddSpraks * The port_pollwkup() will flag the
82011dc39ddSpraks * port_t so that it will not disappear
82111dc39ddSpraks * till port_pollwkdone() is called.
82261b4b1efSpraks */
82311dc39ddSpraks pevents =
82411dc39ddSpraks port_pollwkup(pkevp->portkev_port);
82511dc39ddSpraks if (pevents) {
82611dc39ddSpraks struct plist *t;
82711dc39ddSpraks t = kmem_zalloc(
8285c7544f7SDavid Plauger sizeof (struct plist),
8295c7544f7SDavid Plauger KM_SLEEP);
83011dc39ddSpraks t->pp = pkevp->portkev_port;
83111dc39ddSpraks t->pevents = pevents;
83211dc39ddSpraks if (plhead == NULL) {
83311dc39ddSpraks plhead = t;
83411dc39ddSpraks } else {
83511dc39ddSpraks pltail->next = t;
83611dc39ddSpraks }
83711dc39ddSpraks pltail = t;
83811dc39ddSpraks }
83961b4b1efSpraks } else {
84061b4b1efSpraks mutex_exit(&pkevp->portkev_lock);
8417c478bd9Sstevel@tonic-gate }
8427c478bd9Sstevel@tonic-gate continue;
8437c478bd9Sstevel@tonic-gate }
8447c478bd9Sstevel@tonic-gate
8457c478bd9Sstevel@tonic-gate pcp = pdp->pd_pcache;
8467c478bd9Sstevel@tonic-gate
8477c478bd9Sstevel@tonic-gate /*
8487c478bd9Sstevel@tonic-gate * Try to grab the lock for this thread. If
8497c478bd9Sstevel@tonic-gate * we don't get it then we may deadlock so
8507c478bd9Sstevel@tonic-gate * back out and restart all over again. Note
8517c478bd9Sstevel@tonic-gate * that the failure rate is very very low.
8527c478bd9Sstevel@tonic-gate */
8537c478bd9Sstevel@tonic-gate if (mutex_tryenter(&pcp->pc_lock)) {
8547c478bd9Sstevel@tonic-gate pollnotify(pcp, pdp->pd_fd);
8557c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
8567c478bd9Sstevel@tonic-gate } else {
8577c478bd9Sstevel@tonic-gate /*
8587c478bd9Sstevel@tonic-gate * We are here because:
8597c478bd9Sstevel@tonic-gate * 1) This thread has been woke up
8607c478bd9Sstevel@tonic-gate * and is trying to get out of poll().
8617c478bd9Sstevel@tonic-gate * 2) Some other thread is also here
8627c478bd9Sstevel@tonic-gate * but with a different pollhead lock.
8637c478bd9Sstevel@tonic-gate *
8647c478bd9Sstevel@tonic-gate * So, we need to drop the lock on pollhead
8657c478bd9Sstevel@tonic-gate * because of (1) but we want to prevent
8667c478bd9Sstevel@tonic-gate * that thread from doing lwp_exit() or
8677c478bd9Sstevel@tonic-gate * devpoll close. We want to ensure that
8687c478bd9Sstevel@tonic-gate * the pollcache pointer is still invalid.
8697c478bd9Sstevel@tonic-gate *
8707c478bd9Sstevel@tonic-gate * Solution: Grab the pcp->pc_no_exit lock,
8717c478bd9Sstevel@tonic-gate * increment the pc_busy counter, drop every
8727c478bd9Sstevel@tonic-gate * lock in sight. Get out of the way and wait
8737c478bd9Sstevel@tonic-gate * for type (2) threads to finish.
8747c478bd9Sstevel@tonic-gate */
8757c478bd9Sstevel@tonic-gate
8767c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
8777c478bd9Sstevel@tonic-gate pcp->pc_busy++; /* prevents exit()'s */
8787c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
8797c478bd9Sstevel@tonic-gate
8807c478bd9Sstevel@tonic-gate PH_EXIT(php);
8817c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
8827c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
8837c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
8847c478bd9Sstevel@tonic-gate pcp->pc_busy--;
8857c478bd9Sstevel@tonic-gate if (pcp->pc_busy == 0) {
8867c478bd9Sstevel@tonic-gate /*
8877c478bd9Sstevel@tonic-gate * Wakeup the thread waiting in
8887c478bd9Sstevel@tonic-gate * thread_exit().
8897c478bd9Sstevel@tonic-gate */
8907c478bd9Sstevel@tonic-gate cv_signal(&pcp->pc_busy_cv);
8917c478bd9Sstevel@tonic-gate }
8927c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
8937c478bd9Sstevel@tonic-gate goto retry;
8947c478bd9Sstevel@tonic-gate }
8957c478bd9Sstevel@tonic-gate }
8967c478bd9Sstevel@tonic-gate }
89711dc39ddSpraks
89811dc39ddSpraks
89911dc39ddSpraks /*
90011dc39ddSpraks * Event ports - If this php is of the port on the list,
90111dc39ddSpraks * call port_pollwkdone() to release it. The port_pollwkdone()
90211dc39ddSpraks * needs to be called before dropping the PH lock so that any new
90311dc39ddSpraks * thread attempting to poll this port are blocked. There can be
90411dc39ddSpraks * only one thread here in pollwakeup notifying this port's fd.
90511dc39ddSpraks */
90611dc39ddSpraks if (plhead != NULL && &plhead->pp->port_pollhd == php) {
90711dc39ddSpraks struct plist *t;
90811dc39ddSpraks port_pollwkdone(plhead->pp);
90911dc39ddSpraks t = plhead;
91011dc39ddSpraks plhead = plhead->next;
91111dc39ddSpraks kmem_free(t, sizeof (struct plist));
91211dc39ddSpraks }
9137c478bd9Sstevel@tonic-gate PH_EXIT(php);
91411dc39ddSpraks
91511dc39ddSpraks /*
91611dc39ddSpraks * Event ports - Notify threads polling the event port's fd.
91711dc39ddSpraks * This is normally done in port_send_event() where it calls
91811dc39ddSpraks * pollwakeup() on the port. But, for PORT_SOURCE_FD source alone,
91911dc39ddSpraks * we do it here in pollwakeup() to avoid a recursive call.
92011dc39ddSpraks */
92111dc39ddSpraks if (plhead != NULL) {
92211dc39ddSpraks php = &plhead->pp->port_pollhd;
92311dc39ddSpraks events = plhead->pevents;
92411dc39ddSpraks goto retry;
92511dc39ddSpraks }
9267c478bd9Sstevel@tonic-gate }
9277c478bd9Sstevel@tonic-gate
9287c478bd9Sstevel@tonic-gate /*
929a5eb7107SBryan Cantrill * This function is called to inform a thread (or threads) that an event being
930a5eb7107SBryan Cantrill * polled on has occurred. The pollstate lock on the thread should be held
931a5eb7107SBryan Cantrill * on entry.
9327c478bd9Sstevel@tonic-gate */
9337c478bd9Sstevel@tonic-gate void
pollnotify(pollcache_t * pcp,int fd)9347c478bd9Sstevel@tonic-gate pollnotify(pollcache_t *pcp, int fd)
9357c478bd9Sstevel@tonic-gate {
9367c478bd9Sstevel@tonic-gate ASSERT(fd < pcp->pc_mapsize);
9377c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pcp->pc_lock));
9387c478bd9Sstevel@tonic-gate BT_SET(pcp->pc_bitmap, fd);
939a5eb7107SBryan Cantrill pcp->pc_flag |= PC_POLLWAKE;
940a5eb7107SBryan Cantrill cv_broadcast(&pcp->pc_cv);
941f3bb54f3SPatrick Mooney pcache_wake_parents(pcp);
9427c478bd9Sstevel@tonic-gate }
9437c478bd9Sstevel@tonic-gate
9447c478bd9Sstevel@tonic-gate /*
9452c76d751SPatrick Mooney * Associate a polldat entry with a pollhead (add it to ph_list).
9462c76d751SPatrick Mooney *
9472c76d751SPatrick Mooney * The polldat struct is used by pollwakeup to wake sleeping pollers when polled
9482c76d751SPatrick Mooney * events has happened.
9497c478bd9Sstevel@tonic-gate */
9507c478bd9Sstevel@tonic-gate void
polldat_associate(polldat_t * pdp,pollhead_t * php)9512c76d751SPatrick Mooney polldat_associate(polldat_t *pdp, pollhead_t *php)
9527c478bd9Sstevel@tonic-gate {
9532c76d751SPatrick Mooney ASSERT3P(pdp->pd_php, ==, NULL);
9542c76d751SPatrick Mooney ASSERT3P(pdp->pd_next, ==, NULL);
9552c76d751SPatrick Mooney
9567c478bd9Sstevel@tonic-gate PH_ENTER(php);
9577c478bd9Sstevel@tonic-gate #ifdef DEBUG
9582c76d751SPatrick Mooney /* The polldat should not be already on the list */
9592c76d751SPatrick Mooney for (polldat_t *wp = php->ph_list; wp != NULL; wp = wp->pd_next) {
9602c76d751SPatrick Mooney ASSERT3P(wp, !=, pdp);
9617c478bd9Sstevel@tonic-gate }
9627c478bd9Sstevel@tonic-gate #endif /* DEBUG */
9632c76d751SPatrick Mooney
9647c478bd9Sstevel@tonic-gate pdp->pd_next = php->ph_list;
9657c478bd9Sstevel@tonic-gate php->ph_list = pdp;
9662c76d751SPatrick Mooney pdp->pd_php = php;
9677c478bd9Sstevel@tonic-gate PH_EXIT(php);
9687c478bd9Sstevel@tonic-gate }
9697c478bd9Sstevel@tonic-gate
9707c478bd9Sstevel@tonic-gate /*
9712c76d751SPatrick Mooney * Disassociate a polldat from its pollhead (if such an association exists).
9727c478bd9Sstevel@tonic-gate */
9737c478bd9Sstevel@tonic-gate void
polldat_disassociate(polldat_t * pdp)9742c76d751SPatrick Mooney polldat_disassociate(polldat_t *pdp)
9757c478bd9Sstevel@tonic-gate {
9762c76d751SPatrick Mooney pollhead_t *php;
9777c478bd9Sstevel@tonic-gate
9782c76d751SPatrick Mooney /*
9792c76d751SPatrick Mooney * Acquire the lock for the pollhead which this polldat is associated
9802c76d751SPatrick Mooney * with. This must be done with care, re-checking pd_php after entering
9812c76d751SPatrick Mooney * the pollhead lock, since a racing pollhead_clean() could have already
9822c76d751SPatrick Mooney * performed the disassociation.
9832c76d751SPatrick Mooney */
9842c76d751SPatrick Mooney for (;;) {
9852c76d751SPatrick Mooney php = pdp->pd_php;
9862c76d751SPatrick Mooney if (php == NULL) {
9872c76d751SPatrick Mooney /* polldat is not associated with a pollhead */
9882c76d751SPatrick Mooney return;
9892c76d751SPatrick Mooney }
9902c76d751SPatrick Mooney
9912c76d751SPatrick Mooney /*
9922c76d751SPatrick Mooney * The lock for a given pollhead is not stored in the pollhead
9932c76d751SPatrick Mooney * itself, but is rather a global entry in an array (plocks)
9942c76d751SPatrick Mooney * which the pollhead pointer hashes into (see: PHLOCK()).
9952c76d751SPatrick Mooney */
9962c76d751SPatrick Mooney PH_ENTER(php);
9972c76d751SPatrick Mooney if (pdp->pd_php == php) {
9982c76d751SPatrick Mooney break;
9992c76d751SPatrick Mooney }
10002c76d751SPatrick Mooney PH_EXIT(php);
10012c76d751SPatrick Mooney }
10022c76d751SPatrick Mooney
10032c76d751SPatrick Mooney polldat_t **wpp = &php->ph_list, *wp = php->ph_list;
10042c76d751SPatrick Mooney while (wp != NULL) {
10057c478bd9Sstevel@tonic-gate if (wp == pdp) {
10062c76d751SPatrick Mooney /* Unlink the polldat from the list */
10077c478bd9Sstevel@tonic-gate *wpp = pdp->pd_next;
10087c478bd9Sstevel@tonic-gate pdp->pd_next = NULL;
10097c478bd9Sstevel@tonic-gate break;
10107c478bd9Sstevel@tonic-gate }
10112c76d751SPatrick Mooney wpp = &wp->pd_next;
10122c76d751SPatrick Mooney wp = wp->pd_next;
10137c478bd9Sstevel@tonic-gate }
10142c76d751SPatrick Mooney
10157c478bd9Sstevel@tonic-gate #ifdef DEBUG
10162c76d751SPatrick Mooney /* It would be unexpected if pdp was not in the pollhead list */
10172c76d751SPatrick Mooney ASSERT(wp != NULL);
10182c76d751SPatrick Mooney
10192c76d751SPatrick Mooney /* Assert that pdp is not duplicated somewhere later in the list */
10207c478bd9Sstevel@tonic-gate for (wp = *wpp; wp; wp = wp->pd_next) {
10217c478bd9Sstevel@tonic-gate ASSERT(wp != pdp);
10227c478bd9Sstevel@tonic-gate }
10237c478bd9Sstevel@tonic-gate #endif /* DEBUG */
10242c76d751SPatrick Mooney
10252c76d751SPatrick Mooney pdp->pd_php = NULL;
10267c478bd9Sstevel@tonic-gate PH_EXIT(php);
10277c478bd9Sstevel@tonic-gate }
10287c478bd9Sstevel@tonic-gate
10297c478bd9Sstevel@tonic-gate /*
10307c478bd9Sstevel@tonic-gate * walk through the poll fd lists to see if they are identical. This is an
10317c478bd9Sstevel@tonic-gate * expensive operation and should not be done more than once for each poll()
10327c478bd9Sstevel@tonic-gate * call.
10337c478bd9Sstevel@tonic-gate *
10347c478bd9Sstevel@tonic-gate * As an optimization (i.e., not having to go through the lists more than
10357c478bd9Sstevel@tonic-gate * once), this routine also clear the revents field of pollfd in 'current'.
10367c478bd9Sstevel@tonic-gate * Zeroing out the revents field of each entry in current poll list is
10377c478bd9Sstevel@tonic-gate * required by poll man page.
10387c478bd9Sstevel@tonic-gate *
10397c478bd9Sstevel@tonic-gate * Since the events field of cached list has illegal poll events filtered
10407c478bd9Sstevel@tonic-gate * out, the current list applies the same filtering before comparison.
10417c478bd9Sstevel@tonic-gate *
10427c478bd9Sstevel@tonic-gate * The routine stops when it detects a meaningful difference, or when it
10437c478bd9Sstevel@tonic-gate * exhausts the lists.
10447c478bd9Sstevel@tonic-gate */
10457c478bd9Sstevel@tonic-gate int
pcacheset_cmp(pollfd_t * current,pollfd_t * cached,pollfd_t * newlist,int n)10467c478bd9Sstevel@tonic-gate pcacheset_cmp(pollfd_t *current, pollfd_t *cached, pollfd_t *newlist, int n)
10477c478bd9Sstevel@tonic-gate {
10487c478bd9Sstevel@tonic-gate int ix;
10497c478bd9Sstevel@tonic-gate
10507c478bd9Sstevel@tonic-gate for (ix = 0; ix < n; ix++) {
10515c7544f7SDavid Plauger /* Prefetch 64 bytes worth of 8-byte elements */
10525c7544f7SDavid Plauger if ((ix & 0x7) == 0) {
105394e7edb1Slucy wang - Sun Microsystems - Beijing China prefetch_write_many((caddr_t)¤t[ix + 8]);
105494e7edb1Slucy wang - Sun Microsystems - Beijing China prefetch_write_many((caddr_t)&cached[ix + 8]);
10555c7544f7SDavid Plauger }
10567c478bd9Sstevel@tonic-gate if (current[ix].fd == cached[ix].fd) {
10577c478bd9Sstevel@tonic-gate /*
10587c478bd9Sstevel@tonic-gate * Filter out invalid poll events while we are in
10597c478bd9Sstevel@tonic-gate * inside the loop.
10607c478bd9Sstevel@tonic-gate */
10617c478bd9Sstevel@tonic-gate if (current[ix].events & ~VALID_POLL_EVENTS) {
10627c478bd9Sstevel@tonic-gate current[ix].events &= VALID_POLL_EVENTS;
10637c478bd9Sstevel@tonic-gate if (newlist != NULL)
10647c478bd9Sstevel@tonic-gate newlist[ix].events = current[ix].events;
10657c478bd9Sstevel@tonic-gate }
10667c478bd9Sstevel@tonic-gate if (current[ix].events == cached[ix].events) {
10677c478bd9Sstevel@tonic-gate current[ix].revents = 0;
10687c478bd9Sstevel@tonic-gate continue;
10697c478bd9Sstevel@tonic-gate }
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate if ((current[ix].fd < 0) && (cached[ix].fd < 0)) {
10727c478bd9Sstevel@tonic-gate current[ix].revents = 0;
10737c478bd9Sstevel@tonic-gate continue;
10747c478bd9Sstevel@tonic-gate }
10757c478bd9Sstevel@tonic-gate return (ix);
10767c478bd9Sstevel@tonic-gate }
10777c478bd9Sstevel@tonic-gate return (ix);
10787c478bd9Sstevel@tonic-gate }
10797c478bd9Sstevel@tonic-gate
10807c478bd9Sstevel@tonic-gate /*
10817c478bd9Sstevel@tonic-gate * This routine returns a pointer to a cached poll fd entry, or NULL if it
10827c478bd9Sstevel@tonic-gate * does not find it in the hash table.
10837c478bd9Sstevel@tonic-gate */
10847c478bd9Sstevel@tonic-gate polldat_t *
pcache_lookup_fd(pollcache_t * pcp,int fd)10857c478bd9Sstevel@tonic-gate pcache_lookup_fd(pollcache_t *pcp, int fd)
10867c478bd9Sstevel@tonic-gate {
10877c478bd9Sstevel@tonic-gate int hashindex;
10887c478bd9Sstevel@tonic-gate polldat_t *pdp;
10897c478bd9Sstevel@tonic-gate
10907c478bd9Sstevel@tonic-gate hashindex = POLLHASH(pcp->pc_hashsize, fd);
10917c478bd9Sstevel@tonic-gate pdp = pcp->pc_hash[hashindex];
10927c478bd9Sstevel@tonic-gate while (pdp != NULL) {
10937c478bd9Sstevel@tonic-gate if (pdp->pd_fd == fd)
10947c478bd9Sstevel@tonic-gate break;
10957c478bd9Sstevel@tonic-gate pdp = pdp->pd_hashnext;
10967c478bd9Sstevel@tonic-gate }
10977c478bd9Sstevel@tonic-gate return (pdp);
10987c478bd9Sstevel@tonic-gate }
10997c478bd9Sstevel@tonic-gate
11007c478bd9Sstevel@tonic-gate polldat_t *
pcache_alloc_fd(int nsets)11017c478bd9Sstevel@tonic-gate pcache_alloc_fd(int nsets)
11027c478bd9Sstevel@tonic-gate {
11037c478bd9Sstevel@tonic-gate polldat_t *pdp;
11047c478bd9Sstevel@tonic-gate
11057c478bd9Sstevel@tonic-gate pdp = kmem_zalloc(sizeof (polldat_t), KM_SLEEP);
11067c478bd9Sstevel@tonic-gate if (nsets > 0) {
11077c478bd9Sstevel@tonic-gate pdp->pd_ref = kmem_zalloc(sizeof (xref_t) * nsets, KM_SLEEP);
11087c478bd9Sstevel@tonic-gate pdp->pd_nsets = nsets;
11097c478bd9Sstevel@tonic-gate }
11107c478bd9Sstevel@tonic-gate return (pdp);
11117c478bd9Sstevel@tonic-gate }
11127c478bd9Sstevel@tonic-gate
11137c478bd9Sstevel@tonic-gate /*
11147c478bd9Sstevel@tonic-gate * This routine inserts a polldat into the pollcache's hash table. It
11157c478bd9Sstevel@tonic-gate * may be necessary to grow the size of the hash table.
11167c478bd9Sstevel@tonic-gate */
11177c478bd9Sstevel@tonic-gate void
pcache_insert_fd(pollcache_t * pcp,polldat_t * pdp,nfds_t nfds)11187c478bd9Sstevel@tonic-gate pcache_insert_fd(pollcache_t *pcp, polldat_t *pdp, nfds_t nfds)
11197c478bd9Sstevel@tonic-gate {
11207c478bd9Sstevel@tonic-gate int hashindex;
11217c478bd9Sstevel@tonic-gate int fd;
11227c478bd9Sstevel@tonic-gate
11237c478bd9Sstevel@tonic-gate if ((pcp->pc_fdcount > pcp->pc_hashsize * POLLHASHTHRESHOLD) ||
11247c478bd9Sstevel@tonic-gate (nfds > pcp->pc_hashsize * POLLHASHTHRESHOLD)) {
11257c478bd9Sstevel@tonic-gate pcache_grow_hashtbl(pcp, nfds);
11267c478bd9Sstevel@tonic-gate }
11277c478bd9Sstevel@tonic-gate fd = pdp->pd_fd;
11287c478bd9Sstevel@tonic-gate hashindex = POLLHASH(pcp->pc_hashsize, fd);
11297c478bd9Sstevel@tonic-gate pdp->pd_hashnext = pcp->pc_hash[hashindex];
11307c478bd9Sstevel@tonic-gate pcp->pc_hash[hashindex] = pdp;
11317c478bd9Sstevel@tonic-gate pcp->pc_fdcount++;
11327c478bd9Sstevel@tonic-gate
11337c478bd9Sstevel@tonic-gate #ifdef DEBUG
11347c478bd9Sstevel@tonic-gate {
11357c478bd9Sstevel@tonic-gate /*
11367c478bd9Sstevel@tonic-gate * same fd should not appear on a hash list twice
11377c478bd9Sstevel@tonic-gate */
11387c478bd9Sstevel@tonic-gate polldat_t *pdp1;
11397c478bd9Sstevel@tonic-gate for (pdp1 = pdp->pd_hashnext; pdp1; pdp1 = pdp1->pd_hashnext) {
11407c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fd != pdp1->pd_fd);
11417c478bd9Sstevel@tonic-gate }
11427c478bd9Sstevel@tonic-gate }
11437c478bd9Sstevel@tonic-gate #endif /* DEBUG */
11447c478bd9Sstevel@tonic-gate }
11457c478bd9Sstevel@tonic-gate
11467c478bd9Sstevel@tonic-gate /*
11477c478bd9Sstevel@tonic-gate * Grow the hash table -- either double the table size or round it to the
11487c478bd9Sstevel@tonic-gate * nearest multiples of POLLHASHCHUNKSZ, whichever is bigger. Rehash all the
11497c478bd9Sstevel@tonic-gate * elements on the hash table.
11507c478bd9Sstevel@tonic-gate */
11517c478bd9Sstevel@tonic-gate void
pcache_grow_hashtbl(pollcache_t * pcp,nfds_t nfds)11527c478bd9Sstevel@tonic-gate pcache_grow_hashtbl(pollcache_t *pcp, nfds_t nfds)
11537c478bd9Sstevel@tonic-gate {
11547c478bd9Sstevel@tonic-gate int oldsize;
11557c478bd9Sstevel@tonic-gate polldat_t **oldtbl;
11567c478bd9Sstevel@tonic-gate polldat_t *pdp, *pdp1;
11577c478bd9Sstevel@tonic-gate int i;
11587c478bd9Sstevel@tonic-gate #ifdef DEBUG
11597c478bd9Sstevel@tonic-gate int count = 0;
11607c478bd9Sstevel@tonic-gate #endif
11617c478bd9Sstevel@tonic-gate
11627c478bd9Sstevel@tonic-gate ASSERT(pcp->pc_hashsize % POLLHASHCHUNKSZ == 0);
11637c478bd9Sstevel@tonic-gate oldsize = pcp->pc_hashsize;
11647c478bd9Sstevel@tonic-gate oldtbl = pcp->pc_hash;
11657c478bd9Sstevel@tonic-gate if (nfds > pcp->pc_hashsize * POLLHASHINC) {
11667c478bd9Sstevel@tonic-gate pcp->pc_hashsize = (nfds + POLLHASHCHUNKSZ - 1) &
11677c478bd9Sstevel@tonic-gate ~(POLLHASHCHUNKSZ - 1);
11687c478bd9Sstevel@tonic-gate } else {
11697c478bd9Sstevel@tonic-gate pcp->pc_hashsize = pcp->pc_hashsize * POLLHASHINC;
11707c478bd9Sstevel@tonic-gate }
11717c478bd9Sstevel@tonic-gate pcp->pc_hash = kmem_zalloc(pcp->pc_hashsize * sizeof (polldat_t *),
11727c478bd9Sstevel@tonic-gate KM_SLEEP);
11737c478bd9Sstevel@tonic-gate /*
11747c478bd9Sstevel@tonic-gate * rehash existing elements
11757c478bd9Sstevel@tonic-gate */
11767c478bd9Sstevel@tonic-gate pcp->pc_fdcount = 0;
11777c478bd9Sstevel@tonic-gate for (i = 0; i < oldsize; i++) {
11787c478bd9Sstevel@tonic-gate pdp = oldtbl[i];
11797c478bd9Sstevel@tonic-gate while (pdp != NULL) {
11807c478bd9Sstevel@tonic-gate pdp1 = pdp->pd_hashnext;
11817c478bd9Sstevel@tonic-gate pcache_insert_fd(pcp, pdp, nfds);
11827c478bd9Sstevel@tonic-gate pdp = pdp1;
11837c478bd9Sstevel@tonic-gate #ifdef DEBUG
11847c478bd9Sstevel@tonic-gate count++;
11857c478bd9Sstevel@tonic-gate #endif
11867c478bd9Sstevel@tonic-gate }
11877c478bd9Sstevel@tonic-gate }
11887c478bd9Sstevel@tonic-gate kmem_free(oldtbl, oldsize * sizeof (polldat_t *));
11897c478bd9Sstevel@tonic-gate ASSERT(pcp->pc_fdcount == count);
11907c478bd9Sstevel@tonic-gate }
11917c478bd9Sstevel@tonic-gate
11927c478bd9Sstevel@tonic-gate void
pcache_grow_map(pollcache_t * pcp,int fd)11937c478bd9Sstevel@tonic-gate pcache_grow_map(pollcache_t *pcp, int fd)
11947c478bd9Sstevel@tonic-gate {
11952c76d751SPatrick Mooney int newsize;
11967c478bd9Sstevel@tonic-gate ulong_t *newmap;
11977c478bd9Sstevel@tonic-gate
11987c478bd9Sstevel@tonic-gate /*
11997c478bd9Sstevel@tonic-gate * grow to nearest multiple of POLLMAPCHUNK, assuming POLLMAPCHUNK is
12007c478bd9Sstevel@tonic-gate * power of 2.
12017c478bd9Sstevel@tonic-gate */
12027c478bd9Sstevel@tonic-gate newsize = (fd + POLLMAPCHUNK) & ~(POLLMAPCHUNK - 1);
12037c478bd9Sstevel@tonic-gate newmap = kmem_zalloc((newsize / BT_NBIPUL) * sizeof (ulong_t),
12047c478bd9Sstevel@tonic-gate KM_SLEEP);
12057c478bd9Sstevel@tonic-gate /*
12067c478bd9Sstevel@tonic-gate * don't want pollwakeup to set a bit while growing the bitmap.
12077c478bd9Sstevel@tonic-gate */
12087c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&pcp->pc_lock) == 0);
12097c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
12107c478bd9Sstevel@tonic-gate bcopy(pcp->pc_bitmap, newmap,
12117c478bd9Sstevel@tonic-gate (pcp->pc_mapsize / BT_NBIPUL) * sizeof (ulong_t));
12127c478bd9Sstevel@tonic-gate kmem_free(pcp->pc_bitmap,
12137c478bd9Sstevel@tonic-gate (pcp->pc_mapsize /BT_NBIPUL) * sizeof (ulong_t));
12147c478bd9Sstevel@tonic-gate pcp->pc_bitmap = newmap;
12157c478bd9Sstevel@tonic-gate pcp->pc_mapsize = newsize;
12167c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
12177c478bd9Sstevel@tonic-gate }
12187c478bd9Sstevel@tonic-gate
12197c478bd9Sstevel@tonic-gate /*
12207c478bd9Sstevel@tonic-gate * remove all the reference from pollhead list and fpollinfo lists.
12217c478bd9Sstevel@tonic-gate */
12227c478bd9Sstevel@tonic-gate void
pcache_clean(pollcache_t * pcp)12237c478bd9Sstevel@tonic-gate pcache_clean(pollcache_t *pcp)
12247c478bd9Sstevel@tonic-gate {
12257c478bd9Sstevel@tonic-gate int i;
12267c478bd9Sstevel@tonic-gate polldat_t **hashtbl;
12277c478bd9Sstevel@tonic-gate polldat_t *pdp;
12287c478bd9Sstevel@tonic-gate
12297c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&curthread->t_pollstate->ps_lock));
12307c478bd9Sstevel@tonic-gate hashtbl = pcp->pc_hash;
12317c478bd9Sstevel@tonic-gate for (i = 0; i < pcp->pc_hashsize; i++) {
12327c478bd9Sstevel@tonic-gate for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
12332c76d751SPatrick Mooney polldat_disassociate(pdp);
12347c478bd9Sstevel@tonic-gate if (pdp->pd_fp != NULL) {
12357c478bd9Sstevel@tonic-gate delfpollinfo(pdp->pd_fd);
12367c478bd9Sstevel@tonic-gate pdp->pd_fp = NULL;
12377c478bd9Sstevel@tonic-gate }
12387c478bd9Sstevel@tonic-gate }
12397c478bd9Sstevel@tonic-gate }
12407c478bd9Sstevel@tonic-gate }
12417c478bd9Sstevel@tonic-gate
12427c478bd9Sstevel@tonic-gate void
pcacheset_invalidate(pollstate_t * ps,polldat_t * pdp)12437c478bd9Sstevel@tonic-gate pcacheset_invalidate(pollstate_t *ps, polldat_t *pdp)
12447c478bd9Sstevel@tonic-gate {
12452c76d751SPatrick Mooney int i;
12467c478bd9Sstevel@tonic-gate int fd = pdp->pd_fd;
12477c478bd9Sstevel@tonic-gate
12487c478bd9Sstevel@tonic-gate /*
12497c478bd9Sstevel@tonic-gate * we come here because an earlier close() on this cached poll fd.
12507c478bd9Sstevel@tonic-gate */
12517c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fp == NULL);
12527c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
12537c478bd9Sstevel@tonic-gate pdp->pd_events = 0;
12547c478bd9Sstevel@tonic-gate for (i = 0; i < ps->ps_nsets; i++) {
12557c478bd9Sstevel@tonic-gate xref_t *refp;
12567c478bd9Sstevel@tonic-gate pollcacheset_t *pcsp;
12577c478bd9Sstevel@tonic-gate
12587c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
12597c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[i];
12607c478bd9Sstevel@tonic-gate if (refp->xf_refcnt) {
12617c478bd9Sstevel@tonic-gate ASSERT(refp->xf_position >= 0);
12627c478bd9Sstevel@tonic-gate pcsp = &ps->ps_pcacheset[i];
12637c478bd9Sstevel@tonic-gate if (refp->xf_refcnt == 1) {
12647c478bd9Sstevel@tonic-gate pcsp->pcs_pollfd[refp->xf_position].fd = -1;
12657c478bd9Sstevel@tonic-gate refp->xf_refcnt = 0;
12667c478bd9Sstevel@tonic-gate pdp->pd_count--;
12677c478bd9Sstevel@tonic-gate } else if (refp->xf_refcnt > 1) {
12687c478bd9Sstevel@tonic-gate int j;
12697c478bd9Sstevel@tonic-gate
12707c478bd9Sstevel@tonic-gate /*
12717c478bd9Sstevel@tonic-gate * turn off every appearance in pcs_pollfd list
12727c478bd9Sstevel@tonic-gate */
12737c478bd9Sstevel@tonic-gate for (j = refp->xf_position;
12747c478bd9Sstevel@tonic-gate j < pcsp->pcs_nfds; j++) {
12757c478bd9Sstevel@tonic-gate if (pcsp->pcs_pollfd[j].fd == fd) {
12767c478bd9Sstevel@tonic-gate pcsp->pcs_pollfd[j].fd = -1;
12777c478bd9Sstevel@tonic-gate refp->xf_refcnt--;
12787c478bd9Sstevel@tonic-gate pdp->pd_count--;
12797c478bd9Sstevel@tonic-gate }
12807c478bd9Sstevel@tonic-gate }
12817c478bd9Sstevel@tonic-gate }
12827c478bd9Sstevel@tonic-gate ASSERT(refp->xf_refcnt == 0);
12837c478bd9Sstevel@tonic-gate refp->xf_position = POLLPOSINVAL;
12847c478bd9Sstevel@tonic-gate }
12857c478bd9Sstevel@tonic-gate }
12867c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_count == 0);
12877c478bd9Sstevel@tonic-gate }
12887c478bd9Sstevel@tonic-gate
12897c478bd9Sstevel@tonic-gate /*
12907c478bd9Sstevel@tonic-gate * Insert poll fd into the pollcache, and add poll registration.
12917c478bd9Sstevel@tonic-gate * This routine is called after getf() and before releasef(). So the vnode
12927c478bd9Sstevel@tonic-gate * can not disappear even if we block here.
12937c478bd9Sstevel@tonic-gate * If there is an error, the polled fd is not cached.
12947c478bd9Sstevel@tonic-gate */
12957c478bd9Sstevel@tonic-gate int
pcache_insert(pollstate_t * ps,file_t * fp,pollfd_t * pollfdp,int * fdcntp,ssize_t pos,int which)12967c478bd9Sstevel@tonic-gate pcache_insert(pollstate_t *ps, file_t *fp, pollfd_t *pollfdp, int *fdcntp,
12977c478bd9Sstevel@tonic-gate ssize_t pos, int which)
12987c478bd9Sstevel@tonic-gate {
12997c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
13007c478bd9Sstevel@tonic-gate polldat_t *pdp;
13017c478bd9Sstevel@tonic-gate int error;
13027c478bd9Sstevel@tonic-gate int fd;
13037c478bd9Sstevel@tonic-gate pollhead_t *memphp = NULL;
13047c478bd9Sstevel@tonic-gate xref_t *refp;
13057c478bd9Sstevel@tonic-gate int newpollfd = 0;
13067c478bd9Sstevel@tonic-gate
13077c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
13087c478bd9Sstevel@tonic-gate /*
13097c478bd9Sstevel@tonic-gate * The poll caching uses the existing VOP_POLL interface. If there
13107c478bd9Sstevel@tonic-gate * is no polled events, we want the polled device to set its "some
13117c478bd9Sstevel@tonic-gate * one is sleeping in poll" flag. When the polled events happen
13127c478bd9Sstevel@tonic-gate * later, the driver will call pollwakeup(). We achieve this by
13137c478bd9Sstevel@tonic-gate * always passing 0 in the third parameter ("anyyet") when calling
13147c478bd9Sstevel@tonic-gate * VOP_POLL. This parameter is not looked at by drivers when the
13157c478bd9Sstevel@tonic-gate * polled events exist. If a driver chooses to ignore this parameter
13167c478bd9Sstevel@tonic-gate * and call pollwakeup whenever the polled events happen, that will
13177c478bd9Sstevel@tonic-gate * be OK too.
13187c478bd9Sstevel@tonic-gate */
13197c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pollcache == NULL);
13207c478bd9Sstevel@tonic-gate error = VOP_POLL(fp->f_vnode, pollfdp->events, 0, &pollfdp->revents,
1321da6c28aaSamw &memphp, NULL);
13227c478bd9Sstevel@tonic-gate if (error) {
13237c478bd9Sstevel@tonic-gate return (error);
13247c478bd9Sstevel@tonic-gate }
13257c478bd9Sstevel@tonic-gate if (pollfdp->revents) {
13267c478bd9Sstevel@tonic-gate (*fdcntp)++;
13277c478bd9Sstevel@tonic-gate }
13287c478bd9Sstevel@tonic-gate /*
13297c478bd9Sstevel@tonic-gate * polling the underlying device succeeded. Now we can cache it.
13307c478bd9Sstevel@tonic-gate * A close can't come in here because we have not done a releasef()
13317c478bd9Sstevel@tonic-gate * yet.
13327c478bd9Sstevel@tonic-gate */
13337c478bd9Sstevel@tonic-gate fd = pollfdp->fd;
13347c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
13357c478bd9Sstevel@tonic-gate if (pdp == NULL) {
13367c478bd9Sstevel@tonic-gate ASSERT(ps->ps_nsets > 0);
13377c478bd9Sstevel@tonic-gate pdp = pcache_alloc_fd(ps->ps_nsets);
13387c478bd9Sstevel@tonic-gate newpollfd = 1;
13397c478bd9Sstevel@tonic-gate }
13407c478bd9Sstevel@tonic-gate /*
13417c478bd9Sstevel@tonic-gate * If this entry was used to cache a poll fd which was closed, and
13427c478bd9Sstevel@tonic-gate * this entry has not been cleaned, do it now.
13437c478bd9Sstevel@tonic-gate */
13447c478bd9Sstevel@tonic-gate if ((pdp->pd_count > 0) && (pdp->pd_fp == NULL)) {
13457c478bd9Sstevel@tonic-gate pcacheset_invalidate(ps, pdp);
13467c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_next == NULL);
13477c478bd9Sstevel@tonic-gate }
13487c478bd9Sstevel@tonic-gate if (pdp->pd_count == 0) {
13497c478bd9Sstevel@tonic-gate pdp->pd_fd = fd;
13507c478bd9Sstevel@tonic-gate pdp->pd_fp = fp;
13517c478bd9Sstevel@tonic-gate addfpollinfo(fd);
13527c478bd9Sstevel@tonic-gate pdp->pd_thread = curthread;
13537c478bd9Sstevel@tonic-gate pdp->pd_pcache = pcp;
13547c478bd9Sstevel@tonic-gate /*
13557c478bd9Sstevel@tonic-gate * the entry is never used or cleared by removing a cached
13567c478bd9Sstevel@tonic-gate * pollfd (pcache_delete_fd). So all the fields should be clear.
13577c478bd9Sstevel@tonic-gate */
13587c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_next == NULL);
13597c478bd9Sstevel@tonic-gate }
13607c478bd9Sstevel@tonic-gate
13617c478bd9Sstevel@tonic-gate /*
13627c478bd9Sstevel@tonic-gate * A polled fd is considered cached. So there should be a fpollinfo
13637c478bd9Sstevel@tonic-gate * entry on uf_fpollinfo list.
13647c478bd9Sstevel@tonic-gate */
13657c478bd9Sstevel@tonic-gate ASSERT(infpollinfo(fd));
13667c478bd9Sstevel@tonic-gate /*
13677c478bd9Sstevel@tonic-gate * If there is an inconsistency, we want to know it here.
13687c478bd9Sstevel@tonic-gate */
13697c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fp == fp);
13707c478bd9Sstevel@tonic-gate
13717c478bd9Sstevel@tonic-gate /*
13727c478bd9Sstevel@tonic-gate * XXX pd_events is a union of all polled events on this fd, possibly
13737c478bd9Sstevel@tonic-gate * by different threads. Unless this is a new first poll(), pd_events
13747c478bd9Sstevel@tonic-gate * never shrinks. If an event is no longer polled by a process, there
13757c478bd9Sstevel@tonic-gate * is no way to cancel that event. In that case, poll degrade to its
13767c478bd9Sstevel@tonic-gate * old form -- polling on this fd every time poll() is called. The
13777c478bd9Sstevel@tonic-gate * assumption is an app always polls the same type of events.
13787c478bd9Sstevel@tonic-gate */
13797c478bd9Sstevel@tonic-gate pdp->pd_events |= pollfdp->events;
13807c478bd9Sstevel@tonic-gate
13817c478bd9Sstevel@tonic-gate pdp->pd_count++;
13827c478bd9Sstevel@tonic-gate /*
13837c478bd9Sstevel@tonic-gate * There is not much special handling for multiple appearances of
13847c478bd9Sstevel@tonic-gate * same fd other than xf_position always recording the first
13857c478bd9Sstevel@tonic-gate * appearance in poll list. If this is called from pcacheset_cache_list,
13867c478bd9Sstevel@tonic-gate * a VOP_POLL is called on every pollfd entry; therefore each
13877c478bd9Sstevel@tonic-gate * revents and fdcnt should be set correctly. If this is called from
13887c478bd9Sstevel@tonic-gate * pcacheset_resolve, we don't care about fdcnt here. Pollreadmap will
13897c478bd9Sstevel@tonic-gate * pick up the right count and handle revents field of each pollfd
13907c478bd9Sstevel@tonic-gate * entry.
13917c478bd9Sstevel@tonic-gate */
13927c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
13937c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[which];
13947c478bd9Sstevel@tonic-gate if (refp->xf_refcnt == 0) {
13957c478bd9Sstevel@tonic-gate refp->xf_position = pos;
13967c478bd9Sstevel@tonic-gate } else {
13977c478bd9Sstevel@tonic-gate /*
13987c478bd9Sstevel@tonic-gate * xf_position records the fd's first appearance in poll list
13997c478bd9Sstevel@tonic-gate */
14007c478bd9Sstevel@tonic-gate if (pos < refp->xf_position) {
14017c478bd9Sstevel@tonic-gate refp->xf_position = pos;
14027c478bd9Sstevel@tonic-gate }
14037c478bd9Sstevel@tonic-gate }
14047c478bd9Sstevel@tonic-gate ASSERT(pollfdp->fd == ps->ps_pollfd[refp->xf_position].fd);
14057c478bd9Sstevel@tonic-gate refp->xf_refcnt++;
14067c478bd9Sstevel@tonic-gate if (fd >= pcp->pc_mapsize) {
14077c478bd9Sstevel@tonic-gate pcache_grow_map(pcp, fd);
14087c478bd9Sstevel@tonic-gate }
14097c478bd9Sstevel@tonic-gate if (fd > pcp->pc_mapend) {
14107c478bd9Sstevel@tonic-gate pcp->pc_mapend = fd;
14117c478bd9Sstevel@tonic-gate }
14127c478bd9Sstevel@tonic-gate if (newpollfd != 0) {
14137c478bd9Sstevel@tonic-gate pcache_insert_fd(ps->ps_pcache, pdp, ps->ps_nfds);
14147c478bd9Sstevel@tonic-gate }
14157c478bd9Sstevel@tonic-gate if (memphp) {
14167c478bd9Sstevel@tonic-gate if (pdp->pd_php == NULL) {
14172c76d751SPatrick Mooney polldat_associate(pdp, memphp);
14187c478bd9Sstevel@tonic-gate } else {
14197c478bd9Sstevel@tonic-gate if (memphp != pdp->pd_php) {
14207c478bd9Sstevel@tonic-gate /*
14217c478bd9Sstevel@tonic-gate * layered devices (e.g. console driver)
14227c478bd9Sstevel@tonic-gate * may change the vnode and thus the pollhead
14237c478bd9Sstevel@tonic-gate * pointer out from underneath us.
14247c478bd9Sstevel@tonic-gate */
14252c76d751SPatrick Mooney polldat_disassociate(pdp);
14262c76d751SPatrick Mooney polldat_associate(pdp, memphp);
14277c478bd9Sstevel@tonic-gate }
14287c478bd9Sstevel@tonic-gate }
14297c478bd9Sstevel@tonic-gate }
14307c478bd9Sstevel@tonic-gate /*
14317c478bd9Sstevel@tonic-gate * Since there is a considerable window between VOP_POLL and when
14327c478bd9Sstevel@tonic-gate * we actually put the polldat struct on the pollhead list, we could
14337c478bd9Sstevel@tonic-gate * miss a pollwakeup. In the case of polling additional events, we
14347c478bd9Sstevel@tonic-gate * don't update the events until after VOP_POLL. So we could miss
14357c478bd9Sstevel@tonic-gate * pollwakeup there too. So we always set the bit here just to be
14367c478bd9Sstevel@tonic-gate * safe. The real performance gain is in subsequent pcache_poll.
14377c478bd9Sstevel@tonic-gate */
14387c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
14397c478bd9Sstevel@tonic-gate BT_SET(pcp->pc_bitmap, fd);
14407c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14417c478bd9Sstevel@tonic-gate return (0);
14427c478bd9Sstevel@tonic-gate }
14437c478bd9Sstevel@tonic-gate
14447c478bd9Sstevel@tonic-gate /*
14457c478bd9Sstevel@tonic-gate * The entry is not really deleted. The fields are cleared so that the
14467c478bd9Sstevel@tonic-gate * entry is no longer useful, but it will remain in the hash table for reuse
14477c478bd9Sstevel@tonic-gate * later. It will be freed when the polling lwp exits.
14487c478bd9Sstevel@tonic-gate */
14497c478bd9Sstevel@tonic-gate int
pcache_delete_fd(pollstate_t * ps,int fd,size_t pos,int which,uint_t cevent)14507c478bd9Sstevel@tonic-gate pcache_delete_fd(pollstate_t *ps, int fd, size_t pos, int which, uint_t cevent)
14517c478bd9Sstevel@tonic-gate {
14527c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
14537c478bd9Sstevel@tonic-gate polldat_t *pdp;
14547c478bd9Sstevel@tonic-gate xref_t *refp;
14557c478bd9Sstevel@tonic-gate
14567c478bd9Sstevel@tonic-gate ASSERT(fd < pcp->pc_mapsize);
14577c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
14587c478bd9Sstevel@tonic-gate
14597c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
14607c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
14617c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_count > 0);
14627c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
14637c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[which];
14647c478bd9Sstevel@tonic-gate if (pdp->pd_count == 1) {
14657c478bd9Sstevel@tonic-gate pdp->pd_events = 0;
14667c478bd9Sstevel@tonic-gate refp->xf_position = POLLPOSINVAL;
14677c478bd9Sstevel@tonic-gate ASSERT(refp->xf_refcnt == 1);
14687c478bd9Sstevel@tonic-gate refp->xf_refcnt = 0;
14692c76d751SPatrick Mooney
14702c76d751SPatrick Mooney /*
14712c76d751SPatrick Mooney * It is possible for a wakeup thread to get ahead of the
14722c76d751SPatrick Mooney * following polldat_disassociate and set the bit in bitmap.
14732c76d751SPatrick Mooney * That is OK because the bit will be cleared here anyway.
14742c76d751SPatrick Mooney */
14752c76d751SPatrick Mooney polldat_disassociate(pdp);
14762c76d751SPatrick Mooney
14777c478bd9Sstevel@tonic-gate pdp->pd_count = 0;
14787c478bd9Sstevel@tonic-gate if (pdp->pd_fp != NULL) {
14797c478bd9Sstevel@tonic-gate pdp->pd_fp = NULL;
14807c478bd9Sstevel@tonic-gate delfpollinfo(fd);
14817c478bd9Sstevel@tonic-gate }
14827c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
14837c478bd9Sstevel@tonic-gate BT_CLEAR(pcp->pc_bitmap, fd);
14847c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14857c478bd9Sstevel@tonic-gate return (0);
14867c478bd9Sstevel@tonic-gate }
14877c478bd9Sstevel@tonic-gate if ((cevent & POLLCLOSED) == POLLCLOSED) {
14887c478bd9Sstevel@tonic-gate /*
14897c478bd9Sstevel@tonic-gate * fd cached here has been closed. This is the first
14907c478bd9Sstevel@tonic-gate * pcache_delete_fd called after the close. Clean up the
14917c478bd9Sstevel@tonic-gate * entire entry.
14927c478bd9Sstevel@tonic-gate */
14937c478bd9Sstevel@tonic-gate pcacheset_invalidate(ps, pdp);
14947c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_php == NULL);
14957c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
14967c478bd9Sstevel@tonic-gate BT_CLEAR(pcp->pc_bitmap, fd);
14977c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14987c478bd9Sstevel@tonic-gate return (0);
14997c478bd9Sstevel@tonic-gate }
15007c478bd9Sstevel@tonic-gate #ifdef DEBUG
15017c478bd9Sstevel@tonic-gate if (getf(fd) != NULL) {
15027c478bd9Sstevel@tonic-gate ASSERT(infpollinfo(fd));
15037c478bd9Sstevel@tonic-gate releasef(fd);
15047c478bd9Sstevel@tonic-gate }
15057c478bd9Sstevel@tonic-gate #endif /* DEBUG */
15067c478bd9Sstevel@tonic-gate pdp->pd_count--;
15077c478bd9Sstevel@tonic-gate ASSERT(refp->xf_refcnt > 0);
15087c478bd9Sstevel@tonic-gate if (--refp->xf_refcnt == 0) {
15097c478bd9Sstevel@tonic-gate refp->xf_position = POLLPOSINVAL;
15107c478bd9Sstevel@tonic-gate } else {
15117c478bd9Sstevel@tonic-gate ASSERT(pos >= refp->xf_position);
15127c478bd9Sstevel@tonic-gate if (pos == refp->xf_position) {
15137c478bd9Sstevel@tonic-gate /*
15147c478bd9Sstevel@tonic-gate * The xref position is no longer valid.
15157c478bd9Sstevel@tonic-gate * Reset it to a special value and let
15167c478bd9Sstevel@tonic-gate * caller know it needs to updatexref()
15177c478bd9Sstevel@tonic-gate * with a new xf_position value.
15187c478bd9Sstevel@tonic-gate */
15197c478bd9Sstevel@tonic-gate refp->xf_position = POLLPOSTRANS;
15207c478bd9Sstevel@tonic-gate return (1);
15217c478bd9Sstevel@tonic-gate }
15227c478bd9Sstevel@tonic-gate }
15237c478bd9Sstevel@tonic-gate return (0);
15247c478bd9Sstevel@tonic-gate }
15257c478bd9Sstevel@tonic-gate
15267c478bd9Sstevel@tonic-gate void
pcache_update_xref(pollcache_t * pcp,int fd,ssize_t pos,int which)15277c478bd9Sstevel@tonic-gate pcache_update_xref(pollcache_t *pcp, int fd, ssize_t pos, int which)
15287c478bd9Sstevel@tonic-gate {
15297c478bd9Sstevel@tonic-gate polldat_t *pdp;
15307c478bd9Sstevel@tonic-gate
15317c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
15327c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
15337c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
15347c478bd9Sstevel@tonic-gate pdp->pd_ref[which].xf_position = pos;
15357c478bd9Sstevel@tonic-gate }
15367c478bd9Sstevel@tonic-gate
15377c478bd9Sstevel@tonic-gate #ifdef DEBUG
15387c478bd9Sstevel@tonic-gate /*
15397c478bd9Sstevel@tonic-gate * For each polled fd, it's either in the bitmap or cached in
15407c478bd9Sstevel@tonic-gate * pcache hash table. If this routine returns 0, something is wrong.
15417c478bd9Sstevel@tonic-gate */
15427c478bd9Sstevel@tonic-gate static int
pollchecksanity(pollstate_t * ps,nfds_t nfds)15437c478bd9Sstevel@tonic-gate pollchecksanity(pollstate_t *ps, nfds_t nfds)
15447c478bd9Sstevel@tonic-gate {
15452c76d751SPatrick Mooney int i;
15467c478bd9Sstevel@tonic-gate int fd;
15477c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
15487c478bd9Sstevel@tonic-gate polldat_t *pdp;
15497c478bd9Sstevel@tonic-gate pollfd_t *pollfdp = ps->ps_pollfd;
15507c478bd9Sstevel@tonic-gate file_t *fp;
15517c478bd9Sstevel@tonic-gate
15527c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
15537c478bd9Sstevel@tonic-gate for (i = 0; i < nfds; i++) {
15547c478bd9Sstevel@tonic-gate fd = pollfdp[i].fd;
15557c478bd9Sstevel@tonic-gate if (fd < 0) {
15567c478bd9Sstevel@tonic-gate ASSERT(pollfdp[i].revents == 0);
15577c478bd9Sstevel@tonic-gate continue;
15587c478bd9Sstevel@tonic-gate }
15597c478bd9Sstevel@tonic-gate if (pollfdp[i].revents == POLLNVAL)
15607c478bd9Sstevel@tonic-gate continue;
15617c478bd9Sstevel@tonic-gate if ((fp = getf(fd)) == NULL)
15627c478bd9Sstevel@tonic-gate continue;
15637c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
15647c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
15657c478bd9Sstevel@tonic-gate ASSERT(infpollinfo(fd));
15667c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fp == fp);
15677c478bd9Sstevel@tonic-gate releasef(fd);
15687c478bd9Sstevel@tonic-gate if (BT_TEST(pcp->pc_bitmap, fd))
15697c478bd9Sstevel@tonic-gate continue;
15707c478bd9Sstevel@tonic-gate if (pdp->pd_php == NULL)
15717c478bd9Sstevel@tonic-gate return (0);
15727c478bd9Sstevel@tonic-gate }
15737c478bd9Sstevel@tonic-gate return (1);
15747c478bd9Sstevel@tonic-gate }
15757c478bd9Sstevel@tonic-gate #endif /* DEBUG */
15767c478bd9Sstevel@tonic-gate
15777c478bd9Sstevel@tonic-gate /*
15787c478bd9Sstevel@tonic-gate * resolve the difference between the current poll list and a cached one.
15797c478bd9Sstevel@tonic-gate */
15807c478bd9Sstevel@tonic-gate int
pcacheset_resolve(pollstate_t * ps,nfds_t nfds,int * fdcntp,int which)15817c478bd9Sstevel@tonic-gate pcacheset_resolve(pollstate_t *ps, nfds_t nfds, int *fdcntp, int which)
15827c478bd9Sstevel@tonic-gate {
15832c76d751SPatrick Mooney int i;
15847c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
15857c478bd9Sstevel@tonic-gate pollfd_t *newlist = NULL;
15867c478bd9Sstevel@tonic-gate pollfd_t *current = ps->ps_pollfd;
15877c478bd9Sstevel@tonic-gate pollfd_t *cached;
15887c478bd9Sstevel@tonic-gate pollcacheset_t *pcsp;
15897c478bd9Sstevel@tonic-gate int common;
15907c478bd9Sstevel@tonic-gate int count = 0;
15917c478bd9Sstevel@tonic-gate int offset;
15927c478bd9Sstevel@tonic-gate int remain;
15937c478bd9Sstevel@tonic-gate int fd;
15947c478bd9Sstevel@tonic-gate file_t *fp;
15957c478bd9Sstevel@tonic-gate int fdcnt = 0;
15967c478bd9Sstevel@tonic-gate int cnt = 0;
15977c478bd9Sstevel@tonic-gate nfds_t old_nfds;
15987c478bd9Sstevel@tonic-gate int error = 0;
15997c478bd9Sstevel@tonic-gate int mismatch = 0;
16007c478bd9Sstevel@tonic-gate
16017c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
16027c478bd9Sstevel@tonic-gate #ifdef DEBUG
16037c478bd9Sstevel@tonic-gate checkpolldat(ps);
16047c478bd9Sstevel@tonic-gate #endif
16057c478bd9Sstevel@tonic-gate pcsp = &ps->ps_pcacheset[which];
16067c478bd9Sstevel@tonic-gate old_nfds = pcsp->pcs_nfds;
16077c478bd9Sstevel@tonic-gate common = (nfds > old_nfds) ? old_nfds : nfds;
16087c478bd9Sstevel@tonic-gate if (nfds != old_nfds) {
16097c478bd9Sstevel@tonic-gate /*
16107c478bd9Sstevel@tonic-gate * the length of poll list has changed. allocate a new
16117c478bd9Sstevel@tonic-gate * pollfd list.
16127c478bd9Sstevel@tonic-gate */
16137c478bd9Sstevel@tonic-gate newlist = kmem_alloc(nfds * sizeof (pollfd_t), KM_SLEEP);
16147c478bd9Sstevel@tonic-gate bcopy(current, newlist, sizeof (pollfd_t) * nfds);
16157c478bd9Sstevel@tonic-gate }
16167c478bd9Sstevel@tonic-gate /*
16177c478bd9Sstevel@tonic-gate * Compare the overlapping part of the current fd list with the
16187c478bd9Sstevel@tonic-gate * cached one. Whenever a difference is found, resolve it.
16197c478bd9Sstevel@tonic-gate * The comparison is done on the current poll list and the
16207c478bd9Sstevel@tonic-gate * cached list. But we may be setting up the newlist to be the
16217c478bd9Sstevel@tonic-gate * cached list for next poll.
16227c478bd9Sstevel@tonic-gate */
16237c478bd9Sstevel@tonic-gate cached = pcsp->pcs_pollfd;
16247c478bd9Sstevel@tonic-gate remain = common;
16257c478bd9Sstevel@tonic-gate
16267c478bd9Sstevel@tonic-gate while (count < common) {
16277c478bd9Sstevel@tonic-gate int tmpfd;
16287c478bd9Sstevel@tonic-gate pollfd_t *np;
16297c478bd9Sstevel@tonic-gate
16307c478bd9Sstevel@tonic-gate np = (newlist != NULL) ? &newlist[count] : NULL;
16317c478bd9Sstevel@tonic-gate offset = pcacheset_cmp(¤t[count], &cached[count], np,
16327c478bd9Sstevel@tonic-gate remain);
16337c478bd9Sstevel@tonic-gate /*
16347c478bd9Sstevel@tonic-gate * Collect stats. If lists are completed the first time,
16357c478bd9Sstevel@tonic-gate * it's a hit. Otherwise, it's a partial hit or miss.
16367c478bd9Sstevel@tonic-gate */
16377c478bd9Sstevel@tonic-gate if ((count == 0) && (offset == common)) {
16387c478bd9Sstevel@tonic-gate pollstats.pollcachehit.value.ui64++;
16397c478bd9Sstevel@tonic-gate } else {
16407c478bd9Sstevel@tonic-gate mismatch++;
16417c478bd9Sstevel@tonic-gate }
16427c478bd9Sstevel@tonic-gate count += offset;
16437c478bd9Sstevel@tonic-gate if (offset < remain) {
16447c478bd9Sstevel@tonic-gate ASSERT(count < common);
16457c478bd9Sstevel@tonic-gate ASSERT((current[count].fd != cached[count].fd) ||
16467c478bd9Sstevel@tonic-gate (current[count].events != cached[count].events));
16477c478bd9Sstevel@tonic-gate /*
16487c478bd9Sstevel@tonic-gate * Filter out invalid events.
16497c478bd9Sstevel@tonic-gate */
16507c478bd9Sstevel@tonic-gate if (current[count].events & ~VALID_POLL_EVENTS) {
16517c478bd9Sstevel@tonic-gate if (newlist != NULL) {
16527c478bd9Sstevel@tonic-gate newlist[count].events =
16535c7544f7SDavid Plauger current[count].events &=
16545c7544f7SDavid Plauger VALID_POLL_EVENTS;
16557c478bd9Sstevel@tonic-gate } else {
16567c478bd9Sstevel@tonic-gate current[count].events &=
16575c7544f7SDavid Plauger VALID_POLL_EVENTS;
16587c478bd9Sstevel@tonic-gate }
16597c478bd9Sstevel@tonic-gate }
16607c478bd9Sstevel@tonic-gate /*
16617c478bd9Sstevel@tonic-gate * when resolving a difference, we always remove the
16627c478bd9Sstevel@tonic-gate * fd from cache before inserting one into cache.
16637c478bd9Sstevel@tonic-gate */
16647c478bd9Sstevel@tonic-gate if (cached[count].fd >= 0) {
16657c478bd9Sstevel@tonic-gate tmpfd = cached[count].fd;
16667c478bd9Sstevel@tonic-gate if (pcache_delete_fd(ps, tmpfd, count, which,
16677c478bd9Sstevel@tonic-gate (uint_t)cached[count].events)) {
16687c478bd9Sstevel@tonic-gate /*
16697c478bd9Sstevel@tonic-gate * This should be rare but needed for
16707c478bd9Sstevel@tonic-gate * correctness.
16717c478bd9Sstevel@tonic-gate *
16727c478bd9Sstevel@tonic-gate * The first appearance in cached list
16737c478bd9Sstevel@tonic-gate * is being "turned off". The same fd
16747c478bd9Sstevel@tonic-gate * appear more than once in the cached
16757c478bd9Sstevel@tonic-gate * poll list. Find the next one on the
16767c478bd9Sstevel@tonic-gate * list and update the cached
16777c478bd9Sstevel@tonic-gate * xf_position field.
16787c478bd9Sstevel@tonic-gate */
16797c478bd9Sstevel@tonic-gate for (i = count + 1; i < old_nfds; i++) {
16807c478bd9Sstevel@tonic-gate if (cached[i].fd == tmpfd) {
16817c478bd9Sstevel@tonic-gate pcache_update_xref(pcp,
16827c478bd9Sstevel@tonic-gate tmpfd, (ssize_t)i,
16837c478bd9Sstevel@tonic-gate which);
16845c7544f7SDavid Plauger break;
16857c478bd9Sstevel@tonic-gate }
16867c478bd9Sstevel@tonic-gate }
16877c478bd9Sstevel@tonic-gate ASSERT(i <= old_nfds);
16887c478bd9Sstevel@tonic-gate }
16897c478bd9Sstevel@tonic-gate /*
16907c478bd9Sstevel@tonic-gate * In case a new cache list is allocated,
16917c478bd9Sstevel@tonic-gate * need to keep both cache lists in sync
16927c478bd9Sstevel@tonic-gate * b/c the new one can be freed if we have
16937c478bd9Sstevel@tonic-gate * an error later.
16947c478bd9Sstevel@tonic-gate */
16957c478bd9Sstevel@tonic-gate cached[count].fd = -1;
16967c478bd9Sstevel@tonic-gate if (newlist != NULL) {
16977c478bd9Sstevel@tonic-gate newlist[count].fd = -1;
16987c478bd9Sstevel@tonic-gate }
16997c478bd9Sstevel@tonic-gate }
17007c478bd9Sstevel@tonic-gate if ((tmpfd = current[count].fd) >= 0) {
17017c478bd9Sstevel@tonic-gate /*
17027c478bd9Sstevel@tonic-gate * add to the cached fd tbl and bitmap.
17037c478bd9Sstevel@tonic-gate */
17047c478bd9Sstevel@tonic-gate if ((fp = getf(tmpfd)) == NULL) {
17057c478bd9Sstevel@tonic-gate current[count].revents = POLLNVAL;
17067c478bd9Sstevel@tonic-gate if (newlist != NULL) {
17077c478bd9Sstevel@tonic-gate newlist[count].fd = -1;
17087c478bd9Sstevel@tonic-gate }
17097c478bd9Sstevel@tonic-gate cached[count].fd = -1;
17107c478bd9Sstevel@tonic-gate fdcnt++;
17117c478bd9Sstevel@tonic-gate } else {
17127c478bd9Sstevel@tonic-gate /*
17137c478bd9Sstevel@tonic-gate * Here we don't care about the
17147c478bd9Sstevel@tonic-gate * fdcnt. We will examine the bitmap
17157c478bd9Sstevel@tonic-gate * later and pick up the correct
17167c478bd9Sstevel@tonic-gate * fdcnt there. So we never bother
17177c478bd9Sstevel@tonic-gate * to check value of 'cnt'.
17187c478bd9Sstevel@tonic-gate */
17197c478bd9Sstevel@tonic-gate error = pcache_insert(ps, fp,
17207c478bd9Sstevel@tonic-gate ¤t[count], &cnt,
17217c478bd9Sstevel@tonic-gate (ssize_t)count, which);
17227c478bd9Sstevel@tonic-gate /*
17237c478bd9Sstevel@tonic-gate * if no error, we want to do releasef
17247c478bd9Sstevel@tonic-gate * after we updated cache poll list
17257c478bd9Sstevel@tonic-gate * entry so that close() won't race
17267c478bd9Sstevel@tonic-gate * us.
17277c478bd9Sstevel@tonic-gate */
17287c478bd9Sstevel@tonic-gate if (error) {
17297c478bd9Sstevel@tonic-gate /*
17307c478bd9Sstevel@tonic-gate * If we encountered an error,
17317c478bd9Sstevel@tonic-gate * we have invalidated an
17327c478bd9Sstevel@tonic-gate * entry in cached poll list
17337c478bd9Sstevel@tonic-gate * (in pcache_delete_fd() above)
17347c478bd9Sstevel@tonic-gate * but failed to add one here.
17357c478bd9Sstevel@tonic-gate * This is OK b/c what's in the
17367c478bd9Sstevel@tonic-gate * cached list is consistent
17377c478bd9Sstevel@tonic-gate * with content of cache.
17387c478bd9Sstevel@tonic-gate * It will not have any ill
17397c478bd9Sstevel@tonic-gate * effect on next poll().
17407c478bd9Sstevel@tonic-gate */
17417c478bd9Sstevel@tonic-gate releasef(tmpfd);
17427c478bd9Sstevel@tonic-gate if (newlist != NULL) {
17437c478bd9Sstevel@tonic-gate kmem_free(newlist,
17447c478bd9Sstevel@tonic-gate nfds *
17457c478bd9Sstevel@tonic-gate sizeof (pollfd_t));
17467c478bd9Sstevel@tonic-gate }
17477c478bd9Sstevel@tonic-gate return (error);
17487c478bd9Sstevel@tonic-gate }
17497c478bd9Sstevel@tonic-gate /*
17507c478bd9Sstevel@tonic-gate * If we have allocated a new(temp)
17517c478bd9Sstevel@tonic-gate * cache list, we need to keep both
17527c478bd9Sstevel@tonic-gate * in sync b/c the new one can be freed
17537c478bd9Sstevel@tonic-gate * if we have an error later.
17547c478bd9Sstevel@tonic-gate */
17557c478bd9Sstevel@tonic-gate if (newlist != NULL) {
17567c478bd9Sstevel@tonic-gate newlist[count].fd =
17577c478bd9Sstevel@tonic-gate current[count].fd;
17587c478bd9Sstevel@tonic-gate newlist[count].events =
17597c478bd9Sstevel@tonic-gate current[count].events;
17607c478bd9Sstevel@tonic-gate }
17617c478bd9Sstevel@tonic-gate cached[count].fd = current[count].fd;
17627c478bd9Sstevel@tonic-gate cached[count].events =
17637c478bd9Sstevel@tonic-gate current[count].events;
17647c478bd9Sstevel@tonic-gate releasef(tmpfd);
17657c478bd9Sstevel@tonic-gate }
17667c478bd9Sstevel@tonic-gate } else {
17677c478bd9Sstevel@tonic-gate current[count].revents = 0;
17687c478bd9Sstevel@tonic-gate }
17697c478bd9Sstevel@tonic-gate count++;
17707c478bd9Sstevel@tonic-gate remain = common - count;
17717c478bd9Sstevel@tonic-gate }
17727c478bd9Sstevel@tonic-gate }
17737c478bd9Sstevel@tonic-gate if (mismatch != 0) {
17747c478bd9Sstevel@tonic-gate if (mismatch == common) {
17757c478bd9Sstevel@tonic-gate pollstats.pollcachemiss.value.ui64++;
17767c478bd9Sstevel@tonic-gate } else {
17777c478bd9Sstevel@tonic-gate pollstats.pollcachephit.value.ui64++;
17787c478bd9Sstevel@tonic-gate }
17797c478bd9Sstevel@tonic-gate }
17807c478bd9Sstevel@tonic-gate /*
17817c478bd9Sstevel@tonic-gate * take care of the non overlapping part of a list
17827c478bd9Sstevel@tonic-gate */
17837c478bd9Sstevel@tonic-gate if (nfds > old_nfds) {
17847c478bd9Sstevel@tonic-gate ASSERT(newlist != NULL);
17857c478bd9Sstevel@tonic-gate for (i = old_nfds; i < nfds; i++) {
17867c478bd9Sstevel@tonic-gate /* filter out invalid events */
17877c478bd9Sstevel@tonic-gate if (current[i].events & ~VALID_POLL_EVENTS) {
17887c478bd9Sstevel@tonic-gate newlist[i].events = current[i].events =
17895c7544f7SDavid Plauger current[i].events & VALID_POLL_EVENTS;
17907c478bd9Sstevel@tonic-gate }
17917c478bd9Sstevel@tonic-gate if ((fd = current[i].fd) < 0) {
17927c478bd9Sstevel@tonic-gate current[i].revents = 0;
17937c478bd9Sstevel@tonic-gate continue;
17947c478bd9Sstevel@tonic-gate }
17957c478bd9Sstevel@tonic-gate /*
17967c478bd9Sstevel@tonic-gate * add to the cached fd tbl and bitmap.
17977c478bd9Sstevel@tonic-gate */
17987c478bd9Sstevel@tonic-gate if ((fp = getf(fd)) == NULL) {
17997c478bd9Sstevel@tonic-gate current[i].revents = POLLNVAL;
18007c478bd9Sstevel@tonic-gate newlist[i].fd = -1;
18017c478bd9Sstevel@tonic-gate fdcnt++;
18027c478bd9Sstevel@tonic-gate continue;
18037c478bd9Sstevel@tonic-gate }
18047c478bd9Sstevel@tonic-gate /*
18057c478bd9Sstevel@tonic-gate * Here we don't care about the
18067c478bd9Sstevel@tonic-gate * fdcnt. We will examine the bitmap
18077c478bd9Sstevel@tonic-gate * later and pick up the correct
18087c478bd9Sstevel@tonic-gate * fdcnt there. So we never bother to
18097c478bd9Sstevel@tonic-gate * check 'cnt'.
18107c478bd9Sstevel@tonic-gate */
18117c478bd9Sstevel@tonic-gate error = pcache_insert(ps, fp, ¤t[i], &cnt,
18127c478bd9Sstevel@tonic-gate (ssize_t)i, which);
18137c478bd9Sstevel@tonic-gate releasef(fd);
18147c478bd9Sstevel@tonic-gate if (error) {
18157c478bd9Sstevel@tonic-gate /*
18167c478bd9Sstevel@tonic-gate * Here we are half way through adding newly
18177c478bd9Sstevel@tonic-gate * polled fd. Undo enough to keep the cache
18187c478bd9Sstevel@tonic-gate * list consistent with the cache content.
18197c478bd9Sstevel@tonic-gate */
18207c478bd9Sstevel@tonic-gate pcacheset_remove_list(ps, current, old_nfds,
18217c478bd9Sstevel@tonic-gate i, which, 0);
18227c478bd9Sstevel@tonic-gate kmem_free(newlist, nfds * sizeof (pollfd_t));
18237c478bd9Sstevel@tonic-gate return (error);
18247c478bd9Sstevel@tonic-gate }
18257c478bd9Sstevel@tonic-gate }
18267c478bd9Sstevel@tonic-gate }
18277c478bd9Sstevel@tonic-gate if (old_nfds > nfds) {
18287c478bd9Sstevel@tonic-gate /*
18297c478bd9Sstevel@tonic-gate * remove the fd's which are no longer polled.
18307c478bd9Sstevel@tonic-gate */
18317c478bd9Sstevel@tonic-gate pcacheset_remove_list(ps, pcsp->pcs_pollfd, nfds, old_nfds,
18327c478bd9Sstevel@tonic-gate which, 1);
18337c478bd9Sstevel@tonic-gate }
18347c478bd9Sstevel@tonic-gate /*
18357c478bd9Sstevel@tonic-gate * set difference resolved. update nfds and cachedlist
18367c478bd9Sstevel@tonic-gate * in pollstate struct.
18377c478bd9Sstevel@tonic-gate */
18387c478bd9Sstevel@tonic-gate if (newlist != NULL) {
18397c478bd9Sstevel@tonic-gate kmem_free(pcsp->pcs_pollfd, old_nfds * sizeof (pollfd_t));
18407c478bd9Sstevel@tonic-gate /*
18417c478bd9Sstevel@tonic-gate * By now, the pollfd.revents field should
18427c478bd9Sstevel@tonic-gate * all be zeroed.
18437c478bd9Sstevel@tonic-gate */
18447c478bd9Sstevel@tonic-gate pcsp->pcs_pollfd = newlist;
18457c478bd9Sstevel@tonic-gate pcsp->pcs_nfds = nfds;
18467c478bd9Sstevel@tonic-gate }
18477c478bd9Sstevel@tonic-gate ASSERT(*fdcntp == 0);
18487c478bd9Sstevel@tonic-gate *fdcntp = fdcnt;
18497c478bd9Sstevel@tonic-gate /*
18507c478bd9Sstevel@tonic-gate * By now for every fd in pollfdp, one of the following should be
18517c478bd9Sstevel@tonic-gate * true. Otherwise we will miss a polled event.
18527c478bd9Sstevel@tonic-gate *
18537c478bd9Sstevel@tonic-gate * 1. the bit corresponding to the fd in bitmap is set. So VOP_POLL
18547c478bd9Sstevel@tonic-gate * will be called on this fd in next poll.
18557c478bd9Sstevel@tonic-gate * 2. the fd is cached in the pcache (i.e. pd_php is set). So
18567c478bd9Sstevel@tonic-gate * pollnotify will happen.
18577c478bd9Sstevel@tonic-gate */
18587c478bd9Sstevel@tonic-gate ASSERT(pollchecksanity(ps, nfds));
18597c478bd9Sstevel@tonic-gate /*
18607c478bd9Sstevel@tonic-gate * make sure cross reference between cached poll lists and cached
18617c478bd9Sstevel@tonic-gate * poll fds are correct.
18627c478bd9Sstevel@tonic-gate */
18637c478bd9Sstevel@tonic-gate ASSERT(pollcheckxref(ps, which));
18647c478bd9Sstevel@tonic-gate /*
18657c478bd9Sstevel@tonic-gate * ensure each polldat in pollcache reference a polled fd in
18667c478bd9Sstevel@tonic-gate * pollcacheset.
18677c478bd9Sstevel@tonic-gate */
18687c478bd9Sstevel@tonic-gate #ifdef DEBUG
18697c478bd9Sstevel@tonic-gate checkpolldat(ps);
18707c478bd9Sstevel@tonic-gate #endif
18717c478bd9Sstevel@tonic-gate return (0);
18727c478bd9Sstevel@tonic-gate }
18737c478bd9Sstevel@tonic-gate
18747c478bd9Sstevel@tonic-gate #ifdef DEBUG
18757c478bd9Sstevel@tonic-gate static int
pollscanrevents(pollcache_t * pcp,pollfd_t * pollfdp,nfds_t nfds)18767c478bd9Sstevel@tonic-gate pollscanrevents(pollcache_t *pcp, pollfd_t *pollfdp, nfds_t nfds)
18777c478bd9Sstevel@tonic-gate {
18787c478bd9Sstevel@tonic-gate int i;
18797c478bd9Sstevel@tonic-gate int reventcnt = 0;
18807c478bd9Sstevel@tonic-gate
18817c478bd9Sstevel@tonic-gate for (i = 0; i < nfds; i++) {
18827c478bd9Sstevel@tonic-gate if (pollfdp[i].fd < 0) {
18837c478bd9Sstevel@tonic-gate ASSERT(pollfdp[i].revents == 0);
18847c478bd9Sstevel@tonic-gate continue;
18857c478bd9Sstevel@tonic-gate }
18867c478bd9Sstevel@tonic-gate if (pollfdp[i].revents) {
18877c478bd9Sstevel@tonic-gate reventcnt++;
18887c478bd9Sstevel@tonic-gate }
18897c478bd9Sstevel@tonic-gate if (pollfdp[i].revents && (pollfdp[i].revents != POLLNVAL)) {
18907c478bd9Sstevel@tonic-gate ASSERT(BT_TEST(pcp->pc_bitmap, pollfdp[i].fd));
18917c478bd9Sstevel@tonic-gate }
18927c478bd9Sstevel@tonic-gate }
18937c478bd9Sstevel@tonic-gate return (reventcnt);
18947c478bd9Sstevel@tonic-gate }
18957c478bd9Sstevel@tonic-gate #endif /* DEBUG */
18967c478bd9Sstevel@tonic-gate
18977c478bd9Sstevel@tonic-gate /*
18987c478bd9Sstevel@tonic-gate * read the bitmap and poll on fds corresponding to the '1' bits. The ps_lock
18997c478bd9Sstevel@tonic-gate * is held upon entry.
19007c478bd9Sstevel@tonic-gate */
19017c478bd9Sstevel@tonic-gate int
pcache_poll(pollfd_t * pollfdp,pollstate_t * ps,nfds_t nfds,int * fdcntp,int which)19027c478bd9Sstevel@tonic-gate pcache_poll(pollfd_t *pollfdp, pollstate_t *ps, nfds_t nfds, int *fdcntp,
19037c478bd9Sstevel@tonic-gate int which)
19047c478bd9Sstevel@tonic-gate {
19057c478bd9Sstevel@tonic-gate int i;
19067c478bd9Sstevel@tonic-gate pollcache_t *pcp;
19072c76d751SPatrick Mooney int fd;
19082c76d751SPatrick Mooney int begin, end, done;
19097c478bd9Sstevel@tonic-gate pollhead_t *php;
19107c478bd9Sstevel@tonic-gate int fdcnt;
19117c478bd9Sstevel@tonic-gate int error = 0;
19127c478bd9Sstevel@tonic-gate file_t *fp;
19137c478bd9Sstevel@tonic-gate polldat_t *pdp;
19147c478bd9Sstevel@tonic-gate xref_t *refp;
19157c478bd9Sstevel@tonic-gate int entry;
19167c478bd9Sstevel@tonic-gate
19177c478bd9Sstevel@tonic-gate pcp = ps->ps_pcache;
19187c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
19197c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pcp->pc_lock));
19207c478bd9Sstevel@tonic-gate retry:
19217c478bd9Sstevel@tonic-gate done = 0;
19227c478bd9Sstevel@tonic-gate begin = 0;
19237c478bd9Sstevel@tonic-gate fdcnt = 0;
19247c478bd9Sstevel@tonic-gate end = pcp->pc_mapend;
19257c478bd9Sstevel@tonic-gate while ((fdcnt < nfds) && !done) {
19267c478bd9Sstevel@tonic-gate php = NULL;
19277c478bd9Sstevel@tonic-gate /*
19287c478bd9Sstevel@tonic-gate * only poll fds which may have events
19297c478bd9Sstevel@tonic-gate */
19307c478bd9Sstevel@tonic-gate fd = bt_getlowbit(pcp->pc_bitmap, begin, end);
19317c478bd9Sstevel@tonic-gate ASSERT(fd <= end);
19327c478bd9Sstevel@tonic-gate if (fd >= 0) {
19337c478bd9Sstevel@tonic-gate ASSERT(pollcheckrevents(ps, begin, fd, which));
19347c478bd9Sstevel@tonic-gate /*
19357c478bd9Sstevel@tonic-gate * adjust map pointers for next round
19367c478bd9Sstevel@tonic-gate */
19377c478bd9Sstevel@tonic-gate if (fd == end) {
19387c478bd9Sstevel@tonic-gate done = 1;
19397c478bd9Sstevel@tonic-gate } else {
19407c478bd9Sstevel@tonic-gate begin = fd + 1;
19417c478bd9Sstevel@tonic-gate }
19427c478bd9Sstevel@tonic-gate /*
19437c478bd9Sstevel@tonic-gate * A bitmap caches poll state information of
19447c478bd9Sstevel@tonic-gate * multiple poll lists. Call VOP_POLL only if
19457c478bd9Sstevel@tonic-gate * the bit corresponds to an fd in this poll
19467c478bd9Sstevel@tonic-gate * list.
19477c478bd9Sstevel@tonic-gate */
19487c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
19497c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
19507c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
19517c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[which];
19527c478bd9Sstevel@tonic-gate if (refp->xf_refcnt == 0)
19537c478bd9Sstevel@tonic-gate continue;
19547c478bd9Sstevel@tonic-gate entry = refp->xf_position;
19557c478bd9Sstevel@tonic-gate ASSERT((entry >= 0) && (entry < nfds));
19567c478bd9Sstevel@tonic-gate ASSERT(pollfdp[entry].fd == fd);
19577c478bd9Sstevel@tonic-gate /*
19587c478bd9Sstevel@tonic-gate * we are in this routine implies that we have
19597c478bd9Sstevel@tonic-gate * successfully polled this fd in the past.
19607c478bd9Sstevel@tonic-gate * Check to see this fd is closed while we are
19617c478bd9Sstevel@tonic-gate * blocked in poll. This ensures that we don't
19627c478bd9Sstevel@tonic-gate * miss a close on the fd in the case this fd is
19637c478bd9Sstevel@tonic-gate * reused.
19647c478bd9Sstevel@tonic-gate */
19657c478bd9Sstevel@tonic-gate if (pdp->pd_fp == NULL) {
19667c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_count > 0);
19677c478bd9Sstevel@tonic-gate pollfdp[entry].revents = POLLNVAL;
19687c478bd9Sstevel@tonic-gate fdcnt++;
19697c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
19707c478bd9Sstevel@tonic-gate /*
19717c478bd9Sstevel@tonic-gate * this fd appeared multiple time
19727c478bd9Sstevel@tonic-gate * in the poll list. Find all of them.
19737c478bd9Sstevel@tonic-gate */
19747c478bd9Sstevel@tonic-gate for (i = entry + 1; i < nfds; i++) {
19757c478bd9Sstevel@tonic-gate if (pollfdp[i].fd == fd) {
19767c478bd9Sstevel@tonic-gate pollfdp[i].revents =
19777c478bd9Sstevel@tonic-gate POLLNVAL;
19787c478bd9Sstevel@tonic-gate fdcnt++;
19797c478bd9Sstevel@tonic-gate }
19807c478bd9Sstevel@tonic-gate }
19817c478bd9Sstevel@tonic-gate }
19827c478bd9Sstevel@tonic-gate pcacheset_invalidate(ps, pdp);
19837c478bd9Sstevel@tonic-gate continue;
19847c478bd9Sstevel@tonic-gate }
19857c478bd9Sstevel@tonic-gate /*
19867c478bd9Sstevel@tonic-gate * We can be here polling a device that is being
19877c478bd9Sstevel@tonic-gate * closed (i.e. the file pointer is set to NULL,
19887c478bd9Sstevel@tonic-gate * but pollcacheclean has not happened yet).
19897c478bd9Sstevel@tonic-gate */
19907c478bd9Sstevel@tonic-gate if ((fp = getf(fd)) == NULL) {
19917c478bd9Sstevel@tonic-gate pollfdp[entry].revents = POLLNVAL;
19927c478bd9Sstevel@tonic-gate fdcnt++;
19937c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
19947c478bd9Sstevel@tonic-gate /*
19957c478bd9Sstevel@tonic-gate * this fd appeared multiple time
19967c478bd9Sstevel@tonic-gate * in the poll list. Find all of them.
19977c478bd9Sstevel@tonic-gate */
19987c478bd9Sstevel@tonic-gate for (i = entry + 1; i < nfds; i++) {
19997c478bd9Sstevel@tonic-gate if (pollfdp[i].fd == fd) {
20007c478bd9Sstevel@tonic-gate pollfdp[i].revents =
20017c478bd9Sstevel@tonic-gate POLLNVAL;
20027c478bd9Sstevel@tonic-gate fdcnt++;
20037c478bd9Sstevel@tonic-gate }
20047c478bd9Sstevel@tonic-gate }
20057c478bd9Sstevel@tonic-gate }
20067c478bd9Sstevel@tonic-gate continue;
20077c478bd9Sstevel@tonic-gate }
20087c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fp == fp);
20097c478bd9Sstevel@tonic-gate ASSERT(infpollinfo(fd));
20107c478bd9Sstevel@tonic-gate /*
20117c478bd9Sstevel@tonic-gate * Since we no longer hold poll head lock across
20127c478bd9Sstevel@tonic-gate * VOP_POLL, pollunlock logic can be simplifed.
20137c478bd9Sstevel@tonic-gate */
20147c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_php == NULL ||
20157c478bd9Sstevel@tonic-gate MUTEX_NOT_HELD(PHLOCK(pdp->pd_php)));
20167c478bd9Sstevel@tonic-gate /*
20177c478bd9Sstevel@tonic-gate * underlying file systems may set a "pollpending"
20187c478bd9Sstevel@tonic-gate * flag when it sees the poll may block. Pollwakeup()
20197c478bd9Sstevel@tonic-gate * is called by wakeup thread if pollpending is set.
20207c478bd9Sstevel@tonic-gate * Pass a 0 fdcnt so that the underlying file system
20217c478bd9Sstevel@tonic-gate * will set the "pollpending" flag set when there is
20227c478bd9Sstevel@tonic-gate * no polled events.
20237c478bd9Sstevel@tonic-gate *
20247c478bd9Sstevel@tonic-gate * Use pollfdp[].events for actual polling because
20257c478bd9Sstevel@tonic-gate * the pd_events is union of all cached poll events
20267c478bd9Sstevel@tonic-gate * on this fd. The events parameter also affects
20277c478bd9Sstevel@tonic-gate * how the polled device sets the "poll pending"
20287c478bd9Sstevel@tonic-gate * flag.
20297c478bd9Sstevel@tonic-gate */
20307c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pollcache == NULL);
20317c478bd9Sstevel@tonic-gate error = VOP_POLL(fp->f_vnode, pollfdp[entry].events, 0,
2032da6c28aaSamw &pollfdp[entry].revents, &php, NULL);
20337c478bd9Sstevel@tonic-gate /*
20347c478bd9Sstevel@tonic-gate * releasef after completely done with this cached
20357c478bd9Sstevel@tonic-gate * poll entry. To prevent close() coming in to clear
20367c478bd9Sstevel@tonic-gate * this entry.
20377c478bd9Sstevel@tonic-gate */
20387c478bd9Sstevel@tonic-gate if (error) {
20397c478bd9Sstevel@tonic-gate releasef(fd);
20407c478bd9Sstevel@tonic-gate break;
20417c478bd9Sstevel@tonic-gate }
20427c478bd9Sstevel@tonic-gate /*
20437c478bd9Sstevel@tonic-gate * layered devices (e.g. console driver)
20447c478bd9Sstevel@tonic-gate * may change the vnode and thus the pollhead
20457c478bd9Sstevel@tonic-gate * pointer out from underneath us.
20467c478bd9Sstevel@tonic-gate */
20477c478bd9Sstevel@tonic-gate if (php != NULL && pdp->pd_php != NULL &&
20487c478bd9Sstevel@tonic-gate php != pdp->pd_php) {
20497c478bd9Sstevel@tonic-gate releasef(fd);
20502c76d751SPatrick Mooney polldat_disassociate(pdp);
20512c76d751SPatrick Mooney polldat_associate(pdp, php);
20527c478bd9Sstevel@tonic-gate /*
20537c478bd9Sstevel@tonic-gate * We could have missed a wakeup on the new
20547c478bd9Sstevel@tonic-gate * target device. Make sure the new target
20557c478bd9Sstevel@tonic-gate * gets polled once.
20567c478bd9Sstevel@tonic-gate */
20577c478bd9Sstevel@tonic-gate BT_SET(pcp->pc_bitmap, fd);
20587c478bd9Sstevel@tonic-gate goto retry;
20597c478bd9Sstevel@tonic-gate }
20607c478bd9Sstevel@tonic-gate
20617c478bd9Sstevel@tonic-gate if (pollfdp[entry].revents) {
20627c478bd9Sstevel@tonic-gate ASSERT(refp->xf_refcnt >= 1);
20637c478bd9Sstevel@tonic-gate fdcnt++;
20647c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
20657c478bd9Sstevel@tonic-gate /*
20667c478bd9Sstevel@tonic-gate * this fd appeared multiple time
20677c478bd9Sstevel@tonic-gate * in the poll list. This is rare but
20687c478bd9Sstevel@tonic-gate * we have to look at all of them for
20697c478bd9Sstevel@tonic-gate * correctness.
20707c478bd9Sstevel@tonic-gate */
20717c478bd9Sstevel@tonic-gate error = plist_chkdupfd(fp, pdp, ps,
20727c478bd9Sstevel@tonic-gate pollfdp, entry, &fdcnt);
20737c478bd9Sstevel@tonic-gate if (error > 0) {
20747c478bd9Sstevel@tonic-gate releasef(fd);
20757c478bd9Sstevel@tonic-gate break;
20767c478bd9Sstevel@tonic-gate }
20777c478bd9Sstevel@tonic-gate if (error < 0) {
20787c478bd9Sstevel@tonic-gate goto retry;
20797c478bd9Sstevel@tonic-gate }
20807c478bd9Sstevel@tonic-gate }
20817c478bd9Sstevel@tonic-gate releasef(fd);
20827c478bd9Sstevel@tonic-gate } else {
20837c478bd9Sstevel@tonic-gate /*
20847c478bd9Sstevel@tonic-gate * VOP_POLL didn't return any revents. We can
20857c478bd9Sstevel@tonic-gate * clear the bit in bitmap only if we have the
20867c478bd9Sstevel@tonic-gate * pollhead ptr cached and no other cached
20877c478bd9Sstevel@tonic-gate * entry is polling different events on this fd.
20887c478bd9Sstevel@tonic-gate * VOP_POLL may have dropped the ps_lock. Make
20897c478bd9Sstevel@tonic-gate * sure pollwakeup has not happened before clear
20907c478bd9Sstevel@tonic-gate * the bit.
20917c478bd9Sstevel@tonic-gate */
20927c478bd9Sstevel@tonic-gate if ((pdp->pd_php != NULL) &&
20937c478bd9Sstevel@tonic-gate (pollfdp[entry].events == pdp->pd_events) &&
2094a5eb7107SBryan Cantrill ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
20957c478bd9Sstevel@tonic-gate BT_CLEAR(pcp->pc_bitmap, fd);
20967c478bd9Sstevel@tonic-gate }
20977c478bd9Sstevel@tonic-gate /*
20987c478bd9Sstevel@tonic-gate * if the fd can be cached now but not before,
20997c478bd9Sstevel@tonic-gate * do it now.
21007c478bd9Sstevel@tonic-gate */
21017c478bd9Sstevel@tonic-gate if ((pdp->pd_php == NULL) && (php != NULL)) {
21022c76d751SPatrick Mooney polldat_associate(pdp, php);
21037c478bd9Sstevel@tonic-gate /*
21047c478bd9Sstevel@tonic-gate * We are inserting a polldat struct for
21057c478bd9Sstevel@tonic-gate * the first time. We may have missed a
21067c478bd9Sstevel@tonic-gate * wakeup on this device. Re-poll once.
21077c478bd9Sstevel@tonic-gate * This should be a rare event.
21087c478bd9Sstevel@tonic-gate */
21097c478bd9Sstevel@tonic-gate releasef(fd);
21107c478bd9Sstevel@tonic-gate goto retry;
21117c478bd9Sstevel@tonic-gate }
21127c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
21137c478bd9Sstevel@tonic-gate /*
21147c478bd9Sstevel@tonic-gate * this fd appeared multiple time
21157c478bd9Sstevel@tonic-gate * in the poll list. This is rare but
21167c478bd9Sstevel@tonic-gate * we have to look at all of them for
21177c478bd9Sstevel@tonic-gate * correctness.
21187c478bd9Sstevel@tonic-gate */
21197c478bd9Sstevel@tonic-gate error = plist_chkdupfd(fp, pdp, ps,
21207c478bd9Sstevel@tonic-gate pollfdp, entry, &fdcnt);
21217c478bd9Sstevel@tonic-gate if (error > 0) {
21227c478bd9Sstevel@tonic-gate releasef(fd);
21237c478bd9Sstevel@tonic-gate break;
21247c478bd9Sstevel@tonic-gate }
21257c478bd9Sstevel@tonic-gate if (error < 0) {
21267c478bd9Sstevel@tonic-gate goto retry;
21277c478bd9Sstevel@tonic-gate }
21287c478bd9Sstevel@tonic-gate }
21297c478bd9Sstevel@tonic-gate releasef(fd);
21307c478bd9Sstevel@tonic-gate }
21317c478bd9Sstevel@tonic-gate } else {
21327c478bd9Sstevel@tonic-gate done = 1;
21337c478bd9Sstevel@tonic-gate ASSERT(pollcheckrevents(ps, begin, end + 1, which));
21347c478bd9Sstevel@tonic-gate }
21357c478bd9Sstevel@tonic-gate }
21367c478bd9Sstevel@tonic-gate if (!error) {
21377c478bd9Sstevel@tonic-gate ASSERT(*fdcntp + fdcnt == pollscanrevents(pcp, pollfdp, nfds));
21387c478bd9Sstevel@tonic-gate *fdcntp += fdcnt;
21397c478bd9Sstevel@tonic-gate }
21407c478bd9Sstevel@tonic-gate return (error);
21417c478bd9Sstevel@tonic-gate }
21427c478bd9Sstevel@tonic-gate
21437c478bd9Sstevel@tonic-gate /*
21447c478bd9Sstevel@tonic-gate * Going through the poll list without much locking. Poll all fds and
21457c478bd9Sstevel@tonic-gate * cache all valid fds in the pollcache.
21467c478bd9Sstevel@tonic-gate */
21477c478bd9Sstevel@tonic-gate int
pcacheset_cache_list(pollstate_t * ps,pollfd_t * fds,int * fdcntp,int which)21487c478bd9Sstevel@tonic-gate pcacheset_cache_list(pollstate_t *ps, pollfd_t *fds, int *fdcntp, int which)
21497c478bd9Sstevel@tonic-gate {
21507c478bd9Sstevel@tonic-gate pollfd_t *pollfdp = ps->ps_pollfd;
21517c478bd9Sstevel@tonic-gate pollcacheset_t *pcacheset = ps->ps_pcacheset;
21527c478bd9Sstevel@tonic-gate pollfd_t *newfdlist;
21537c478bd9Sstevel@tonic-gate int i;
21547c478bd9Sstevel@tonic-gate int fd;
21557c478bd9Sstevel@tonic-gate file_t *fp;
21567c478bd9Sstevel@tonic-gate int error = 0;
21577c478bd9Sstevel@tonic-gate
21587c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
21597c478bd9Sstevel@tonic-gate ASSERT(which < ps->ps_nsets);
21607c478bd9Sstevel@tonic-gate ASSERT(pcacheset != NULL);
21617c478bd9Sstevel@tonic-gate ASSERT(pcacheset[which].pcs_pollfd == NULL);
21627c478bd9Sstevel@tonic-gate newfdlist = kmem_alloc(ps->ps_nfds * sizeof (pollfd_t), KM_SLEEP);
21637c478bd9Sstevel@tonic-gate /*
21647c478bd9Sstevel@tonic-gate * cache the new poll list in pollcachset.
21657c478bd9Sstevel@tonic-gate */
21667c478bd9Sstevel@tonic-gate bcopy(pollfdp, newfdlist, sizeof (pollfd_t) * ps->ps_nfds);
21677c478bd9Sstevel@tonic-gate
21687c478bd9Sstevel@tonic-gate pcacheset[which].pcs_pollfd = newfdlist;
21697c478bd9Sstevel@tonic-gate pcacheset[which].pcs_nfds = ps->ps_nfds;
21707c478bd9Sstevel@tonic-gate pcacheset[which].pcs_usradr = (uintptr_t)fds;
21717c478bd9Sstevel@tonic-gate
21727c478bd9Sstevel@tonic-gate /*
21737c478bd9Sstevel@tonic-gate * We have saved a copy of current poll fd list in one pollcacheset.
21747c478bd9Sstevel@tonic-gate * The 'revents' field of the new list is not yet set to 0. Loop
21757c478bd9Sstevel@tonic-gate * through the new list just to do that is expensive. We do that
21767c478bd9Sstevel@tonic-gate * while polling the list.
21777c478bd9Sstevel@tonic-gate */
21787c478bd9Sstevel@tonic-gate for (i = 0; i < ps->ps_nfds; i++) {
21797c478bd9Sstevel@tonic-gate fd = pollfdp[i].fd;
21807c478bd9Sstevel@tonic-gate /*
21817c478bd9Sstevel@tonic-gate * We also filter out the illegal poll events in the event
21827c478bd9Sstevel@tonic-gate * field for the cached poll list/set.
21837c478bd9Sstevel@tonic-gate */
21847c478bd9Sstevel@tonic-gate if (pollfdp[i].events & ~VALID_POLL_EVENTS) {
21857c478bd9Sstevel@tonic-gate newfdlist[i].events = pollfdp[i].events =
21865c7544f7SDavid Plauger pollfdp[i].events & VALID_POLL_EVENTS;
21877c478bd9Sstevel@tonic-gate }
21887c478bd9Sstevel@tonic-gate if (fd < 0) {
21897c478bd9Sstevel@tonic-gate pollfdp[i].revents = 0;
21907c478bd9Sstevel@tonic-gate continue;
21917c478bd9Sstevel@tonic-gate }
21927c478bd9Sstevel@tonic-gate if ((fp = getf(fd)) == NULL) {
21937c478bd9Sstevel@tonic-gate pollfdp[i].revents = POLLNVAL;
21947c478bd9Sstevel@tonic-gate /*
21957c478bd9Sstevel@tonic-gate * invalidate this cache entry in the cached poll list
21967c478bd9Sstevel@tonic-gate */
21977c478bd9Sstevel@tonic-gate newfdlist[i].fd = -1;
21987c478bd9Sstevel@tonic-gate (*fdcntp)++;
21997c478bd9Sstevel@tonic-gate continue;
22007c478bd9Sstevel@tonic-gate }
22017c478bd9Sstevel@tonic-gate /*
22027c478bd9Sstevel@tonic-gate * cache this fd.
22037c478bd9Sstevel@tonic-gate */
22047c478bd9Sstevel@tonic-gate error = pcache_insert(ps, fp, &pollfdp[i], fdcntp, (ssize_t)i,
22057c478bd9Sstevel@tonic-gate which);
22067c478bd9Sstevel@tonic-gate releasef(fd);
22077c478bd9Sstevel@tonic-gate if (error) {
22087c478bd9Sstevel@tonic-gate /*
22097c478bd9Sstevel@tonic-gate * Here we are half way through caching a new
22107c478bd9Sstevel@tonic-gate * poll list. Undo every thing.
22117c478bd9Sstevel@tonic-gate */
22127c478bd9Sstevel@tonic-gate pcacheset_remove_list(ps, pollfdp, 0, i, which, 0);
22137c478bd9Sstevel@tonic-gate kmem_free(newfdlist, ps->ps_nfds * sizeof (pollfd_t));
22147c478bd9Sstevel@tonic-gate pcacheset[which].pcs_pollfd = NULL;
22157e12ceb3SToomas Soome pcacheset[which].pcs_usradr = (uintptr_t)NULL;
22167c478bd9Sstevel@tonic-gate break;
22177c478bd9Sstevel@tonic-gate }
22187c478bd9Sstevel@tonic-gate }
22197c478bd9Sstevel@tonic-gate return (error);
22207c478bd9Sstevel@tonic-gate }
22217c478bd9Sstevel@tonic-gate
22227c478bd9Sstevel@tonic-gate /*
22237c478bd9Sstevel@tonic-gate * called by pollcacheclean() to set the fp NULL. It also sets polled events
22247c478bd9Sstevel@tonic-gate * in pcacheset entries to a special events 'POLLCLOSED'. Do a pollwakeup to
22257c478bd9Sstevel@tonic-gate * wake any sleeping poller, then remove the polldat from the driver.
22267c478bd9Sstevel@tonic-gate * The routine is called with ps_pcachelock held.
22277c478bd9Sstevel@tonic-gate */
22287c478bd9Sstevel@tonic-gate void
pcache_clean_entry(pollstate_t * ps,int fd)22297c478bd9Sstevel@tonic-gate pcache_clean_entry(pollstate_t *ps, int fd)
22307c478bd9Sstevel@tonic-gate {
22317c478bd9Sstevel@tonic-gate pollcache_t *pcp;
22327c478bd9Sstevel@tonic-gate polldat_t *pdp;
22337c478bd9Sstevel@tonic-gate int i;
22347c478bd9Sstevel@tonic-gate
22357c478bd9Sstevel@tonic-gate ASSERT(ps != NULL);
22367c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
22377c478bd9Sstevel@tonic-gate pcp = ps->ps_pcache;
22387c478bd9Sstevel@tonic-gate ASSERT(pcp);
22397c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
22407c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
22417c478bd9Sstevel@tonic-gate /*
22427c478bd9Sstevel@tonic-gate * the corresponding fpollinfo in fi_list has been removed by
22437c478bd9Sstevel@tonic-gate * a close on this fd. Reset the cached fp ptr here.
22447c478bd9Sstevel@tonic-gate */
22457c478bd9Sstevel@tonic-gate pdp->pd_fp = NULL;
22467c478bd9Sstevel@tonic-gate /*
22477c478bd9Sstevel@tonic-gate * XXX - This routine also touches data in pcacheset struct.
22487c478bd9Sstevel@tonic-gate *
22497c478bd9Sstevel@tonic-gate * set the event in cached poll lists to POLLCLOSED. This invalidate
22507c478bd9Sstevel@tonic-gate * the cached poll fd entry in that poll list, which will force a
22517c478bd9Sstevel@tonic-gate * removal of this cached entry in next poll(). The cleanup is done
22527c478bd9Sstevel@tonic-gate * at the removal time.
22537c478bd9Sstevel@tonic-gate */
22547c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
22557c478bd9Sstevel@tonic-gate for (i = 0; i < ps->ps_nsets; i++) {
22567c478bd9Sstevel@tonic-gate xref_t *refp;
22577c478bd9Sstevel@tonic-gate pollcacheset_t *pcsp;
22587c478bd9Sstevel@tonic-gate
22597c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[i];
22607c478bd9Sstevel@tonic-gate if (refp->xf_refcnt) {
22617c478bd9Sstevel@tonic-gate ASSERT(refp->xf_position >= 0);
22627c478bd9Sstevel@tonic-gate pcsp = &ps->ps_pcacheset[i];
22637c478bd9Sstevel@tonic-gate if (refp->xf_refcnt == 1) {
22647c478bd9Sstevel@tonic-gate pcsp->pcs_pollfd[refp->xf_position].events =
22657c478bd9Sstevel@tonic-gate (short)POLLCLOSED;
22667c478bd9Sstevel@tonic-gate }
22677c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
22687c478bd9Sstevel@tonic-gate int j;
22697c478bd9Sstevel@tonic-gate /*
22707c478bd9Sstevel@tonic-gate * mark every matching entry in pcs_pollfd
22717c478bd9Sstevel@tonic-gate */
22727c478bd9Sstevel@tonic-gate for (j = refp->xf_position;
22737c478bd9Sstevel@tonic-gate j < pcsp->pcs_nfds; j++) {
22747c478bd9Sstevel@tonic-gate if (pcsp->pcs_pollfd[j].fd == fd) {
22757c478bd9Sstevel@tonic-gate pcsp->pcs_pollfd[j].events =
22767c478bd9Sstevel@tonic-gate (short)POLLCLOSED;
22777c478bd9Sstevel@tonic-gate }
22787c478bd9Sstevel@tonic-gate }
22797c478bd9Sstevel@tonic-gate }
22807c478bd9Sstevel@tonic-gate }
22817c478bd9Sstevel@tonic-gate }
22827c478bd9Sstevel@tonic-gate if (pdp->pd_php) {
22832c76d751SPatrick Mooney /*
22842c76d751SPatrick Mooney * Using pdp->pd_php is a bit risky here, as we lack any
22852c76d751SPatrick Mooney * protection from a racing close operation which could free
22862c76d751SPatrick Mooney * that pollhead prior to pollwakeup() acquiring the locks
22872c76d751SPatrick Mooney * necessary to make it safe.
22882c76d751SPatrick Mooney */
22897c478bd9Sstevel@tonic-gate pollwakeup(pdp->pd_php, POLLHUP);
22902c76d751SPatrick Mooney polldat_disassociate(pdp);
22917c478bd9Sstevel@tonic-gate }
22927c478bd9Sstevel@tonic-gate }
22937c478bd9Sstevel@tonic-gate
2294f3bb54f3SPatrick Mooney void
pcache_wake_parents(pollcache_t * pcp)2295f3bb54f3SPatrick Mooney pcache_wake_parents(pollcache_t *pcp)
2296f3bb54f3SPatrick Mooney {
2297f3bb54f3SPatrick Mooney pcachelink_t *pl, *pln;
2298f3bb54f3SPatrick Mooney
2299f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&pcp->pc_lock));
2300f3bb54f3SPatrick Mooney
2301f3bb54f3SPatrick Mooney for (pl = pcp->pc_parents; pl != NULL; pl = pln) {
2302f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
2303f3bb54f3SPatrick Mooney if (pl->pcl_state == PCL_VALID) {
2304f3bb54f3SPatrick Mooney ASSERT(pl->pcl_parent_pc != NULL);
2305f3bb54f3SPatrick Mooney cv_broadcast(&pl->pcl_parent_pc->pc_cv);
2306f3bb54f3SPatrick Mooney }
2307f3bb54f3SPatrick Mooney pln = pl->pcl_parent_next;
2308f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
2309f3bb54f3SPatrick Mooney }
2310f3bb54f3SPatrick Mooney }
2311f3bb54f3SPatrick Mooney
23127c478bd9Sstevel@tonic-gate /*
2313f3bb54f3SPatrick Mooney * Initialize thread pollstate structure.
2314f3bb54f3SPatrick Mooney * It will persist for the life of the thread, until it calls pollcleanup().
23157c478bd9Sstevel@tonic-gate */
23167c478bd9Sstevel@tonic-gate pollstate_t *
pollstate_create()2317f3bb54f3SPatrick Mooney pollstate_create()
23187c478bd9Sstevel@tonic-gate {
2319f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
23207c478bd9Sstevel@tonic-gate
2321f3bb54f3SPatrick Mooney if (ps == NULL) {
2322f3bb54f3SPatrick Mooney /*
2323f3bb54f3SPatrick Mooney * This is the first time this thread has ever polled, so we
2324f3bb54f3SPatrick Mooney * have to create its pollstate structure.
2325f3bb54f3SPatrick Mooney */
2326f3bb54f3SPatrick Mooney ps = kmem_zalloc(sizeof (pollstate_t), KM_SLEEP);
2327f3bb54f3SPatrick Mooney ps->ps_nsets = POLLFDSETS;
2328f3bb54f3SPatrick Mooney ps->ps_pcacheset = pcacheset_create(ps->ps_nsets);
2329f3bb54f3SPatrick Mooney curthread->t_pollstate = ps;
2330f3bb54f3SPatrick Mooney } else {
2331f3bb54f3SPatrick Mooney ASSERT(ps->ps_depth == 0);
2332f3bb54f3SPatrick Mooney ASSERT(ps->ps_flags == 0);
2333f3bb54f3SPatrick Mooney ASSERT(ps->ps_pc_stack[0] == 0);
2334f3bb54f3SPatrick Mooney }
23357c478bd9Sstevel@tonic-gate return (ps);
23367c478bd9Sstevel@tonic-gate }
23377c478bd9Sstevel@tonic-gate
23387c478bd9Sstevel@tonic-gate void
pollstate_destroy(pollstate_t * ps)23397c478bd9Sstevel@tonic-gate pollstate_destroy(pollstate_t *ps)
23407c478bd9Sstevel@tonic-gate {
23417c478bd9Sstevel@tonic-gate if (ps->ps_pollfd != NULL) {
23427c478bd9Sstevel@tonic-gate kmem_free(ps->ps_pollfd, ps->ps_nfds * sizeof (pollfd_t));
23437c478bd9Sstevel@tonic-gate ps->ps_pollfd = NULL;
23447c478bd9Sstevel@tonic-gate }
23457c478bd9Sstevel@tonic-gate if (ps->ps_pcache != NULL) {
23467c478bd9Sstevel@tonic-gate pcache_destroy(ps->ps_pcache);
23477c478bd9Sstevel@tonic-gate ps->ps_pcache = NULL;
23487c478bd9Sstevel@tonic-gate }
23497c478bd9Sstevel@tonic-gate pcacheset_destroy(ps->ps_pcacheset, ps->ps_nsets);
23507c478bd9Sstevel@tonic-gate ps->ps_pcacheset = NULL;
23517c478bd9Sstevel@tonic-gate if (ps->ps_dpbuf != NULL) {
2352a5eb7107SBryan Cantrill kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
23537c478bd9Sstevel@tonic-gate ps->ps_dpbuf = NULL;
23547c478bd9Sstevel@tonic-gate }
23557c478bd9Sstevel@tonic-gate mutex_destroy(&ps->ps_lock);
23567c478bd9Sstevel@tonic-gate kmem_free(ps, sizeof (pollstate_t));
23577c478bd9Sstevel@tonic-gate }
23587c478bd9Sstevel@tonic-gate
2359f3bb54f3SPatrick Mooney static int
pollstate_contend(pollstate_t * ps,pollcache_t * pcp)2360f3bb54f3SPatrick Mooney pollstate_contend(pollstate_t *ps, pollcache_t *pcp)
2361f3bb54f3SPatrick Mooney {
2362f3bb54f3SPatrick Mooney pollstate_t *rem, *next;
2363f3bb54f3SPatrick Mooney pollcache_t *desired_pc;
2364f3bb54f3SPatrick Mooney int result = 0, depth_total;
2365f3bb54f3SPatrick Mooney
2366f3bb54f3SPatrick Mooney mutex_enter(&pollstate_contenders_lock);
2367f3bb54f3SPatrick Mooney /*
2368f3bb54f3SPatrick Mooney * There is a small chance that the pollcache of interest became
2369f3bb54f3SPatrick Mooney * available while we were waiting on the contenders lock.
2370f3bb54f3SPatrick Mooney */
2371f3bb54f3SPatrick Mooney if (mutex_tryenter(&pcp->pc_lock) != 0) {
2372f3bb54f3SPatrick Mooney goto out;
2373f3bb54f3SPatrick Mooney }
2374f3bb54f3SPatrick Mooney
2375f3bb54f3SPatrick Mooney /*
2376f3bb54f3SPatrick Mooney * Walk the list of contended pollstates, searching for evidence of a
2377f3bb54f3SPatrick Mooney * deadlock condition.
2378f3bb54f3SPatrick Mooney */
2379f3bb54f3SPatrick Mooney depth_total = ps->ps_depth;
2380f3bb54f3SPatrick Mooney desired_pc = pcp;
2381f3bb54f3SPatrick Mooney for (rem = pollstate_contenders; rem != NULL; rem = next) {
2382f3bb54f3SPatrick Mooney int i, j;
2383f3bb54f3SPatrick Mooney next = rem->ps_contend_nextp;
2384f3bb54f3SPatrick Mooney
2385f3bb54f3SPatrick Mooney /* Is this pollstate holding the pollcache of interest? */
2386f3bb54f3SPatrick Mooney for (i = 0; i < rem->ps_depth; i++) {
2387f3bb54f3SPatrick Mooney if (rem->ps_pc_stack[i] != desired_pc) {
2388f3bb54f3SPatrick Mooney continue;
2389f3bb54f3SPatrick Mooney }
2390f3bb54f3SPatrick Mooney
2391f3bb54f3SPatrick Mooney /*
2392f3bb54f3SPatrick Mooney * The remote pollstate holds the pollcache lock we
2393f3bb54f3SPatrick Mooney * desire. If it is waiting on a pollcache we hold,
2394f3bb54f3SPatrick Mooney * then we can report the obvious deadlock.
2395f3bb54f3SPatrick Mooney */
2396f3bb54f3SPatrick Mooney ASSERT(rem->ps_contend_pc != NULL);
2397f3bb54f3SPatrick Mooney for (j = 0; j < ps->ps_depth; j++) {
2398f3bb54f3SPatrick Mooney if (rem->ps_contend_pc == ps->ps_pc_stack[j]) {
2399f3bb54f3SPatrick Mooney rem->ps_flags |= POLLSTATE_STALEMATE;
2400f3bb54f3SPatrick Mooney result = -1;
2401f3bb54f3SPatrick Mooney goto out;
2402f3bb54f3SPatrick Mooney }
2403f3bb54f3SPatrick Mooney }
2404f3bb54f3SPatrick Mooney
2405f3bb54f3SPatrick Mooney /*
2406f3bb54f3SPatrick Mooney * The remote pollstate is not blocking on a pollcache
2407f3bb54f3SPatrick Mooney * which would deadlock against us. That pollcache
2408f3bb54f3SPatrick Mooney * may, however, be held by a pollstate which would
2409f3bb54f3SPatrick Mooney * result in a deadlock.
2410f3bb54f3SPatrick Mooney *
2411f3bb54f3SPatrick Mooney * To detect such a condition, we continue walking
2412f3bb54f3SPatrick Mooney * through the list using the pollcache blocking the
2413f3bb54f3SPatrick Mooney * remote thread as our new search target.
2414f3bb54f3SPatrick Mooney *
2415f3bb54f3SPatrick Mooney * Return to the front of pollstate_contenders since it
2416f3bb54f3SPatrick Mooney * is not ordered to guarantee complete dependency
2417f3bb54f3SPatrick Mooney * traversal. The below depth tracking places an upper
2418f3bb54f3SPatrick Mooney * bound on iterations.
2419f3bb54f3SPatrick Mooney */
2420f3bb54f3SPatrick Mooney desired_pc = rem->ps_contend_pc;
2421f3bb54f3SPatrick Mooney next = pollstate_contenders;
2422f3bb54f3SPatrick Mooney
2423f3bb54f3SPatrick Mooney /*
2424f3bb54f3SPatrick Mooney * The recursion depth of the remote pollstate is used
2425f3bb54f3SPatrick Mooney * to calculate a final depth for the local /dev/poll
2426f3bb54f3SPatrick Mooney * recursion, since those locks will be acquired
2427f3bb54f3SPatrick Mooney * eventually. If that value exceeds the defined
2428f3bb54f3SPatrick Mooney * limit, we can report the failure now instead of
2429f3bb54f3SPatrick Mooney * recursing to that failure depth.
2430f3bb54f3SPatrick Mooney */
2431f3bb54f3SPatrick Mooney depth_total += (rem->ps_depth - i);
2432f3bb54f3SPatrick Mooney if (depth_total >= POLLMAXDEPTH) {
2433f3bb54f3SPatrick Mooney result = -1;
2434f3bb54f3SPatrick Mooney goto out;
2435f3bb54f3SPatrick Mooney }
2436f3bb54f3SPatrick Mooney }
2437f3bb54f3SPatrick Mooney }
2438f3bb54f3SPatrick Mooney
2439f3bb54f3SPatrick Mooney /*
2440f3bb54f3SPatrick Mooney * No deadlock partner was found. The only course of action is to
2441f3bb54f3SPatrick Mooney * record ourself as a contended pollstate and wait for the pollcache
2442f3bb54f3SPatrick Mooney * mutex to become available.
2443f3bb54f3SPatrick Mooney */
2444f3bb54f3SPatrick Mooney ps->ps_contend_pc = pcp;
2445f3bb54f3SPatrick Mooney ps->ps_contend_nextp = pollstate_contenders;
2446f3bb54f3SPatrick Mooney ps->ps_contend_pnextp = &pollstate_contenders;
2447f3bb54f3SPatrick Mooney if (pollstate_contenders != NULL) {
2448f3bb54f3SPatrick Mooney pollstate_contenders->ps_contend_pnextp =
2449f3bb54f3SPatrick Mooney &ps->ps_contend_nextp;
2450f3bb54f3SPatrick Mooney }
2451f3bb54f3SPatrick Mooney pollstate_contenders = ps;
2452f3bb54f3SPatrick Mooney
2453f3bb54f3SPatrick Mooney mutex_exit(&pollstate_contenders_lock);
2454f3bb54f3SPatrick Mooney mutex_enter(&pcp->pc_lock);
2455f3bb54f3SPatrick Mooney mutex_enter(&pollstate_contenders_lock);
2456f3bb54f3SPatrick Mooney
2457f3bb54f3SPatrick Mooney /*
2458f3bb54f3SPatrick Mooney * Our acquisition of the pollcache mutex may be due to another thread
2459f3bb54f3SPatrick Mooney * giving up in the face of deadlock with us. If that is the case,
2460f3bb54f3SPatrick Mooney * we too should report the failure.
2461f3bb54f3SPatrick Mooney */
2462f3bb54f3SPatrick Mooney if ((ps->ps_flags & POLLSTATE_STALEMATE) != 0) {
2463f3bb54f3SPatrick Mooney result = -1;
2464f3bb54f3SPatrick Mooney ps->ps_flags &= ~POLLSTATE_STALEMATE;
2465f3bb54f3SPatrick Mooney mutex_exit(&pcp->pc_lock);
2466f3bb54f3SPatrick Mooney }
2467f3bb54f3SPatrick Mooney
2468f3bb54f3SPatrick Mooney /* Remove ourself from the contenders list. */
2469f3bb54f3SPatrick Mooney if (ps->ps_contend_nextp != NULL) {
2470f3bb54f3SPatrick Mooney ps->ps_contend_nextp->ps_contend_pnextp =
2471f3bb54f3SPatrick Mooney ps->ps_contend_pnextp;
2472f3bb54f3SPatrick Mooney }
2473f3bb54f3SPatrick Mooney *ps->ps_contend_pnextp = ps->ps_contend_nextp;
2474f3bb54f3SPatrick Mooney ps->ps_contend_pc = NULL;
2475f3bb54f3SPatrick Mooney ps->ps_contend_nextp = NULL;
2476f3bb54f3SPatrick Mooney ps->ps_contend_pnextp = NULL;
2477f3bb54f3SPatrick Mooney
2478f3bb54f3SPatrick Mooney out:
2479f3bb54f3SPatrick Mooney mutex_exit(&pollstate_contenders_lock);
2480f3bb54f3SPatrick Mooney return (result);
2481f3bb54f3SPatrick Mooney }
2482f3bb54f3SPatrick Mooney
2483f3bb54f3SPatrick Mooney int
pollstate_enter(pollcache_t * pcp)2484f3bb54f3SPatrick Mooney pollstate_enter(pollcache_t *pcp)
2485f3bb54f3SPatrick Mooney {
2486f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
2487f3bb54f3SPatrick Mooney int i;
2488f3bb54f3SPatrick Mooney
2489f3bb54f3SPatrick Mooney if (ps == NULL) {
2490f3bb54f3SPatrick Mooney /*
2491f3bb54f3SPatrick Mooney * The thread pollstate may not be initialized if VOP_POLL is
2492f3bb54f3SPatrick Mooney * called on a recursion-enabled /dev/poll handle from outside
2493f3bb54f3SPatrick Mooney * the poll() or /dev/poll codepaths.
2494f3bb54f3SPatrick Mooney */
2495f3bb54f3SPatrick Mooney return (PSE_FAIL_POLLSTATE);
2496f3bb54f3SPatrick Mooney }
2497f3bb54f3SPatrick Mooney if (ps->ps_depth >= POLLMAXDEPTH) {
2498f3bb54f3SPatrick Mooney return (PSE_FAIL_DEPTH);
2499f3bb54f3SPatrick Mooney }
2500f3bb54f3SPatrick Mooney /*
2501f3bb54f3SPatrick Mooney * Check the desired pollcache against pollcaches we already have
2502f3bb54f3SPatrick Mooney * locked. Such a loop is the most simple deadlock scenario.
2503f3bb54f3SPatrick Mooney */
2504f3bb54f3SPatrick Mooney for (i = 0; i < ps->ps_depth; i++) {
2505f3bb54f3SPatrick Mooney if (ps->ps_pc_stack[i] == pcp) {
2506f3bb54f3SPatrick Mooney return (PSE_FAIL_LOOP);
2507f3bb54f3SPatrick Mooney }
2508f3bb54f3SPatrick Mooney }
2509f3bb54f3SPatrick Mooney ASSERT(ps->ps_pc_stack[i] == NULL);
2510f3bb54f3SPatrick Mooney
2511f3bb54f3SPatrick Mooney if (ps->ps_depth == 0) {
2512f3bb54f3SPatrick Mooney /* Locking initial the pollcache requires no caution */
2513f3bb54f3SPatrick Mooney mutex_enter(&pcp->pc_lock);
2514f3bb54f3SPatrick Mooney } else if (mutex_tryenter(&pcp->pc_lock) == 0) {
2515f3bb54f3SPatrick Mooney if (pollstate_contend(ps, pcp) != 0) {
2516f3bb54f3SPatrick Mooney /* This pollcache cannot safely be locked. */
2517f3bb54f3SPatrick Mooney return (PSE_FAIL_DEADLOCK);
2518f3bb54f3SPatrick Mooney }
2519f3bb54f3SPatrick Mooney }
2520f3bb54f3SPatrick Mooney
2521f3bb54f3SPatrick Mooney ps->ps_pc_stack[ps->ps_depth++] = pcp;
2522f3bb54f3SPatrick Mooney return (PSE_SUCCESS);
2523f3bb54f3SPatrick Mooney }
2524f3bb54f3SPatrick Mooney
2525f3bb54f3SPatrick Mooney void
pollstate_exit(pollcache_t * pcp)2526f3bb54f3SPatrick Mooney pollstate_exit(pollcache_t *pcp)
2527f3bb54f3SPatrick Mooney {
2528f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
2529f3bb54f3SPatrick Mooney
2530f3bb54f3SPatrick Mooney VERIFY(ps != NULL);
2531f3bb54f3SPatrick Mooney VERIFY(ps->ps_pc_stack[ps->ps_depth - 1] == pcp);
2532f3bb54f3SPatrick Mooney
2533f3bb54f3SPatrick Mooney mutex_exit(&pcp->pc_lock);
2534f3bb54f3SPatrick Mooney ps->ps_pc_stack[--ps->ps_depth] = NULL;
2535f3bb54f3SPatrick Mooney VERIFY(ps->ps_depth >= 0);
2536f3bb54f3SPatrick Mooney }
2537f3bb54f3SPatrick Mooney
2538f3bb54f3SPatrick Mooney
25397c478bd9Sstevel@tonic-gate /*
25407c478bd9Sstevel@tonic-gate * We are holding the appropriate uf_lock entering this routine.
25417c478bd9Sstevel@tonic-gate * Bump up the ps_busy count to prevent the thread from exiting.
25427c478bd9Sstevel@tonic-gate */
25437c478bd9Sstevel@tonic-gate void
pollblockexit(fpollinfo_t * fpip)25447c478bd9Sstevel@tonic-gate pollblockexit(fpollinfo_t *fpip)
25457c478bd9Sstevel@tonic-gate {
25467c478bd9Sstevel@tonic-gate for (; fpip; fpip = fpip->fp_next) {
25477c478bd9Sstevel@tonic-gate pollcache_t *pcp = fpip->fp_thread->t_pollstate->ps_pcache;
25487c478bd9Sstevel@tonic-gate
25497c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
25507c478bd9Sstevel@tonic-gate pcp->pc_busy++; /* prevents exit()'s */
25517c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
25527c478bd9Sstevel@tonic-gate }
25537c478bd9Sstevel@tonic-gate }
25547c478bd9Sstevel@tonic-gate
25557c478bd9Sstevel@tonic-gate /*
25567c478bd9Sstevel@tonic-gate * Complete phase 2 of cached poll fd cleanup. Call pcache_clean_entry to mark
25577c478bd9Sstevel@tonic-gate * the pcacheset events field POLLCLOSED to force the next poll() to remove
25587c478bd9Sstevel@tonic-gate * this cache entry. We can't clean the polldat entry clean up here because
25597c478bd9Sstevel@tonic-gate * lwp block in poll() needs the info to return. Wakeup anyone blocked in
25607c478bd9Sstevel@tonic-gate * poll and let exiting lwp go. No lock is help upon entry. So it's OK for
25617c478bd9Sstevel@tonic-gate * pcache_clean_entry to call pollwakeup().
25627c478bd9Sstevel@tonic-gate */
25637c478bd9Sstevel@tonic-gate void
pollcacheclean(fpollinfo_t * fip,int fd)25647c478bd9Sstevel@tonic-gate pollcacheclean(fpollinfo_t *fip, int fd)
25657c478bd9Sstevel@tonic-gate {
25667c478bd9Sstevel@tonic-gate struct fpollinfo *fpip, *fpip2;
25677c478bd9Sstevel@tonic-gate
25687c478bd9Sstevel@tonic-gate fpip = fip;
25697c478bd9Sstevel@tonic-gate while (fpip) {
25707c478bd9Sstevel@tonic-gate pollstate_t *ps = fpip->fp_thread->t_pollstate;
25717c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
25727c478bd9Sstevel@tonic-gate
25737c478bd9Sstevel@tonic-gate mutex_enter(&ps->ps_lock);
25747c478bd9Sstevel@tonic-gate pcache_clean_entry(ps, fd);
25757c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
25767c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
25777c478bd9Sstevel@tonic-gate pcp->pc_busy--;
25787c478bd9Sstevel@tonic-gate if (pcp->pc_busy == 0) {
25797c478bd9Sstevel@tonic-gate /*
25807c478bd9Sstevel@tonic-gate * Wakeup the thread waiting in
25817c478bd9Sstevel@tonic-gate * thread_exit().
25827c478bd9Sstevel@tonic-gate */
25837c478bd9Sstevel@tonic-gate cv_signal(&pcp->pc_busy_cv);
25847c478bd9Sstevel@tonic-gate }
25857c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
25867c478bd9Sstevel@tonic-gate
25877c478bd9Sstevel@tonic-gate fpip2 = fpip;
25887c478bd9Sstevel@tonic-gate fpip = fpip->fp_next;
25897c478bd9Sstevel@tonic-gate kmem_free(fpip2, sizeof (fpollinfo_t));
25907c478bd9Sstevel@tonic-gate }
25917c478bd9Sstevel@tonic-gate }
25927c478bd9Sstevel@tonic-gate
25937c478bd9Sstevel@tonic-gate /*
25947c478bd9Sstevel@tonic-gate * one of the cache line's counter is wrapping around. Reset all cache line
25957c478bd9Sstevel@tonic-gate * counters to zero except one. This is simplistic, but probably works
25967c478bd9Sstevel@tonic-gate * effectively.
25977c478bd9Sstevel@tonic-gate */
25987c478bd9Sstevel@tonic-gate void
pcacheset_reset_count(pollstate_t * ps,int index)25997c478bd9Sstevel@tonic-gate pcacheset_reset_count(pollstate_t *ps, int index)
26007c478bd9Sstevel@tonic-gate {
26017c478bd9Sstevel@tonic-gate int i;
26027c478bd9Sstevel@tonic-gate
26037c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
26047c478bd9Sstevel@tonic-gate for (i = 0; i < ps->ps_nsets; i++) {
26057c478bd9Sstevel@tonic-gate if (ps->ps_pcacheset[i].pcs_pollfd != NULL) {
26067c478bd9Sstevel@tonic-gate ps->ps_pcacheset[i].pcs_count = 0;
26077c478bd9Sstevel@tonic-gate }
26087c478bd9Sstevel@tonic-gate }
26097c478bd9Sstevel@tonic-gate ps->ps_pcacheset[index].pcs_count = 1;
26107c478bd9Sstevel@tonic-gate }
26117c478bd9Sstevel@tonic-gate
26127c478bd9Sstevel@tonic-gate /*
26137c478bd9Sstevel@tonic-gate * this routine implements poll cache list replacement policy.
26147c478bd9Sstevel@tonic-gate * It is currently choose the "least used".
26157c478bd9Sstevel@tonic-gate */
26167c478bd9Sstevel@tonic-gate int
pcacheset_replace(pollstate_t * ps)26177c478bd9Sstevel@tonic-gate pcacheset_replace(pollstate_t *ps)
26187c478bd9Sstevel@tonic-gate {
26197c478bd9Sstevel@tonic-gate int i;
26207c478bd9Sstevel@tonic-gate int index = 0;
26217c478bd9Sstevel@tonic-gate
26227c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
26237c478bd9Sstevel@tonic-gate for (i = 1; i < ps->ps_nsets; i++) {
26247c478bd9Sstevel@tonic-gate if (ps->ps_pcacheset[index].pcs_count >
26257c478bd9Sstevel@tonic-gate ps->ps_pcacheset[i].pcs_count) {
26267c478bd9Sstevel@tonic-gate index = i;
26277c478bd9Sstevel@tonic-gate }
26287c478bd9Sstevel@tonic-gate }
26297c478bd9Sstevel@tonic-gate ps->ps_pcacheset[index].pcs_count = 0;
26307c478bd9Sstevel@tonic-gate return (index);
26317c478bd9Sstevel@tonic-gate }
26327c478bd9Sstevel@tonic-gate
26337c478bd9Sstevel@tonic-gate /*
26347c478bd9Sstevel@tonic-gate * this routine is called by strclose to remove remaining polldat struct on
26357c478bd9Sstevel@tonic-gate * the pollhead list of the device being closed. There are two reasons as why
26367c478bd9Sstevel@tonic-gate * the polldat structures still remain on the pollhead list:
26377c478bd9Sstevel@tonic-gate *
26387c478bd9Sstevel@tonic-gate * (1) The layered device(e.g.the console driver).
26397c478bd9Sstevel@tonic-gate * In this case, the existence of a polldat implies that the thread putting
26407c478bd9Sstevel@tonic-gate * the polldat on this list has not exited yet. Before the thread exits, it
26417c478bd9Sstevel@tonic-gate * will have to hold this pollhead lock to remove the polldat. So holding the
26427c478bd9Sstevel@tonic-gate * pollhead lock here effectively prevents the thread which put the polldat
26437c478bd9Sstevel@tonic-gate * on this list from exiting.
26447c478bd9Sstevel@tonic-gate *
26457c478bd9Sstevel@tonic-gate * (2) /dev/poll.
26467c478bd9Sstevel@tonic-gate * When a polled fd is cached in /dev/poll, its polldat will remain on the
26477c478bd9Sstevel@tonic-gate * pollhead list if the process has not done a POLLREMOVE before closing the
26487c478bd9Sstevel@tonic-gate * polled fd. We just unlink it here.
26497c478bd9Sstevel@tonic-gate */
26507c478bd9Sstevel@tonic-gate void
pollhead_clean(pollhead_t * php)26517c478bd9Sstevel@tonic-gate pollhead_clean(pollhead_t *php)
26527c478bd9Sstevel@tonic-gate {
26537c478bd9Sstevel@tonic-gate polldat_t *pdp;
26547c478bd9Sstevel@tonic-gate
26557c478bd9Sstevel@tonic-gate /*
26567c478bd9Sstevel@tonic-gate * In case(1), while we must prevent the thread in question from
26577c478bd9Sstevel@tonic-gate * exiting, we must also obey the proper locking order, i.e.
26587c478bd9Sstevel@tonic-gate * (ps_lock -> phlock).
26597c478bd9Sstevel@tonic-gate */
26607c478bd9Sstevel@tonic-gate PH_ENTER(php);
26617c478bd9Sstevel@tonic-gate while (php->ph_list != NULL) {
26627c478bd9Sstevel@tonic-gate pollstate_t *ps;
26637c478bd9Sstevel@tonic-gate pollcache_t *pcp;
26647c478bd9Sstevel@tonic-gate
26657c478bd9Sstevel@tonic-gate pdp = php->ph_list;
26667c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_php == php);
26677c478bd9Sstevel@tonic-gate if (pdp->pd_thread == NULL) {
26687c478bd9Sstevel@tonic-gate /*
26697c478bd9Sstevel@tonic-gate * This is case(2). Since the ph_lock is sufficient
26707c478bd9Sstevel@tonic-gate * to synchronize this lwp with any other /dev/poll
26717c478bd9Sstevel@tonic-gate * lwp, just unlink the polldat.
26727c478bd9Sstevel@tonic-gate */
26737c478bd9Sstevel@tonic-gate php->ph_list = pdp->pd_next;
26747c478bd9Sstevel@tonic-gate pdp->pd_php = NULL;
26757c478bd9Sstevel@tonic-gate pdp->pd_next = NULL;
26767c478bd9Sstevel@tonic-gate continue;
26777c478bd9Sstevel@tonic-gate }
26787c478bd9Sstevel@tonic-gate ps = pdp->pd_thread->t_pollstate;
26797c478bd9Sstevel@tonic-gate ASSERT(ps != NULL);
26807c478bd9Sstevel@tonic-gate pcp = pdp->pd_pcache;
26817c478bd9Sstevel@tonic-gate ASSERT(pcp != NULL);
26827c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
26837c478bd9Sstevel@tonic-gate pcp->pc_busy++; /* prevents exit()'s */
26847c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
26857c478bd9Sstevel@tonic-gate /*
26867c478bd9Sstevel@tonic-gate * Now get the locks in proper order to avoid deadlock.
26877c478bd9Sstevel@tonic-gate */
26887c478bd9Sstevel@tonic-gate PH_EXIT(php);
26897c478bd9Sstevel@tonic-gate mutex_enter(&ps->ps_lock);
26907c478bd9Sstevel@tonic-gate /*
26917c478bd9Sstevel@tonic-gate * while we dropped the pollhead lock, the element could be
26927c478bd9Sstevel@tonic-gate * taken off the list already.
26937c478bd9Sstevel@tonic-gate */
26947c478bd9Sstevel@tonic-gate PH_ENTER(php);
26957c478bd9Sstevel@tonic-gate if (pdp->pd_php == php) {
26967c478bd9Sstevel@tonic-gate ASSERT(pdp == php->ph_list);
26977c478bd9Sstevel@tonic-gate php->ph_list = pdp->pd_next;
26987c478bd9Sstevel@tonic-gate pdp->pd_php = NULL;
26997c478bd9Sstevel@tonic-gate pdp->pd_next = NULL;
27007c478bd9Sstevel@tonic-gate }
27017c478bd9Sstevel@tonic-gate PH_EXIT(php);
27027c478bd9Sstevel@tonic-gate mutex_exit(&ps->ps_lock);
27037c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
27047c478bd9Sstevel@tonic-gate pcp->pc_busy--;
27057c478bd9Sstevel@tonic-gate if (pcp->pc_busy == 0) {
27067c478bd9Sstevel@tonic-gate /*
27077c478bd9Sstevel@tonic-gate * Wakeup the thread waiting in
27087c478bd9Sstevel@tonic-gate * thread_exit().
27097c478bd9Sstevel@tonic-gate */
27107c478bd9Sstevel@tonic-gate cv_signal(&pcp->pc_busy_cv);
27117c478bd9Sstevel@tonic-gate }
27127c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
27137c478bd9Sstevel@tonic-gate PH_ENTER(php);
27147c478bd9Sstevel@tonic-gate }
27157c478bd9Sstevel@tonic-gate PH_EXIT(php);
27167c478bd9Sstevel@tonic-gate }
27177c478bd9Sstevel@tonic-gate
27187c478bd9Sstevel@tonic-gate /*
27197c478bd9Sstevel@tonic-gate * The remove_list is called to cleanup a partially cached 'current' list or
27207c478bd9Sstevel@tonic-gate * to remove a partial list which is no longer cached. The flag value of 1
27217c478bd9Sstevel@tonic-gate * indicates the second case.
27227c478bd9Sstevel@tonic-gate */
27237c478bd9Sstevel@tonic-gate void
pcacheset_remove_list(pollstate_t * ps,pollfd_t * pollfdp,int start,int end,int cacheindex,int flag)27247c478bd9Sstevel@tonic-gate pcacheset_remove_list(pollstate_t *ps, pollfd_t *pollfdp, int start, int end,
27257c478bd9Sstevel@tonic-gate int cacheindex, int flag)
27267c478bd9Sstevel@tonic-gate {
27277c478bd9Sstevel@tonic-gate int i;
27287c478bd9Sstevel@tonic-gate
27297c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&ps->ps_lock));
27307c478bd9Sstevel@tonic-gate for (i = start; i < end; i++) {
27317c478bd9Sstevel@tonic-gate if ((pollfdp[i].fd >= 0) &&
27327c478bd9Sstevel@tonic-gate (flag || !(pollfdp[i].revents & POLLNVAL))) {
27337c478bd9Sstevel@tonic-gate if (pcache_delete_fd(ps, pollfdp[i].fd, i, cacheindex,
27347c478bd9Sstevel@tonic-gate (uint_t)pollfdp[i].events)) {
27357c478bd9Sstevel@tonic-gate int j;
27367c478bd9Sstevel@tonic-gate int fd = pollfdp[i].fd;
27377c478bd9Sstevel@tonic-gate
27387c478bd9Sstevel@tonic-gate for (j = i + 1; j < end; j++) {
27397c478bd9Sstevel@tonic-gate if (pollfdp[j].fd == fd) {
27407c478bd9Sstevel@tonic-gate pcache_update_xref(
27417c478bd9Sstevel@tonic-gate ps->ps_pcache, fd,
27427c478bd9Sstevel@tonic-gate (ssize_t)j, cacheindex);
27437c478bd9Sstevel@tonic-gate break;
27447c478bd9Sstevel@tonic-gate }
27457c478bd9Sstevel@tonic-gate }
27467c478bd9Sstevel@tonic-gate ASSERT(j <= end);
27477c478bd9Sstevel@tonic-gate }
27487c478bd9Sstevel@tonic-gate }
27497c478bd9Sstevel@tonic-gate }
27507c478bd9Sstevel@tonic-gate }
27517c478bd9Sstevel@tonic-gate
27527c478bd9Sstevel@tonic-gate #ifdef DEBUG
27537c478bd9Sstevel@tonic-gate
27547c478bd9Sstevel@tonic-gate #include<sys/strsubr.h>
27557c478bd9Sstevel@tonic-gate /*
27567c478bd9Sstevel@tonic-gate * make sure curthread is not on anyone's pollhead list any more.
27577c478bd9Sstevel@tonic-gate */
27587c478bd9Sstevel@tonic-gate static void
pollcheckphlist()27597c478bd9Sstevel@tonic-gate pollcheckphlist()
27607c478bd9Sstevel@tonic-gate {
27617c478bd9Sstevel@tonic-gate int i;
27627c478bd9Sstevel@tonic-gate file_t *fp;
27637c478bd9Sstevel@tonic-gate uf_entry_t *ufp;
27647c478bd9Sstevel@tonic-gate uf_info_t *fip = P_FINFO(curproc);
27657c478bd9Sstevel@tonic-gate struct stdata *stp;
27667c478bd9Sstevel@tonic-gate polldat_t *pdp;
27677c478bd9Sstevel@tonic-gate
27687c478bd9Sstevel@tonic-gate mutex_enter(&fip->fi_lock);
27697c478bd9Sstevel@tonic-gate for (i = 0; i < fip->fi_nfiles; i++) {
27707c478bd9Sstevel@tonic-gate UF_ENTER(ufp, fip, i);
27717c478bd9Sstevel@tonic-gate if ((fp = ufp->uf_file) != NULL) {
27727c478bd9Sstevel@tonic-gate if ((stp = fp->f_vnode->v_stream) != NULL) {
27737c478bd9Sstevel@tonic-gate PH_ENTER(&stp->sd_pollist);
27747c478bd9Sstevel@tonic-gate pdp = stp->sd_pollist.ph_list;
27757c478bd9Sstevel@tonic-gate while (pdp) {
27767c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_thread != curthread);
27777c478bd9Sstevel@tonic-gate pdp = pdp->pd_next;
27787c478bd9Sstevel@tonic-gate }
27797c478bd9Sstevel@tonic-gate PH_EXIT(&stp->sd_pollist);
27807c478bd9Sstevel@tonic-gate }
27817c478bd9Sstevel@tonic-gate }
27827c478bd9Sstevel@tonic-gate UF_EXIT(ufp);
27837c478bd9Sstevel@tonic-gate }
27847c478bd9Sstevel@tonic-gate mutex_exit(&fip->fi_lock);
27857c478bd9Sstevel@tonic-gate }
27867c478bd9Sstevel@tonic-gate
27877c478bd9Sstevel@tonic-gate /*
27887c478bd9Sstevel@tonic-gate * for resolved set poll list, the xref info in the pcache should be
27897c478bd9Sstevel@tonic-gate * consistent with this poll list.
27907c478bd9Sstevel@tonic-gate */
27917c478bd9Sstevel@tonic-gate static int
pollcheckxref(pollstate_t * ps,int cacheindex)27927c478bd9Sstevel@tonic-gate pollcheckxref(pollstate_t *ps, int cacheindex)
27937c478bd9Sstevel@tonic-gate {
27947c478bd9Sstevel@tonic-gate pollfd_t *pollfdp = ps->ps_pcacheset[cacheindex].pcs_pollfd;
27957c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
27967c478bd9Sstevel@tonic-gate polldat_t *pdp;
27977c478bd9Sstevel@tonic-gate int i;
27987c478bd9Sstevel@tonic-gate xref_t *refp;
27997c478bd9Sstevel@tonic-gate
28007c478bd9Sstevel@tonic-gate for (i = 0; i < ps->ps_pcacheset[cacheindex].pcs_nfds; i++) {
28017c478bd9Sstevel@tonic-gate if (pollfdp[i].fd < 0) {
28027c478bd9Sstevel@tonic-gate continue;
28037c478bd9Sstevel@tonic-gate }
28047c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, pollfdp[i].fd);
28057c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
28067c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
28077c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[cacheindex];
28087c478bd9Sstevel@tonic-gate if (refp->xf_position >= 0) {
28097c478bd9Sstevel@tonic-gate ASSERT(refp->xf_refcnt >= 1);
28107c478bd9Sstevel@tonic-gate ASSERT(pollfdp[refp->xf_position].fd == pdp->pd_fd);
28117c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
28127c478bd9Sstevel@tonic-gate int j;
28137c478bd9Sstevel@tonic-gate int count = 0;
28147c478bd9Sstevel@tonic-gate
28157c478bd9Sstevel@tonic-gate for (j = refp->xf_position;
28167c478bd9Sstevel@tonic-gate j < ps->ps_pcacheset[cacheindex].pcs_nfds;
28177c478bd9Sstevel@tonic-gate j++) {
28187c478bd9Sstevel@tonic-gate if (pollfdp[j].fd == pdp->pd_fd) {
28197c478bd9Sstevel@tonic-gate count++;
28207c478bd9Sstevel@tonic-gate }
28217c478bd9Sstevel@tonic-gate }
28227c478bd9Sstevel@tonic-gate ASSERT(count == refp->xf_refcnt);
28237c478bd9Sstevel@tonic-gate }
28247c478bd9Sstevel@tonic-gate }
28257c478bd9Sstevel@tonic-gate }
28267c478bd9Sstevel@tonic-gate return (1);
28277c478bd9Sstevel@tonic-gate }
28287c478bd9Sstevel@tonic-gate
28297c478bd9Sstevel@tonic-gate /*
28307c478bd9Sstevel@tonic-gate * For every cached pollfd, its polldat struct should be consistent with
28317c478bd9Sstevel@tonic-gate * what is in the pcacheset lists.
28327c478bd9Sstevel@tonic-gate */
28337c478bd9Sstevel@tonic-gate static void
checkpolldat(pollstate_t * ps)28347c478bd9Sstevel@tonic-gate checkpolldat(pollstate_t *ps)
28357c478bd9Sstevel@tonic-gate {
28367c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
28377c478bd9Sstevel@tonic-gate polldat_t **hashtbl;
28387c478bd9Sstevel@tonic-gate int i;
28397c478bd9Sstevel@tonic-gate
28407c478bd9Sstevel@tonic-gate hashtbl = pcp->pc_hash;
28417c478bd9Sstevel@tonic-gate for (i = 0; i < pcp->pc_hashsize; i++) {
28427c478bd9Sstevel@tonic-gate polldat_t *pdp;
28437c478bd9Sstevel@tonic-gate
28447c478bd9Sstevel@tonic-gate for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
28457c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
28467c478bd9Sstevel@tonic-gate if (pdp->pd_count > 0) {
28477c478bd9Sstevel@tonic-gate xref_t *refp;
28487c478bd9Sstevel@tonic-gate int j;
28497c478bd9Sstevel@tonic-gate pollcacheset_t *pcsp;
28507c478bd9Sstevel@tonic-gate pollfd_t *pollfd;
28517c478bd9Sstevel@tonic-gate
28527c478bd9Sstevel@tonic-gate for (j = 0; j < ps->ps_nsets; j++) {
28537c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[j];
28547c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 0) {
28557c478bd9Sstevel@tonic-gate pcsp = &ps->ps_pcacheset[j];
28563df2e8b2SRobert Mustacchi ASSERT(refp->xf_position <
28573df2e8b2SRobert Mustacchi pcsp->pcs_nfds);
28587c478bd9Sstevel@tonic-gate pollfd = pcsp->pcs_pollfd;
28593df2e8b2SRobert Mustacchi ASSERT(pdp->pd_fd ==
28603df2e8b2SRobert Mustacchi pollfd[refp->xf_position].
28613df2e8b2SRobert Mustacchi fd);
28627c478bd9Sstevel@tonic-gate }
28637c478bd9Sstevel@tonic-gate }
28647c478bd9Sstevel@tonic-gate }
28657c478bd9Sstevel@tonic-gate }
28667c478bd9Sstevel@tonic-gate }
28677c478bd9Sstevel@tonic-gate }
28687c478bd9Sstevel@tonic-gate
28697c478bd9Sstevel@tonic-gate /*
28707c478bd9Sstevel@tonic-gate * every wfd element on ph_list must have a corresponding fpollinfo on the
28717c478bd9Sstevel@tonic-gate * uf_fpollinfo list. This is a variation of infpollinfo() w/o holding locks.
28727c478bd9Sstevel@tonic-gate */
28737c478bd9Sstevel@tonic-gate void
checkwfdlist(vnode_t * vp,fpollinfo_t * fpip)28747c478bd9Sstevel@tonic-gate checkwfdlist(vnode_t *vp, fpollinfo_t *fpip)
28757c478bd9Sstevel@tonic-gate {
28767c478bd9Sstevel@tonic-gate stdata_t *stp;
28777c478bd9Sstevel@tonic-gate polldat_t *pdp;
28787c478bd9Sstevel@tonic-gate fpollinfo_t *fpip2;
28797c478bd9Sstevel@tonic-gate
28807c478bd9Sstevel@tonic-gate if ((stp = vp->v_stream) == NULL) {
28817c478bd9Sstevel@tonic-gate return;
28827c478bd9Sstevel@tonic-gate }
28837c478bd9Sstevel@tonic-gate PH_ENTER(&stp->sd_pollist);
28847c478bd9Sstevel@tonic-gate for (pdp = stp->sd_pollist.ph_list; pdp; pdp = pdp->pd_next) {
2885d61dd0c1SPrakash Sangappa if (pdp->pd_thread != NULL &&
2886d61dd0c1SPrakash Sangappa pdp->pd_thread->t_procp == curthread->t_procp) {
28877c478bd9Sstevel@tonic-gate for (fpip2 = fpip; fpip2; fpip2 = fpip2->fp_next) {
28887c478bd9Sstevel@tonic-gate if (pdp->pd_thread == fpip2->fp_thread) {
28897c478bd9Sstevel@tonic-gate break;
28907c478bd9Sstevel@tonic-gate }
28917c478bd9Sstevel@tonic-gate }
28927c478bd9Sstevel@tonic-gate ASSERT(fpip2 != NULL);
28937c478bd9Sstevel@tonic-gate }
28947c478bd9Sstevel@tonic-gate }
28957c478bd9Sstevel@tonic-gate PH_EXIT(&stp->sd_pollist);
28967c478bd9Sstevel@tonic-gate }
28977c478bd9Sstevel@tonic-gate
28987c478bd9Sstevel@tonic-gate /*
28997c478bd9Sstevel@tonic-gate * For each cached fd whose bit is not set in bitmap, its revents field in
29007c478bd9Sstevel@tonic-gate * current poll list should be 0.
29017c478bd9Sstevel@tonic-gate */
29027c478bd9Sstevel@tonic-gate static int
pollcheckrevents(pollstate_t * ps,int begin,int end,int cacheindex)29037c478bd9Sstevel@tonic-gate pollcheckrevents(pollstate_t *ps, int begin, int end, int cacheindex)
29047c478bd9Sstevel@tonic-gate {
29057c478bd9Sstevel@tonic-gate pollcache_t *pcp = ps->ps_pcache;
29067c478bd9Sstevel@tonic-gate pollfd_t *pollfdp = ps->ps_pollfd;
29077c478bd9Sstevel@tonic-gate int i;
29087c478bd9Sstevel@tonic-gate
29097c478bd9Sstevel@tonic-gate for (i = begin; i < end; i++) {
29107c478bd9Sstevel@tonic-gate polldat_t *pdp;
29117c478bd9Sstevel@tonic-gate
29127c478bd9Sstevel@tonic-gate ASSERT(!BT_TEST(pcp->pc_bitmap, i));
29137c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, i);
29147c478bd9Sstevel@tonic-gate if (pdp && pdp->pd_fp != NULL) {
29157c478bd9Sstevel@tonic-gate xref_t *refp;
29167c478bd9Sstevel@tonic-gate int entry;
29177c478bd9Sstevel@tonic-gate
29187c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_ref != NULL);
29197c478bd9Sstevel@tonic-gate refp = &pdp->pd_ref[cacheindex];
29207c478bd9Sstevel@tonic-gate if (refp->xf_refcnt == 0) {
29217c478bd9Sstevel@tonic-gate continue;
29227c478bd9Sstevel@tonic-gate }
29237c478bd9Sstevel@tonic-gate entry = refp->xf_position;
29247c478bd9Sstevel@tonic-gate ASSERT(entry >= 0);
29257c478bd9Sstevel@tonic-gate ASSERT(pollfdp[entry].revents == 0);
29267c478bd9Sstevel@tonic-gate if (refp->xf_refcnt > 1) {
29277c478bd9Sstevel@tonic-gate int j;
29287c478bd9Sstevel@tonic-gate
29297c478bd9Sstevel@tonic-gate for (j = entry + 1; j < ps->ps_nfds; j++) {
29307c478bd9Sstevel@tonic-gate if (pollfdp[j].fd == i) {
29317c478bd9Sstevel@tonic-gate ASSERT(pollfdp[j].revents == 0);
29327c478bd9Sstevel@tonic-gate }
29337c478bd9Sstevel@tonic-gate }
29347c478bd9Sstevel@tonic-gate }
29357c478bd9Sstevel@tonic-gate }
29367c478bd9Sstevel@tonic-gate }
29377c478bd9Sstevel@tonic-gate return (1);
29387c478bd9Sstevel@tonic-gate }
29397c478bd9Sstevel@tonic-gate
29407c478bd9Sstevel@tonic-gate #endif /* DEBUG */
29417c478bd9Sstevel@tonic-gate
29427c478bd9Sstevel@tonic-gate pollcache_t *
pcache_alloc()29437c478bd9Sstevel@tonic-gate pcache_alloc()
29447c478bd9Sstevel@tonic-gate {
29457c478bd9Sstevel@tonic-gate return (kmem_zalloc(sizeof (pollcache_t), KM_SLEEP));
29467c478bd9Sstevel@tonic-gate }
29477c478bd9Sstevel@tonic-gate
29487c478bd9Sstevel@tonic-gate void
pcache_create(pollcache_t * pcp,nfds_t nfds)29497c478bd9Sstevel@tonic-gate pcache_create(pollcache_t *pcp, nfds_t nfds)
29507c478bd9Sstevel@tonic-gate {
29517c478bd9Sstevel@tonic-gate size_t mapsize;
29527c478bd9Sstevel@tonic-gate
29537c478bd9Sstevel@tonic-gate /*
29547c478bd9Sstevel@tonic-gate * allocate enough bits for the poll fd list
29557c478bd9Sstevel@tonic-gate */
29567c478bd9Sstevel@tonic-gate if ((mapsize = POLLMAPCHUNK) <= nfds) {
29577c478bd9Sstevel@tonic-gate mapsize = (nfds + POLLMAPCHUNK - 1) & ~(POLLMAPCHUNK - 1);
29587c478bd9Sstevel@tonic-gate }
29597c478bd9Sstevel@tonic-gate pcp->pc_bitmap = kmem_zalloc((mapsize / BT_NBIPUL) * sizeof (ulong_t),
29607c478bd9Sstevel@tonic-gate KM_SLEEP);
29617c478bd9Sstevel@tonic-gate pcp->pc_mapsize = mapsize;
29627c478bd9Sstevel@tonic-gate /*
29637c478bd9Sstevel@tonic-gate * The hash size is at least POLLHASHCHUNKSZ. If user polls a large
29647c478bd9Sstevel@tonic-gate * number of fd to start with, allocate a bigger hash table (to the
29657c478bd9Sstevel@tonic-gate * nearest multiple of POLLHASHCHUNKSZ) because dynamically growing a
29667c478bd9Sstevel@tonic-gate * hash table is expensive.
29677c478bd9Sstevel@tonic-gate */
29687c478bd9Sstevel@tonic-gate if (nfds < POLLHASHCHUNKSZ) {
29697c478bd9Sstevel@tonic-gate pcp->pc_hashsize = POLLHASHCHUNKSZ;
29707c478bd9Sstevel@tonic-gate } else {
29717c478bd9Sstevel@tonic-gate pcp->pc_hashsize = (nfds + POLLHASHCHUNKSZ - 1) &
29727c478bd9Sstevel@tonic-gate ~(POLLHASHCHUNKSZ - 1);
29737c478bd9Sstevel@tonic-gate }
29747c478bd9Sstevel@tonic-gate pcp->pc_hash = kmem_zalloc(pcp->pc_hashsize * sizeof (polldat_t *),
29757c478bd9Sstevel@tonic-gate KM_SLEEP);
29767c478bd9Sstevel@tonic-gate }
29777c478bd9Sstevel@tonic-gate
29787c478bd9Sstevel@tonic-gate void
pcache_destroy(pollcache_t * pcp)29797c478bd9Sstevel@tonic-gate pcache_destroy(pollcache_t *pcp)
29807c478bd9Sstevel@tonic-gate {
29817c478bd9Sstevel@tonic-gate polldat_t **hashtbl;
29827c478bd9Sstevel@tonic-gate int i;
29837c478bd9Sstevel@tonic-gate
29847c478bd9Sstevel@tonic-gate hashtbl = pcp->pc_hash;
29857c478bd9Sstevel@tonic-gate for (i = 0; i < pcp->pc_hashsize; i++) {
29867c478bd9Sstevel@tonic-gate if (hashtbl[i] != NULL) {
29877c478bd9Sstevel@tonic-gate polldat_t *pdp, *pdp2;
29887c478bd9Sstevel@tonic-gate
29897c478bd9Sstevel@tonic-gate pdp = hashtbl[i];
29907c478bd9Sstevel@tonic-gate while (pdp != NULL) {
29917c478bd9Sstevel@tonic-gate pdp2 = pdp->pd_hashnext;
29927c478bd9Sstevel@tonic-gate if (pdp->pd_ref != NULL) {
29937c478bd9Sstevel@tonic-gate kmem_free(pdp->pd_ref, sizeof (xref_t) *
29947c478bd9Sstevel@tonic-gate pdp->pd_nsets);
29957c478bd9Sstevel@tonic-gate }
29967c478bd9Sstevel@tonic-gate kmem_free(pdp, sizeof (polldat_t));
29977c478bd9Sstevel@tonic-gate pdp = pdp2;
29987c478bd9Sstevel@tonic-gate pcp->pc_fdcount--;
29997c478bd9Sstevel@tonic-gate }
30007c478bd9Sstevel@tonic-gate }
30017c478bd9Sstevel@tonic-gate }
30027c478bd9Sstevel@tonic-gate ASSERT(pcp->pc_fdcount == 0);
30037c478bd9Sstevel@tonic-gate kmem_free(pcp->pc_hash, sizeof (polldat_t *) * pcp->pc_hashsize);
30047c478bd9Sstevel@tonic-gate kmem_free(pcp->pc_bitmap,
30057c478bd9Sstevel@tonic-gate sizeof (ulong_t) * (pcp->pc_mapsize/BT_NBIPUL));
30067c478bd9Sstevel@tonic-gate mutex_destroy(&pcp->pc_no_exit);
30077c478bd9Sstevel@tonic-gate mutex_destroy(&pcp->pc_lock);
30087c478bd9Sstevel@tonic-gate cv_destroy(&pcp->pc_cv);
30097c478bd9Sstevel@tonic-gate cv_destroy(&pcp->pc_busy_cv);
30107c478bd9Sstevel@tonic-gate kmem_free(pcp, sizeof (pollcache_t));
30117c478bd9Sstevel@tonic-gate }
30127c478bd9Sstevel@tonic-gate
30137c478bd9Sstevel@tonic-gate pollcacheset_t *
pcacheset_create(int nsets)30147c478bd9Sstevel@tonic-gate pcacheset_create(int nsets)
30157c478bd9Sstevel@tonic-gate {
30167c478bd9Sstevel@tonic-gate return (kmem_zalloc(sizeof (pollcacheset_t) * nsets, KM_SLEEP));
30177c478bd9Sstevel@tonic-gate }
30187c478bd9Sstevel@tonic-gate
30197c478bd9Sstevel@tonic-gate void
pcacheset_destroy(pollcacheset_t * pcsp,int nsets)30207c478bd9Sstevel@tonic-gate pcacheset_destroy(pollcacheset_t *pcsp, int nsets)
30217c478bd9Sstevel@tonic-gate {
30227c478bd9Sstevel@tonic-gate int i;
30237c478bd9Sstevel@tonic-gate
30247c478bd9Sstevel@tonic-gate for (i = 0; i < nsets; i++) {
30257c478bd9Sstevel@tonic-gate if (pcsp[i].pcs_pollfd != NULL) {
30267c478bd9Sstevel@tonic-gate kmem_free(pcsp[i].pcs_pollfd, pcsp[i].pcs_nfds *
30277c478bd9Sstevel@tonic-gate sizeof (pollfd_t));
30287c478bd9Sstevel@tonic-gate }
30297c478bd9Sstevel@tonic-gate }
30307c478bd9Sstevel@tonic-gate kmem_free(pcsp, sizeof (pollcacheset_t) * nsets);
30317c478bd9Sstevel@tonic-gate }
30327c478bd9Sstevel@tonic-gate
30337c478bd9Sstevel@tonic-gate /*
30347c478bd9Sstevel@tonic-gate * Check each duplicated poll fd in the poll list. It may be necessary to
30357c478bd9Sstevel@tonic-gate * VOP_POLL the same fd again using different poll events. getf() has been
30367c478bd9Sstevel@tonic-gate * done by caller. This routine returns 0 if it can sucessfully process the
30377c478bd9Sstevel@tonic-gate * entire poll fd list. It returns -1 if underlying vnode has changed during
30387c478bd9Sstevel@tonic-gate * a VOP_POLL, in which case the caller has to repoll. It returns a positive
30397c478bd9Sstevel@tonic-gate * value if VOP_POLL failed.
30407c478bd9Sstevel@tonic-gate */
30417c478bd9Sstevel@tonic-gate static int
plist_chkdupfd(file_t * fp,polldat_t * pdp,pollstate_t * psp,pollfd_t * pollfdp,int entry,int * fdcntp)30427c478bd9Sstevel@tonic-gate plist_chkdupfd(file_t *fp, polldat_t *pdp, pollstate_t *psp, pollfd_t *pollfdp,
30437c478bd9Sstevel@tonic-gate int entry, int *fdcntp)
30447c478bd9Sstevel@tonic-gate {
30457c478bd9Sstevel@tonic-gate int i;
30467c478bd9Sstevel@tonic-gate int fd;
30477c478bd9Sstevel@tonic-gate nfds_t nfds = psp->ps_nfds;
30487c478bd9Sstevel@tonic-gate
30497c478bd9Sstevel@tonic-gate fd = pollfdp[entry].fd;
30507c478bd9Sstevel@tonic-gate for (i = entry + 1; i < nfds; i++) {
30517c478bd9Sstevel@tonic-gate if (pollfdp[i].fd == fd) {
30527c478bd9Sstevel@tonic-gate if (pollfdp[i].events == pollfdp[entry].events) {
30537c478bd9Sstevel@tonic-gate if ((pollfdp[i].revents =
30547c478bd9Sstevel@tonic-gate pollfdp[entry].revents) != 0) {
30557c478bd9Sstevel@tonic-gate (*fdcntp)++;
30567c478bd9Sstevel@tonic-gate }
30577c478bd9Sstevel@tonic-gate } else {
30587c478bd9Sstevel@tonic-gate
30597c478bd9Sstevel@tonic-gate int error;
30607c478bd9Sstevel@tonic-gate pollhead_t *php;
30617c478bd9Sstevel@tonic-gate pollcache_t *pcp = psp->ps_pcache;
30627c478bd9Sstevel@tonic-gate
30637c478bd9Sstevel@tonic-gate /*
30647c478bd9Sstevel@tonic-gate * the events are different. VOP_POLL on this
30657c478bd9Sstevel@tonic-gate * fd so that we don't miss any revents.
30667c478bd9Sstevel@tonic-gate */
30677c478bd9Sstevel@tonic-gate php = NULL;
30687c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pollcache == NULL);
30697c478bd9Sstevel@tonic-gate error = VOP_POLL(fp->f_vnode,
30707c478bd9Sstevel@tonic-gate pollfdp[i].events, 0,
3071da6c28aaSamw &pollfdp[i].revents, &php, NULL);
30727c478bd9Sstevel@tonic-gate if (error) {
30737c478bd9Sstevel@tonic-gate return (error);
30747c478bd9Sstevel@tonic-gate }
30757c478bd9Sstevel@tonic-gate /*
30767c478bd9Sstevel@tonic-gate * layered devices(e.g. console driver)
30777c478bd9Sstevel@tonic-gate * may change the vnode and thus the pollhead
30787c478bd9Sstevel@tonic-gate * pointer out from underneath us.
30797c478bd9Sstevel@tonic-gate */
30807c478bd9Sstevel@tonic-gate if (php != NULL && pdp->pd_php != NULL &&
30817c478bd9Sstevel@tonic-gate php != pdp->pd_php) {
30822c76d751SPatrick Mooney polldat_disassociate(pdp);
30832c76d751SPatrick Mooney polldat_associate(pdp, php);
30847c478bd9Sstevel@tonic-gate /*
30857c478bd9Sstevel@tonic-gate * We could have missed a wakeup on the
30867c478bd9Sstevel@tonic-gate * new target device. Make sure the new
30877c478bd9Sstevel@tonic-gate * target gets polled once.
30887c478bd9Sstevel@tonic-gate */
30897c478bd9Sstevel@tonic-gate BT_SET(pcp->pc_bitmap, fd);
30907c478bd9Sstevel@tonic-gate return (-1);
30917c478bd9Sstevel@tonic-gate }
30927c478bd9Sstevel@tonic-gate if (pollfdp[i].revents) {
30937c478bd9Sstevel@tonic-gate (*fdcntp)++;
30947c478bd9Sstevel@tonic-gate }
30957c478bd9Sstevel@tonic-gate }
30967c478bd9Sstevel@tonic-gate }
30977c478bd9Sstevel@tonic-gate }
30987c478bd9Sstevel@tonic-gate return (0);
30997c478bd9Sstevel@tonic-gate }
3100