17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
55f684e24Ssp * Common Development and Distribution License (the "License").
65f684e24Ssp * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22a85084caSmeem * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
26cd1c8b85SMatthew Ahrens /*
27cd1c8b85SMatthew Ahrens * Copyright (c) 2012 by Delphix. All rights reserved.
2895e434b5SPatrick Mooney * Copyright 2019 Joyent, Inc.
29*2c76d751SPatrick Mooney * Copyright 2022 Oxide Computer Company
30cd1c8b85SMatthew Ahrens */
317c478bd9Sstevel@tonic-gate
327c478bd9Sstevel@tonic-gate #include <sys/types.h>
337c478bd9Sstevel@tonic-gate #include <sys/devops.h>
347c478bd9Sstevel@tonic-gate #include <sys/conf.h>
357c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
367c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
377c478bd9Sstevel@tonic-gate #include <sys/stat.h>
387c478bd9Sstevel@tonic-gate #include <sys/poll_impl.h>
397c478bd9Sstevel@tonic-gate #include <sys/errno.h>
407c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
417c478bd9Sstevel@tonic-gate #include <sys/mkdev.h>
427c478bd9Sstevel@tonic-gate #include <sys/debug.h>
437c478bd9Sstevel@tonic-gate #include <sys/file.h>
447c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
457c478bd9Sstevel@tonic-gate #include <sys/systm.h>
467c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
477c478bd9Sstevel@tonic-gate #include <sys/devpoll.h>
487c478bd9Sstevel@tonic-gate #include <sys/rctl.h>
497c478bd9Sstevel@tonic-gate #include <sys/resource.h>
50a5eb7107SBryan Cantrill #include <sys/schedctl.h>
51a5eb7107SBryan Cantrill #include <sys/epoll.h>
527c478bd9Sstevel@tonic-gate
537c478bd9Sstevel@tonic-gate #define RESERVED 1
547c478bd9Sstevel@tonic-gate
557c478bd9Sstevel@tonic-gate /* local data struct */
56fe234e7cSMatt Amdur static dp_entry_t **devpolltbl; /* dev poll entries */
577c478bd9Sstevel@tonic-gate static size_t dptblsize;
587c478bd9Sstevel@tonic-gate
597c478bd9Sstevel@tonic-gate static kmutex_t devpoll_lock; /* lock protecting dev tbl */
607c478bd9Sstevel@tonic-gate int devpoll_init; /* is /dev/poll initialized already */
617c478bd9Sstevel@tonic-gate
627c478bd9Sstevel@tonic-gate /* device local functions */
637c478bd9Sstevel@tonic-gate
647c478bd9Sstevel@tonic-gate static int dpopen(dev_t *devp, int flag, int otyp, cred_t *credp);
657c478bd9Sstevel@tonic-gate static int dpwrite(dev_t dev, struct uio *uiop, cred_t *credp);
667c478bd9Sstevel@tonic-gate static int dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
677c478bd9Sstevel@tonic-gate int *rvalp);
687c478bd9Sstevel@tonic-gate static int dppoll(dev_t dev, short events, int anyyet, short *reventsp,
697c478bd9Sstevel@tonic-gate struct pollhead **phpp);
707c478bd9Sstevel@tonic-gate static int dpclose(dev_t dev, int flag, int otyp, cred_t *credp);
717c478bd9Sstevel@tonic-gate static dev_info_t *dpdevi;
727c478bd9Sstevel@tonic-gate
737c478bd9Sstevel@tonic-gate
747c478bd9Sstevel@tonic-gate static struct cb_ops dp_cb_ops = {
757c478bd9Sstevel@tonic-gate dpopen, /* open */
767c478bd9Sstevel@tonic-gate dpclose, /* close */
777c478bd9Sstevel@tonic-gate nodev, /* strategy */
787c478bd9Sstevel@tonic-gate nodev, /* print */
797c478bd9Sstevel@tonic-gate nodev, /* dump */
807c478bd9Sstevel@tonic-gate nodev, /* read */
817c478bd9Sstevel@tonic-gate dpwrite, /* write */
827c478bd9Sstevel@tonic-gate dpioctl, /* ioctl */
837c478bd9Sstevel@tonic-gate nodev, /* devmap */
847c478bd9Sstevel@tonic-gate nodev, /* mmap */
857c478bd9Sstevel@tonic-gate nodev, /* segmap */
867c478bd9Sstevel@tonic-gate dppoll, /* poll */
87a913d554Scth ddi_prop_op, /* prop_op */
887c478bd9Sstevel@tonic-gate (struct streamtab *)0, /* streamtab */
89a913d554Scth D_MP, /* flags */
90a913d554Scth CB_REV, /* cb_ops revision */
91a913d554Scth nodev, /* aread */
92a913d554Scth nodev /* awrite */
937c478bd9Sstevel@tonic-gate };
947c478bd9Sstevel@tonic-gate
957c478bd9Sstevel@tonic-gate static int dpattach(dev_info_t *, ddi_attach_cmd_t);
967c478bd9Sstevel@tonic-gate static int dpdetach(dev_info_t *, ddi_detach_cmd_t);
977c478bd9Sstevel@tonic-gate static int dpinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
987c478bd9Sstevel@tonic-gate
997c478bd9Sstevel@tonic-gate static struct dev_ops dp_ops = {
1007c478bd9Sstevel@tonic-gate DEVO_REV, /* devo_rev */
1017c478bd9Sstevel@tonic-gate 0, /* refcnt */
1027c478bd9Sstevel@tonic-gate dpinfo, /* info */
1037c478bd9Sstevel@tonic-gate nulldev, /* identify */
1047c478bd9Sstevel@tonic-gate nulldev, /* probe */
1057c478bd9Sstevel@tonic-gate dpattach, /* attach */
1067c478bd9Sstevel@tonic-gate dpdetach, /* detach */
1077c478bd9Sstevel@tonic-gate nodev, /* reset */
1087c478bd9Sstevel@tonic-gate &dp_cb_ops, /* driver operations */
1097c478bd9Sstevel@tonic-gate (struct bus_ops *)NULL, /* bus operations */
11019397407SSherry Moore nulldev, /* power */
11119397407SSherry Moore ddi_quiesce_not_needed, /* quiesce */
1127c478bd9Sstevel@tonic-gate };
1137c478bd9Sstevel@tonic-gate
1147c478bd9Sstevel@tonic-gate
1157c478bd9Sstevel@tonic-gate static struct modldrv modldrv = {
1167c478bd9Sstevel@tonic-gate &mod_driverops, /* type of module - a driver */
117a85084caSmeem "/dev/poll driver",
1187c478bd9Sstevel@tonic-gate &dp_ops,
1197c478bd9Sstevel@tonic-gate };
1207c478bd9Sstevel@tonic-gate
1217c478bd9Sstevel@tonic-gate static struct modlinkage modlinkage = {
1227c478bd9Sstevel@tonic-gate MODREV_1,
1237c478bd9Sstevel@tonic-gate (void *)&modldrv,
1247c478bd9Sstevel@tonic-gate NULL
1257c478bd9Sstevel@tonic-gate };
1267c478bd9Sstevel@tonic-gate
127f3bb54f3SPatrick Mooney static void pcachelink_assoc(pollcache_t *, pollcache_t *);
128f3bb54f3SPatrick Mooney static void pcachelink_mark_stale(pollcache_t *);
129f3bb54f3SPatrick Mooney static void pcachelink_purge_stale(pollcache_t *);
130f3bb54f3SPatrick Mooney static void pcachelink_purge_all(pollcache_t *);
131f3bb54f3SPatrick Mooney
132f3bb54f3SPatrick Mooney
1337c478bd9Sstevel@tonic-gate /*
1347c478bd9Sstevel@tonic-gate * Locking Design
1357c478bd9Sstevel@tonic-gate *
1367c478bd9Sstevel@tonic-gate * The /dev/poll driver shares most of its code with poll sys call whose
1377c478bd9Sstevel@tonic-gate * code is in common/syscall/poll.c. In poll(2) design, the pollcache
1387c478bd9Sstevel@tonic-gate * structure is per lwp. An implicit assumption is made there that some
1397c478bd9Sstevel@tonic-gate * portion of pollcache will never be touched by other lwps. E.g., in
1407c478bd9Sstevel@tonic-gate * poll(2) design, no lwp will ever need to grow bitmap of other lwp.
1417c478bd9Sstevel@tonic-gate * This assumption is not true for /dev/poll; hence the need for extra
1427c478bd9Sstevel@tonic-gate * locking.
1437c478bd9Sstevel@tonic-gate *
144da6c28aaSamw * To allow more parallelism, each /dev/poll file descriptor (indexed by
1457c478bd9Sstevel@tonic-gate * minor number) has its own lock. Since read (dpioctl) is a much more
1467c478bd9Sstevel@tonic-gate * frequent operation than write, we want to allow multiple reads on same
1477c478bd9Sstevel@tonic-gate * /dev/poll fd. However, we prevent writes from being starved by giving
1487c478bd9Sstevel@tonic-gate * priority to write operation. Theoretically writes can starve reads as
149da6c28aaSamw * well. But in practical sense this is not important because (1) writes
1507c478bd9Sstevel@tonic-gate * happens less often than reads, and (2) write operation defines the
1517c478bd9Sstevel@tonic-gate * content of poll fd a cache set. If writes happens so often that they
1527c478bd9Sstevel@tonic-gate * can starve reads, that means the cached set is very unstable. It may
1537c478bd9Sstevel@tonic-gate * not make sense to read an unstable cache set anyway. Therefore, the
1547c478bd9Sstevel@tonic-gate * writers starving readers case is not handled in this design.
1557c478bd9Sstevel@tonic-gate */
1567c478bd9Sstevel@tonic-gate
1577c478bd9Sstevel@tonic-gate int
_init()1587c478bd9Sstevel@tonic-gate _init()
1597c478bd9Sstevel@tonic-gate {
1607c478bd9Sstevel@tonic-gate int error;
1617c478bd9Sstevel@tonic-gate
1627c478bd9Sstevel@tonic-gate dptblsize = DEVPOLLSIZE;
1637c478bd9Sstevel@tonic-gate devpolltbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
1647c478bd9Sstevel@tonic-gate mutex_init(&devpoll_lock, NULL, MUTEX_DEFAULT, NULL);
1657c478bd9Sstevel@tonic-gate devpoll_init = 1;
1667c478bd9Sstevel@tonic-gate if ((error = mod_install(&modlinkage)) != 0) {
1677c478bd9Sstevel@tonic-gate kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
1687c478bd9Sstevel@tonic-gate devpoll_init = 0;
1697c478bd9Sstevel@tonic-gate }
1707c478bd9Sstevel@tonic-gate return (error);
1717c478bd9Sstevel@tonic-gate }
1727c478bd9Sstevel@tonic-gate
1737c478bd9Sstevel@tonic-gate int
_fini()1747c478bd9Sstevel@tonic-gate _fini()
1757c478bd9Sstevel@tonic-gate {
1767c478bd9Sstevel@tonic-gate int error;
1777c478bd9Sstevel@tonic-gate
1787c478bd9Sstevel@tonic-gate if ((error = mod_remove(&modlinkage)) != 0) {
1797c478bd9Sstevel@tonic-gate return (error);
1807c478bd9Sstevel@tonic-gate }
1817c478bd9Sstevel@tonic-gate mutex_destroy(&devpoll_lock);
1827c478bd9Sstevel@tonic-gate kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize);
1837c478bd9Sstevel@tonic-gate return (0);
1847c478bd9Sstevel@tonic-gate }
1857c478bd9Sstevel@tonic-gate
1867c478bd9Sstevel@tonic-gate int
_info(struct modinfo * modinfop)1877c478bd9Sstevel@tonic-gate _info(struct modinfo *modinfop)
1887c478bd9Sstevel@tonic-gate {
1897c478bd9Sstevel@tonic-gate return (mod_info(&modlinkage, modinfop));
1907c478bd9Sstevel@tonic-gate }
1917c478bd9Sstevel@tonic-gate
1927c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1937c478bd9Sstevel@tonic-gate static int
dpattach(dev_info_t * devi,ddi_attach_cmd_t cmd)1947c478bd9Sstevel@tonic-gate dpattach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1957c478bd9Sstevel@tonic-gate {
1964ff62ce9SToomas Soome if (ddi_create_minor_node(devi, "poll", S_IFCHR, 0, DDI_PSEUDO, 0)
1977c478bd9Sstevel@tonic-gate == DDI_FAILURE) {
1987c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL);
1997c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
2007c478bd9Sstevel@tonic-gate }
2017c478bd9Sstevel@tonic-gate dpdevi = devi;
2027c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2037c478bd9Sstevel@tonic-gate }
2047c478bd9Sstevel@tonic-gate
2057c478bd9Sstevel@tonic-gate static int
dpdetach(dev_info_t * devi,ddi_detach_cmd_t cmd)2067c478bd9Sstevel@tonic-gate dpdetach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2077c478bd9Sstevel@tonic-gate {
2087c478bd9Sstevel@tonic-gate if (cmd != DDI_DETACH)
2097c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
2107c478bd9Sstevel@tonic-gate
2117c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL);
2127c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2137c478bd9Sstevel@tonic-gate }
2147c478bd9Sstevel@tonic-gate
2157c478bd9Sstevel@tonic-gate /* ARGSUSED */
2167c478bd9Sstevel@tonic-gate static int
dpinfo(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)2177c478bd9Sstevel@tonic-gate dpinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2187c478bd9Sstevel@tonic-gate {
2197c478bd9Sstevel@tonic-gate int error;
2207c478bd9Sstevel@tonic-gate
2217c478bd9Sstevel@tonic-gate switch (infocmd) {
2227c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2DEVINFO:
2237c478bd9Sstevel@tonic-gate *result = (void *)dpdevi;
2247c478bd9Sstevel@tonic-gate error = DDI_SUCCESS;
2257c478bd9Sstevel@tonic-gate break;
2267c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2INSTANCE:
2277c478bd9Sstevel@tonic-gate *result = (void *)0;
2287c478bd9Sstevel@tonic-gate error = DDI_SUCCESS;
2297c478bd9Sstevel@tonic-gate break;
2307c478bd9Sstevel@tonic-gate default:
2317c478bd9Sstevel@tonic-gate error = DDI_FAILURE;
2327c478bd9Sstevel@tonic-gate }
2337c478bd9Sstevel@tonic-gate return (error);
2347c478bd9Sstevel@tonic-gate }
2357c478bd9Sstevel@tonic-gate
2367c478bd9Sstevel@tonic-gate /*
2377c478bd9Sstevel@tonic-gate * dp_pcache_poll has similar logic to pcache_poll() in poll.c. The major
2387c478bd9Sstevel@tonic-gate * differences are: (1) /dev/poll requires scanning the bitmap starting at
2397c478bd9Sstevel@tonic-gate * where it was stopped last time, instead of always starting from 0,
2407c478bd9Sstevel@tonic-gate * (2) since user may not have cleaned up the cached fds when they are
2417c478bd9Sstevel@tonic-gate * closed, some polldats in cache may refer to closed or reused fds. We
2427c478bd9Sstevel@tonic-gate * need to check for those cases.
2437c478bd9Sstevel@tonic-gate *
2447c478bd9Sstevel@tonic-gate * NOTE: Upon closing an fd, automatic poll cache cleanup is done for
2457c478bd9Sstevel@tonic-gate * poll(2) caches but NOT for /dev/poll caches. So expect some
2467c478bd9Sstevel@tonic-gate * stale entries!
2477c478bd9Sstevel@tonic-gate */
2487c478bd9Sstevel@tonic-gate static int
dp_pcache_poll(dp_entry_t * dpep,void * dpbuf,pollcache_t * pcp,nfds_t nfds,int * fdcntp)249086d9687SPatrick Mooney dp_pcache_poll(dp_entry_t *dpep, void *dpbuf, pollcache_t *pcp, nfds_t nfds,
250086d9687SPatrick Mooney int *fdcntp)
2517c478bd9Sstevel@tonic-gate {
252086d9687SPatrick Mooney int start, ostart, end, fdcnt, error = 0;
253086d9687SPatrick Mooney boolean_t done, no_wrap;
254a5eb7107SBryan Cantrill pollfd_t *pfdp;
255a5eb7107SBryan Cantrill epoll_event_t *epoll;
256086d9687SPatrick Mooney const short mask = POLLRDHUP | POLLWRBAND;
257086d9687SPatrick Mooney const boolean_t is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
2587c478bd9Sstevel@tonic-gate
2597c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&pcp->pc_lock));
2607c478bd9Sstevel@tonic-gate if (pcp->pc_bitmap == NULL) {
261086d9687SPatrick Mooney /* No Need to search because no poll fd has been cached. */
262086d9687SPatrick Mooney return (0);
2637c478bd9Sstevel@tonic-gate }
264a5eb7107SBryan Cantrill
265f3bb54f3SPatrick Mooney if (is_epoll) {
266a5eb7107SBryan Cantrill pfdp = NULL;
267a5eb7107SBryan Cantrill epoll = (epoll_event_t *)dpbuf;
268a5eb7107SBryan Cantrill } else {
269a5eb7107SBryan Cantrill pfdp = (pollfd_t *)dpbuf;
270a5eb7107SBryan Cantrill epoll = NULL;
271a5eb7107SBryan Cantrill }
2727c478bd9Sstevel@tonic-gate retry:
2737c478bd9Sstevel@tonic-gate start = ostart = pcp->pc_mapstart;
2747c478bd9Sstevel@tonic-gate end = pcp->pc_mapend;
2757c478bd9Sstevel@tonic-gate
2767c478bd9Sstevel@tonic-gate if (start == 0) {
2777c478bd9Sstevel@tonic-gate /*
2787c478bd9Sstevel@tonic-gate * started from every begining, no need to wrap around.
2797c478bd9Sstevel@tonic-gate */
2807c478bd9Sstevel@tonic-gate no_wrap = B_TRUE;
2817c478bd9Sstevel@tonic-gate } else {
2827c478bd9Sstevel@tonic-gate no_wrap = B_FALSE;
2837c478bd9Sstevel@tonic-gate }
2847c478bd9Sstevel@tonic-gate done = B_FALSE;
2857c478bd9Sstevel@tonic-gate fdcnt = 0;
2867c478bd9Sstevel@tonic-gate while ((fdcnt < nfds) && !done) {
287086d9687SPatrick Mooney pollhead_t *php = NULL;
288086d9687SPatrick Mooney short revent = 0;
289086d9687SPatrick Mooney uf_entry_gen_t gen;
290086d9687SPatrick Mooney int fd;
291086d9687SPatrick Mooney
2927c478bd9Sstevel@tonic-gate /*
2937c478bd9Sstevel@tonic-gate * Examine the bit map in a circular fashion
2947c478bd9Sstevel@tonic-gate * to avoid starvation. Always resume from
2957c478bd9Sstevel@tonic-gate * last stop. Scan till end of the map. Then
2967c478bd9Sstevel@tonic-gate * wrap around.
2977c478bd9Sstevel@tonic-gate */
2987c478bd9Sstevel@tonic-gate fd = bt_getlowbit(pcp->pc_bitmap, start, end);
2997c478bd9Sstevel@tonic-gate ASSERT(fd <= end);
3007c478bd9Sstevel@tonic-gate if (fd >= 0) {
301086d9687SPatrick Mooney file_t *fp;
302086d9687SPatrick Mooney polldat_t *pdp;
303086d9687SPatrick Mooney
3047c478bd9Sstevel@tonic-gate if (fd == end) {
3057c478bd9Sstevel@tonic-gate if (no_wrap) {
3067c478bd9Sstevel@tonic-gate done = B_TRUE;
3077c478bd9Sstevel@tonic-gate } else {
3087c478bd9Sstevel@tonic-gate start = 0;
3097c478bd9Sstevel@tonic-gate end = ostart - 1;
3107c478bd9Sstevel@tonic-gate no_wrap = B_TRUE;
3117c478bd9Sstevel@tonic-gate }
3127c478bd9Sstevel@tonic-gate } else {
3137c478bd9Sstevel@tonic-gate start = fd + 1;
3147c478bd9Sstevel@tonic-gate }
3157c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
316a85084caSmeem repoll:
3177c478bd9Sstevel@tonic-gate ASSERT(pdp != NULL);
3187c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fd == fd);
3197c478bd9Sstevel@tonic-gate if (pdp->pd_fp == NULL) {
3207c478bd9Sstevel@tonic-gate /*
3217c478bd9Sstevel@tonic-gate * The fd is POLLREMOVed. This fd is
3227c478bd9Sstevel@tonic-gate * logically no longer cached. So move
3237c478bd9Sstevel@tonic-gate * on to the next one.
3247c478bd9Sstevel@tonic-gate */
3257c478bd9Sstevel@tonic-gate continue;
3267c478bd9Sstevel@tonic-gate }
327086d9687SPatrick Mooney if ((fp = getf_gen(fd, &gen)) == NULL) {
328f3bb54f3SPatrick Mooney if (is_epoll) {
329086d9687SPatrick Mooney /*
330086d9687SPatrick Mooney * In the epoll compatibility case, we
331086d9687SPatrick Mooney * actually perform the implicit
332086d9687SPatrick Mooney * removal to remain closer to the
333086d9687SPatrick Mooney * epoll semantics.
334086d9687SPatrick Mooney */
335f3bb54f3SPatrick Mooney pdp->pd_fp = NULL;
336f3bb54f3SPatrick Mooney pdp->pd_events = 0;
337a5eb7107SBryan Cantrill
338*2c76d751SPatrick Mooney polldat_disassociate(pdp);
339a5eb7107SBryan Cantrill
340f3bb54f3SPatrick Mooney BT_CLEAR(pcp->pc_bitmap, fd);
341086d9687SPatrick Mooney } else if (pfdp != NULL) {
342086d9687SPatrick Mooney /*
343086d9687SPatrick Mooney * The fd has been closed, but user has
344086d9687SPatrick Mooney * not done a POLLREMOVE on this fd
345086d9687SPatrick Mooney * yet. Instead of cleaning it here
346086d9687SPatrick Mooney * implicitly, we return POLLNVAL. This
347086d9687SPatrick Mooney * is consistent with poll(2) polling a
348086d9687SPatrick Mooney * closed fd. Hope this will remind
349086d9687SPatrick Mooney * user to do a POLLREMOVE.
350086d9687SPatrick Mooney */
351086d9687SPatrick Mooney pfdp[fdcnt].fd = fd;
352086d9687SPatrick Mooney pfdp[fdcnt].revents = POLLNVAL;
353086d9687SPatrick Mooney fdcnt++;
354a5eb7107SBryan Cantrill }
355086d9687SPatrick Mooney continue;
3567c478bd9Sstevel@tonic-gate }
357a5eb7107SBryan Cantrill
358086d9687SPatrick Mooney /*
359086d9687SPatrick Mooney * Detect a change to the resource underlying a cached
360086d9687SPatrick Mooney * file descriptor. While the fd generation comparison
361086d9687SPatrick Mooney * will catch nearly all cases, the file_t comparison
362086d9687SPatrick Mooney * is maintained as a failsafe as well.
363086d9687SPatrick Mooney */
364086d9687SPatrick Mooney if (gen != pdp->pd_gen || fp != pdp->pd_fp) {
3657c478bd9Sstevel@tonic-gate /*
36680d5689fSPatrick Mooney * The user is polling on a cached fd which was
36780d5689fSPatrick Mooney * closed and then reused. Unfortunately there
36880d5689fSPatrick Mooney * is no good way to communicate this fact to
36980d5689fSPatrick Mooney * the consumer.
37080d5689fSPatrick Mooney *
37180d5689fSPatrick Mooney * When this situation has been detected, it's
37280d5689fSPatrick Mooney * likely that any existing pollhead is
37380d5689fSPatrick Mooney * ill-suited to perform proper wake-ups.
37480d5689fSPatrick Mooney *
37580d5689fSPatrick Mooney * Clean up the old entry under the expectation
37680d5689fSPatrick Mooney * that a valid one will be provided as part of
37780d5689fSPatrick Mooney * the later VOP_POLL.
37880d5689fSPatrick Mooney */
379*2c76d751SPatrick Mooney polldat_disassociate(pdp);
380086d9687SPatrick Mooney
381086d9687SPatrick Mooney /*
382086d9687SPatrick Mooney * Since epoll is expected to act on the
383086d9687SPatrick Mooney * underlying 'struct file' (in Linux terms,
384086d9687SPatrick Mooney * our vnode_t would be a closer analog) rather
385086d9687SPatrick Mooney * than the fd itself, an implicit remove
386086d9687SPatrick Mooney * is necessary under these circumstances to
387086d9687SPatrick Mooney * suppress any results (or errors) from the
388086d9687SPatrick Mooney * new resource occupying the fd.
389086d9687SPatrick Mooney */
390086d9687SPatrick Mooney if (is_epoll) {
391086d9687SPatrick Mooney pdp->pd_fp = NULL;
392086d9687SPatrick Mooney pdp->pd_events = 0;
393086d9687SPatrick Mooney BT_CLEAR(pcp->pc_bitmap, fd);
394086d9687SPatrick Mooney releasef(fd);
395086d9687SPatrick Mooney continue;
396086d9687SPatrick Mooney } else {
397086d9687SPatrick Mooney /*
398086d9687SPatrick Mooney * Regular /dev/poll is unbothered
399086d9687SPatrick Mooney * about the fd reassignment.
400086d9687SPatrick Mooney */
401086d9687SPatrick Mooney pdp->pd_fp = fp;
402086d9687SPatrick Mooney pdp->pd_gen = gen;
403086d9687SPatrick Mooney }
4047c478bd9Sstevel@tonic-gate }
40595e434b5SPatrick Mooney
40695e434b5SPatrick Mooney /*
40795e434b5SPatrick Mooney * Skip entries marked with the sentinal value for
40895e434b5SPatrick Mooney * having already fired under oneshot conditions.
40995e434b5SPatrick Mooney */
41095e434b5SPatrick Mooney if (pdp->pd_events == POLLONESHOT) {
41195e434b5SPatrick Mooney releasef(fd);
41295e434b5SPatrick Mooney BT_CLEAR(pcp->pc_bitmap, fd);
41395e434b5SPatrick Mooney continue;
41495e434b5SPatrick Mooney }
41595e434b5SPatrick Mooney
4167c478bd9Sstevel@tonic-gate /*
4177c478bd9Sstevel@tonic-gate * XXX - pollrelock() logic needs to know which
4187c478bd9Sstevel@tonic-gate * which pollcache lock to grab. It'd be a
4197c478bd9Sstevel@tonic-gate * cleaner solution if we could pass pcp as
4207c478bd9Sstevel@tonic-gate * an arguement in VOP_POLL interface instead
4217c478bd9Sstevel@tonic-gate * of implicitly passing it using thread_t
4227c478bd9Sstevel@tonic-gate * struct. On the other hand, changing VOP_POLL
4237c478bd9Sstevel@tonic-gate * interface will require all driver/file system
4247c478bd9Sstevel@tonic-gate * poll routine to change. May want to revisit
4257c478bd9Sstevel@tonic-gate * the tradeoff later.
4267c478bd9Sstevel@tonic-gate */
4277c478bd9Sstevel@tonic-gate curthread->t_pollcache = pcp;
4287c478bd9Sstevel@tonic-gate error = VOP_POLL(fp->f_vnode, pdp->pd_events, 0,
429da6c28aaSamw &revent, &php, NULL);
43080d5689fSPatrick Mooney
43180d5689fSPatrick Mooney /*
43280d5689fSPatrick Mooney * Recheck edge-triggered descriptors which lack a
43380d5689fSPatrick Mooney * pollhead. While this check is performed when an fd
43480d5689fSPatrick Mooney * is added to the pollcache in dpwrite(), subsequent
43580d5689fSPatrick Mooney * descriptor manipulation could cause a different
43680d5689fSPatrick Mooney * resource to be present now.
43780d5689fSPatrick Mooney */
43880d5689fSPatrick Mooney if ((pdp->pd_events & POLLET) && error == 0 &&
43980d5689fSPatrick Mooney pdp->pd_php == NULL && php == NULL && revent != 0) {
44080d5689fSPatrick Mooney short levent = 0;
44180d5689fSPatrick Mooney
44280d5689fSPatrick Mooney /*
44380d5689fSPatrick Mooney * The same POLLET-only VOP_POLL is used in an
44480d5689fSPatrick Mooney * attempt to coax a pollhead from older
44580d5689fSPatrick Mooney * driver logic.
44680d5689fSPatrick Mooney */
44780d5689fSPatrick Mooney error = VOP_POLL(fp->f_vnode, POLLET,
44880d5689fSPatrick Mooney 0, &levent, &php, NULL);
44980d5689fSPatrick Mooney }
45080d5689fSPatrick Mooney
4517c478bd9Sstevel@tonic-gate curthread->t_pollcache = NULL;
4527c478bd9Sstevel@tonic-gate releasef(fd);
4537c478bd9Sstevel@tonic-gate if (error != 0) {
4547c478bd9Sstevel@tonic-gate break;
4557c478bd9Sstevel@tonic-gate }
456f3bb54f3SPatrick Mooney
4577c478bd9Sstevel@tonic-gate /*
4587c478bd9Sstevel@tonic-gate * layered devices (e.g. console driver)
4597c478bd9Sstevel@tonic-gate * may change the vnode and thus the pollhead
4607c478bd9Sstevel@tonic-gate * pointer out from underneath us.
4617c478bd9Sstevel@tonic-gate */
4627c478bd9Sstevel@tonic-gate if (php != NULL && pdp->pd_php != NULL &&
4637c478bd9Sstevel@tonic-gate php != pdp->pd_php) {
464*2c76d751SPatrick Mooney polldat_disassociate(pdp);
465*2c76d751SPatrick Mooney polldat_associate(pdp, php);
4667c478bd9Sstevel@tonic-gate /*
4677c478bd9Sstevel@tonic-gate * The bit should still be set.
4687c478bd9Sstevel@tonic-gate */
4697c478bd9Sstevel@tonic-gate ASSERT(BT_TEST(pcp->pc_bitmap, fd));
4707c478bd9Sstevel@tonic-gate goto retry;
4717c478bd9Sstevel@tonic-gate }
4727c478bd9Sstevel@tonic-gate
4737c478bd9Sstevel@tonic-gate if (revent != 0) {
474a5eb7107SBryan Cantrill if (pfdp != NULL) {
475a5eb7107SBryan Cantrill pfdp[fdcnt].fd = fd;
476a5eb7107SBryan Cantrill pfdp[fdcnt].events = pdp->pd_events;
477a5eb7107SBryan Cantrill pfdp[fdcnt].revents = revent;
478f3bb54f3SPatrick Mooney } else if (epoll != NULL) {
479a5eb7107SBryan Cantrill epoll_event_t *ep = &epoll[fdcnt];
480a5eb7107SBryan Cantrill
481a5eb7107SBryan Cantrill ASSERT(epoll != NULL);
482a5eb7107SBryan Cantrill ep->data.u64 = pdp->pd_epolldata;
483a5eb7107SBryan Cantrill
48480d5689fSPatrick Mooney /*
48580d5689fSPatrick Mooney * Since POLLNVAL is a legal event for
48680d5689fSPatrick Mooney * VOP_POLL handlers to emit, it must
48780d5689fSPatrick Mooney * be translated epoll-legal.
48880d5689fSPatrick Mooney */
48980d5689fSPatrick Mooney if (revent & POLLNVAL) {
49080d5689fSPatrick Mooney revent &= ~POLLNVAL;
49180d5689fSPatrick Mooney revent |= POLLERR;
49280d5689fSPatrick Mooney }
49380d5689fSPatrick Mooney
494a5eb7107SBryan Cantrill /*
495a5eb7107SBryan Cantrill * If any of the event bits are set for
496a5eb7107SBryan Cantrill * which poll and epoll representations
497a5eb7107SBryan Cantrill * differ, swizzle in the native epoll
498a5eb7107SBryan Cantrill * values.
499a5eb7107SBryan Cantrill */
500a5eb7107SBryan Cantrill if (revent & mask) {
501a5eb7107SBryan Cantrill ep->events = (revent & ~mask) |
502a5eb7107SBryan Cantrill ((revent & POLLRDHUP) ?
503a5eb7107SBryan Cantrill EPOLLRDHUP : 0) |
504a5eb7107SBryan Cantrill ((revent & POLLWRBAND) ?
505a5eb7107SBryan Cantrill EPOLLWRBAND : 0);
506a5eb7107SBryan Cantrill } else {
507a5eb7107SBryan Cantrill ep->events = revent;
508a5eb7107SBryan Cantrill }
509a5eb7107SBryan Cantrill
510a5eb7107SBryan Cantrill /*
511a5eb7107SBryan Cantrill * We define POLLWRNORM to be POLLOUT,
512a5eb7107SBryan Cantrill * but epoll has separate definitions
513a5eb7107SBryan Cantrill * for them; if POLLOUT is set and the
514a5eb7107SBryan Cantrill * user has asked for EPOLLWRNORM, set
515a5eb7107SBryan Cantrill * that as well.
516a5eb7107SBryan Cantrill */
517a5eb7107SBryan Cantrill if ((revent & POLLOUT) &&
518a5eb7107SBryan Cantrill (pdp->pd_events & EPOLLWRNORM)) {
519a5eb7107SBryan Cantrill ep->events |= EPOLLWRNORM;
520a5eb7107SBryan Cantrill }
521f3bb54f3SPatrick Mooney } else {
522f3bb54f3SPatrick Mooney pollstate_t *ps =
523f3bb54f3SPatrick Mooney curthread->t_pollstate;
524f3bb54f3SPatrick Mooney /*
525f3bb54f3SPatrick Mooney * The devpoll handle itself is being
526f3bb54f3SPatrick Mooney * polled. Notify the caller of any
527f3bb54f3SPatrick Mooney * readable event(s), leaving as much
528f3bb54f3SPatrick Mooney * state as possible untouched.
529f3bb54f3SPatrick Mooney */
530f3bb54f3SPatrick Mooney VERIFY(fdcnt == 0);
531f3bb54f3SPatrick Mooney VERIFY(ps != NULL);
532f3bb54f3SPatrick Mooney
533f3bb54f3SPatrick Mooney /*
534f3bb54f3SPatrick Mooney * If a call to pollunlock() fails
535f3bb54f3SPatrick Mooney * during VOP_POLL, skip over the fd
536f3bb54f3SPatrick Mooney * and continue polling.
537f3bb54f3SPatrick Mooney *
538f3bb54f3SPatrick Mooney * Otherwise, report that there is an
539f3bb54f3SPatrick Mooney * event pending.
540f3bb54f3SPatrick Mooney */
541f3bb54f3SPatrick Mooney if ((ps->ps_flags & POLLSTATE_ULFAIL)
542f3bb54f3SPatrick Mooney != 0) {
543f3bb54f3SPatrick Mooney ps->ps_flags &=
544f3bb54f3SPatrick Mooney ~POLLSTATE_ULFAIL;
545f3bb54f3SPatrick Mooney continue;
546f3bb54f3SPatrick Mooney } else {
547f3bb54f3SPatrick Mooney fdcnt++;
548f3bb54f3SPatrick Mooney break;
549f3bb54f3SPatrick Mooney }
550a5eb7107SBryan Cantrill }
551a5eb7107SBryan Cantrill
55280d5689fSPatrick Mooney /* Handle special polling modes. */
553a5eb7107SBryan Cantrill if (pdp->pd_events & POLLONESHOT) {
55480d5689fSPatrick Mooney /*
55595e434b5SPatrick Mooney * Entries operating under POLLONESHOT
55695e434b5SPatrick Mooney * will be marked with a sentinel value
55795e434b5SPatrick Mooney * to indicate that they have "fired"
55895e434b5SPatrick Mooney * when emitting an event. This will
55995e434b5SPatrick Mooney * disable them from polling until a
56095e434b5SPatrick Mooney * later add/modify event rearms them.
56180d5689fSPatrick Mooney */
56295e434b5SPatrick Mooney pdp->pd_events = POLLONESHOT;
563*2c76d751SPatrick Mooney polldat_disassociate(pdp);
564a5eb7107SBryan Cantrill BT_CLEAR(pcp->pc_bitmap, fd);
56580d5689fSPatrick Mooney } else if (pdp->pd_events & POLLET) {
56680d5689fSPatrick Mooney /*
56780d5689fSPatrick Mooney * Wire up the pollhead which should
56880d5689fSPatrick Mooney * have been provided. Edge-triggered
56980d5689fSPatrick Mooney * polling cannot function properly
57080d5689fSPatrick Mooney * with drivers which do not emit one.
57180d5689fSPatrick Mooney */
57280d5689fSPatrick Mooney if (php != NULL &&
57380d5689fSPatrick Mooney pdp->pd_php == NULL) {
574*2c76d751SPatrick Mooney polldat_associate(pdp, php);
57580d5689fSPatrick Mooney }
57680d5689fSPatrick Mooney
57780d5689fSPatrick Mooney /*
57880d5689fSPatrick Mooney * If the driver has emitted a pollhead,
57980d5689fSPatrick Mooney * clear the bit in the bitmap which
58080d5689fSPatrick Mooney * effectively latches the edge on a
58180d5689fSPatrick Mooney * pollwakeup() from the driver.
58280d5689fSPatrick Mooney */
58380d5689fSPatrick Mooney if (pdp->pd_php != NULL) {
58480d5689fSPatrick Mooney BT_CLEAR(pcp->pc_bitmap, fd);
58580d5689fSPatrick Mooney }
586a5eb7107SBryan Cantrill }
587a5eb7107SBryan Cantrill
5887c478bd9Sstevel@tonic-gate fdcnt++;
5897c478bd9Sstevel@tonic-gate } else if (php != NULL) {
5907c478bd9Sstevel@tonic-gate /*
5917c478bd9Sstevel@tonic-gate * We clear a bit or cache a poll fd if
5927c478bd9Sstevel@tonic-gate * the driver returns a poll head ptr,
5937c478bd9Sstevel@tonic-gate * which is expected in the case of 0
5947c478bd9Sstevel@tonic-gate * revents. Some buggy driver may return
5957c478bd9Sstevel@tonic-gate * NULL php pointer with 0 revents. In
5967c478bd9Sstevel@tonic-gate * this case, we just treat the driver as
5977c478bd9Sstevel@tonic-gate * "noncachable" and not clearing the bit
5987c478bd9Sstevel@tonic-gate * in bitmap.
5997c478bd9Sstevel@tonic-gate */
6007c478bd9Sstevel@tonic-gate if ((pdp->pd_php != NULL) &&
601a5eb7107SBryan Cantrill ((pcp->pc_flag & PC_POLLWAKE) == 0)) {
6027c478bd9Sstevel@tonic-gate BT_CLEAR(pcp->pc_bitmap, fd);
6037c478bd9Sstevel@tonic-gate }
6047c478bd9Sstevel@tonic-gate if (pdp->pd_php == NULL) {
605*2c76d751SPatrick Mooney polldat_associate(pdp, php);
606a85084caSmeem /*
607a85084caSmeem * An event of interest may have
608a85084caSmeem * arrived between the VOP_POLL() and
609*2c76d751SPatrick Mooney * the polldat_associate(), so we
610*2c76d751SPatrick Mooney * must check again.
611a85084caSmeem */
612a85084caSmeem goto repoll;
6137c478bd9Sstevel@tonic-gate }
6147c478bd9Sstevel@tonic-gate }
6157c478bd9Sstevel@tonic-gate } else {
6167c478bd9Sstevel@tonic-gate /*
6177c478bd9Sstevel@tonic-gate * No bit set in the range. Check for wrap around.
6187c478bd9Sstevel@tonic-gate */
6197c478bd9Sstevel@tonic-gate if (!no_wrap) {
6207c478bd9Sstevel@tonic-gate start = 0;
6217c478bd9Sstevel@tonic-gate end = ostart - 1;
6227c478bd9Sstevel@tonic-gate no_wrap = B_TRUE;
6237c478bd9Sstevel@tonic-gate } else {
6247c478bd9Sstevel@tonic-gate done = B_TRUE;
6257c478bd9Sstevel@tonic-gate }
6267c478bd9Sstevel@tonic-gate }
6277c478bd9Sstevel@tonic-gate }
6287c478bd9Sstevel@tonic-gate
6297c478bd9Sstevel@tonic-gate if (!done) {
6307c478bd9Sstevel@tonic-gate pcp->pc_mapstart = start;
6317c478bd9Sstevel@tonic-gate }
6327c478bd9Sstevel@tonic-gate ASSERT(*fdcntp == 0);
6337c478bd9Sstevel@tonic-gate *fdcntp = fdcnt;
6347c478bd9Sstevel@tonic-gate return (error);
6357c478bd9Sstevel@tonic-gate }
6367c478bd9Sstevel@tonic-gate
6377c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6387c478bd9Sstevel@tonic-gate static int
dpopen(dev_t * devp,int flag,int otyp,cred_t * credp)6397c478bd9Sstevel@tonic-gate dpopen(dev_t *devp, int flag, int otyp, cred_t *credp)
6407c478bd9Sstevel@tonic-gate {
6417c478bd9Sstevel@tonic-gate minor_t minordev;
6427c478bd9Sstevel@tonic-gate dp_entry_t *dpep;
6437c478bd9Sstevel@tonic-gate pollcache_t *pcp;
6447c478bd9Sstevel@tonic-gate
6457c478bd9Sstevel@tonic-gate ASSERT(devpoll_init);
6467c478bd9Sstevel@tonic-gate ASSERT(dptblsize <= MAXMIN);
6477c478bd9Sstevel@tonic-gate mutex_enter(&devpoll_lock);
6487c478bd9Sstevel@tonic-gate for (minordev = 0; minordev < dptblsize; minordev++) {
6497c478bd9Sstevel@tonic-gate if (devpolltbl[minordev] == NULL) {
6507c478bd9Sstevel@tonic-gate devpolltbl[minordev] = (dp_entry_t *)RESERVED;
6517c478bd9Sstevel@tonic-gate break;
6527c478bd9Sstevel@tonic-gate }
6537c478bd9Sstevel@tonic-gate }
6547c478bd9Sstevel@tonic-gate if (minordev == dptblsize) {
6557c478bd9Sstevel@tonic-gate dp_entry_t **newtbl;
6567c478bd9Sstevel@tonic-gate size_t oldsize;
6577c478bd9Sstevel@tonic-gate
6587c478bd9Sstevel@tonic-gate /*
6597c478bd9Sstevel@tonic-gate * Used up every entry in the existing devpoll table.
6607c478bd9Sstevel@tonic-gate * Grow the table by DEVPOLLSIZE.
6617c478bd9Sstevel@tonic-gate */
6627c478bd9Sstevel@tonic-gate if ((oldsize = dptblsize) >= MAXMIN) {
6637c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
6647c478bd9Sstevel@tonic-gate return (ENXIO);
6657c478bd9Sstevel@tonic-gate }
6667c478bd9Sstevel@tonic-gate dptblsize += DEVPOLLSIZE;
6677c478bd9Sstevel@tonic-gate if (dptblsize > MAXMIN) {
6687c478bd9Sstevel@tonic-gate dptblsize = MAXMIN;
6697c478bd9Sstevel@tonic-gate }
6707c478bd9Sstevel@tonic-gate newtbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP);
6717c478bd9Sstevel@tonic-gate bcopy(devpolltbl, newtbl, sizeof (caddr_t) * oldsize);
6727c478bd9Sstevel@tonic-gate kmem_free(devpolltbl, sizeof (caddr_t) * oldsize);
6737c478bd9Sstevel@tonic-gate devpolltbl = newtbl;
6747c478bd9Sstevel@tonic-gate devpolltbl[minordev] = (dp_entry_t *)RESERVED;
6757c478bd9Sstevel@tonic-gate }
6767c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
6777c478bd9Sstevel@tonic-gate
6787c478bd9Sstevel@tonic-gate dpep = kmem_zalloc(sizeof (dp_entry_t), KM_SLEEP);
6797c478bd9Sstevel@tonic-gate /*
6807c478bd9Sstevel@tonic-gate * allocate a pollcache skeleton here. Delay allocating bitmap
6817c478bd9Sstevel@tonic-gate * structures until dpwrite() time, since we don't know the
682a5eb7107SBryan Cantrill * optimal size yet. We also delay setting the pid until either
683a5eb7107SBryan Cantrill * dpwrite() or attempt to poll on the instance, allowing parents
684a5eb7107SBryan Cantrill * to create instances of /dev/poll for their children. (In the
685a5eb7107SBryan Cantrill * epoll compatibility case, this check isn't performed to maintain
686a5eb7107SBryan Cantrill * semantic compatibility.)
6877c478bd9Sstevel@tonic-gate */
6887c478bd9Sstevel@tonic-gate pcp = pcache_alloc();
6897c478bd9Sstevel@tonic-gate dpep->dpe_pcache = pcp;
690a5eb7107SBryan Cantrill pcp->pc_pid = -1;
6917c478bd9Sstevel@tonic-gate *devp = makedevice(getmajor(*devp), minordev); /* clone the driver */
6927c478bd9Sstevel@tonic-gate mutex_enter(&devpoll_lock);
6937c478bd9Sstevel@tonic-gate ASSERT(minordev < dptblsize);
6947c478bd9Sstevel@tonic-gate ASSERT(devpolltbl[minordev] == (dp_entry_t *)RESERVED);
6957c478bd9Sstevel@tonic-gate devpolltbl[minordev] = dpep;
6967c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
6977c478bd9Sstevel@tonic-gate return (0);
6987c478bd9Sstevel@tonic-gate }
6997c478bd9Sstevel@tonic-gate
7007c478bd9Sstevel@tonic-gate /*
7017c478bd9Sstevel@tonic-gate * Write to dev/poll add/remove fd's to/from a cached poll fd set,
7027c478bd9Sstevel@tonic-gate * or change poll events for a watched fd.
7037c478bd9Sstevel@tonic-gate */
7047c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7057c478bd9Sstevel@tonic-gate static int
dpwrite(dev_t dev,struct uio * uiop,cred_t * credp)7067c478bd9Sstevel@tonic-gate dpwrite(dev_t dev, struct uio *uiop, cred_t *credp)
7077c478bd9Sstevel@tonic-gate {
708fe234e7cSMatt Amdur minor_t minor;
7097c478bd9Sstevel@tonic-gate dp_entry_t *dpep;
7107c478bd9Sstevel@tonic-gate pollcache_t *pcp;
7117c478bd9Sstevel@tonic-gate pollfd_t *pollfdp, *pfdp;
712a5eb7107SBryan Cantrill dvpoll_epollfd_t *epfdp;
713a5eb7107SBryan Cantrill uintptr_t limit;
714086d9687SPatrick Mooney int error;
715086d9687SPatrick Mooney uint_t size;
716086d9687SPatrick Mooney size_t copysize, uiosize;
7177c478bd9Sstevel@tonic-gate nfds_t pollfdnum;
718f3bb54f3SPatrick Mooney boolean_t is_epoll, fds_added = B_FALSE;
7197c478bd9Sstevel@tonic-gate
7207c478bd9Sstevel@tonic-gate minor = getminor(dev);
7217c478bd9Sstevel@tonic-gate
7227c478bd9Sstevel@tonic-gate mutex_enter(&devpoll_lock);
7237c478bd9Sstevel@tonic-gate ASSERT(minor < dptblsize);
7247c478bd9Sstevel@tonic-gate dpep = devpolltbl[minor];
7257c478bd9Sstevel@tonic-gate ASSERT(dpep != NULL);
7267c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
727f3bb54f3SPatrick Mooney
728f3bb54f3SPatrick Mooney mutex_enter(&dpep->dpe_lock);
7297c478bd9Sstevel@tonic-gate pcp = dpep->dpe_pcache;
730f3bb54f3SPatrick Mooney is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
731f3bb54f3SPatrick Mooney size = (is_epoll) ? sizeof (dvpoll_epollfd_t) : sizeof (pollfd_t);
732f3bb54f3SPatrick Mooney mutex_exit(&dpep->dpe_lock);
733a5eb7107SBryan Cantrill
734f3bb54f3SPatrick Mooney if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
735f3bb54f3SPatrick Mooney if (pcp->pc_pid != -1) {
736a5eb7107SBryan Cantrill return (EACCES);
737f3bb54f3SPatrick Mooney }
738a5eb7107SBryan Cantrill
739a5eb7107SBryan Cantrill pcp->pc_pid = curproc->p_pid;
7407c478bd9Sstevel@tonic-gate }
741a5eb7107SBryan Cantrill
742086d9687SPatrick Mooney if (uiop->uio_resid < 0) {
743086d9687SPatrick Mooney /* No one else is this careful, but maybe they should be. */
744086d9687SPatrick Mooney return (EINVAL);
745086d9687SPatrick Mooney }
746086d9687SPatrick Mooney
747086d9687SPatrick Mooney uiosize = (size_t)uiop->uio_resid;
748a5eb7107SBryan Cantrill pollfdnum = uiosize / size;
749158dfbe4SBryan Cantrill
7503c2d4f39SPatrick Mooney /*
7513c2d4f39SPatrick Mooney * For epoll-enabled handles, restrict the allowed write size to 2.
7523c2d4f39SPatrick Mooney * This corresponds to an epoll_ctl(3C) performing an EPOLL_CTL_MOD
7533c2d4f39SPatrick Mooney * operation which is expanded into two operations (DEL and ADD).
7543c2d4f39SPatrick Mooney *
7553c2d4f39SPatrick Mooney * All other operations performed through epoll_ctl(3C) will consist of
7563c2d4f39SPatrick Mooney * a single entry.
7573c2d4f39SPatrick Mooney */
7583c2d4f39SPatrick Mooney if (is_epoll && pollfdnum > 2) {
7593c2d4f39SPatrick Mooney return (EINVAL);
7603c2d4f39SPatrick Mooney }
7613c2d4f39SPatrick Mooney
762158dfbe4SBryan Cantrill /*
763158dfbe4SBryan Cantrill * We want to make sure that pollfdnum isn't large enough to DoS us,
764158dfbe4SBryan Cantrill * but we also don't want to grab p_lock unnecessarily -- so we
765158dfbe4SBryan Cantrill * perform the full check against our resource limits if and only if
766158dfbe4SBryan Cantrill * pollfdnum is larger than the known-to-be-sane value of UINT8_MAX.
767158dfbe4SBryan Cantrill */
768158dfbe4SBryan Cantrill if (pollfdnum > UINT8_MAX) {
769158dfbe4SBryan Cantrill mutex_enter(&curproc->p_lock);
770158dfbe4SBryan Cantrill if (pollfdnum >
771158dfbe4SBryan Cantrill (uint_t)rctl_enforced_value(rctlproc_legacy[RLIMIT_NOFILE],
772158dfbe4SBryan Cantrill curproc->p_rctls, curproc)) {
773158dfbe4SBryan Cantrill (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
774158dfbe4SBryan Cantrill curproc->p_rctls, curproc, RCA_SAFE);
775158dfbe4SBryan Cantrill mutex_exit(&curproc->p_lock);
776158dfbe4SBryan Cantrill return (EINVAL);
777158dfbe4SBryan Cantrill }
7787c478bd9Sstevel@tonic-gate mutex_exit(&curproc->p_lock);
7797c478bd9Sstevel@tonic-gate }
780158dfbe4SBryan Cantrill
7817c478bd9Sstevel@tonic-gate /*
7827c478bd9Sstevel@tonic-gate * Copy in the pollfd array. Walk through the array and add
7837c478bd9Sstevel@tonic-gate * each polled fd to the cached set.
7847c478bd9Sstevel@tonic-gate */
7857c478bd9Sstevel@tonic-gate pollfdp = kmem_alloc(uiosize, KM_SLEEP);
786a5eb7107SBryan Cantrill limit = (uintptr_t)pollfdp + (pollfdnum * size);
7877c478bd9Sstevel@tonic-gate
7887c478bd9Sstevel@tonic-gate /*
7897c478bd9Sstevel@tonic-gate * Although /dev/poll uses the write(2) interface to cache fds, it's
7907c478bd9Sstevel@tonic-gate * not supposed to function as a seekable device. To prevent offset
7917c478bd9Sstevel@tonic-gate * from growing and eventually exceed the maximum, reset the offset
7927c478bd9Sstevel@tonic-gate * here for every call.
7937c478bd9Sstevel@tonic-gate */
7947c478bd9Sstevel@tonic-gate uiop->uio_loffset = 0;
79557a0264bSPatrick Mooney
79657a0264bSPatrick Mooney /*
79757a0264bSPatrick Mooney * Use uiocopy instead of uiomove when populating pollfdp, keeping
79857a0264bSPatrick Mooney * uio_resid untouched for now. Write syscalls will translate EINTR
79957a0264bSPatrick Mooney * into a success if they detect "successfully transfered" data via an
80057a0264bSPatrick Mooney * updated uio_resid. Falsely suppressing such errors is disastrous.
80157a0264bSPatrick Mooney */
80257a0264bSPatrick Mooney if ((error = uiocopy((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop,
80357a0264bSPatrick Mooney ©size)) != 0) {
8047c478bd9Sstevel@tonic-gate kmem_free(pollfdp, uiosize);
8057c478bd9Sstevel@tonic-gate return (error);
8067c478bd9Sstevel@tonic-gate }
80757a0264bSPatrick Mooney
8087c478bd9Sstevel@tonic-gate /*
8097c478bd9Sstevel@tonic-gate * We are about to enter the core portion of dpwrite(). Make sure this
8107c478bd9Sstevel@tonic-gate * write has exclusive access in this portion of the code, i.e., no
811f3bb54f3SPatrick Mooney * other writers in this code.
812f3bb54f3SPatrick Mooney *
813f3bb54f3SPatrick Mooney * Waiting for all readers to drop their references to the dpe is
814f3bb54f3SPatrick Mooney * unecessary since the pollcache itself is protected by pc_lock.
8157c478bd9Sstevel@tonic-gate */
8167c478bd9Sstevel@tonic-gate mutex_enter(&dpep->dpe_lock);
8177c478bd9Sstevel@tonic-gate dpep->dpe_writerwait++;
818f3bb54f3SPatrick Mooney while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0) {
819f3bb54f3SPatrick Mooney ASSERT(dpep->dpe_refcnt != 0);
820a5eb7107SBryan Cantrill
8213c2d4f39SPatrick Mooney /*
8223c2d4f39SPatrick Mooney * The epoll API does not allow EINTR as a result when making
8233c2d4f39SPatrick Mooney * modifications to the set of polled fds. Given that write
8243c2d4f39SPatrick Mooney * activity is relatively quick and the size of accepted writes
8253c2d4f39SPatrick Mooney * is limited above to two entries, a signal-ignorant wait is
8263c2d4f39SPatrick Mooney * used here to avoid the EINTR.
8273c2d4f39SPatrick Mooney */
8283c2d4f39SPatrick Mooney if (is_epoll) {
8293c2d4f39SPatrick Mooney cv_wait(&dpep->dpe_cv, &dpep->dpe_lock);
8303c2d4f39SPatrick Mooney continue;
8313c2d4f39SPatrick Mooney }
8323c2d4f39SPatrick Mooney
8333c2d4f39SPatrick Mooney /*
8343c2d4f39SPatrick Mooney * Non-epoll writers to /dev/poll handles can tolerate EINTR.
8353c2d4f39SPatrick Mooney */
8367c478bd9Sstevel@tonic-gate if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
8377c478bd9Sstevel@tonic-gate dpep->dpe_writerwait--;
8387c478bd9Sstevel@tonic-gate mutex_exit(&dpep->dpe_lock);
8397c478bd9Sstevel@tonic-gate kmem_free(pollfdp, uiosize);
840f3bb54f3SPatrick Mooney return (EINTR);
8417c478bd9Sstevel@tonic-gate }
8427c478bd9Sstevel@tonic-gate }
8437c478bd9Sstevel@tonic-gate dpep->dpe_writerwait--;
8447c478bd9Sstevel@tonic-gate dpep->dpe_flag |= DP_WRITER_PRESENT;
8457c478bd9Sstevel@tonic-gate dpep->dpe_refcnt++;
846a5eb7107SBryan Cantrill
847f3bb54f3SPatrick Mooney if (!is_epoll && (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0) {
848f3bb54f3SPatrick Mooney /*
849f3bb54f3SPatrick Mooney * The epoll compat mode was enabled while we were waiting to
850f3bb54f3SPatrick Mooney * establish write access. It is not safe to continue since
851f3bb54f3SPatrick Mooney * state was prepared for non-epoll operation.
852f3bb54f3SPatrick Mooney */
853f3bb54f3SPatrick Mooney error = EBUSY;
854f3bb54f3SPatrick Mooney goto bypass;
855f3bb54f3SPatrick Mooney }
8567c478bd9Sstevel@tonic-gate mutex_exit(&dpep->dpe_lock);
8577c478bd9Sstevel@tonic-gate
858f3bb54f3SPatrick Mooney /*
859f3bb54f3SPatrick Mooney * Since the dpwrite() may recursively walk an added /dev/poll handle,
860f3bb54f3SPatrick Mooney * pollstate_enter() deadlock and loop detection must be used.
861f3bb54f3SPatrick Mooney */
862f3bb54f3SPatrick Mooney (void) pollstate_create();
863f3bb54f3SPatrick Mooney VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
864a5eb7107SBryan Cantrill
8657c478bd9Sstevel@tonic-gate if (pcp->pc_bitmap == NULL) {
8667c478bd9Sstevel@tonic-gate pcache_create(pcp, pollfdnum);
8677c478bd9Sstevel@tonic-gate }
868a5eb7107SBryan Cantrill for (pfdp = pollfdp; (uintptr_t)pfdp < limit;
869a5eb7107SBryan Cantrill pfdp = (pollfd_t *)((uintptr_t)pfdp + size)) {
870086d9687SPatrick Mooney int fd = pfdp->fd;
871086d9687SPatrick Mooney polldat_t *pdp;
872086d9687SPatrick Mooney
873a5eb7107SBryan Cantrill if ((uint_t)fd >= P_FINFO(curproc)->fi_nfiles) {
874a5eb7107SBryan Cantrill /*
875a5eb7107SBryan Cantrill * epoll semantics demand that we return EBADF if our
876a5eb7107SBryan Cantrill * specified fd is invalid.
877a5eb7107SBryan Cantrill */
878f3bb54f3SPatrick Mooney if (is_epoll) {
879a5eb7107SBryan Cantrill error = EBADF;
880a5eb7107SBryan Cantrill break;
881a5eb7107SBryan Cantrill }
882a5eb7107SBryan Cantrill
8837c478bd9Sstevel@tonic-gate continue;
884a5eb7107SBryan Cantrill }
885a5eb7107SBryan Cantrill
8867c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, fd);
8877c478bd9Sstevel@tonic-gate if (pfdp->events != POLLREMOVE) {
888086d9687SPatrick Mooney uf_entry_gen_t gen;
889086d9687SPatrick Mooney file_t *fp = NULL;
890086d9687SPatrick Mooney struct pollhead *php = NULL;
891a5eb7107SBryan Cantrill
892086d9687SPatrick Mooney /*
893086d9687SPatrick Mooney * If we're in epoll compatibility mode, check that the
894086d9687SPatrick Mooney * fd is valid before allocating anything for it; epoll
895086d9687SPatrick Mooney * semantics demand that we return EBADF if our
896086d9687SPatrick Mooney * specified fd is invalid.
897086d9687SPatrick Mooney */
898086d9687SPatrick Mooney if (is_epoll) {
899086d9687SPatrick Mooney if ((fp = getf_gen(fd, &gen)) == NULL) {
900086d9687SPatrick Mooney error = EBADF;
901086d9687SPatrick Mooney break;
902a5eb7107SBryan Cantrill }
903086d9687SPatrick Mooney }
904086d9687SPatrick Mooney if (pdp == NULL) {
9057c478bd9Sstevel@tonic-gate pdp = pcache_alloc_fd(0);
9067c478bd9Sstevel@tonic-gate pdp->pd_fd = fd;
9077c478bd9Sstevel@tonic-gate pdp->pd_pcache = pcp;
9087c478bd9Sstevel@tonic-gate pcache_insert_fd(pcp, pdp, pollfdnum);
909086d9687SPatrick Mooney }
910086d9687SPatrick Mooney
911086d9687SPatrick Mooney if (is_epoll) {
912a5eb7107SBryan Cantrill /*
913086d9687SPatrick Mooney * If the fd is already a member of the epoll
914086d9687SPatrick Mooney * set, error emission is needed only when the
915086d9687SPatrick Mooney * fd assignment generation matches the one
916086d9687SPatrick Mooney * recorded in the polldat_t. Absence of such
917086d9687SPatrick Mooney * a generation match indicates that a new
918086d9687SPatrick Mooney * resource has been assigned at that fd.
919086d9687SPatrick Mooney *
920086d9687SPatrick Mooney * Caveat: It is possible to force a generation
921086d9687SPatrick Mooney * update while keeping the same backing
922086d9687SPatrick Mooney * resource. This is possible via dup2, but
923086d9687SPatrick Mooney * does not represent real-world use cases,
924086d9687SPatrick Mooney * making the lack of error acceptable.
925a5eb7107SBryan Cantrill */
926086d9687SPatrick Mooney if (pdp->pd_fp != NULL && pdp->pd_gen == gen) {
927086d9687SPatrick Mooney error = EEXIST;
928086d9687SPatrick Mooney releasef(fd);
929086d9687SPatrick Mooney break;
930a5eb7107SBryan Cantrill }
931a5eb7107SBryan Cantrill
932086d9687SPatrick Mooney /*
933086d9687SPatrick Mooney * We have decided that the cached information
93495e434b5SPatrick Mooney * was stale. Reset pd_events to assure that
935086d9687SPatrick Mooney * we don't mistakenly operate on cached event
93695e434b5SPatrick Mooney * disposition. This configures the implicit
93795e434b5SPatrick Mooney * subscription to HUP and ERR events which
93895e434b5SPatrick Mooney * epoll features.
939086d9687SPatrick Mooney */
94095e434b5SPatrick Mooney pdp->pd_events = POLLERR|POLLHUP;
941086d9687SPatrick Mooney
942a5eb7107SBryan Cantrill epfdp = (dvpoll_epollfd_t *)pfdp;
943a5eb7107SBryan Cantrill pdp->pd_epolldata = epfdp->dpep_data;
944a5eb7107SBryan Cantrill }
945a5eb7107SBryan Cantrill
9467c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fd == fd);
9477c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_pcache == pcp);
9487c478bd9Sstevel@tonic-gate if (fd >= pcp->pc_mapsize) {
9497c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
9507c478bd9Sstevel@tonic-gate pcache_grow_map(pcp, fd);
9517c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
9527c478bd9Sstevel@tonic-gate }
9537c478bd9Sstevel@tonic-gate if (fd > pcp->pc_mapend) {
9547c478bd9Sstevel@tonic-gate pcp->pc_mapend = fd;
9557c478bd9Sstevel@tonic-gate }
956a5eb7107SBryan Cantrill
957086d9687SPatrick Mooney if (!is_epoll) {
958086d9687SPatrick Mooney ASSERT(fp == NULL);
959a5eb7107SBryan Cantrill
960086d9687SPatrick Mooney if ((fp = getf_gen(fd, &gen)) == NULL) {
961086d9687SPatrick Mooney /*
962086d9687SPatrick Mooney * The fd is not valid. Since we can't
963086d9687SPatrick Mooney * pass this error back in the write()
964086d9687SPatrick Mooney * call, set the bit in bitmap to force
965086d9687SPatrick Mooney * DP_POLL ioctl to examine it.
966086d9687SPatrick Mooney */
967086d9687SPatrick Mooney BT_SET(pcp->pc_bitmap, fd);
968086d9687SPatrick Mooney pdp->pd_events |= pfdp->events;
969086d9687SPatrick Mooney continue;
970086d9687SPatrick Mooney }
9717c478bd9Sstevel@tonic-gate /*
972086d9687SPatrick Mooney * Don't do VOP_POLL for an already cached fd
973086d9687SPatrick Mooney * with same poll events.
9747c478bd9Sstevel@tonic-gate */
975086d9687SPatrick Mooney if ((pdp->pd_events == pfdp->events) &&
976086d9687SPatrick Mooney (pdp->pd_fp == fp)) {
977086d9687SPatrick Mooney /*
978086d9687SPatrick Mooney * the events are already cached
979086d9687SPatrick Mooney */
980086d9687SPatrick Mooney releasef(fd);
981086d9687SPatrick Mooney continue;
982086d9687SPatrick Mooney }
9837c478bd9Sstevel@tonic-gate }
9847c478bd9Sstevel@tonic-gate
985086d9687SPatrick Mooney
9867c478bd9Sstevel@tonic-gate /*
9877c478bd9Sstevel@tonic-gate * do VOP_POLL and cache this poll fd.
9887c478bd9Sstevel@tonic-gate */
9897c478bd9Sstevel@tonic-gate /*
9907c478bd9Sstevel@tonic-gate * XXX - pollrelock() logic needs to know which
9917c478bd9Sstevel@tonic-gate * which pollcache lock to grab. It'd be a
9927c478bd9Sstevel@tonic-gate * cleaner solution if we could pass pcp as
9937c478bd9Sstevel@tonic-gate * an arguement in VOP_POLL interface instead
9947c478bd9Sstevel@tonic-gate * of implicitly passing it using thread_t
9957c478bd9Sstevel@tonic-gate * struct. On the other hand, changing VOP_POLL
9967c478bd9Sstevel@tonic-gate * interface will require all driver/file system
9977c478bd9Sstevel@tonic-gate * poll routine to change. May want to revisit
9987c478bd9Sstevel@tonic-gate * the tradeoff later.
9997c478bd9Sstevel@tonic-gate */
10007c478bd9Sstevel@tonic-gate curthread->t_pollcache = pcp;
10017c478bd9Sstevel@tonic-gate error = VOP_POLL(fp->f_vnode, pfdp->events, 0,
1002da6c28aaSamw &pfdp->revents, &php, NULL);
100380d5689fSPatrick Mooney
100480d5689fSPatrick Mooney /*
100580d5689fSPatrick Mooney * Edge-triggered polling requires a pollhead in order
100680d5689fSPatrick Mooney * to initiate wake-ups properly. Drivers which are
100780d5689fSPatrick Mooney * savvy to POLLET presence, which should include
100880d5689fSPatrick Mooney * everything in-gate, will always emit one, regardless
100980d5689fSPatrick Mooney * of revent status. Older drivers which only emit a
101080d5689fSPatrick Mooney * pollhead if 'revents == 0' are given a second chance
101180d5689fSPatrick Mooney * here via a second VOP_POLL, with only POLLET set in
101280d5689fSPatrick Mooney * the events of interest. These circumstances should
101380d5689fSPatrick Mooney * induce any cacheable drivers to emit a pollhead for
101480d5689fSPatrick Mooney * wake-ups.
101580d5689fSPatrick Mooney *
101680d5689fSPatrick Mooney * Drivers which never emit a pollhead will simply
101795e434b5SPatrick Mooney * disobey the expectation of edge-triggered behavior.
101880d5689fSPatrick Mooney * This includes recursive epoll which, even on Linux,
101980d5689fSPatrick Mooney * yields its events in a level-triggered fashion only.
102080d5689fSPatrick Mooney */
102195e434b5SPatrick Mooney if ((pfdp->events & POLLET) != 0 && error == 0 &&
102280d5689fSPatrick Mooney php == NULL) {
102380d5689fSPatrick Mooney short levent = 0;
102480d5689fSPatrick Mooney
102580d5689fSPatrick Mooney error = VOP_POLL(fp->f_vnode, POLLET, 0,
102680d5689fSPatrick Mooney &levent, &php, NULL);
102780d5689fSPatrick Mooney }
102880d5689fSPatrick Mooney
10297c478bd9Sstevel@tonic-gate curthread->t_pollcache = NULL;
10307c478bd9Sstevel@tonic-gate /*
1031a85084caSmeem * We always set the bit when this fd is cached;
1032a85084caSmeem * this forces the first DP_POLL to poll this fd.
10337c478bd9Sstevel@tonic-gate * Real performance gain comes from subsequent
1034*2c76d751SPatrick Mooney * DP_POLL. We also attempt a polldat_associate();
1035a85084caSmeem * if it's not possible, we'll do it in dpioctl().
10367c478bd9Sstevel@tonic-gate */
10377c478bd9Sstevel@tonic-gate BT_SET(pcp->pc_bitmap, fd);
10387c478bd9Sstevel@tonic-gate if (error != 0) {
10397c478bd9Sstevel@tonic-gate releasef(fd);
10407c478bd9Sstevel@tonic-gate break;
10417c478bd9Sstevel@tonic-gate }
10427c478bd9Sstevel@tonic-gate pdp->pd_fp = fp;
1043086d9687SPatrick Mooney pdp->pd_gen = gen;
10447c478bd9Sstevel@tonic-gate pdp->pd_events |= pfdp->events;
10457c478bd9Sstevel@tonic-gate if (php != NULL) {
10467c478bd9Sstevel@tonic-gate if (pdp->pd_php == NULL) {
1047*2c76d751SPatrick Mooney polldat_associate(pdp, php);
10487c478bd9Sstevel@tonic-gate } else {
10497c478bd9Sstevel@tonic-gate if (pdp->pd_php != php) {
1050*2c76d751SPatrick Mooney polldat_disassociate(pdp);
1051*2c76d751SPatrick Mooney polldat_associate(pdp, php);
10527c478bd9Sstevel@tonic-gate }
10537c478bd9Sstevel@tonic-gate }
10547c478bd9Sstevel@tonic-gate }
1055f3bb54f3SPatrick Mooney fds_added = B_TRUE;
10567c478bd9Sstevel@tonic-gate releasef(fd);
10577c478bd9Sstevel@tonic-gate } else {
1058a5eb7107SBryan Cantrill if (pdp == NULL || pdp->pd_fp == NULL) {
1059f3bb54f3SPatrick Mooney if (is_epoll) {
1060a5eb7107SBryan Cantrill /*
1061a5eb7107SBryan Cantrill * As with the add case (above), epoll
1062a5eb7107SBryan Cantrill * semantics demand that we error out
1063a5eb7107SBryan Cantrill * in this case.
1064a5eb7107SBryan Cantrill */
1065a5eb7107SBryan Cantrill error = ENOENT;
1066a5eb7107SBryan Cantrill break;
1067a5eb7107SBryan Cantrill }
1068a5eb7107SBryan Cantrill
10697c478bd9Sstevel@tonic-gate continue;
10707c478bd9Sstevel@tonic-gate }
10717c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_fd == fd);
10727c478bd9Sstevel@tonic-gate pdp->pd_fp = NULL;
10737c478bd9Sstevel@tonic-gate pdp->pd_events = 0;
10747c478bd9Sstevel@tonic-gate ASSERT(pdp->pd_thread == NULL);
1075*2c76d751SPatrick Mooney polldat_disassociate(pdp);
10767c478bd9Sstevel@tonic-gate BT_CLEAR(pcp->pc_bitmap, fd);
10777c478bd9Sstevel@tonic-gate }
10787c478bd9Sstevel@tonic-gate }
1079f3bb54f3SPatrick Mooney /*
1080bf75909aSPatrick Mooney * Wake any pollcache waiters so they can check the new descriptors.
1081bf75909aSPatrick Mooney *
1082f3bb54f3SPatrick Mooney * Any fds added to an recursive-capable pollcache could themselves be
1083f3bb54f3SPatrick Mooney * /dev/poll handles. To ensure that proper event propagation occurs,
1084bf75909aSPatrick Mooney * parent pollcaches are woken too, so that they can create any needed
1085f3bb54f3SPatrick Mooney * pollcache links.
1086f3bb54f3SPatrick Mooney */
1087f3bb54f3SPatrick Mooney if (fds_added) {
1088bf75909aSPatrick Mooney cv_broadcast(&pcp->pc_cv);
1089f3bb54f3SPatrick Mooney pcache_wake_parents(pcp);
1090f3bb54f3SPatrick Mooney }
1091f3bb54f3SPatrick Mooney pollstate_exit(pcp);
10927c478bd9Sstevel@tonic-gate mutex_enter(&dpep->dpe_lock);
1093f3bb54f3SPatrick Mooney bypass:
10947c478bd9Sstevel@tonic-gate dpep->dpe_flag &= ~DP_WRITER_PRESENT;
10957c478bd9Sstevel@tonic-gate dpep->dpe_refcnt--;
10967c478bd9Sstevel@tonic-gate cv_broadcast(&dpep->dpe_cv);
10977c478bd9Sstevel@tonic-gate mutex_exit(&dpep->dpe_lock);
10987c478bd9Sstevel@tonic-gate kmem_free(pollfdp, uiosize);
109957a0264bSPatrick Mooney if (error == 0) {
110057a0264bSPatrick Mooney /*
110157a0264bSPatrick Mooney * The state of uio_resid is updated only after the pollcache
110257a0264bSPatrick Mooney * is successfully modified.
110357a0264bSPatrick Mooney */
110457a0264bSPatrick Mooney uioskip(uiop, copysize);
110557a0264bSPatrick Mooney }
11067c478bd9Sstevel@tonic-gate return (error);
11077c478bd9Sstevel@tonic-gate }
11087c478bd9Sstevel@tonic-gate
1109a5eb7107SBryan Cantrill #define DP_SIGMASK_RESTORE(ksetp) { \
1110a5eb7107SBryan Cantrill if (ksetp != NULL) { \
1111a5eb7107SBryan Cantrill mutex_enter(&p->p_lock); \
1112a5eb7107SBryan Cantrill if (lwp->lwp_cursig == 0) { \
1113a5eb7107SBryan Cantrill t->t_hold = lwp->lwp_sigoldmask; \
1114a5eb7107SBryan Cantrill t->t_flag &= ~T_TOMASK; \
1115a5eb7107SBryan Cantrill } \
1116a5eb7107SBryan Cantrill mutex_exit(&p->p_lock); \
1117a5eb7107SBryan Cantrill } \
1118a5eb7107SBryan Cantrill }
1119a5eb7107SBryan Cantrill
11207c478bd9Sstevel@tonic-gate /*ARGSUSED*/
11217c478bd9Sstevel@tonic-gate static int
dpioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)11227c478bd9Sstevel@tonic-gate dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
11237c478bd9Sstevel@tonic-gate {
1124fe234e7cSMatt Amdur minor_t minor;
11257c478bd9Sstevel@tonic-gate dp_entry_t *dpep;
11267c478bd9Sstevel@tonic-gate pollcache_t *pcp;
1127cd1c8b85SMatthew Ahrens hrtime_t now;
1128fe234e7cSMatt Amdur int error = 0;
1129f3bb54f3SPatrick Mooney boolean_t is_epoll;
11307c478bd9Sstevel@tonic-gate STRUCT_DECL(dvpoll, dvpoll);
11317c478bd9Sstevel@tonic-gate
1132a5eb7107SBryan Cantrill if (cmd == DP_POLL || cmd == DP_PPOLL) {
1133cd1c8b85SMatthew Ahrens /* do this now, before we sleep on DP_WRITER_PRESENT */
1134cd1c8b85SMatthew Ahrens now = gethrtime();
1135cd1c8b85SMatthew Ahrens }
1136cd1c8b85SMatthew Ahrens
11377c478bd9Sstevel@tonic-gate minor = getminor(dev);
11387c478bd9Sstevel@tonic-gate mutex_enter(&devpoll_lock);
11397c478bd9Sstevel@tonic-gate ASSERT(minor < dptblsize);
11407c478bd9Sstevel@tonic-gate dpep = devpolltbl[minor];
11417c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
11427c478bd9Sstevel@tonic-gate ASSERT(dpep != NULL);
11437c478bd9Sstevel@tonic-gate pcp = dpep->dpe_pcache;
11447c478bd9Sstevel@tonic-gate
11457c478bd9Sstevel@tonic-gate mutex_enter(&dpep->dpe_lock);
1146f3bb54f3SPatrick Mooney is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0;
1147a5eb7107SBryan Cantrill
1148a5eb7107SBryan Cantrill if (cmd == DP_EPOLLCOMPAT) {
1149a5eb7107SBryan Cantrill if (dpep->dpe_refcnt != 0) {
1150a5eb7107SBryan Cantrill /*
1151a5eb7107SBryan Cantrill * We can't turn on epoll compatibility while there
1152a5eb7107SBryan Cantrill * are outstanding operations.
1153a5eb7107SBryan Cantrill */
1154a5eb7107SBryan Cantrill mutex_exit(&dpep->dpe_lock);
1155a5eb7107SBryan Cantrill return (EBUSY);
1156a5eb7107SBryan Cantrill }
1157a5eb7107SBryan Cantrill
1158a5eb7107SBryan Cantrill /*
1159a5eb7107SBryan Cantrill * epoll compatibility is a one-way street: there's no way
1160a5eb7107SBryan Cantrill * to turn it off for a particular open.
1161a5eb7107SBryan Cantrill */
1162a5eb7107SBryan Cantrill dpep->dpe_flag |= DP_ISEPOLLCOMPAT;
1163a5eb7107SBryan Cantrill
116487bfe94cSPatrick Mooney /* Record the epoll-enabled nature in the pollcache too */
116587bfe94cSPatrick Mooney mutex_enter(&pcp->pc_lock);
116687bfe94cSPatrick Mooney pcp->pc_flag |= PC_EPOLL;
116787bfe94cSPatrick Mooney mutex_exit(&pcp->pc_lock);
116887bfe94cSPatrick Mooney
116987bfe94cSPatrick Mooney mutex_exit(&dpep->dpe_lock);
1170a5eb7107SBryan Cantrill return (0);
1171a5eb7107SBryan Cantrill }
1172a5eb7107SBryan Cantrill
1173f3bb54f3SPatrick Mooney if (!is_epoll && curproc->p_pid != pcp->pc_pid) {
1174a5eb7107SBryan Cantrill if (pcp->pc_pid != -1) {
1175a5eb7107SBryan Cantrill mutex_exit(&dpep->dpe_lock);
1176a5eb7107SBryan Cantrill return (EACCES);
1177a5eb7107SBryan Cantrill }
1178a5eb7107SBryan Cantrill
1179a5eb7107SBryan Cantrill pcp->pc_pid = curproc->p_pid;
1180a5eb7107SBryan Cantrill }
1181a5eb7107SBryan Cantrill
1182f3bb54f3SPatrick Mooney /* Wait until all writers have cleared the handle before continuing */
1183f3bb54f3SPatrick Mooney while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0 ||
11847c478bd9Sstevel@tonic-gate (dpep->dpe_writerwait != 0)) {
11857c478bd9Sstevel@tonic-gate if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) {
11867c478bd9Sstevel@tonic-gate mutex_exit(&dpep->dpe_lock);
11877c478bd9Sstevel@tonic-gate return (EINTR);
11887c478bd9Sstevel@tonic-gate }
11897c478bd9Sstevel@tonic-gate }
11907c478bd9Sstevel@tonic-gate dpep->dpe_refcnt++;
11917c478bd9Sstevel@tonic-gate mutex_exit(&dpep->dpe_lock);
11927c478bd9Sstevel@tonic-gate
11937c478bd9Sstevel@tonic-gate switch (cmd) {
11947c478bd9Sstevel@tonic-gate case DP_POLL:
1195a5eb7107SBryan Cantrill case DP_PPOLL:
11967c478bd9Sstevel@tonic-gate {
1197fe234e7cSMatt Amdur pollstate_t *ps;
1198fe234e7cSMatt Amdur nfds_t nfds;
1199fe234e7cSMatt Amdur int fdcnt = 0;
1200a5eb7107SBryan Cantrill size_t size, fdsize, dpsize;
1201cd1c8b85SMatthew Ahrens hrtime_t deadline = 0;
1202a5eb7107SBryan Cantrill k_sigset_t *ksetp = NULL;
1203a5eb7107SBryan Cantrill k_sigset_t kset;
1204a5eb7107SBryan Cantrill sigset_t set;
1205a5eb7107SBryan Cantrill kthread_t *t = curthread;
1206a5eb7107SBryan Cantrill klwp_t *lwp = ttolwp(t);
1207a5eb7107SBryan Cantrill struct proc *p = ttoproc(curthread);
12087c478bd9Sstevel@tonic-gate
12097c478bd9Sstevel@tonic-gate STRUCT_INIT(dvpoll, mode);
1210a5eb7107SBryan Cantrill
1211a5eb7107SBryan Cantrill /*
1212a5eb7107SBryan Cantrill * The dp_setp member is only required/consumed for DP_PPOLL,
1213a5eb7107SBryan Cantrill * which otherwise uses the same structure as DP_POLL.
1214a5eb7107SBryan Cantrill */
1215a5eb7107SBryan Cantrill if (cmd == DP_POLL) {
1216a5eb7107SBryan Cantrill dpsize = (uintptr_t)STRUCT_FADDR(dvpoll, dp_setp) -
1217a5eb7107SBryan Cantrill (uintptr_t)STRUCT_FADDR(dvpoll, dp_fds);
1218a5eb7107SBryan Cantrill } else {
1219a5eb7107SBryan Cantrill ASSERT(cmd == DP_PPOLL);
1220a5eb7107SBryan Cantrill dpsize = STRUCT_SIZE(dvpoll);
1221a5eb7107SBryan Cantrill }
1222a5eb7107SBryan Cantrill
1223a5eb7107SBryan Cantrill if ((mode & FKIOCTL) != 0) {
1224a5eb7107SBryan Cantrill /* Kernel-internal ioctl call */
1225a5eb7107SBryan Cantrill bcopy((caddr_t)arg, STRUCT_BUF(dvpoll), dpsize);
1226a5eb7107SBryan Cantrill error = 0;
1227a5eb7107SBryan Cantrill } else {
1228a5eb7107SBryan Cantrill error = copyin((caddr_t)arg, STRUCT_BUF(dvpoll),
1229a5eb7107SBryan Cantrill dpsize);
1230a5eb7107SBryan Cantrill }
1231a5eb7107SBryan Cantrill
12327c478bd9Sstevel@tonic-gate if (error) {
12337c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
12347c478bd9Sstevel@tonic-gate return (EFAULT);
12357c478bd9Sstevel@tonic-gate }
12367c478bd9Sstevel@tonic-gate
1237cd1c8b85SMatthew Ahrens deadline = STRUCT_FGET(dvpoll, dp_timeout);
1238cd1c8b85SMatthew Ahrens if (deadline > 0) {
12397c478bd9Sstevel@tonic-gate /*
1240cd1c8b85SMatthew Ahrens * Convert the deadline from relative milliseconds
1241cd1c8b85SMatthew Ahrens * to absolute nanoseconds. They must wait for at
1242cd1c8b85SMatthew Ahrens * least a tick.
12437c478bd9Sstevel@tonic-gate */
124419449258SJosef 'Jeff' Sipek deadline = MSEC2NSEC(deadline);
1245cd1c8b85SMatthew Ahrens deadline = MAX(deadline, nsec_per_tick);
1246cd1c8b85SMatthew Ahrens deadline += now;
12477c478bd9Sstevel@tonic-gate }
12487c478bd9Sstevel@tonic-gate
1249a5eb7107SBryan Cantrill if (cmd == DP_PPOLL) {
1250a5eb7107SBryan Cantrill void *setp = STRUCT_FGETP(dvpoll, dp_setp);
1251a5eb7107SBryan Cantrill
1252a5eb7107SBryan Cantrill if (setp != NULL) {
125357a0264bSPatrick Mooney if ((mode & FKIOCTL) != 0) {
125457a0264bSPatrick Mooney /* Use the signal set directly */
125557a0264bSPatrick Mooney ksetp = (k_sigset_t *)setp;
125657a0264bSPatrick Mooney } else {
125757a0264bSPatrick Mooney if (copyin(setp, &set, sizeof (set))) {
125857a0264bSPatrick Mooney DP_REFRELE(dpep);
125957a0264bSPatrick Mooney return (EFAULT);
126057a0264bSPatrick Mooney }
126157a0264bSPatrick Mooney sigutok(&set, &kset);
126257a0264bSPatrick Mooney ksetp = &kset;
1263a5eb7107SBryan Cantrill }
1264a5eb7107SBryan Cantrill
1265a5eb7107SBryan Cantrill mutex_enter(&p->p_lock);
1266a5eb7107SBryan Cantrill schedctl_finish_sigblock(t);
1267a5eb7107SBryan Cantrill lwp->lwp_sigoldmask = t->t_hold;
1268a5eb7107SBryan Cantrill t->t_hold = *ksetp;
1269a5eb7107SBryan Cantrill t->t_flag |= T_TOMASK;
1270a5eb7107SBryan Cantrill
1271a5eb7107SBryan Cantrill /*
1272a5eb7107SBryan Cantrill * Like ppoll() with a non-NULL sigset, we'll
1273a5eb7107SBryan Cantrill * call cv_reltimedwait_sig() just to check for
1274a5eb7107SBryan Cantrill * signals. This call will return immediately
1275a5eb7107SBryan Cantrill * with either 0 (signalled) or -1 (no signal).
1276a5eb7107SBryan Cantrill * There are some conditions whereby we can
1277a5eb7107SBryan Cantrill * get 0 from cv_reltimedwait_sig() without
1278a5eb7107SBryan Cantrill * a true signal (e.g., a directed stop), so
1279a5eb7107SBryan Cantrill * we restore our signal mask in the unlikely
1280a5eb7107SBryan Cantrill * event that lwp_cursig is 0.
1281a5eb7107SBryan Cantrill */
1282a5eb7107SBryan Cantrill if (!cv_reltimedwait_sig(&t->t_delay_cv,
1283a5eb7107SBryan Cantrill &p->p_lock, 0, TR_CLOCK_TICK)) {
1284a5eb7107SBryan Cantrill if (lwp->lwp_cursig == 0) {
1285a5eb7107SBryan Cantrill t->t_hold = lwp->lwp_sigoldmask;
1286a5eb7107SBryan Cantrill t->t_flag &= ~T_TOMASK;
1287a5eb7107SBryan Cantrill }
1288a5eb7107SBryan Cantrill
1289a5eb7107SBryan Cantrill mutex_exit(&p->p_lock);
1290a5eb7107SBryan Cantrill
1291a5eb7107SBryan Cantrill DP_REFRELE(dpep);
1292a5eb7107SBryan Cantrill return (EINTR);
1293a5eb7107SBryan Cantrill }
1294a5eb7107SBryan Cantrill
1295a5eb7107SBryan Cantrill mutex_exit(&p->p_lock);
1296a5eb7107SBryan Cantrill }
1297a5eb7107SBryan Cantrill }
1298a5eb7107SBryan Cantrill
12997c478bd9Sstevel@tonic-gate if ((nfds = STRUCT_FGET(dvpoll, dp_nfds)) == 0) {
13007c478bd9Sstevel@tonic-gate /*
13017c478bd9Sstevel@tonic-gate * We are just using DP_POLL to sleep, so
13027c478bd9Sstevel@tonic-gate * we don't any of the devpoll apparatus.
13037c478bd9Sstevel@tonic-gate * Do not check for signals if we have a zero timeout.
13047c478bd9Sstevel@tonic-gate */
13057c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
1306a5eb7107SBryan Cantrill if (deadline == 0) {
1307a5eb7107SBryan Cantrill DP_SIGMASK_RESTORE(ksetp);
13087c478bd9Sstevel@tonic-gate return (0);
1309a5eb7107SBryan Cantrill }
1310a5eb7107SBryan Cantrill
13117c478bd9Sstevel@tonic-gate mutex_enter(&curthread->t_delay_lock);
1312cd1c8b85SMatthew Ahrens while ((error =
1313cd1c8b85SMatthew Ahrens cv_timedwait_sig_hrtime(&curthread->t_delay_cv,
1314cd1c8b85SMatthew Ahrens &curthread->t_delay_lock, deadline)) > 0)
13157c478bd9Sstevel@tonic-gate continue;
13167c478bd9Sstevel@tonic-gate mutex_exit(&curthread->t_delay_lock);
1317a5eb7107SBryan Cantrill
1318a5eb7107SBryan Cantrill DP_SIGMASK_RESTORE(ksetp);
1319a5eb7107SBryan Cantrill
1320cd1c8b85SMatthew Ahrens return (error == 0 ? EINTR : 0);
13217c478bd9Sstevel@tonic-gate }
13227c478bd9Sstevel@tonic-gate
1323f3bb54f3SPatrick Mooney if (is_epoll) {
1324a5eb7107SBryan Cantrill size = nfds * (fdsize = sizeof (epoll_event_t));
1325a5eb7107SBryan Cantrill } else {
1326a5eb7107SBryan Cantrill size = nfds * (fdsize = sizeof (pollfd_t));
1327a5eb7107SBryan Cantrill }
1328a5eb7107SBryan Cantrill
13297c478bd9Sstevel@tonic-gate /*
1330fe234e7cSMatt Amdur * XXX It would be nice not to have to alloc each time, but it
1331fe234e7cSMatt Amdur * requires another per thread structure hook. This can be
1332fe234e7cSMatt Amdur * implemented later if data suggests that it's necessary.
13337c478bd9Sstevel@tonic-gate */
1334f3bb54f3SPatrick Mooney ps = pollstate_create();
1335a5eb7107SBryan Cantrill
1336a5eb7107SBryan Cantrill if (ps->ps_dpbufsize < size) {
13377c478bd9Sstevel@tonic-gate /*
1338a5eb7107SBryan Cantrill * If nfds is larger than twice the current maximum
1339a5eb7107SBryan Cantrill * open file count, we'll silently clamp it. This
1340a5eb7107SBryan Cantrill * only limits our exposure to allocating an
1341a5eb7107SBryan Cantrill * inordinate amount of kernel memory; it doesn't
1342a5eb7107SBryan Cantrill * otherwise affect the semantics. (We have this
1343a5eb7107SBryan Cantrill * check at twice the maximum instead of merely the
1344a5eb7107SBryan Cantrill * maximum because some applications pass an nfds that
1345a5eb7107SBryan Cantrill * is only slightly larger than their limit.)
13467c478bd9Sstevel@tonic-gate */
13477c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock);
1348a5eb7107SBryan Cantrill if ((nfds >> 1) > p->p_fno_ctl) {
1349a5eb7107SBryan Cantrill nfds = p->p_fno_ctl;
1350a5eb7107SBryan Cantrill size = nfds * fdsize;
13517c478bd9Sstevel@tonic-gate }
13527c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock);
1353a5eb7107SBryan Cantrill
1354a5eb7107SBryan Cantrill if (ps->ps_dpbufsize < size) {
1355a5eb7107SBryan Cantrill kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize);
1356a5eb7107SBryan Cantrill ps->ps_dpbuf = kmem_zalloc(size, KM_SLEEP);
1357a5eb7107SBryan Cantrill ps->ps_dpbufsize = size;
1358a5eb7107SBryan Cantrill }
13597c478bd9Sstevel@tonic-gate }
13607c478bd9Sstevel@tonic-gate
1361f3bb54f3SPatrick Mooney VERIFY(pollstate_enter(pcp) == PSE_SUCCESS);
13627c478bd9Sstevel@tonic-gate for (;;) {
1363a5eb7107SBryan Cantrill pcp->pc_flag &= ~PC_POLLWAKE;
1364a5eb7107SBryan Cantrill
1365f3bb54f3SPatrick Mooney /*
1366f3bb54f3SPatrick Mooney * Mark all child pcachelinks as stale.
1367f3bb54f3SPatrick Mooney * Those which are still part of the tree will be
1368f3bb54f3SPatrick Mooney * marked as valid during the poll.
1369f3bb54f3SPatrick Mooney */
1370f3bb54f3SPatrick Mooney pcachelink_mark_stale(pcp);
1371f3bb54f3SPatrick Mooney
1372a5eb7107SBryan Cantrill error = dp_pcache_poll(dpep, ps->ps_dpbuf,
1373a5eb7107SBryan Cantrill pcp, nfds, &fdcnt);
13747c478bd9Sstevel@tonic-gate if (fdcnt > 0 || error != 0)
13757c478bd9Sstevel@tonic-gate break;
13767c478bd9Sstevel@tonic-gate
1377f3bb54f3SPatrick Mooney /* Purge still-stale child pcachelinks */
1378f3bb54f3SPatrick Mooney pcachelink_purge_stale(pcp);
1379f3bb54f3SPatrick Mooney
13807c478bd9Sstevel@tonic-gate /*
13817c478bd9Sstevel@tonic-gate * A pollwake has happened since we polled cache.
13827c478bd9Sstevel@tonic-gate */
1383a5eb7107SBryan Cantrill if (pcp->pc_flag & PC_POLLWAKE)
13847c478bd9Sstevel@tonic-gate continue;
13857c478bd9Sstevel@tonic-gate
13867c478bd9Sstevel@tonic-gate /*
1387da6c28aaSamw * Sleep until we are notified, signaled, or timed out.
13887c478bd9Sstevel@tonic-gate */
1389cd1c8b85SMatthew Ahrens if (deadline == 0) {
1390cd1c8b85SMatthew Ahrens /* immediate timeout; do not check signals */
13917c478bd9Sstevel@tonic-gate break;
1392cd1c8b85SMatthew Ahrens }
1393a5eb7107SBryan Cantrill
1394f3bb54f3SPatrick Mooney error = cv_timedwait_sig_hrtime(&pcp->pc_cv,
1395f3bb54f3SPatrick Mooney &pcp->pc_lock, deadline);
1396a5eb7107SBryan Cantrill
13977c478bd9Sstevel@tonic-gate /*
1398f3bb54f3SPatrick Mooney * If we were awakened by a signal or timeout then
1399f3bb54f3SPatrick Mooney * break the loop, else poll again.
14007c478bd9Sstevel@tonic-gate */
1401cd1c8b85SMatthew Ahrens if (error <= 0) {
1402cd1c8b85SMatthew Ahrens error = (error == 0) ? EINTR : 0;
14037c478bd9Sstevel@tonic-gate break;
1404cd1c8b85SMatthew Ahrens } else {
1405cd1c8b85SMatthew Ahrens error = 0;
14067c478bd9Sstevel@tonic-gate }
14077c478bd9Sstevel@tonic-gate }
1408f3bb54f3SPatrick Mooney pollstate_exit(pcp);
14097c478bd9Sstevel@tonic-gate
1410a5eb7107SBryan Cantrill DP_SIGMASK_RESTORE(ksetp);
1411a5eb7107SBryan Cantrill
14127c478bd9Sstevel@tonic-gate if (error == 0 && fdcnt > 0) {
141357a0264bSPatrick Mooney /*
141457a0264bSPatrick Mooney * It should be noted that FKIOCTL does not influence
141557a0264bSPatrick Mooney * the copyout (vs bcopy) of dp_fds at this time.
141657a0264bSPatrick Mooney */
1417a5eb7107SBryan Cantrill if (copyout(ps->ps_dpbuf,
1418a5eb7107SBryan Cantrill STRUCT_FGETP(dvpoll, dp_fds), fdcnt * fdsize)) {
14197c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14207c478bd9Sstevel@tonic-gate return (EFAULT);
14217c478bd9Sstevel@tonic-gate }
14227c478bd9Sstevel@tonic-gate *rvalp = fdcnt;
14237c478bd9Sstevel@tonic-gate }
14247c478bd9Sstevel@tonic-gate break;
14257c478bd9Sstevel@tonic-gate }
14267c478bd9Sstevel@tonic-gate
14277c478bd9Sstevel@tonic-gate case DP_ISPOLLED:
14287c478bd9Sstevel@tonic-gate {
14297c478bd9Sstevel@tonic-gate pollfd_t pollfd;
14307c478bd9Sstevel@tonic-gate polldat_t *pdp;
14317c478bd9Sstevel@tonic-gate
14327c478bd9Sstevel@tonic-gate STRUCT_INIT(dvpoll, mode);
14337c478bd9Sstevel@tonic-gate error = copyin((caddr_t)arg, &pollfd, sizeof (pollfd_t));
14347c478bd9Sstevel@tonic-gate if (error) {
14357c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14367c478bd9Sstevel@tonic-gate return (EFAULT);
14377c478bd9Sstevel@tonic-gate }
14387c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_lock);
14397c478bd9Sstevel@tonic-gate if (pcp->pc_hash == NULL) {
14407c478bd9Sstevel@tonic-gate /*
14417c478bd9Sstevel@tonic-gate * No Need to search because no poll fd
14427c478bd9Sstevel@tonic-gate * has been cached.
14437c478bd9Sstevel@tonic-gate */
14447c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14457c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14467c478bd9Sstevel@tonic-gate return (0);
14477c478bd9Sstevel@tonic-gate }
14487c478bd9Sstevel@tonic-gate if (pollfd.fd < 0) {
14497c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14507c478bd9Sstevel@tonic-gate break;
14517c478bd9Sstevel@tonic-gate }
14527c478bd9Sstevel@tonic-gate pdp = pcache_lookup_fd(pcp, pollfd.fd);
14537c478bd9Sstevel@tonic-gate if ((pdp != NULL) && (pdp->pd_fd == pollfd.fd) &&
14547c478bd9Sstevel@tonic-gate (pdp->pd_fp != NULL)) {
14557c478bd9Sstevel@tonic-gate pollfd.revents = pdp->pd_events;
14567c478bd9Sstevel@tonic-gate if (copyout(&pollfd, (caddr_t)arg, sizeof (pollfd_t))) {
14577c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14587c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14597c478bd9Sstevel@tonic-gate return (EFAULT);
14607c478bd9Sstevel@tonic-gate }
14617c478bd9Sstevel@tonic-gate *rvalp = 1;
14627c478bd9Sstevel@tonic-gate }
14637c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_lock);
14647c478bd9Sstevel@tonic-gate break;
14657c478bd9Sstevel@tonic-gate }
14667c478bd9Sstevel@tonic-gate
14677c478bd9Sstevel@tonic-gate default:
14687c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14697c478bd9Sstevel@tonic-gate return (EINVAL);
14707c478bd9Sstevel@tonic-gate }
14717c478bd9Sstevel@tonic-gate DP_REFRELE(dpep);
14727c478bd9Sstevel@tonic-gate return (error);
14737c478bd9Sstevel@tonic-gate }
14747c478bd9Sstevel@tonic-gate
1475f3bb54f3SPatrick Mooney /*
1476f3bb54f3SPatrick Mooney * Overview of Recursive Polling
1477f3bb54f3SPatrick Mooney *
1478f3bb54f3SPatrick Mooney * It is possible for /dev/poll to poll for events on file descriptors which
1479f3bb54f3SPatrick Mooney * themselves are /dev/poll handles. Pending events in the child handle are
1480f3bb54f3SPatrick Mooney * represented as readable data via the POLLIN flag. To limit surface area,
1481f3bb54f3SPatrick Mooney * this recursion is presently allowed on only /dev/poll handles which have
1482f3bb54f3SPatrick Mooney * been placed in epoll mode via the DP_EPOLLCOMPAT ioctl. Recursion depth is
1483f3bb54f3SPatrick Mooney * limited to 5 in order to be consistent with Linux epoll.
1484f3bb54f3SPatrick Mooney *
1485f3bb54f3SPatrick Mooney * Extending dppoll() for VOP_POLL:
1486f3bb54f3SPatrick Mooney *
1487f3bb54f3SPatrick Mooney * The recursive /dev/poll implementation begins by extending dppoll() to
1488f3bb54f3SPatrick Mooney * report when resources contained in the pollcache have relevant event state.
1489f3bb54f3SPatrick Mooney * At the highest level, it means calling dp_pcache_poll() so it indicates if
1490f3bb54f3SPatrick Mooney * fd events are present without consuming them or altering the pollcache
1491f3bb54f3SPatrick Mooney * bitmap. This ensures that a subsequent DP_POLL operation on the bitmap will
1492f3bb54f3SPatrick Mooney * yield the initiating event. Additionally, the VOP_POLL should return in
1493f3bb54f3SPatrick Mooney * such a way that dp_pcache_poll() does not clear the parent bitmap entry
1494f3bb54f3SPatrick Mooney * which corresponds to the child /dev/poll fd. This means that child
1495f3bb54f3SPatrick Mooney * pollcaches will be checked during every poll which facilitates wake-up
1496f3bb54f3SPatrick Mooney * behavior detailed below.
1497f3bb54f3SPatrick Mooney *
1498f3bb54f3SPatrick Mooney * Pollcache Links and Wake Events:
1499f3bb54f3SPatrick Mooney *
1500f3bb54f3SPatrick Mooney * Recursive /dev/poll avoids complicated pollcache locking constraints during
1501f3bb54f3SPatrick Mooney * pollwakeup events by eschewing the traditional pollhead mechanism in favor
1502f3bb54f3SPatrick Mooney * of a different approach. For each pollcache at the root of a recursive
1503f3bb54f3SPatrick Mooney * /dev/poll "tree", pcachelink_t structures are established to all child
1504f3bb54f3SPatrick Mooney * /dev/poll pollcaches. During pollnotify() in a child pollcache, the
1505f3bb54f3SPatrick Mooney * linked list of pcachelink_t entries is walked, where those marked as valid
1506f3bb54f3SPatrick Mooney * incur a cv_broadcast to their parent pollcache. Most notably, these
1507f3bb54f3SPatrick Mooney * pcachelink_t cv wakeups are performed without acquiring pc_lock on the
1508f3bb54f3SPatrick Mooney * parent pollcache (which would require careful deadlock avoidance). This
1509f3bb54f3SPatrick Mooney * still allows the woken poll on the parent to discover the pertinent events
1510f3bb54f3SPatrick Mooney * due to the fact that bitmap entires for the child pollcache are always
1511f3bb54f3SPatrick Mooney * maintained by the dppoll() logic above.
1512f3bb54f3SPatrick Mooney *
1513f3bb54f3SPatrick Mooney * Depth Limiting and Loop Prevention:
1514f3bb54f3SPatrick Mooney *
1515f3bb54f3SPatrick Mooney * As each pollcache is encountered (either via DP_POLL or dppoll()), depth and
1516f3bb54f3SPatrick Mooney * loop constraints are enforced via pollstate_enter(). The pollcache_t
1517f3bb54f3SPatrick Mooney * pointer is compared against any existing entries in ps_pc_stack and is added
1518f3bb54f3SPatrick Mooney * to the end if no match (and therefore loop) is found. Once poll operations
1519f3bb54f3SPatrick Mooney * for a given pollcache_t are complete, pollstate_exit() clears the pointer
1520f3bb54f3SPatrick Mooney * from the list. The pollstate_enter() and pollstate_exit() functions are
1521f3bb54f3SPatrick Mooney * responsible for acquiring and releasing pc_lock, respectively.
1522f3bb54f3SPatrick Mooney *
1523f3bb54f3SPatrick Mooney * Deadlock Safety:
1524f3bb54f3SPatrick Mooney *
1525f3bb54f3SPatrick Mooney * Descending through a tree of recursive /dev/poll handles involves the tricky
1526f3bb54f3SPatrick Mooney * business of sequentially entering multiple pollcache locks. This tree
1527f3bb54f3SPatrick Mooney * topology cannot define a lock acquisition order in such a way that it is
1528f3bb54f3SPatrick Mooney * immune to deadlocks between threads. The pollstate_enter() and
1529f3bb54f3SPatrick Mooney * pollstate_exit() functions provide an interface for recursive /dev/poll
1530f3bb54f3SPatrick Mooney * operations to safely lock pollcaches while failing gracefully in the face of
1531f3bb54f3SPatrick Mooney * deadlocking topologies. (See pollstate_contend() for more detail about how
1532f3bb54f3SPatrick Mooney * deadlocks are detected and resolved.)
1533f3bb54f3SPatrick Mooney */
1534f3bb54f3SPatrick Mooney
15357c478bd9Sstevel@tonic-gate /*ARGSUSED*/
15367c478bd9Sstevel@tonic-gate static int
dppoll(dev_t dev,short events,int anyyet,short * reventsp,struct pollhead ** phpp)15377c478bd9Sstevel@tonic-gate dppoll(dev_t dev, short events, int anyyet, short *reventsp,
15387c478bd9Sstevel@tonic-gate struct pollhead **phpp)
15397c478bd9Sstevel@tonic-gate {
1540a5eb7107SBryan Cantrill minor_t minor;
1541a5eb7107SBryan Cantrill dp_entry_t *dpep;
1542f3bb54f3SPatrick Mooney pollcache_t *pcp;
1543f3bb54f3SPatrick Mooney int res, rc = 0;
1544a5eb7107SBryan Cantrill
1545a5eb7107SBryan Cantrill minor = getminor(dev);
1546a5eb7107SBryan Cantrill mutex_enter(&devpoll_lock);
1547f3bb54f3SPatrick Mooney ASSERT(minor < dptblsize);
1548a5eb7107SBryan Cantrill dpep = devpolltbl[minor];
1549a5eb7107SBryan Cantrill ASSERT(dpep != NULL);
1550a5eb7107SBryan Cantrill mutex_exit(&devpoll_lock);
1551a5eb7107SBryan Cantrill
1552f3bb54f3SPatrick Mooney mutex_enter(&dpep->dpe_lock);
1553f3bb54f3SPatrick Mooney if ((dpep->dpe_flag & DP_ISEPOLLCOMPAT) == 0) {
1554f3bb54f3SPatrick Mooney /* Poll recursion is not yet supported for non-epoll handles */
1555a5eb7107SBryan Cantrill *reventsp = POLLERR;
1556f3bb54f3SPatrick Mooney mutex_exit(&dpep->dpe_lock);
1557f3bb54f3SPatrick Mooney return (0);
1558f3bb54f3SPatrick Mooney } else {
1559f3bb54f3SPatrick Mooney dpep->dpe_refcnt++;
1560f3bb54f3SPatrick Mooney pcp = dpep->dpe_pcache;
1561f3bb54f3SPatrick Mooney mutex_exit(&dpep->dpe_lock);
1562a5eb7107SBryan Cantrill }
1563f3bb54f3SPatrick Mooney
1564f3bb54f3SPatrick Mooney res = pollstate_enter(pcp);
1565f3bb54f3SPatrick Mooney if (res == PSE_SUCCESS) {
1566f3bb54f3SPatrick Mooney nfds_t nfds = 1;
1567f3bb54f3SPatrick Mooney int fdcnt = 0;
1568f3bb54f3SPatrick Mooney pollstate_t *ps = curthread->t_pollstate;
1569f3bb54f3SPatrick Mooney
157080d5689fSPatrick Mooney /*
157180d5689fSPatrick Mooney * Recursive polling will only emit certain events. Skip a
157280d5689fSPatrick Mooney * scan of the pollcache if those events are not of interest.
157380d5689fSPatrick Mooney */
157480d5689fSPatrick Mooney if (events & (POLLIN|POLLRDNORM)) {
157580d5689fSPatrick Mooney rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt);
157680d5689fSPatrick Mooney } else {
157780d5689fSPatrick Mooney rc = 0;
157880d5689fSPatrick Mooney fdcnt = 0;
157980d5689fSPatrick Mooney }
158080d5689fSPatrick Mooney
158180d5689fSPatrick Mooney if (rc == 0 && fdcnt > 0) {
158280d5689fSPatrick Mooney *reventsp = POLLIN|POLLRDNORM;
158380d5689fSPatrick Mooney } else {
158480d5689fSPatrick Mooney *reventsp = 0;
1585f3bb54f3SPatrick Mooney }
1586f3bb54f3SPatrick Mooney pcachelink_assoc(pcp, ps->ps_pc_stack[0]);
1587f3bb54f3SPatrick Mooney pollstate_exit(pcp);
1588f3bb54f3SPatrick Mooney } else {
1589f3bb54f3SPatrick Mooney switch (res) {
1590f3bb54f3SPatrick Mooney case PSE_FAIL_DEPTH:
1591f3bb54f3SPatrick Mooney rc = EINVAL;
1592f3bb54f3SPatrick Mooney break;
1593f3bb54f3SPatrick Mooney case PSE_FAIL_LOOP:
1594f3bb54f3SPatrick Mooney case PSE_FAIL_DEADLOCK:
1595f3bb54f3SPatrick Mooney rc = ELOOP;
1596f3bb54f3SPatrick Mooney break;
1597f3bb54f3SPatrick Mooney default:
1598f3bb54f3SPatrick Mooney /*
1599f3bb54f3SPatrick Mooney * If anything else has gone awry, such as being polled
1600f3bb54f3SPatrick Mooney * from an unexpected context, fall back to the
1601f3bb54f3SPatrick Mooney * recursion-intolerant response.
1602f3bb54f3SPatrick Mooney */
1603f3bb54f3SPatrick Mooney *reventsp = POLLERR;
1604f3bb54f3SPatrick Mooney rc = 0;
1605f3bb54f3SPatrick Mooney break;
1606f3bb54f3SPatrick Mooney }
1607f3bb54f3SPatrick Mooney }
1608f3bb54f3SPatrick Mooney
1609f3bb54f3SPatrick Mooney DP_REFRELE(dpep);
1610f3bb54f3SPatrick Mooney return (rc);
16117c478bd9Sstevel@tonic-gate }
16127c478bd9Sstevel@tonic-gate
16137c478bd9Sstevel@tonic-gate /*
16147c478bd9Sstevel@tonic-gate * devpoll close should do enough clean up before the pollcache is deleted,
16157c478bd9Sstevel@tonic-gate * i.e., it should ensure no one still references the pollcache later.
16167c478bd9Sstevel@tonic-gate * There is no "permission" check in here. Any process having the last
16177c478bd9Sstevel@tonic-gate * reference of this /dev/poll fd can close.
16187c478bd9Sstevel@tonic-gate */
16197c478bd9Sstevel@tonic-gate /*ARGSUSED*/
16207c478bd9Sstevel@tonic-gate static int
dpclose(dev_t dev,int flag,int otyp,cred_t * credp)16217c478bd9Sstevel@tonic-gate dpclose(dev_t dev, int flag, int otyp, cred_t *credp)
16227c478bd9Sstevel@tonic-gate {
1623fe234e7cSMatt Amdur minor_t minor;
16247c478bd9Sstevel@tonic-gate dp_entry_t *dpep;
16257c478bd9Sstevel@tonic-gate pollcache_t *pcp;
16267c478bd9Sstevel@tonic-gate int i;
16277c478bd9Sstevel@tonic-gate polldat_t **hashtbl;
16287c478bd9Sstevel@tonic-gate polldat_t *pdp;
16297c478bd9Sstevel@tonic-gate
16307c478bd9Sstevel@tonic-gate minor = getminor(dev);
16317c478bd9Sstevel@tonic-gate
16327c478bd9Sstevel@tonic-gate mutex_enter(&devpoll_lock);
16337c478bd9Sstevel@tonic-gate dpep = devpolltbl[minor];
16347c478bd9Sstevel@tonic-gate ASSERT(dpep != NULL);
16357c478bd9Sstevel@tonic-gate devpolltbl[minor] = NULL;
16367c478bd9Sstevel@tonic-gate mutex_exit(&devpoll_lock);
16377c478bd9Sstevel@tonic-gate pcp = dpep->dpe_pcache;
16387c478bd9Sstevel@tonic-gate ASSERT(pcp != NULL);
16397c478bd9Sstevel@tonic-gate /*
16407c478bd9Sstevel@tonic-gate * At this point, no other lwp can access this pollcache via the
16417c478bd9Sstevel@tonic-gate * /dev/poll fd. This pollcache is going away, so do the clean
16427c478bd9Sstevel@tonic-gate * up without the pc_lock.
16437c478bd9Sstevel@tonic-gate */
16447c478bd9Sstevel@tonic-gate hashtbl = pcp->pc_hash;
16457c478bd9Sstevel@tonic-gate for (i = 0; i < pcp->pc_hashsize; i++) {
16467c478bd9Sstevel@tonic-gate for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) {
1647*2c76d751SPatrick Mooney polldat_disassociate(pdp);
1648*2c76d751SPatrick Mooney pdp->pd_fp = NULL;
16497c478bd9Sstevel@tonic-gate }
16507c478bd9Sstevel@tonic-gate }
16517c478bd9Sstevel@tonic-gate /*
16527c478bd9Sstevel@tonic-gate * pollwakeup() may still interact with this pollcache. Wait until
16537c478bd9Sstevel@tonic-gate * it is done.
16547c478bd9Sstevel@tonic-gate */
16557c478bd9Sstevel@tonic-gate mutex_enter(&pcp->pc_no_exit);
16567c478bd9Sstevel@tonic-gate ASSERT(pcp->pc_busy >= 0);
16577c478bd9Sstevel@tonic-gate while (pcp->pc_busy > 0)
16587c478bd9Sstevel@tonic-gate cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit);
16597c478bd9Sstevel@tonic-gate mutex_exit(&pcp->pc_no_exit);
1660f3bb54f3SPatrick Mooney
1661f3bb54f3SPatrick Mooney /* Clean up any pollcache links created via recursive /dev/poll */
1662f3bb54f3SPatrick Mooney if (pcp->pc_parents != NULL || pcp->pc_children != NULL) {
1663f3bb54f3SPatrick Mooney /*
1664f3bb54f3SPatrick Mooney * Because of the locking rules for pcachelink manipulation,
1665f3bb54f3SPatrick Mooney * acquring pc_lock is required for this step.
1666f3bb54f3SPatrick Mooney */
1667f3bb54f3SPatrick Mooney mutex_enter(&pcp->pc_lock);
1668f3bb54f3SPatrick Mooney pcachelink_purge_all(pcp);
1669f3bb54f3SPatrick Mooney mutex_exit(&pcp->pc_lock);
1670f3bb54f3SPatrick Mooney }
1671f3bb54f3SPatrick Mooney
16727c478bd9Sstevel@tonic-gate pcache_destroy(pcp);
16737c478bd9Sstevel@tonic-gate ASSERT(dpep->dpe_refcnt == 0);
16747c478bd9Sstevel@tonic-gate kmem_free(dpep, sizeof (dp_entry_t));
16757c478bd9Sstevel@tonic-gate return (0);
16767c478bd9Sstevel@tonic-gate }
1677f3bb54f3SPatrick Mooney
1678f3bb54f3SPatrick Mooney static void
pcachelink_locked_rele(pcachelink_t * pl)1679f3bb54f3SPatrick Mooney pcachelink_locked_rele(pcachelink_t *pl)
1680f3bb54f3SPatrick Mooney {
1681f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&pl->pcl_lock));
1682f3bb54f3SPatrick Mooney VERIFY(pl->pcl_refcnt >= 1);
1683f3bb54f3SPatrick Mooney
1684f3bb54f3SPatrick Mooney pl->pcl_refcnt--;
1685f3bb54f3SPatrick Mooney if (pl->pcl_refcnt == 0) {
1686f3bb54f3SPatrick Mooney VERIFY(pl->pcl_state == PCL_INVALID);
1687f3bb54f3SPatrick Mooney ASSERT(pl->pcl_parent_pc == NULL);
1688f3bb54f3SPatrick Mooney ASSERT(pl->pcl_child_pc == NULL);
1689f3bb54f3SPatrick Mooney ASSERT(pl->pcl_parent_next == NULL);
1690f3bb54f3SPatrick Mooney ASSERT(pl->pcl_child_next == NULL);
1691f3bb54f3SPatrick Mooney
1692f3bb54f3SPatrick Mooney pl->pcl_state = PCL_FREE;
1693f3bb54f3SPatrick Mooney mutex_destroy(&pl->pcl_lock);
1694f3bb54f3SPatrick Mooney kmem_free(pl, sizeof (pcachelink_t));
1695f3bb54f3SPatrick Mooney } else {
1696f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
1697f3bb54f3SPatrick Mooney }
1698f3bb54f3SPatrick Mooney }
1699f3bb54f3SPatrick Mooney
1700f3bb54f3SPatrick Mooney /*
1701f3bb54f3SPatrick Mooney * Associate parent and child pollcaches via a pcachelink_t. If an existing
1702f3bb54f3SPatrick Mooney * link (stale or valid) between the two is found, it will be reused. If a
1703f3bb54f3SPatrick Mooney * suitable link is not found for reuse, a new one will be allocated.
1704f3bb54f3SPatrick Mooney */
1705f3bb54f3SPatrick Mooney static void
pcachelink_assoc(pollcache_t * child,pollcache_t * parent)1706f3bb54f3SPatrick Mooney pcachelink_assoc(pollcache_t *child, pollcache_t *parent)
1707f3bb54f3SPatrick Mooney {
1708f3bb54f3SPatrick Mooney pcachelink_t *pl, **plpn;
1709f3bb54f3SPatrick Mooney
1710f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&child->pc_lock));
1711f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&parent->pc_lock));
1712f3bb54f3SPatrick Mooney
1713f3bb54f3SPatrick Mooney /* Search for an existing link we can reuse. */
1714f3bb54f3SPatrick Mooney plpn = &child->pc_parents;
1715f3bb54f3SPatrick Mooney for (pl = child->pc_parents; pl != NULL; pl = *plpn) {
1716f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
1717f3bb54f3SPatrick Mooney if (pl->pcl_state == PCL_INVALID) {
1718f3bb54f3SPatrick Mooney /* Clean any invalid links while walking the list */
1719f3bb54f3SPatrick Mooney *plpn = pl->pcl_parent_next;
1720f3bb54f3SPatrick Mooney pl->pcl_child_pc = NULL;
1721f3bb54f3SPatrick Mooney pl->pcl_parent_next = NULL;
1722f3bb54f3SPatrick Mooney pcachelink_locked_rele(pl);
1723f3bb54f3SPatrick Mooney } else if (pl->pcl_parent_pc == parent) {
1724f3bb54f3SPatrick Mooney /* Successfully found parent link */
1725f3bb54f3SPatrick Mooney ASSERT(pl->pcl_state == PCL_VALID ||
1726f3bb54f3SPatrick Mooney pl->pcl_state == PCL_STALE);
1727f3bb54f3SPatrick Mooney pl->pcl_state = PCL_VALID;
1728f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
1729f3bb54f3SPatrick Mooney return;
1730f3bb54f3SPatrick Mooney } else {
1731f3bb54f3SPatrick Mooney plpn = &pl->pcl_parent_next;
1732f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
1733f3bb54f3SPatrick Mooney }
1734f3bb54f3SPatrick Mooney }
1735f3bb54f3SPatrick Mooney
1736f3bb54f3SPatrick Mooney /* No existing link to the parent was found. Create a fresh one. */
1737f3bb54f3SPatrick Mooney pl = kmem_zalloc(sizeof (pcachelink_t), KM_SLEEP);
1738f3bb54f3SPatrick Mooney mutex_init(&pl->pcl_lock, NULL, MUTEX_DEFAULT, NULL);
1739f3bb54f3SPatrick Mooney
1740f3bb54f3SPatrick Mooney pl->pcl_parent_pc = parent;
1741f3bb54f3SPatrick Mooney pl->pcl_child_next = parent->pc_children;
1742f3bb54f3SPatrick Mooney parent->pc_children = pl;
1743f3bb54f3SPatrick Mooney pl->pcl_refcnt++;
1744f3bb54f3SPatrick Mooney
1745f3bb54f3SPatrick Mooney pl->pcl_child_pc = child;
1746f3bb54f3SPatrick Mooney pl->pcl_parent_next = child->pc_parents;
1747f3bb54f3SPatrick Mooney child->pc_parents = pl;
1748f3bb54f3SPatrick Mooney pl->pcl_refcnt++;
1749f3bb54f3SPatrick Mooney
1750f3bb54f3SPatrick Mooney pl->pcl_state = PCL_VALID;
1751f3bb54f3SPatrick Mooney }
1752f3bb54f3SPatrick Mooney
1753f3bb54f3SPatrick Mooney /*
1754f3bb54f3SPatrick Mooney * Mark all child links in a pollcache as stale. Any invalid child links found
1755f3bb54f3SPatrick Mooney * during iteration are purged.
1756f3bb54f3SPatrick Mooney */
1757f3bb54f3SPatrick Mooney static void
pcachelink_mark_stale(pollcache_t * pcp)1758f3bb54f3SPatrick Mooney pcachelink_mark_stale(pollcache_t *pcp)
1759f3bb54f3SPatrick Mooney {
1760f3bb54f3SPatrick Mooney pcachelink_t *pl, **plpn;
1761f3bb54f3SPatrick Mooney
1762f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&pcp->pc_lock));
1763f3bb54f3SPatrick Mooney
1764f3bb54f3SPatrick Mooney plpn = &pcp->pc_children;
1765f3bb54f3SPatrick Mooney for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1766f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
1767f3bb54f3SPatrick Mooney if (pl->pcl_state == PCL_INVALID) {
1768f3bb54f3SPatrick Mooney /*
1769f3bb54f3SPatrick Mooney * Remove any invalid links while we are going to the
1770f3bb54f3SPatrick Mooney * trouble of walking the list.
1771f3bb54f3SPatrick Mooney */
1772f3bb54f3SPatrick Mooney *plpn = pl->pcl_child_next;
1773f3bb54f3SPatrick Mooney pl->pcl_parent_pc = NULL;
1774f3bb54f3SPatrick Mooney pl->pcl_child_next = NULL;
1775f3bb54f3SPatrick Mooney pcachelink_locked_rele(pl);
1776f3bb54f3SPatrick Mooney } else {
1777f3bb54f3SPatrick Mooney pl->pcl_state = PCL_STALE;
1778f3bb54f3SPatrick Mooney plpn = &pl->pcl_child_next;
1779f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
1780f3bb54f3SPatrick Mooney }
1781f3bb54f3SPatrick Mooney }
1782f3bb54f3SPatrick Mooney }
1783f3bb54f3SPatrick Mooney
1784f3bb54f3SPatrick Mooney /*
1785f3bb54f3SPatrick Mooney * Purge all stale (or invalid) child links from a pollcache.
1786f3bb54f3SPatrick Mooney */
1787f3bb54f3SPatrick Mooney static void
pcachelink_purge_stale(pollcache_t * pcp)1788f3bb54f3SPatrick Mooney pcachelink_purge_stale(pollcache_t *pcp)
1789f3bb54f3SPatrick Mooney {
1790f3bb54f3SPatrick Mooney pcachelink_t *pl, **plpn;
1791f3bb54f3SPatrick Mooney
1792f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&pcp->pc_lock));
1793f3bb54f3SPatrick Mooney
1794f3bb54f3SPatrick Mooney plpn = &pcp->pc_children;
1795f3bb54f3SPatrick Mooney for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1796f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
1797f3bb54f3SPatrick Mooney switch (pl->pcl_state) {
1798f3bb54f3SPatrick Mooney case PCL_STALE:
1799f3bb54f3SPatrick Mooney pl->pcl_state = PCL_INVALID;
1800f3bb54f3SPatrick Mooney /* FALLTHROUGH */
1801f3bb54f3SPatrick Mooney case PCL_INVALID:
1802f3bb54f3SPatrick Mooney *plpn = pl->pcl_child_next;
1803f3bb54f3SPatrick Mooney pl->pcl_parent_pc = NULL;
1804f3bb54f3SPatrick Mooney pl->pcl_child_next = NULL;
1805f3bb54f3SPatrick Mooney pcachelink_locked_rele(pl);
1806f3bb54f3SPatrick Mooney break;
1807f3bb54f3SPatrick Mooney default:
1808f3bb54f3SPatrick Mooney plpn = &pl->pcl_child_next;
1809f3bb54f3SPatrick Mooney mutex_exit(&pl->pcl_lock);
1810f3bb54f3SPatrick Mooney }
1811f3bb54f3SPatrick Mooney }
1812f3bb54f3SPatrick Mooney }
1813f3bb54f3SPatrick Mooney
1814f3bb54f3SPatrick Mooney /*
1815f3bb54f3SPatrick Mooney * Purge all child and parent links from a pollcache, regardless of status.
1816f3bb54f3SPatrick Mooney */
1817f3bb54f3SPatrick Mooney static void
pcachelink_purge_all(pollcache_t * pcp)1818f3bb54f3SPatrick Mooney pcachelink_purge_all(pollcache_t *pcp)
1819f3bb54f3SPatrick Mooney {
1820f3bb54f3SPatrick Mooney pcachelink_t *pl, **plpn;
1821f3bb54f3SPatrick Mooney
1822f3bb54f3SPatrick Mooney ASSERT(MUTEX_HELD(&pcp->pc_lock));
1823f3bb54f3SPatrick Mooney
1824f3bb54f3SPatrick Mooney plpn = &pcp->pc_parents;
1825f3bb54f3SPatrick Mooney for (pl = pcp->pc_parents; pl != NULL; pl = *plpn) {
1826f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
1827f3bb54f3SPatrick Mooney pl->pcl_state = PCL_INVALID;
1828f3bb54f3SPatrick Mooney *plpn = pl->pcl_parent_next;
1829f3bb54f3SPatrick Mooney pl->pcl_child_pc = NULL;
1830f3bb54f3SPatrick Mooney pl->pcl_parent_next = NULL;
1831f3bb54f3SPatrick Mooney pcachelink_locked_rele(pl);
1832f3bb54f3SPatrick Mooney }
1833f3bb54f3SPatrick Mooney
1834f3bb54f3SPatrick Mooney plpn = &pcp->pc_children;
1835f3bb54f3SPatrick Mooney for (pl = pcp->pc_children; pl != NULL; pl = *plpn) {
1836f3bb54f3SPatrick Mooney mutex_enter(&pl->pcl_lock);
1837f3bb54f3SPatrick Mooney pl->pcl_state = PCL_INVALID;
1838f3bb54f3SPatrick Mooney *plpn = pl->pcl_child_next;
1839f3bb54f3SPatrick Mooney pl->pcl_parent_pc = NULL;
1840f3bb54f3SPatrick Mooney pl->pcl_child_next = NULL;
1841f3bb54f3SPatrick Mooney pcachelink_locked_rele(pl);
1842f3bb54f3SPatrick Mooney }
1843f3bb54f3SPatrick Mooney
1844f3bb54f3SPatrick Mooney ASSERT(pcp->pc_parents == NULL);
1845f3bb54f3SPatrick Mooney ASSERT(pcp->pc_children == NULL);
1846f3bb54f3SPatrick Mooney }
1847