1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * Copyright (c) 2012 by Delphix. All rights reserved. 28 * Copyright 2017 Joyent, Inc. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/devops.h> 33 #include <sys/conf.h> 34 #include <sys/modctl.h> 35 #include <sys/sunddi.h> 36 #include <sys/stat.h> 37 #include <sys/poll_impl.h> 38 #include <sys/errno.h> 39 #include <sys/kmem.h> 40 #include <sys/mkdev.h> 41 #include <sys/debug.h> 42 #include <sys/file.h> 43 #include <sys/sysmacros.h> 44 #include <sys/systm.h> 45 #include <sys/bitmap.h> 46 #include <sys/devpoll.h> 47 #include <sys/rctl.h> 48 #include <sys/resource.h> 49 #include <sys/schedctl.h> 50 #include <sys/epoll.h> 51 52 #define RESERVED 1 53 54 /* local data struct */ 55 static dp_entry_t **devpolltbl; /* dev poll entries */ 56 static size_t dptblsize; 57 58 static kmutex_t devpoll_lock; /* lock protecting dev tbl */ 59 int devpoll_init; /* is /dev/poll initialized already */ 60 61 /* device local functions */ 62 63 static int dpopen(dev_t *devp, int flag, int otyp, cred_t *credp); 64 static int dpwrite(dev_t dev, struct uio *uiop, cred_t *credp); 65 static int dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, 66 int *rvalp); 67 static int dppoll(dev_t dev, short events, int anyyet, short *reventsp, 68 struct pollhead **phpp); 69 static int dpclose(dev_t dev, int flag, int otyp, cred_t *credp); 70 static dev_info_t *dpdevi; 71 72 73 static struct cb_ops dp_cb_ops = { 74 dpopen, /* open */ 75 dpclose, /* close */ 76 nodev, /* strategy */ 77 nodev, /* print */ 78 nodev, /* dump */ 79 nodev, /* read */ 80 dpwrite, /* write */ 81 dpioctl, /* ioctl */ 82 nodev, /* devmap */ 83 nodev, /* mmap */ 84 nodev, /* segmap */ 85 dppoll, /* poll */ 86 ddi_prop_op, /* prop_op */ 87 (struct streamtab *)0, /* streamtab */ 88 D_MP, /* flags */ 89 CB_REV, /* cb_ops revision */ 90 nodev, /* aread */ 91 nodev /* awrite */ 92 }; 93 94 static int dpattach(dev_info_t *, ddi_attach_cmd_t); 95 static int dpdetach(dev_info_t *, ddi_detach_cmd_t); 96 static int dpinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 97 98 static struct dev_ops dp_ops = { 99 DEVO_REV, /* devo_rev */ 100 0, /* refcnt */ 101 dpinfo, /* info */ 102 nulldev, /* identify */ 103 nulldev, /* probe */ 104 dpattach, /* attach */ 105 dpdetach, /* detach */ 106 nodev, /* reset */ 107 &dp_cb_ops, /* driver operations */ 108 (struct bus_ops *)NULL, /* bus operations */ 109 nulldev, /* power */ 110 ddi_quiesce_not_needed, /* quiesce */ 111 }; 112 113 114 static struct modldrv modldrv = { 115 &mod_driverops, /* type of module - a driver */ 116 "/dev/poll driver", 117 &dp_ops, 118 }; 119 120 static struct modlinkage modlinkage = { 121 MODREV_1, 122 (void *)&modldrv, 123 NULL 124 }; 125 126 static void pcachelink_assoc(pollcache_t *, pollcache_t *); 127 static void pcachelink_mark_stale(pollcache_t *); 128 static void pcachelink_purge_stale(pollcache_t *); 129 static void pcachelink_purge_all(pollcache_t *); 130 131 132 /* 133 * Locking Design 134 * 135 * The /dev/poll driver shares most of its code with poll sys call whose 136 * code is in common/syscall/poll.c. In poll(2) design, the pollcache 137 * structure is per lwp. An implicit assumption is made there that some 138 * portion of pollcache will never be touched by other lwps. E.g., in 139 * poll(2) design, no lwp will ever need to grow bitmap of other lwp. 140 * This assumption is not true for /dev/poll; hence the need for extra 141 * locking. 142 * 143 * To allow more parallelism, each /dev/poll file descriptor (indexed by 144 * minor number) has its own lock. Since read (dpioctl) is a much more 145 * frequent operation than write, we want to allow multiple reads on same 146 * /dev/poll fd. However, we prevent writes from being starved by giving 147 * priority to write operation. Theoretically writes can starve reads as 148 * well. But in practical sense this is not important because (1) writes 149 * happens less often than reads, and (2) write operation defines the 150 * content of poll fd a cache set. If writes happens so often that they 151 * can starve reads, that means the cached set is very unstable. It may 152 * not make sense to read an unstable cache set anyway. Therefore, the 153 * writers starving readers case is not handled in this design. 154 */ 155 156 int 157 _init() 158 { 159 int error; 160 161 dptblsize = DEVPOLLSIZE; 162 devpolltbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP); 163 mutex_init(&devpoll_lock, NULL, MUTEX_DEFAULT, NULL); 164 devpoll_init = 1; 165 if ((error = mod_install(&modlinkage)) != 0) { 166 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize); 167 devpoll_init = 0; 168 } 169 return (error); 170 } 171 172 int 173 _fini() 174 { 175 int error; 176 177 if ((error = mod_remove(&modlinkage)) != 0) { 178 return (error); 179 } 180 mutex_destroy(&devpoll_lock); 181 kmem_free(devpolltbl, sizeof (caddr_t) * dptblsize); 182 return (0); 183 } 184 185 int 186 _info(struct modinfo *modinfop) 187 { 188 return (mod_info(&modlinkage, modinfop)); 189 } 190 191 /*ARGSUSED*/ 192 static int 193 dpattach(dev_info_t *devi, ddi_attach_cmd_t cmd) 194 { 195 if (ddi_create_minor_node(devi, "poll", S_IFCHR, 0, DDI_PSEUDO, 0) 196 == DDI_FAILURE) { 197 ddi_remove_minor_node(devi, NULL); 198 return (DDI_FAILURE); 199 } 200 dpdevi = devi; 201 return (DDI_SUCCESS); 202 } 203 204 static int 205 dpdetach(dev_info_t *devi, ddi_detach_cmd_t cmd) 206 { 207 if (cmd != DDI_DETACH) 208 return (DDI_FAILURE); 209 210 ddi_remove_minor_node(devi, NULL); 211 return (DDI_SUCCESS); 212 } 213 214 /* ARGSUSED */ 215 static int 216 dpinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 217 { 218 int error; 219 220 switch (infocmd) { 221 case DDI_INFO_DEVT2DEVINFO: 222 *result = (void *)dpdevi; 223 error = DDI_SUCCESS; 224 break; 225 case DDI_INFO_DEVT2INSTANCE: 226 *result = (void *)0; 227 error = DDI_SUCCESS; 228 break; 229 default: 230 error = DDI_FAILURE; 231 } 232 return (error); 233 } 234 235 /* 236 * dp_pcache_poll has similar logic to pcache_poll() in poll.c. The major 237 * differences are: (1) /dev/poll requires scanning the bitmap starting at 238 * where it was stopped last time, instead of always starting from 0, 239 * (2) since user may not have cleaned up the cached fds when they are 240 * closed, some polldats in cache may refer to closed or reused fds. We 241 * need to check for those cases. 242 * 243 * NOTE: Upon closing an fd, automatic poll cache cleanup is done for 244 * poll(2) caches but NOT for /dev/poll caches. So expect some 245 * stale entries! 246 */ 247 static int 248 dp_pcache_poll(dp_entry_t *dpep, void *dpbuf, 249 pollcache_t *pcp, nfds_t nfds, int *fdcntp) 250 { 251 int start, ostart, end; 252 int fdcnt, fd; 253 boolean_t done; 254 file_t *fp; 255 short revent; 256 boolean_t no_wrap; 257 pollhead_t *php; 258 polldat_t *pdp; 259 pollfd_t *pfdp; 260 epoll_event_t *epoll; 261 int error = 0; 262 short mask = POLLRDHUP | POLLWRBAND; 263 boolean_t is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0; 264 265 ASSERT(MUTEX_HELD(&pcp->pc_lock)); 266 if (pcp->pc_bitmap == NULL) { 267 /* 268 * No Need to search because no poll fd 269 * has been cached. 270 */ 271 return (error); 272 } 273 274 if (is_epoll) { 275 pfdp = NULL; 276 epoll = (epoll_event_t *)dpbuf; 277 } else { 278 pfdp = (pollfd_t *)dpbuf; 279 epoll = NULL; 280 } 281 retry: 282 start = ostart = pcp->pc_mapstart; 283 end = pcp->pc_mapend; 284 php = NULL; 285 286 if (start == 0) { 287 /* 288 * started from every begining, no need to wrap around. 289 */ 290 no_wrap = B_TRUE; 291 } else { 292 no_wrap = B_FALSE; 293 } 294 done = B_FALSE; 295 fdcnt = 0; 296 while ((fdcnt < nfds) && !done) { 297 php = NULL; 298 revent = 0; 299 /* 300 * Examine the bit map in a circular fashion 301 * to avoid starvation. Always resume from 302 * last stop. Scan till end of the map. Then 303 * wrap around. 304 */ 305 fd = bt_getlowbit(pcp->pc_bitmap, start, end); 306 ASSERT(fd <= end); 307 if (fd >= 0) { 308 if (fd == end) { 309 if (no_wrap) { 310 done = B_TRUE; 311 } else { 312 start = 0; 313 end = ostart - 1; 314 no_wrap = B_TRUE; 315 } 316 } else { 317 start = fd + 1; 318 } 319 pdp = pcache_lookup_fd(pcp, fd); 320 repoll: 321 ASSERT(pdp != NULL); 322 ASSERT(pdp->pd_fd == fd); 323 if (pdp->pd_fp == NULL) { 324 /* 325 * The fd is POLLREMOVed. This fd is 326 * logically no longer cached. So move 327 * on to the next one. 328 */ 329 continue; 330 } 331 if ((fp = getf(fd)) == NULL) { 332 /* 333 * The fd has been closed, but user has not 334 * done a POLLREMOVE on this fd yet. Instead 335 * of cleaning it here implicitly, we return 336 * POLLNVAL. This is consistent with poll(2) 337 * polling a closed fd. Hope this will remind 338 * user to do a POLLREMOVE. 339 */ 340 if (!is_epoll && pfdp != NULL) { 341 pfdp[fdcnt].fd = fd; 342 pfdp[fdcnt].revents = POLLNVAL; 343 fdcnt++; 344 continue; 345 } 346 347 /* 348 * In the epoll compatibility case, we actually 349 * perform the implicit removal to remain 350 * closer to the epoll semantics. 351 */ 352 if (is_epoll) { 353 pdp->pd_fp = NULL; 354 pdp->pd_events = 0; 355 356 if (pdp->pd_php != NULL) { 357 pollhead_delete(pdp->pd_php, 358 pdp); 359 pdp->pd_php = NULL; 360 } 361 362 BT_CLEAR(pcp->pc_bitmap, fd); 363 continue; 364 } 365 } 366 367 if (fp != pdp->pd_fp) { 368 /* 369 * The user is polling on a cached fd which was 370 * closed and then reused. Unfortunately there 371 * is no good way to communicate this fact to 372 * the consumer. 373 * 374 * If the file struct is also reused, we may 375 * not be able to detect the fd reuse at all. 376 * As long as this does not cause system 377 * failure and/or memory leaks, we will play 378 * along. The man page states that if the user 379 * does not clean up closed fds, polling 380 * results will be indeterministic. 381 * 382 * XXX: perhaps log the detection of fd reuse? 383 */ 384 pdp->pd_fp = fp; 385 386 /* 387 * When this situation has been detected, it's 388 * likely that any existing pollhead is 389 * ill-suited to perform proper wake-ups. 390 * 391 * Clean up the old entry under the expectation 392 * that a valid one will be provided as part of 393 * the later VOP_POLL. 394 */ 395 if (pdp->pd_php != NULL) { 396 pollhead_delete(pdp->pd_php, pdp); 397 pdp->pd_php = NULL; 398 } 399 } 400 /* 401 * XXX - pollrelock() logic needs to know which 402 * which pollcache lock to grab. It'd be a 403 * cleaner solution if we could pass pcp as 404 * an arguement in VOP_POLL interface instead 405 * of implicitly passing it using thread_t 406 * struct. On the other hand, changing VOP_POLL 407 * interface will require all driver/file system 408 * poll routine to change. May want to revisit 409 * the tradeoff later. 410 */ 411 curthread->t_pollcache = pcp; 412 error = VOP_POLL(fp->f_vnode, pdp->pd_events, 0, 413 &revent, &php, NULL); 414 415 /* 416 * Recheck edge-triggered descriptors which lack a 417 * pollhead. While this check is performed when an fd 418 * is added to the pollcache in dpwrite(), subsequent 419 * descriptor manipulation could cause a different 420 * resource to be present now. 421 */ 422 if ((pdp->pd_events & POLLET) && error == 0 && 423 pdp->pd_php == NULL && php == NULL && revent != 0) { 424 short levent = 0; 425 426 /* 427 * The same POLLET-only VOP_POLL is used in an 428 * attempt to coax a pollhead from older 429 * driver logic. 430 */ 431 error = VOP_POLL(fp->f_vnode, POLLET, 432 0, &levent, &php, NULL); 433 } 434 435 curthread->t_pollcache = NULL; 436 releasef(fd); 437 if (error != 0) { 438 break; 439 } 440 441 /* 442 * layered devices (e.g. console driver) 443 * may change the vnode and thus the pollhead 444 * pointer out from underneath us. 445 */ 446 if (php != NULL && pdp->pd_php != NULL && 447 php != pdp->pd_php) { 448 pollhead_delete(pdp->pd_php, pdp); 449 pdp->pd_php = php; 450 pollhead_insert(php, pdp); 451 /* 452 * The bit should still be set. 453 */ 454 ASSERT(BT_TEST(pcp->pc_bitmap, fd)); 455 goto retry; 456 } 457 458 if (revent != 0) { 459 if (pfdp != NULL) { 460 pfdp[fdcnt].fd = fd; 461 pfdp[fdcnt].events = pdp->pd_events; 462 pfdp[fdcnt].revents = revent; 463 } else if (epoll != NULL) { 464 epoll_event_t *ep = &epoll[fdcnt]; 465 466 ASSERT(epoll != NULL); 467 ep->data.u64 = pdp->pd_epolldata; 468 469 /* 470 * Since POLLNVAL is a legal event for 471 * VOP_POLL handlers to emit, it must 472 * be translated epoll-legal. 473 */ 474 if (revent & POLLNVAL) { 475 revent &= ~POLLNVAL; 476 revent |= POLLERR; 477 } 478 479 /* 480 * If any of the event bits are set for 481 * which poll and epoll representations 482 * differ, swizzle in the native epoll 483 * values. 484 */ 485 if (revent & mask) { 486 ep->events = (revent & ~mask) | 487 ((revent & POLLRDHUP) ? 488 EPOLLRDHUP : 0) | 489 ((revent & POLLWRBAND) ? 490 EPOLLWRBAND : 0); 491 } else { 492 ep->events = revent; 493 } 494 495 /* 496 * We define POLLWRNORM to be POLLOUT, 497 * but epoll has separate definitions 498 * for them; if POLLOUT is set and the 499 * user has asked for EPOLLWRNORM, set 500 * that as well. 501 */ 502 if ((revent & POLLOUT) && 503 (pdp->pd_events & EPOLLWRNORM)) { 504 ep->events |= EPOLLWRNORM; 505 } 506 } else { 507 pollstate_t *ps = 508 curthread->t_pollstate; 509 /* 510 * The devpoll handle itself is being 511 * polled. Notify the caller of any 512 * readable event(s), leaving as much 513 * state as possible untouched. 514 */ 515 VERIFY(fdcnt == 0); 516 VERIFY(ps != NULL); 517 518 /* 519 * If a call to pollunlock() fails 520 * during VOP_POLL, skip over the fd 521 * and continue polling. 522 * 523 * Otherwise, report that there is an 524 * event pending. 525 */ 526 if ((ps->ps_flags & POLLSTATE_ULFAIL) 527 != 0) { 528 ps->ps_flags &= 529 ~POLLSTATE_ULFAIL; 530 continue; 531 } else { 532 fdcnt++; 533 break; 534 } 535 } 536 537 /* Handle special polling modes. */ 538 if (pdp->pd_events & POLLONESHOT) { 539 /* 540 * If POLLONESHOT is set, perform the 541 * implicit POLLREMOVE. 542 */ 543 pdp->pd_fp = NULL; 544 pdp->pd_events = 0; 545 546 if (pdp->pd_php != NULL) { 547 pollhead_delete(pdp->pd_php, 548 pdp); 549 pdp->pd_php = NULL; 550 } 551 552 BT_CLEAR(pcp->pc_bitmap, fd); 553 } else if (pdp->pd_events & POLLET) { 554 /* 555 * Wire up the pollhead which should 556 * have been provided. Edge-triggered 557 * polling cannot function properly 558 * with drivers which do not emit one. 559 */ 560 if (php != NULL && 561 pdp->pd_php == NULL) { 562 pollhead_insert(php, pdp); 563 pdp->pd_php = php; 564 } 565 566 /* 567 * If the driver has emitted a pollhead, 568 * clear the bit in the bitmap which 569 * effectively latches the edge on a 570 * pollwakeup() from the driver. 571 */ 572 if (pdp->pd_php != NULL) { 573 BT_CLEAR(pcp->pc_bitmap, fd); 574 } 575 } 576 577 fdcnt++; 578 } else if (php != NULL) { 579 /* 580 * We clear a bit or cache a poll fd if 581 * the driver returns a poll head ptr, 582 * which is expected in the case of 0 583 * revents. Some buggy driver may return 584 * NULL php pointer with 0 revents. In 585 * this case, we just treat the driver as 586 * "noncachable" and not clearing the bit 587 * in bitmap. 588 */ 589 if ((pdp->pd_php != NULL) && 590 ((pcp->pc_flag & PC_POLLWAKE) == 0)) { 591 BT_CLEAR(pcp->pc_bitmap, fd); 592 } 593 if (pdp->pd_php == NULL) { 594 pollhead_insert(php, pdp); 595 pdp->pd_php = php; 596 /* 597 * An event of interest may have 598 * arrived between the VOP_POLL() and 599 * the pollhead_insert(); check again. 600 */ 601 goto repoll; 602 } 603 } 604 } else { 605 /* 606 * No bit set in the range. Check for wrap around. 607 */ 608 if (!no_wrap) { 609 start = 0; 610 end = ostart - 1; 611 no_wrap = B_TRUE; 612 } else { 613 done = B_TRUE; 614 } 615 } 616 } 617 618 if (!done) { 619 pcp->pc_mapstart = start; 620 } 621 ASSERT(*fdcntp == 0); 622 *fdcntp = fdcnt; 623 return (error); 624 } 625 626 /*ARGSUSED*/ 627 static int 628 dpopen(dev_t *devp, int flag, int otyp, cred_t *credp) 629 { 630 minor_t minordev; 631 dp_entry_t *dpep; 632 pollcache_t *pcp; 633 634 ASSERT(devpoll_init); 635 ASSERT(dptblsize <= MAXMIN); 636 mutex_enter(&devpoll_lock); 637 for (minordev = 0; minordev < dptblsize; minordev++) { 638 if (devpolltbl[minordev] == NULL) { 639 devpolltbl[minordev] = (dp_entry_t *)RESERVED; 640 break; 641 } 642 } 643 if (minordev == dptblsize) { 644 dp_entry_t **newtbl; 645 size_t oldsize; 646 647 /* 648 * Used up every entry in the existing devpoll table. 649 * Grow the table by DEVPOLLSIZE. 650 */ 651 if ((oldsize = dptblsize) >= MAXMIN) { 652 mutex_exit(&devpoll_lock); 653 return (ENXIO); 654 } 655 dptblsize += DEVPOLLSIZE; 656 if (dptblsize > MAXMIN) { 657 dptblsize = MAXMIN; 658 } 659 newtbl = kmem_zalloc(sizeof (caddr_t) * dptblsize, KM_SLEEP); 660 bcopy(devpolltbl, newtbl, sizeof (caddr_t) * oldsize); 661 kmem_free(devpolltbl, sizeof (caddr_t) * oldsize); 662 devpolltbl = newtbl; 663 devpolltbl[minordev] = (dp_entry_t *)RESERVED; 664 } 665 mutex_exit(&devpoll_lock); 666 667 dpep = kmem_zalloc(sizeof (dp_entry_t), KM_SLEEP); 668 /* 669 * allocate a pollcache skeleton here. Delay allocating bitmap 670 * structures until dpwrite() time, since we don't know the 671 * optimal size yet. We also delay setting the pid until either 672 * dpwrite() or attempt to poll on the instance, allowing parents 673 * to create instances of /dev/poll for their children. (In the 674 * epoll compatibility case, this check isn't performed to maintain 675 * semantic compatibility.) 676 */ 677 pcp = pcache_alloc(); 678 dpep->dpe_pcache = pcp; 679 pcp->pc_pid = -1; 680 *devp = makedevice(getmajor(*devp), minordev); /* clone the driver */ 681 mutex_enter(&devpoll_lock); 682 ASSERT(minordev < dptblsize); 683 ASSERT(devpolltbl[minordev] == (dp_entry_t *)RESERVED); 684 devpolltbl[minordev] = dpep; 685 mutex_exit(&devpoll_lock); 686 return (0); 687 } 688 689 /* 690 * Write to dev/poll add/remove fd's to/from a cached poll fd set, 691 * or change poll events for a watched fd. 692 */ 693 /*ARGSUSED*/ 694 static int 695 dpwrite(dev_t dev, struct uio *uiop, cred_t *credp) 696 { 697 minor_t minor; 698 dp_entry_t *dpep; 699 pollcache_t *pcp; 700 pollfd_t *pollfdp, *pfdp; 701 dvpoll_epollfd_t *epfdp; 702 uintptr_t limit; 703 int error, size; 704 ssize_t uiosize; 705 size_t copysize; 706 nfds_t pollfdnum; 707 struct pollhead *php = NULL; 708 polldat_t *pdp; 709 int fd; 710 file_t *fp; 711 boolean_t is_epoll, fds_added = B_FALSE; 712 713 minor = getminor(dev); 714 715 mutex_enter(&devpoll_lock); 716 ASSERT(minor < dptblsize); 717 dpep = devpolltbl[minor]; 718 ASSERT(dpep != NULL); 719 mutex_exit(&devpoll_lock); 720 721 mutex_enter(&dpep->dpe_lock); 722 pcp = dpep->dpe_pcache; 723 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0; 724 size = (is_epoll) ? sizeof (dvpoll_epollfd_t) : sizeof (pollfd_t); 725 mutex_exit(&dpep->dpe_lock); 726 727 if (!is_epoll && curproc->p_pid != pcp->pc_pid) { 728 if (pcp->pc_pid != -1) { 729 return (EACCES); 730 } 731 732 pcp->pc_pid = curproc->p_pid; 733 } 734 735 uiosize = uiop->uio_resid; 736 pollfdnum = uiosize / size; 737 738 /* 739 * For epoll-enabled handles, restrict the allowed write size to 2. 740 * This corresponds to an epoll_ctl(3C) performing an EPOLL_CTL_MOD 741 * operation which is expanded into two operations (DEL and ADD). 742 * 743 * All other operations performed through epoll_ctl(3C) will consist of 744 * a single entry. 745 */ 746 if (is_epoll && pollfdnum > 2) { 747 return (EINVAL); 748 } 749 750 /* 751 * We want to make sure that pollfdnum isn't large enough to DoS us, 752 * but we also don't want to grab p_lock unnecessarily -- so we 753 * perform the full check against our resource limits if and only if 754 * pollfdnum is larger than the known-to-be-sane value of UINT8_MAX. 755 */ 756 if (pollfdnum > UINT8_MAX) { 757 mutex_enter(&curproc->p_lock); 758 if (pollfdnum > 759 (uint_t)rctl_enforced_value(rctlproc_legacy[RLIMIT_NOFILE], 760 curproc->p_rctls, curproc)) { 761 (void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE], 762 curproc->p_rctls, curproc, RCA_SAFE); 763 mutex_exit(&curproc->p_lock); 764 return (EINVAL); 765 } 766 mutex_exit(&curproc->p_lock); 767 } 768 769 /* 770 * Copy in the pollfd array. Walk through the array and add 771 * each polled fd to the cached set. 772 */ 773 pollfdp = kmem_alloc(uiosize, KM_SLEEP); 774 limit = (uintptr_t)pollfdp + (pollfdnum * size); 775 776 /* 777 * Although /dev/poll uses the write(2) interface to cache fds, it's 778 * not supposed to function as a seekable device. To prevent offset 779 * from growing and eventually exceed the maximum, reset the offset 780 * here for every call. 781 */ 782 uiop->uio_loffset = 0; 783 784 /* 785 * Use uiocopy instead of uiomove when populating pollfdp, keeping 786 * uio_resid untouched for now. Write syscalls will translate EINTR 787 * into a success if they detect "successfully transfered" data via an 788 * updated uio_resid. Falsely suppressing such errors is disastrous. 789 */ 790 if ((error = uiocopy((caddr_t)pollfdp, uiosize, UIO_WRITE, uiop, 791 ©size)) != 0) { 792 kmem_free(pollfdp, uiosize); 793 return (error); 794 } 795 796 /* 797 * We are about to enter the core portion of dpwrite(). Make sure this 798 * write has exclusive access in this portion of the code, i.e., no 799 * other writers in this code. 800 * 801 * Waiting for all readers to drop their references to the dpe is 802 * unecessary since the pollcache itself is protected by pc_lock. 803 */ 804 mutex_enter(&dpep->dpe_lock); 805 dpep->dpe_writerwait++; 806 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0) { 807 ASSERT(dpep->dpe_refcnt != 0); 808 809 /* 810 * The epoll API does not allow EINTR as a result when making 811 * modifications to the set of polled fds. Given that write 812 * activity is relatively quick and the size of accepted writes 813 * is limited above to two entries, a signal-ignorant wait is 814 * used here to avoid the EINTR. 815 */ 816 if (is_epoll) { 817 cv_wait(&dpep->dpe_cv, &dpep->dpe_lock); 818 continue; 819 } 820 821 /* 822 * Non-epoll writers to /dev/poll handles can tolerate EINTR. 823 */ 824 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) { 825 dpep->dpe_writerwait--; 826 mutex_exit(&dpep->dpe_lock); 827 kmem_free(pollfdp, uiosize); 828 return (EINTR); 829 } 830 } 831 dpep->dpe_writerwait--; 832 dpep->dpe_flag |= DP_WRITER_PRESENT; 833 dpep->dpe_refcnt++; 834 835 if (!is_epoll && (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0) { 836 /* 837 * The epoll compat mode was enabled while we were waiting to 838 * establish write access. It is not safe to continue since 839 * state was prepared for non-epoll operation. 840 */ 841 error = EBUSY; 842 goto bypass; 843 } 844 mutex_exit(&dpep->dpe_lock); 845 846 /* 847 * Since the dpwrite() may recursively walk an added /dev/poll handle, 848 * pollstate_enter() deadlock and loop detection must be used. 849 */ 850 (void) pollstate_create(); 851 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS); 852 853 if (pcp->pc_bitmap == NULL) { 854 pcache_create(pcp, pollfdnum); 855 } 856 for (pfdp = pollfdp; (uintptr_t)pfdp < limit; 857 pfdp = (pollfd_t *)((uintptr_t)pfdp + size)) { 858 fd = pfdp->fd; 859 if ((uint_t)fd >= P_FINFO(curproc)->fi_nfiles) { 860 /* 861 * epoll semantics demand that we return EBADF if our 862 * specified fd is invalid. 863 */ 864 if (is_epoll) { 865 error = EBADF; 866 break; 867 } 868 869 continue; 870 } 871 872 pdp = pcache_lookup_fd(pcp, fd); 873 if (pfdp->events != POLLREMOVE) { 874 875 fp = NULL; 876 877 if (pdp == NULL) { 878 /* 879 * If we're in epoll compatibility mode, check 880 * that the fd is valid before allocating 881 * anything for it; epoll semantics demand that 882 * we return EBADF if our specified fd is 883 * invalid. 884 */ 885 if (is_epoll) { 886 if ((fp = getf(fd)) == NULL) { 887 error = EBADF; 888 break; 889 } 890 } 891 892 pdp = pcache_alloc_fd(0); 893 pdp->pd_fd = fd; 894 pdp->pd_pcache = pcp; 895 pcache_insert_fd(pcp, pdp, pollfdnum); 896 } else { 897 /* 898 * epoll semantics demand that we error out if 899 * a file descriptor is added twice, which we 900 * check (imperfectly) by checking if we both 901 * have the file descriptor cached and the 902 * file pointer that correponds to the file 903 * descriptor matches our cached value. If 904 * there is a pointer mismatch, the file 905 * descriptor was closed without being removed. 906 * The converse is clearly not true, however, 907 * so to narrow the window by which a spurious 908 * EEXIST may be returned, we also check if 909 * this fp has been added to an epoll control 910 * descriptor in the past; if it hasn't, we 911 * know that this is due to fp reuse -- it's 912 * not a true EEXIST case. (By performing this 913 * additional check, we limit the window of 914 * spurious EEXIST to situations where a single 915 * file descriptor is being used across two or 916 * more epoll control descriptors -- and even 917 * then, the file descriptor must be closed and 918 * reused in a relatively tight time span.) 919 */ 920 if (is_epoll) { 921 if (pdp->pd_fp != NULL && 922 (fp = getf(fd)) != NULL && 923 fp == pdp->pd_fp && 924 (fp->f_flag2 & FEPOLLED)) { 925 error = EEXIST; 926 releasef(fd); 927 break; 928 } 929 930 /* 931 * We have decided that the cached 932 * information was stale: it either 933 * didn't match, or the fp had never 934 * actually been epoll()'d on before. 935 * We need to now clear our pd_events 936 * to assure that we don't mistakenly 937 * operate on cached event disposition. 938 */ 939 pdp->pd_events = 0; 940 } 941 } 942 943 if (is_epoll) { 944 epfdp = (dvpoll_epollfd_t *)pfdp; 945 pdp->pd_epolldata = epfdp->dpep_data; 946 } 947 948 ASSERT(pdp->pd_fd == fd); 949 ASSERT(pdp->pd_pcache == pcp); 950 if (fd >= pcp->pc_mapsize) { 951 mutex_exit(&pcp->pc_lock); 952 pcache_grow_map(pcp, fd); 953 mutex_enter(&pcp->pc_lock); 954 } 955 if (fd > pcp->pc_mapend) { 956 pcp->pc_mapend = fd; 957 } 958 if (fp == NULL && (fp = getf(fd)) == NULL) { 959 /* 960 * The fd is not valid. Since we can't pass 961 * this error back in the write() call, set 962 * the bit in bitmap to force DP_POLL ioctl 963 * to examine it. 964 */ 965 BT_SET(pcp->pc_bitmap, fd); 966 pdp->pd_events |= pfdp->events; 967 continue; 968 } 969 970 /* 971 * To (greatly) reduce EEXIST false positives, we 972 * denote that this fp has been epoll()'d. We do this 973 * regardless of epoll compatibility mode, as the flag 974 * is harmless if not in epoll compatibility mode. 975 */ 976 fp->f_flag2 |= FEPOLLED; 977 978 /* 979 * Don't do VOP_POLL for an already cached fd with 980 * same poll events. 981 */ 982 if ((pdp->pd_events == pfdp->events) && 983 (pdp->pd_fp == fp)) { 984 /* 985 * the events are already cached 986 */ 987 releasef(fd); 988 continue; 989 } 990 991 /* 992 * do VOP_POLL and cache this poll fd. 993 */ 994 /* 995 * XXX - pollrelock() logic needs to know which 996 * which pollcache lock to grab. It'd be a 997 * cleaner solution if we could pass pcp as 998 * an arguement in VOP_POLL interface instead 999 * of implicitly passing it using thread_t 1000 * struct. On the other hand, changing VOP_POLL 1001 * interface will require all driver/file system 1002 * poll routine to change. May want to revisit 1003 * the tradeoff later. 1004 */ 1005 curthread->t_pollcache = pcp; 1006 error = VOP_POLL(fp->f_vnode, pfdp->events, 0, 1007 &pfdp->revents, &php, NULL); 1008 1009 /* 1010 * Edge-triggered polling requires a pollhead in order 1011 * to initiate wake-ups properly. Drivers which are 1012 * savvy to POLLET presence, which should include 1013 * everything in-gate, will always emit one, regardless 1014 * of revent status. Older drivers which only emit a 1015 * pollhead if 'revents == 0' are given a second chance 1016 * here via a second VOP_POLL, with only POLLET set in 1017 * the events of interest. These circumstances should 1018 * induce any cacheable drivers to emit a pollhead for 1019 * wake-ups. 1020 * 1021 * Drivers which never emit a pollhead will simply 1022 * disobey the exectation of edge-triggered behavior. 1023 * This includes recursive epoll which, even on Linux, 1024 * yields its events in a level-triggered fashion only. 1025 */ 1026 if ((pdp->pd_events & POLLET) && error == 0 && 1027 php == NULL) { 1028 short levent = 0; 1029 1030 error = VOP_POLL(fp->f_vnode, POLLET, 0, 1031 &levent, &php, NULL); 1032 } 1033 1034 curthread->t_pollcache = NULL; 1035 /* 1036 * We always set the bit when this fd is cached; 1037 * this forces the first DP_POLL to poll this fd. 1038 * Real performance gain comes from subsequent 1039 * DP_POLL. We also attempt a pollhead_insert(); 1040 * if it's not possible, we'll do it in dpioctl(). 1041 */ 1042 BT_SET(pcp->pc_bitmap, fd); 1043 if (error != 0) { 1044 releasef(fd); 1045 break; 1046 } 1047 pdp->pd_fp = fp; 1048 pdp->pd_events |= pfdp->events; 1049 if (php != NULL) { 1050 if (pdp->pd_php == NULL) { 1051 pollhead_insert(php, pdp); 1052 pdp->pd_php = php; 1053 } else { 1054 if (pdp->pd_php != php) { 1055 pollhead_delete(pdp->pd_php, 1056 pdp); 1057 pollhead_insert(php, pdp); 1058 pdp->pd_php = php; 1059 } 1060 } 1061 } 1062 fds_added = B_TRUE; 1063 releasef(fd); 1064 } else { 1065 if (pdp == NULL || pdp->pd_fp == NULL) { 1066 if (is_epoll) { 1067 /* 1068 * As with the add case (above), epoll 1069 * semantics demand that we error out 1070 * in this case. 1071 */ 1072 error = ENOENT; 1073 break; 1074 } 1075 1076 continue; 1077 } 1078 ASSERT(pdp->pd_fd == fd); 1079 pdp->pd_fp = NULL; 1080 pdp->pd_events = 0; 1081 ASSERT(pdp->pd_thread == NULL); 1082 if (pdp->pd_php != NULL) { 1083 pollhead_delete(pdp->pd_php, pdp); 1084 pdp->pd_php = NULL; 1085 } 1086 BT_CLEAR(pcp->pc_bitmap, fd); 1087 } 1088 } 1089 /* 1090 * Wake any pollcache waiters so they can check the new descriptors. 1091 * 1092 * Any fds added to an recursive-capable pollcache could themselves be 1093 * /dev/poll handles. To ensure that proper event propagation occurs, 1094 * parent pollcaches are woken too, so that they can create any needed 1095 * pollcache links. 1096 */ 1097 if (fds_added) { 1098 cv_broadcast(&pcp->pc_cv); 1099 pcache_wake_parents(pcp); 1100 } 1101 pollstate_exit(pcp); 1102 mutex_enter(&dpep->dpe_lock); 1103 bypass: 1104 dpep->dpe_flag &= ~DP_WRITER_PRESENT; 1105 dpep->dpe_refcnt--; 1106 cv_broadcast(&dpep->dpe_cv); 1107 mutex_exit(&dpep->dpe_lock); 1108 kmem_free(pollfdp, uiosize); 1109 if (error == 0) { 1110 /* 1111 * The state of uio_resid is updated only after the pollcache 1112 * is successfully modified. 1113 */ 1114 uioskip(uiop, copysize); 1115 } 1116 return (error); 1117 } 1118 1119 #define DP_SIGMASK_RESTORE(ksetp) { \ 1120 if (ksetp != NULL) { \ 1121 mutex_enter(&p->p_lock); \ 1122 if (lwp->lwp_cursig == 0) { \ 1123 t->t_hold = lwp->lwp_sigoldmask; \ 1124 t->t_flag &= ~T_TOMASK; \ 1125 } \ 1126 mutex_exit(&p->p_lock); \ 1127 } \ 1128 } 1129 1130 /*ARGSUSED*/ 1131 static int 1132 dpioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp) 1133 { 1134 minor_t minor; 1135 dp_entry_t *dpep; 1136 pollcache_t *pcp; 1137 hrtime_t now; 1138 int error = 0; 1139 boolean_t is_epoll; 1140 STRUCT_DECL(dvpoll, dvpoll); 1141 1142 if (cmd == DP_POLL || cmd == DP_PPOLL) { 1143 /* do this now, before we sleep on DP_WRITER_PRESENT */ 1144 now = gethrtime(); 1145 } 1146 1147 minor = getminor(dev); 1148 mutex_enter(&devpoll_lock); 1149 ASSERT(minor < dptblsize); 1150 dpep = devpolltbl[minor]; 1151 mutex_exit(&devpoll_lock); 1152 ASSERT(dpep != NULL); 1153 pcp = dpep->dpe_pcache; 1154 1155 mutex_enter(&dpep->dpe_lock); 1156 is_epoll = (dpep->dpe_flag & DP_ISEPOLLCOMPAT) != 0; 1157 1158 if (cmd == DP_EPOLLCOMPAT) { 1159 if (dpep->dpe_refcnt != 0) { 1160 /* 1161 * We can't turn on epoll compatibility while there 1162 * are outstanding operations. 1163 */ 1164 mutex_exit(&dpep->dpe_lock); 1165 return (EBUSY); 1166 } 1167 1168 /* 1169 * epoll compatibility is a one-way street: there's no way 1170 * to turn it off for a particular open. 1171 */ 1172 dpep->dpe_flag |= DP_ISEPOLLCOMPAT; 1173 1174 /* Record the epoll-enabled nature in the pollcache too */ 1175 mutex_enter(&pcp->pc_lock); 1176 pcp->pc_flag |= PC_EPOLL; 1177 mutex_exit(&pcp->pc_lock); 1178 1179 mutex_exit(&dpep->dpe_lock); 1180 return (0); 1181 } 1182 1183 if (!is_epoll && curproc->p_pid != pcp->pc_pid) { 1184 if (pcp->pc_pid != -1) { 1185 mutex_exit(&dpep->dpe_lock); 1186 return (EACCES); 1187 } 1188 1189 pcp->pc_pid = curproc->p_pid; 1190 } 1191 1192 /* Wait until all writers have cleared the handle before continuing */ 1193 while ((dpep->dpe_flag & DP_WRITER_PRESENT) != 0 || 1194 (dpep->dpe_writerwait != 0)) { 1195 if (!cv_wait_sig_swap(&dpep->dpe_cv, &dpep->dpe_lock)) { 1196 mutex_exit(&dpep->dpe_lock); 1197 return (EINTR); 1198 } 1199 } 1200 dpep->dpe_refcnt++; 1201 mutex_exit(&dpep->dpe_lock); 1202 1203 switch (cmd) { 1204 case DP_POLL: 1205 case DP_PPOLL: 1206 { 1207 pollstate_t *ps; 1208 nfds_t nfds; 1209 int fdcnt = 0; 1210 size_t size, fdsize, dpsize; 1211 hrtime_t deadline = 0; 1212 k_sigset_t *ksetp = NULL; 1213 k_sigset_t kset; 1214 sigset_t set; 1215 kthread_t *t = curthread; 1216 klwp_t *lwp = ttolwp(t); 1217 struct proc *p = ttoproc(curthread); 1218 1219 STRUCT_INIT(dvpoll, mode); 1220 1221 /* 1222 * The dp_setp member is only required/consumed for DP_PPOLL, 1223 * which otherwise uses the same structure as DP_POLL. 1224 */ 1225 if (cmd == DP_POLL) { 1226 dpsize = (uintptr_t)STRUCT_FADDR(dvpoll, dp_setp) - 1227 (uintptr_t)STRUCT_FADDR(dvpoll, dp_fds); 1228 } else { 1229 ASSERT(cmd == DP_PPOLL); 1230 dpsize = STRUCT_SIZE(dvpoll); 1231 } 1232 1233 if ((mode & FKIOCTL) != 0) { 1234 /* Kernel-internal ioctl call */ 1235 bcopy((caddr_t)arg, STRUCT_BUF(dvpoll), dpsize); 1236 error = 0; 1237 } else { 1238 error = copyin((caddr_t)arg, STRUCT_BUF(dvpoll), 1239 dpsize); 1240 } 1241 1242 if (error) { 1243 DP_REFRELE(dpep); 1244 return (EFAULT); 1245 } 1246 1247 deadline = STRUCT_FGET(dvpoll, dp_timeout); 1248 if (deadline > 0) { 1249 /* 1250 * Convert the deadline from relative milliseconds 1251 * to absolute nanoseconds. They must wait for at 1252 * least a tick. 1253 */ 1254 deadline = MSEC2NSEC(deadline); 1255 deadline = MAX(deadline, nsec_per_tick); 1256 deadline += now; 1257 } 1258 1259 if (cmd == DP_PPOLL) { 1260 void *setp = STRUCT_FGETP(dvpoll, dp_setp); 1261 1262 if (setp != NULL) { 1263 if ((mode & FKIOCTL) != 0) { 1264 /* Use the signal set directly */ 1265 ksetp = (k_sigset_t *)setp; 1266 } else { 1267 if (copyin(setp, &set, sizeof (set))) { 1268 DP_REFRELE(dpep); 1269 return (EFAULT); 1270 } 1271 sigutok(&set, &kset); 1272 ksetp = &kset; 1273 } 1274 1275 mutex_enter(&p->p_lock); 1276 schedctl_finish_sigblock(t); 1277 lwp->lwp_sigoldmask = t->t_hold; 1278 t->t_hold = *ksetp; 1279 t->t_flag |= T_TOMASK; 1280 1281 /* 1282 * Like ppoll() with a non-NULL sigset, we'll 1283 * call cv_reltimedwait_sig() just to check for 1284 * signals. This call will return immediately 1285 * with either 0 (signalled) or -1 (no signal). 1286 * There are some conditions whereby we can 1287 * get 0 from cv_reltimedwait_sig() without 1288 * a true signal (e.g., a directed stop), so 1289 * we restore our signal mask in the unlikely 1290 * event that lwp_cursig is 0. 1291 */ 1292 if (!cv_reltimedwait_sig(&t->t_delay_cv, 1293 &p->p_lock, 0, TR_CLOCK_TICK)) { 1294 if (lwp->lwp_cursig == 0) { 1295 t->t_hold = lwp->lwp_sigoldmask; 1296 t->t_flag &= ~T_TOMASK; 1297 } 1298 1299 mutex_exit(&p->p_lock); 1300 1301 DP_REFRELE(dpep); 1302 return (EINTR); 1303 } 1304 1305 mutex_exit(&p->p_lock); 1306 } 1307 } 1308 1309 if ((nfds = STRUCT_FGET(dvpoll, dp_nfds)) == 0) { 1310 /* 1311 * We are just using DP_POLL to sleep, so 1312 * we don't any of the devpoll apparatus. 1313 * Do not check for signals if we have a zero timeout. 1314 */ 1315 DP_REFRELE(dpep); 1316 if (deadline == 0) { 1317 DP_SIGMASK_RESTORE(ksetp); 1318 return (0); 1319 } 1320 1321 mutex_enter(&curthread->t_delay_lock); 1322 while ((error = 1323 cv_timedwait_sig_hrtime(&curthread->t_delay_cv, 1324 &curthread->t_delay_lock, deadline)) > 0) 1325 continue; 1326 mutex_exit(&curthread->t_delay_lock); 1327 1328 DP_SIGMASK_RESTORE(ksetp); 1329 1330 return (error == 0 ? EINTR : 0); 1331 } 1332 1333 if (is_epoll) { 1334 size = nfds * (fdsize = sizeof (epoll_event_t)); 1335 } else { 1336 size = nfds * (fdsize = sizeof (pollfd_t)); 1337 } 1338 1339 /* 1340 * XXX It would be nice not to have to alloc each time, but it 1341 * requires another per thread structure hook. This can be 1342 * implemented later if data suggests that it's necessary. 1343 */ 1344 ps = pollstate_create(); 1345 1346 if (ps->ps_dpbufsize < size) { 1347 /* 1348 * If nfds is larger than twice the current maximum 1349 * open file count, we'll silently clamp it. This 1350 * only limits our exposure to allocating an 1351 * inordinate amount of kernel memory; it doesn't 1352 * otherwise affect the semantics. (We have this 1353 * check at twice the maximum instead of merely the 1354 * maximum because some applications pass an nfds that 1355 * is only slightly larger than their limit.) 1356 */ 1357 mutex_enter(&p->p_lock); 1358 if ((nfds >> 1) > p->p_fno_ctl) { 1359 nfds = p->p_fno_ctl; 1360 size = nfds * fdsize; 1361 } 1362 mutex_exit(&p->p_lock); 1363 1364 if (ps->ps_dpbufsize < size) { 1365 kmem_free(ps->ps_dpbuf, ps->ps_dpbufsize); 1366 ps->ps_dpbuf = kmem_zalloc(size, KM_SLEEP); 1367 ps->ps_dpbufsize = size; 1368 } 1369 } 1370 1371 VERIFY(pollstate_enter(pcp) == PSE_SUCCESS); 1372 for (;;) { 1373 pcp->pc_flag &= ~PC_POLLWAKE; 1374 1375 /* 1376 * Mark all child pcachelinks as stale. 1377 * Those which are still part of the tree will be 1378 * marked as valid during the poll. 1379 */ 1380 pcachelink_mark_stale(pcp); 1381 1382 error = dp_pcache_poll(dpep, ps->ps_dpbuf, 1383 pcp, nfds, &fdcnt); 1384 if (fdcnt > 0 || error != 0) 1385 break; 1386 1387 /* Purge still-stale child pcachelinks */ 1388 pcachelink_purge_stale(pcp); 1389 1390 /* 1391 * A pollwake has happened since we polled cache. 1392 */ 1393 if (pcp->pc_flag & PC_POLLWAKE) 1394 continue; 1395 1396 /* 1397 * Sleep until we are notified, signaled, or timed out. 1398 */ 1399 if (deadline == 0) { 1400 /* immediate timeout; do not check signals */ 1401 break; 1402 } 1403 1404 error = cv_timedwait_sig_hrtime(&pcp->pc_cv, 1405 &pcp->pc_lock, deadline); 1406 1407 /* 1408 * If we were awakened by a signal or timeout then 1409 * break the loop, else poll again. 1410 */ 1411 if (error <= 0) { 1412 error = (error == 0) ? EINTR : 0; 1413 break; 1414 } else { 1415 error = 0; 1416 } 1417 } 1418 pollstate_exit(pcp); 1419 1420 DP_SIGMASK_RESTORE(ksetp); 1421 1422 if (error == 0 && fdcnt > 0) { 1423 /* 1424 * It should be noted that FKIOCTL does not influence 1425 * the copyout (vs bcopy) of dp_fds at this time. 1426 */ 1427 if (copyout(ps->ps_dpbuf, 1428 STRUCT_FGETP(dvpoll, dp_fds), fdcnt * fdsize)) { 1429 DP_REFRELE(dpep); 1430 return (EFAULT); 1431 } 1432 *rvalp = fdcnt; 1433 } 1434 break; 1435 } 1436 1437 case DP_ISPOLLED: 1438 { 1439 pollfd_t pollfd; 1440 polldat_t *pdp; 1441 1442 STRUCT_INIT(dvpoll, mode); 1443 error = copyin((caddr_t)arg, &pollfd, sizeof (pollfd_t)); 1444 if (error) { 1445 DP_REFRELE(dpep); 1446 return (EFAULT); 1447 } 1448 mutex_enter(&pcp->pc_lock); 1449 if (pcp->pc_hash == NULL) { 1450 /* 1451 * No Need to search because no poll fd 1452 * has been cached. 1453 */ 1454 mutex_exit(&pcp->pc_lock); 1455 DP_REFRELE(dpep); 1456 return (0); 1457 } 1458 if (pollfd.fd < 0) { 1459 mutex_exit(&pcp->pc_lock); 1460 break; 1461 } 1462 pdp = pcache_lookup_fd(pcp, pollfd.fd); 1463 if ((pdp != NULL) && (pdp->pd_fd == pollfd.fd) && 1464 (pdp->pd_fp != NULL)) { 1465 pollfd.revents = pdp->pd_events; 1466 if (copyout(&pollfd, (caddr_t)arg, sizeof (pollfd_t))) { 1467 mutex_exit(&pcp->pc_lock); 1468 DP_REFRELE(dpep); 1469 return (EFAULT); 1470 } 1471 *rvalp = 1; 1472 } 1473 mutex_exit(&pcp->pc_lock); 1474 break; 1475 } 1476 1477 default: 1478 DP_REFRELE(dpep); 1479 return (EINVAL); 1480 } 1481 DP_REFRELE(dpep); 1482 return (error); 1483 } 1484 1485 /* 1486 * Overview of Recursive Polling 1487 * 1488 * It is possible for /dev/poll to poll for events on file descriptors which 1489 * themselves are /dev/poll handles. Pending events in the child handle are 1490 * represented as readable data via the POLLIN flag. To limit surface area, 1491 * this recursion is presently allowed on only /dev/poll handles which have 1492 * been placed in epoll mode via the DP_EPOLLCOMPAT ioctl. Recursion depth is 1493 * limited to 5 in order to be consistent with Linux epoll. 1494 * 1495 * Extending dppoll() for VOP_POLL: 1496 * 1497 * The recursive /dev/poll implementation begins by extending dppoll() to 1498 * report when resources contained in the pollcache have relevant event state. 1499 * At the highest level, it means calling dp_pcache_poll() so it indicates if 1500 * fd events are present without consuming them or altering the pollcache 1501 * bitmap. This ensures that a subsequent DP_POLL operation on the bitmap will 1502 * yield the initiating event. Additionally, the VOP_POLL should return in 1503 * such a way that dp_pcache_poll() does not clear the parent bitmap entry 1504 * which corresponds to the child /dev/poll fd. This means that child 1505 * pollcaches will be checked during every poll which facilitates wake-up 1506 * behavior detailed below. 1507 * 1508 * Pollcache Links and Wake Events: 1509 * 1510 * Recursive /dev/poll avoids complicated pollcache locking constraints during 1511 * pollwakeup events by eschewing the traditional pollhead mechanism in favor 1512 * of a different approach. For each pollcache at the root of a recursive 1513 * /dev/poll "tree", pcachelink_t structures are established to all child 1514 * /dev/poll pollcaches. During pollnotify() in a child pollcache, the 1515 * linked list of pcachelink_t entries is walked, where those marked as valid 1516 * incur a cv_broadcast to their parent pollcache. Most notably, these 1517 * pcachelink_t cv wakeups are performed without acquiring pc_lock on the 1518 * parent pollcache (which would require careful deadlock avoidance). This 1519 * still allows the woken poll on the parent to discover the pertinent events 1520 * due to the fact that bitmap entires for the child pollcache are always 1521 * maintained by the dppoll() logic above. 1522 * 1523 * Depth Limiting and Loop Prevention: 1524 * 1525 * As each pollcache is encountered (either via DP_POLL or dppoll()), depth and 1526 * loop constraints are enforced via pollstate_enter(). The pollcache_t 1527 * pointer is compared against any existing entries in ps_pc_stack and is added 1528 * to the end if no match (and therefore loop) is found. Once poll operations 1529 * for a given pollcache_t are complete, pollstate_exit() clears the pointer 1530 * from the list. The pollstate_enter() and pollstate_exit() functions are 1531 * responsible for acquiring and releasing pc_lock, respectively. 1532 * 1533 * Deadlock Safety: 1534 * 1535 * Descending through a tree of recursive /dev/poll handles involves the tricky 1536 * business of sequentially entering multiple pollcache locks. This tree 1537 * topology cannot define a lock acquisition order in such a way that it is 1538 * immune to deadlocks between threads. The pollstate_enter() and 1539 * pollstate_exit() functions provide an interface for recursive /dev/poll 1540 * operations to safely lock pollcaches while failing gracefully in the face of 1541 * deadlocking topologies. (See pollstate_contend() for more detail about how 1542 * deadlocks are detected and resolved.) 1543 */ 1544 1545 /*ARGSUSED*/ 1546 static int 1547 dppoll(dev_t dev, short events, int anyyet, short *reventsp, 1548 struct pollhead **phpp) 1549 { 1550 minor_t minor; 1551 dp_entry_t *dpep; 1552 pollcache_t *pcp; 1553 int res, rc = 0; 1554 1555 minor = getminor(dev); 1556 mutex_enter(&devpoll_lock); 1557 ASSERT(minor < dptblsize); 1558 dpep = devpolltbl[minor]; 1559 ASSERT(dpep != NULL); 1560 mutex_exit(&devpoll_lock); 1561 1562 mutex_enter(&dpep->dpe_lock); 1563 if ((dpep->dpe_flag & DP_ISEPOLLCOMPAT) == 0) { 1564 /* Poll recursion is not yet supported for non-epoll handles */ 1565 *reventsp = POLLERR; 1566 mutex_exit(&dpep->dpe_lock); 1567 return (0); 1568 } else { 1569 dpep->dpe_refcnt++; 1570 pcp = dpep->dpe_pcache; 1571 mutex_exit(&dpep->dpe_lock); 1572 } 1573 1574 res = pollstate_enter(pcp); 1575 if (res == PSE_SUCCESS) { 1576 nfds_t nfds = 1; 1577 int fdcnt = 0; 1578 pollstate_t *ps = curthread->t_pollstate; 1579 1580 /* 1581 * Recursive polling will only emit certain events. Skip a 1582 * scan of the pollcache if those events are not of interest. 1583 */ 1584 if (events & (POLLIN|POLLRDNORM)) { 1585 rc = dp_pcache_poll(dpep, NULL, pcp, nfds, &fdcnt); 1586 } else { 1587 rc = 0; 1588 fdcnt = 0; 1589 } 1590 1591 if (rc == 0 && fdcnt > 0) { 1592 *reventsp = POLLIN|POLLRDNORM; 1593 } else { 1594 *reventsp = 0; 1595 } 1596 pcachelink_assoc(pcp, ps->ps_pc_stack[0]); 1597 pollstate_exit(pcp); 1598 } else { 1599 switch (res) { 1600 case PSE_FAIL_DEPTH: 1601 rc = EINVAL; 1602 break; 1603 case PSE_FAIL_LOOP: 1604 case PSE_FAIL_DEADLOCK: 1605 rc = ELOOP; 1606 break; 1607 default: 1608 /* 1609 * If anything else has gone awry, such as being polled 1610 * from an unexpected context, fall back to the 1611 * recursion-intolerant response. 1612 */ 1613 *reventsp = POLLERR; 1614 rc = 0; 1615 break; 1616 } 1617 } 1618 1619 DP_REFRELE(dpep); 1620 return (rc); 1621 } 1622 1623 /* 1624 * devpoll close should do enough clean up before the pollcache is deleted, 1625 * i.e., it should ensure no one still references the pollcache later. 1626 * There is no "permission" check in here. Any process having the last 1627 * reference of this /dev/poll fd can close. 1628 */ 1629 /*ARGSUSED*/ 1630 static int 1631 dpclose(dev_t dev, int flag, int otyp, cred_t *credp) 1632 { 1633 minor_t minor; 1634 dp_entry_t *dpep; 1635 pollcache_t *pcp; 1636 int i; 1637 polldat_t **hashtbl; 1638 polldat_t *pdp; 1639 1640 minor = getminor(dev); 1641 1642 mutex_enter(&devpoll_lock); 1643 dpep = devpolltbl[minor]; 1644 ASSERT(dpep != NULL); 1645 devpolltbl[minor] = NULL; 1646 mutex_exit(&devpoll_lock); 1647 pcp = dpep->dpe_pcache; 1648 ASSERT(pcp != NULL); 1649 /* 1650 * At this point, no other lwp can access this pollcache via the 1651 * /dev/poll fd. This pollcache is going away, so do the clean 1652 * up without the pc_lock. 1653 */ 1654 hashtbl = pcp->pc_hash; 1655 for (i = 0; i < pcp->pc_hashsize; i++) { 1656 for (pdp = hashtbl[i]; pdp; pdp = pdp->pd_hashnext) { 1657 if (pdp->pd_php != NULL) { 1658 pollhead_delete(pdp->pd_php, pdp); 1659 pdp->pd_php = NULL; 1660 pdp->pd_fp = NULL; 1661 } 1662 } 1663 } 1664 /* 1665 * pollwakeup() may still interact with this pollcache. Wait until 1666 * it is done. 1667 */ 1668 mutex_enter(&pcp->pc_no_exit); 1669 ASSERT(pcp->pc_busy >= 0); 1670 while (pcp->pc_busy > 0) 1671 cv_wait(&pcp->pc_busy_cv, &pcp->pc_no_exit); 1672 mutex_exit(&pcp->pc_no_exit); 1673 1674 /* Clean up any pollcache links created via recursive /dev/poll */ 1675 if (pcp->pc_parents != NULL || pcp->pc_children != NULL) { 1676 /* 1677 * Because of the locking rules for pcachelink manipulation, 1678 * acquring pc_lock is required for this step. 1679 */ 1680 mutex_enter(&pcp->pc_lock); 1681 pcachelink_purge_all(pcp); 1682 mutex_exit(&pcp->pc_lock); 1683 } 1684 1685 pcache_destroy(pcp); 1686 ASSERT(dpep->dpe_refcnt == 0); 1687 kmem_free(dpep, sizeof (dp_entry_t)); 1688 return (0); 1689 } 1690 1691 static void 1692 pcachelink_locked_rele(pcachelink_t *pl) 1693 { 1694 ASSERT(MUTEX_HELD(&pl->pcl_lock)); 1695 VERIFY(pl->pcl_refcnt >= 1); 1696 1697 pl->pcl_refcnt--; 1698 if (pl->pcl_refcnt == 0) { 1699 VERIFY(pl->pcl_state == PCL_INVALID); 1700 ASSERT(pl->pcl_parent_pc == NULL); 1701 ASSERT(pl->pcl_child_pc == NULL); 1702 ASSERT(pl->pcl_parent_next == NULL); 1703 ASSERT(pl->pcl_child_next == NULL); 1704 1705 pl->pcl_state = PCL_FREE; 1706 mutex_destroy(&pl->pcl_lock); 1707 kmem_free(pl, sizeof (pcachelink_t)); 1708 } else { 1709 mutex_exit(&pl->pcl_lock); 1710 } 1711 } 1712 1713 /* 1714 * Associate parent and child pollcaches via a pcachelink_t. If an existing 1715 * link (stale or valid) between the two is found, it will be reused. If a 1716 * suitable link is not found for reuse, a new one will be allocated. 1717 */ 1718 static void 1719 pcachelink_assoc(pollcache_t *child, pollcache_t *parent) 1720 { 1721 pcachelink_t *pl, **plpn; 1722 1723 ASSERT(MUTEX_HELD(&child->pc_lock)); 1724 ASSERT(MUTEX_HELD(&parent->pc_lock)); 1725 1726 /* Search for an existing link we can reuse. */ 1727 plpn = &child->pc_parents; 1728 for (pl = child->pc_parents; pl != NULL; pl = *plpn) { 1729 mutex_enter(&pl->pcl_lock); 1730 if (pl->pcl_state == PCL_INVALID) { 1731 /* Clean any invalid links while walking the list */ 1732 *plpn = pl->pcl_parent_next; 1733 pl->pcl_child_pc = NULL; 1734 pl->pcl_parent_next = NULL; 1735 pcachelink_locked_rele(pl); 1736 } else if (pl->pcl_parent_pc == parent) { 1737 /* Successfully found parent link */ 1738 ASSERT(pl->pcl_state == PCL_VALID || 1739 pl->pcl_state == PCL_STALE); 1740 pl->pcl_state = PCL_VALID; 1741 mutex_exit(&pl->pcl_lock); 1742 return; 1743 } else { 1744 plpn = &pl->pcl_parent_next; 1745 mutex_exit(&pl->pcl_lock); 1746 } 1747 } 1748 1749 /* No existing link to the parent was found. Create a fresh one. */ 1750 pl = kmem_zalloc(sizeof (pcachelink_t), KM_SLEEP); 1751 mutex_init(&pl->pcl_lock, NULL, MUTEX_DEFAULT, NULL); 1752 1753 pl->pcl_parent_pc = parent; 1754 pl->pcl_child_next = parent->pc_children; 1755 parent->pc_children = pl; 1756 pl->pcl_refcnt++; 1757 1758 pl->pcl_child_pc = child; 1759 pl->pcl_parent_next = child->pc_parents; 1760 child->pc_parents = pl; 1761 pl->pcl_refcnt++; 1762 1763 pl->pcl_state = PCL_VALID; 1764 } 1765 1766 /* 1767 * Mark all child links in a pollcache as stale. Any invalid child links found 1768 * during iteration are purged. 1769 */ 1770 static void 1771 pcachelink_mark_stale(pollcache_t *pcp) 1772 { 1773 pcachelink_t *pl, **plpn; 1774 1775 ASSERT(MUTEX_HELD(&pcp->pc_lock)); 1776 1777 plpn = &pcp->pc_children; 1778 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) { 1779 mutex_enter(&pl->pcl_lock); 1780 if (pl->pcl_state == PCL_INVALID) { 1781 /* 1782 * Remove any invalid links while we are going to the 1783 * trouble of walking the list. 1784 */ 1785 *plpn = pl->pcl_child_next; 1786 pl->pcl_parent_pc = NULL; 1787 pl->pcl_child_next = NULL; 1788 pcachelink_locked_rele(pl); 1789 } else { 1790 pl->pcl_state = PCL_STALE; 1791 plpn = &pl->pcl_child_next; 1792 mutex_exit(&pl->pcl_lock); 1793 } 1794 } 1795 } 1796 1797 /* 1798 * Purge all stale (or invalid) child links from a pollcache. 1799 */ 1800 static void 1801 pcachelink_purge_stale(pollcache_t *pcp) 1802 { 1803 pcachelink_t *pl, **plpn; 1804 1805 ASSERT(MUTEX_HELD(&pcp->pc_lock)); 1806 1807 plpn = &pcp->pc_children; 1808 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) { 1809 mutex_enter(&pl->pcl_lock); 1810 switch (pl->pcl_state) { 1811 case PCL_STALE: 1812 pl->pcl_state = PCL_INVALID; 1813 /* FALLTHROUGH */ 1814 case PCL_INVALID: 1815 *plpn = pl->pcl_child_next; 1816 pl->pcl_parent_pc = NULL; 1817 pl->pcl_child_next = NULL; 1818 pcachelink_locked_rele(pl); 1819 break; 1820 default: 1821 plpn = &pl->pcl_child_next; 1822 mutex_exit(&pl->pcl_lock); 1823 } 1824 } 1825 } 1826 1827 /* 1828 * Purge all child and parent links from a pollcache, regardless of status. 1829 */ 1830 static void 1831 pcachelink_purge_all(pollcache_t *pcp) 1832 { 1833 pcachelink_t *pl, **plpn; 1834 1835 ASSERT(MUTEX_HELD(&pcp->pc_lock)); 1836 1837 plpn = &pcp->pc_parents; 1838 for (pl = pcp->pc_parents; pl != NULL; pl = *plpn) { 1839 mutex_enter(&pl->pcl_lock); 1840 pl->pcl_state = PCL_INVALID; 1841 *plpn = pl->pcl_parent_next; 1842 pl->pcl_child_pc = NULL; 1843 pl->pcl_parent_next = NULL; 1844 pcachelink_locked_rele(pl); 1845 } 1846 1847 plpn = &pcp->pc_children; 1848 for (pl = pcp->pc_children; pl != NULL; pl = *plpn) { 1849 mutex_enter(&pl->pcl_lock); 1850 pl->pcl_state = PCL_INVALID; 1851 *plpn = pl->pcl_child_next; 1852 pl->pcl_parent_pc = NULL; 1853 pl->pcl_child_next = NULL; 1854 pcachelink_locked_rele(pl); 1855 } 1856 1857 ASSERT(pcp->pc_parents == NULL); 1858 ASSERT(pcp->pc_children == NULL); 1859 } 1860