Lines Matching refs:sq
471 #define SQPUT_Q(sq, qp) \ argument
473 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
478 EQUIV(sq->sq_head, sq->sq_tail); \
480 ASSERT(sq == qp->q_syncq); \
482 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \
484 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
485 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
491 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \
493 EQUIV(sq->sq_head, sq->sq_nqueues); \
494 if (sq->sq_head == NULL) { \
495 sq->sq_head = sq->sq_tail = qp; \
496 sq->sq_flags |= SQ_MESSAGES; \
498 qp->q_sqprev = sq->sq_tail; \
499 sq->sq_tail->q_sqnext = qp; \
500 sq->sq_tail = qp; \
506 queue_t **qpp = &sq->sq_tail; \
518 sq->sq_head = qp; \
519 sq->sq_pri = sq->sq_head->q_spri; \
525 sq->sq_nqueues++; \
533 #define SQRM_Q(sq, qp) \ argument
535 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
537 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \
538 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \
540 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \
541 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \
542 ASSERT(sq->sq_nqueues != 0); \
545 sq->sq_head = qp->q_sqnext; \
552 sq->sq_tail = qp->q_sqprev; \
561 if (sq->sq_head != NULL) { \
562 sq->sq_pri = sq->sq_head->q_spri; \
564 sq->sq_flags &= ~SQ_MESSAGES; \
565 sq->sq_pri = 0; \
567 sq->sq_nqueues--; \
568 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \
569 (sq->sq_flags & SQ_QUEUED) == 0); \
599 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ argument
600 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
601 if ((sq)->sq_ciputctrl != NULL) { \
603 int nlocks = (sq)->sq_nciputctrl; \
604 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
605 ASSERT((sq)->sq_type & SQ_CIPUT); \
614 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ argument
615 ASSERT(MUTEX_HELD(SQLOCK(sq))); \
616 if ((sq)->sq_ciputctrl != NULL) { \
618 int nlocks = (sq)->sq_nciputctrl; \
619 ciputctrl_t *cip = (sq)->sq_ciputctrl; \
620 ASSERT((sq)->sq_type & SQ_CIPUT); \
694 syncq_t *sq = &qip->qu_syncq; in queue_constructor() local
726 sq->sq_head = NULL; in queue_constructor()
727 sq->sq_tail = NULL; in queue_constructor()
728 sq->sq_evhead = NULL; in queue_constructor()
729 sq->sq_evtail = NULL; in queue_constructor()
730 sq->sq_callbpend = NULL; in queue_constructor()
731 sq->sq_outer = NULL; in queue_constructor()
732 sq->sq_onext = NULL; in queue_constructor()
733 sq->sq_oprev = NULL; in queue_constructor()
734 sq->sq_next = NULL; in queue_constructor()
735 sq->sq_svcflags = 0; in queue_constructor()
736 sq->sq_servcount = 0; in queue_constructor()
737 sq->sq_needexcl = 0; in queue_constructor()
738 sq->sq_nqueues = 0; in queue_constructor()
739 sq->sq_pri = 0; in queue_constructor()
741 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); in queue_constructor()
742 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); in queue_constructor()
743 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); in queue_constructor()
755 syncq_t *sq = &qip->qu_syncq; in queue_destructor() local
770 mutex_destroy(&sq->sq_lock); in queue_destructor()
771 cv_destroy(&sq->sq_wait); in queue_destructor()
772 cv_destroy(&sq->sq_exitwait); in queue_destructor()
782 syncq_t *sq = buf; in syncq_constructor() local
786 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); in syncq_constructor()
787 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); in syncq_constructor()
788 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); in syncq_constructor()
797 syncq_t *sq = buf; in syncq_destructor() local
799 ASSERT(sq->sq_head == NULL); in syncq_destructor()
800 ASSERT(sq->sq_tail == NULL); in syncq_destructor()
801 ASSERT(sq->sq_evhead == NULL); in syncq_destructor()
802 ASSERT(sq->sq_evtail == NULL); in syncq_destructor()
803 ASSERT(sq->sq_callbpend == NULL); in syncq_destructor()
804 ASSERT(sq->sq_callbflags == 0); in syncq_destructor()
805 ASSERT(sq->sq_outer == NULL); in syncq_destructor()
806 ASSERT(sq->sq_onext == NULL); in syncq_destructor()
807 ASSERT(sq->sq_oprev == NULL); in syncq_destructor()
808 ASSERT(sq->sq_next == NULL); in syncq_destructor()
809 ASSERT(sq->sq_needexcl == 0); in syncq_destructor()
810 ASSERT(sq->sq_svcflags == 0); in syncq_destructor()
811 ASSERT(sq->sq_servcount == 0); in syncq_destructor()
812 ASSERT(sq->sq_nqueues == 0); in syncq_destructor()
813 ASSERT(sq->sq_pri == 0); in syncq_destructor()
814 ASSERT(sq->sq_count == 0); in syncq_destructor()
815 ASSERT(sq->sq_rmqcount == 0); in syncq_destructor()
816 ASSERT(sq->sq_cancelid == 0); in syncq_destructor()
817 ASSERT(sq->sq_ciputctrl == NULL); in syncq_destructor()
818 ASSERT(sq->sq_nciputctrl == 0); in syncq_destructor()
819 ASSERT(sq->sq_type == 0); in syncq_destructor()
820 ASSERT(sq->sq_flags == 0); in syncq_destructor()
822 mutex_destroy(&sq->sq_lock); in syncq_destructor()
823 cv_destroy(&sq->sq_wait); in syncq_destructor()
824 cv_destroy(&sq->sq_exitwait); in syncq_destructor()
1690 wait_syncq(syncq_t *sq) in wait_syncq() argument
1694 mutex_enter(SQLOCK(sq)); in wait_syncq()
1695 count = sq->sq_count; in wait_syncq()
1696 SQ_PUTLOCKS_ENTER(sq); in wait_syncq()
1697 SUM_SQ_PUTCOUNTS(sq, count); in wait_syncq()
1699 sq->sq_flags |= SQ_WANTWAKEUP; in wait_syncq()
1700 SQ_PUTLOCKS_EXIT(sq); in wait_syncq()
1701 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_syncq()
1702 count = sq->sq_count; in wait_syncq()
1703 SQ_PUTLOCKS_ENTER(sq); in wait_syncq()
1704 SUM_SQ_PUTCOUNTS(sq, count); in wait_syncq()
1706 SQ_PUTLOCKS_EXIT(sq); in wait_syncq()
1707 mutex_exit(SQLOCK(sq)); in wait_syncq()
1717 syncq_t *sq = q->q_syncq; in wait_q_syncq() local
1719 mutex_enter(SQLOCK(sq)); in wait_q_syncq()
1721 sq->sq_flags |= SQ_WANTWAKEUP; in wait_q_syncq()
1722 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_q_syncq()
1724 mutex_exit(SQLOCK(sq)); in wait_q_syncq()
2115 syncq_t *sq = wrq->q_syncq; in munlink() local
2117 mutex_enter(SQLOCK(sq)); in munlink()
2119 sq->sq_flags |= SQ_WANTWAKEUP; in munlink()
2120 cv_wait(&sq->sq_wait, SQLOCK(sq)); in munlink()
2122 mutex_exit(SQLOCK(sq)); in munlink()
2127 syncq_t *sq, *outer; in munlink() local
2153 sq = rq->q_syncq; in munlink()
2154 if ((outer = sq->sq_outer) != NULL) { in munlink()
2181 wait_syncq(sq); in munlink()
2193 syncq_t *sq = rq->q_syncq; in munlink() local
2195 mutex_enter(SQLOCK(sq)); in munlink()
2197 sq->sq_flags |= SQ_WANTWAKEUP; in munlink()
2198 cv_wait(&sq->sq_wait, SQLOCK(sq)); in munlink()
2200 mutex_exit(SQLOCK(sq)); in munlink()
2515 syncq_t *sq, *outer; in setq() local
2534 sq = rq->q_syncq; in setq()
2535 outer = sq->sq_outer; in setq()
2542 ASSERT(sq->sq_outer == NULL); in setq()
2543 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in setq()
2545 if (sq != SQ(rq)) { in setq()
2547 free_syncq(sq); in setq()
2552 if (wq->q_syncq != NULL && wq->q_syncq != sq && in setq()
2584 sq = SQ(rq); in setq()
2585 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); in setq()
2586 ASSERT(sq->sq_outer == NULL); in setq()
2587 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in setq()
2593 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); in setq()
2595 rq->q_syncq = wq->q_syncq = sq; in setq()
2596 sq->sq_type = sqtype; in setq()
2597 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); in setq()
2605 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); in setq()
2606 sq->sq_svcflags = 0; in setq()
2626 sq = new_syncq(); in setq()
2627 sq->sq_type = rq->q_syncq->sq_type; in setq()
2628 sq->sq_flags = rq->q_syncq->sq_flags; in setq()
2629 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in setq()
2630 sq->sq_oprev == NULL); in setq()
2631 wq->q_syncq = sq; in setq()
2634 sq = dmp->dm_sq; in setq()
2640 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in setq()
2641 sq->sq_oprev == NULL); in setq()
2642 rq->q_syncq = wq->q_syncq = sq; in setq()
2670 syncq_t *sq; in hold_dm() local
2688 sq = new_syncq(); in hold_dm()
2690 sq->sq_type = sqtype | SQ_PERMOD; in hold_dm()
2691 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; in hold_dm()
2694 sq->sq_onext = sq->sq_oprev = sq; in hold_dm()
2698 dmp->dm_sq = sq; in hold_dm()
2708 free_syncq(sq); in hold_dm()
3378 syncq_t *sq; in allocq() local
3384 sq = &qip->qu_syncq; in allocq()
3392 qp->q_syncq = sq; in allocq()
3408 wqp->q_syncq = sq; in allocq()
3417 sq->sq_count = 0; in allocq()
3418 sq->sq_rmqcount = 0; in allocq()
3419 sq->sq_flags = 0; in allocq()
3420 sq->sq_type = 0; in allocq()
3421 sq->sq_callbflags = 0; in allocq()
3422 sq->sq_cancelid = 0; in allocq()
3423 sq->sq_ciputctrl = NULL; in allocq()
3424 sq->sq_nciputctrl = 0; in allocq()
3425 sq->sq_needexcl = 0; in allocq()
3426 sq->sq_svcflags = 0; in allocq()
3440 syncq_t *sq, *outer; in freeq() local
3490 sq = SQ(qp); in freeq()
3491 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in freeq()
3492 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); in freeq()
3493 ASSERT(sq->sq_outer == NULL); in freeq()
3494 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); in freeq()
3495 ASSERT(sq->sq_callbpend == NULL); in freeq()
3496 ASSERT(sq->sq_needexcl == 0); in freeq()
3498 if (sq->sq_ciputctrl != NULL) { in freeq()
3499 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); in freeq()
3500 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, in freeq()
3501 sq->sq_nciputctrl, 0); in freeq()
3503 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); in freeq()
3504 sq->sq_ciputctrl = NULL; in freeq()
3505 sq->sq_nciputctrl = 0; in freeq()
3786 syncq_t *sq; in streams_sqbkgrnd_service() local
3807 DQ(sq, sqhead, sqtail, sq_next); in streams_sqbkgrnd_service()
3808 ASSERT(sq != NULL); in streams_sqbkgrnd_service()
3809 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); in streams_sqbkgrnd_service()
3811 syncq_service(sq); in streams_sqbkgrnd_service()
3823 wait_sq_svc(syncq_t *sq) in wait_sq_svc() argument
3825 mutex_enter(SQLOCK(sq)); in wait_sq_svc()
3826 sq->sq_svcflags |= SQ_DISABLED; in wait_sq_svc()
3827 if (sq->sq_svcflags & SQ_BGTHREAD) { in wait_sq_svc()
3832 ASSERT(sq->sq_servcount == 1); in wait_sq_svc()
3834 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); in wait_sq_svc()
3837 sq->sq_svcflags &= ~SQ_BGTHREAD; in wait_sq_svc()
3838 sq->sq_servcount = 0; in wait_sq_svc()
3843 while (sq->sq_servcount != 0) { in wait_sq_svc()
3844 sq->sq_flags |= SQ_WANTWAKEUP; in wait_sq_svc()
3845 cv_wait(&sq->sq_wait, SQLOCK(sq)); in wait_sq_svc()
3848 mutex_exit(SQLOCK(sq)); in wait_sq_svc()
3864 sqenable(syncq_t *sq) in sqenable() argument
3872 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sqenable()
3874 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); in sqenable()
3875 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); in sqenable()
3881 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) in sqenable()
3892 if (sq->sq_servcount != 0) { in sqenable()
3893 if (((sq->sq_type & SQ_PERMOD) == 0) || in sqenable()
3894 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { in sqenable()
3900 sq->sq_tstamp = ddi_get_lbolt(); in sqenable()
3904 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, in sqenable()
3905 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); in sqenable()
3906 if (sq->sq_servid != NULL) { in sqenable()
3907 sq->sq_servcount++; in sqenable()
3916 if (sq->sq_servcount != 0) in sqenable()
3926 ENQUEUE(sq, sqhead, sqtail, sq_next); in sqenable()
3927 sq->sq_svcflags |= SQ_BGTHREAD; in sqenable()
3928 sq->sq_servcount = 1; in sqenable()
4647 syncq_t *sq = sql->sql_sq; in strlock() local
4650 mutex_enter(SQLOCK(sq)); in strlock()
4651 count = sq->sq_count; in strlock()
4652 ASSERT(sq->sq_rmqcount <= count); in strlock()
4653 SQ_PUTLOCKS_ENTER(sq); in strlock()
4654 SUM_SQ_PUTCOUNTS(sq, count); in strlock()
4655 if (count == sq->sq_rmqcount) in strlock()
4683 sq->sq_needexcl++; in strlock()
4684 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in strlock()
4685 while (count > sq->sq_rmqcount) { in strlock()
4686 sq->sq_flags |= SQ_WANTWAKEUP; in strlock()
4687 SQ_PUTLOCKS_EXIT(sq); in strlock()
4688 cv_wait(&sq->sq_wait, SQLOCK(sq)); in strlock()
4689 count = sq->sq_count; in strlock()
4690 SQ_PUTLOCKS_ENTER(sq); in strlock()
4691 SUM_SQ_PUTCOUNTS(sq, count); in strlock()
4693 sq->sq_needexcl--; in strlock()
4694 if (sq->sq_needexcl == 0) in strlock()
4695 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in strlock()
4696 SQ_PUTLOCKS_EXIT(sq); in strlock()
4697 ASSERT(count == sq->sq_rmqcount); in strlock()
4698 mutex_exit(SQLOCK(sq)); in strlock()
4884 syncq_t *sq = qp->q_syncq; in removeq() local
4899 mutex_enter(SQLOCK(sq)); in removeq()
4900 if (sq->sq_flags & SQ_WANTWAKEUP) { in removeq()
4901 sq->sq_flags &= ~SQ_WANTWAKEUP; in removeq()
4902 cv_broadcast(&sq->sq_wait); in removeq()
4904 mutex_exit(SQLOCK(sq)); in removeq()
4906 if (wsq != sq) { in removeq()
4930 mutex_enter(SQLOCK(sq)); in removeq()
4931 sq->sq_rmqcount++; in removeq()
4932 if (sq->sq_flags & SQ_WANTWAKEUP) { in removeq()
4933 sq->sq_flags &= ~SQ_WANTWAKEUP; in removeq()
4934 cv_broadcast(&sq->sq_wait); in removeq()
4936 mutex_exit(SQLOCK(sq)); in removeq()
5016 mutex_enter(SQLOCK(sq)); in removeq()
5017 sq->sq_rmqcount--; in removeq()
5018 mutex_exit(SQLOCK(sq)); in removeq()
5034 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) in blocksq() argument
5038 mutex_enter(SQLOCK(sq)); in blocksq()
5047 count = sq->sq_count; in blocksq()
5048 SQ_PUTLOCKS_ENTER(sq); in blocksq()
5049 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in blocksq()
5050 SUM_SQ_PUTCOUNTS(sq, count); in blocksq()
5052 sq->sq_needexcl++; in blocksq()
5053 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in blocksq()
5055 while ((sq->sq_flags & flag) || in blocksq()
5057 sq->sq_flags |= SQ_WANTWAKEUP; in blocksq()
5059 SQ_PUTLOCKS_EXIT(sq); in blocksq()
5061 cv_wait(&sq->sq_wait, SQLOCK(sq)); in blocksq()
5063 count = sq->sq_count; in blocksq()
5064 SQ_PUTLOCKS_ENTER(sq); in blocksq()
5065 SUM_SQ_PUTCOUNTS(sq, count); in blocksq()
5068 sq->sq_needexcl--; in blocksq()
5069 sq->sq_flags |= flag; in blocksq()
5072 if (sq->sq_needexcl == 0) { in blocksq()
5073 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in blocksq()
5075 SQ_PUTLOCKS_EXIT(sq); in blocksq()
5076 } else if (sq->sq_needexcl == 0) { in blocksq()
5077 SQ_PUTCOUNT_SETFAST(sq); in blocksq()
5080 mutex_exit(SQLOCK(sq)); in blocksq()
5096 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) in unblocksq() argument
5100 mutex_enter(SQLOCK(sq)); in unblocksq()
5102 ASSERT(sq->sq_flags & resetflag); in unblocksq()
5103 flags = sq->sq_flags & ~resetflag; in unblocksq()
5104 sq->sq_flags = flags; in unblocksq()
5108 cv_broadcast(&sq->sq_wait); in unblocksq()
5110 sq->sq_flags = flags; in unblocksq()
5114 drain_syncq(sq); in unblocksq()
5119 mutex_exit(SQLOCK(sq)); in unblocksq()
5131 dropsq(syncq_t *sq, uint16_t resetflag) in dropsq() argument
5135 mutex_enter(SQLOCK(sq)); in dropsq()
5136 ASSERT(sq->sq_flags & resetflag); in dropsq()
5137 flags = sq->sq_flags & ~resetflag; in dropsq()
5140 cv_broadcast(&sq->sq_wait); in dropsq()
5142 sq->sq_flags = flags; in dropsq()
5143 mutex_exit(SQLOCK(sq)); in dropsq()
5156 emptysq(syncq_t *sq) in emptysq() argument
5160 mutex_enter(SQLOCK(sq)); in emptysq()
5161 flags = sq->sq_flags; in emptysq()
5167 if (sq->sq_count == 0) { in emptysq()
5169 drain_syncq(sq); in emptysq()
5172 sqenable(sq); in emptysq()
5174 mutex_exit(SQLOCK(sq)); in emptysq()
5405 entersq(syncq_t *sq, int entrypoint) in entersq() argument
5417 ASSERT(sq); in entersq()
5419 mutex_enter(SQLOCK(sq)); in entersq()
5420 flags = sq->sq_flags; in entersq()
5421 type = sq->sq_type; in entersq()
5424 count = sq->sq_count; in entersq()
5425 SQ_PUTLOCKS_ENTER(sq); in entersq()
5426 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in entersq()
5427 SUM_SQ_PUTCOUNTS(sq, count); in entersq()
5428 sq->sq_needexcl++; in entersq()
5429 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in entersq()
5439 sq->sq_flags = flags | SQ_WANTWAKEUP; in entersq()
5441 SQ_PUTLOCKS_EXIT(sq); in entersq()
5443 cv_wait(&sq->sq_wait, SQLOCK(sq)); in entersq()
5445 count = sq->sq_count; in entersq()
5446 SQ_PUTLOCKS_ENTER(sq); in entersq()
5447 SUM_SQ_PUTCOUNTS(sq, count); in entersq()
5449 flags = sq->sq_flags; in entersq()
5453 ASSERT(sq->sq_needexcl > 0); in entersq()
5454 sq->sq_needexcl--; in entersq()
5455 if (sq->sq_needexcl == 0) { in entersq()
5456 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in entersq()
5470 SQ_PUTLOCKS_EXIT(sq); in entersq()
5472 mutex_exit(SQLOCK(sq)); in entersq()
5473 outer_enter(sq->sq_outer, SQ_GOAWAY); in entersq()
5474 mutex_enter(SQLOCK(sq)); in entersq()
5475 flags = sq->sq_flags; in entersq()
5481 count = sq->sq_count; in entersq()
5490 SUMCHECK_SQ_PUTCOUNTS(sq, 0); in entersq()
5494 sq->sq_flags = flags | SQ_WANTWAKEUP; in entersq()
5495 cv_wait(&sq->sq_wait, SQLOCK(sq)); in entersq()
5496 count = sq->sq_count; in entersq()
5497 flags = sq->sq_flags; in entersq()
5501 sq->sq_count++; in entersq()
5502 ASSERT(sq->sq_count != 0); /* Wraparound */ in entersq()
5505 ASSERT(sq->sq_count == 1); in entersq()
5506 sq->sq_flags |= SQ_EXCL; in entersq()
5508 SQ_PUTLOCKS_EXIT(sq); in entersq()
5511 mutex_exit(SQLOCK(sq)); in entersq()
5524 leavesq(syncq_t *sq, int entrypoint) in leavesq() argument
5537 ASSERT(sq); in leavesq()
5539 mutex_enter(SQLOCK(sq)); in leavesq()
5540 flags = sq->sq_flags; in leavesq()
5541 type = sq->sq_type; in leavesq()
5546 cv_broadcast(&sq->sq_wait); in leavesq()
5550 cv_broadcast(&sq->sq_exitwait); in leavesq()
5558 ASSERT(sq->sq_count != 0); in leavesq()
5559 sq->sq_count--; in leavesq()
5561 sq->sq_flags = flags & ~SQ_EXCL; in leavesq()
5562 drain_syncq(sq); in leavesq()
5563 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in leavesq()
5567 outer_exit(sq->sq_outer); in leavesq()
5571 ASSERT(sq->sq_count != 0); in leavesq()
5572 sq->sq_count--; in leavesq()
5574 sq->sq_flags = flags & ~SQ_EXCL; in leavesq()
5575 mutex_exit(SQLOCK(sq)); in leavesq()
5578 if (!(sq->sq_type & c_outer)) in leavesq()
5579 outer_exit(sq->sq_outer); in leavesq()
5591 syncq_t *sq = qp->q_syncq; in claimq() local
5593 mutex_enter(SQLOCK(sq)); in claimq()
5594 sq->sq_count++; in claimq()
5595 ASSERT(sq->sq_count != 0); /* Wraparound */ in claimq()
5596 mutex_exit(SQLOCK(sq)); in claimq()
5608 syncq_t *sq = qp->q_syncq; in releaseq() local
5611 mutex_enter(SQLOCK(sq)); in releaseq()
5612 ASSERT(sq->sq_count > 0); in releaseq()
5613 sq->sq_count--; in releaseq()
5615 flags = sq->sq_flags; in releaseq()
5619 cv_broadcast(&sq->sq_wait); in releaseq()
5621 sq->sq_flags = flags; in releaseq()
5628 if (sq->sq_count == 0) { in releaseq()
5629 drain_syncq(sq); in releaseq()
5632 sqenable(sq); in releaseq()
5635 mutex_exit(SQLOCK(sq)); in releaseq()
5674 free_syncq(syncq_t *sq) in free_syncq() argument
5676 ASSERT(sq->sq_head == NULL); in free_syncq()
5677 ASSERT(sq->sq_outer == NULL); in free_syncq()
5678 ASSERT(sq->sq_callbpend == NULL); in free_syncq()
5679 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || in free_syncq()
5680 (sq->sq_onext == sq && sq->sq_oprev == sq)); in free_syncq()
5682 if (sq->sq_ciputctrl != NULL) { in free_syncq()
5683 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); in free_syncq()
5684 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, in free_syncq()
5685 sq->sq_nciputctrl, 0); in free_syncq()
5687 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); in free_syncq()
5690 sq->sq_tail = NULL; in free_syncq()
5691 sq->sq_evhead = NULL; in free_syncq()
5692 sq->sq_evtail = NULL; in free_syncq()
5693 sq->sq_ciputctrl = NULL; in free_syncq()
5694 sq->sq_nciputctrl = 0; in free_syncq()
5695 sq->sq_count = 0; in free_syncq()
5696 sq->sq_rmqcount = 0; in free_syncq()
5697 sq->sq_callbflags = 0; in free_syncq()
5698 sq->sq_cancelid = 0; in free_syncq()
5699 sq->sq_next = NULL; in free_syncq()
5700 sq->sq_needexcl = 0; in free_syncq()
5701 sq->sq_svcflags = 0; in free_syncq()
5702 sq->sq_nqueues = 0; in free_syncq()
5703 sq->sq_pri = 0; in free_syncq()
5704 sq->sq_onext = NULL; in free_syncq()
5705 sq->sq_oprev = NULL; in free_syncq()
5706 sq->sq_flags = 0; in free_syncq()
5707 sq->sq_type = 0; in free_syncq()
5708 sq->sq_servcount = 0; in free_syncq()
5710 kmem_cache_free(syncq_cache, sq); in free_syncq()
5752 syncq_t *sq; in outer_enter() local
5777 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in outer_enter()
5778 mutex_enter(SQLOCK(sq)); in outer_enter()
5779 count = sq->sq_count; in outer_enter()
5780 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5781 sq->sq_flags |= SQ_WRITER; in outer_enter()
5782 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5785 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5786 mutex_exit(SQLOCK(sq)); in outer_enter()
5802 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in outer_enter()
5803 mutex_enter(SQLOCK(sq)); in outer_enter()
5804 count = sq->sq_count; in outer_enter()
5805 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5806 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5808 sq->sq_flags |= SQ_WANTWAKEUP; in outer_enter()
5809 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5810 cv_wait(&sq->sq_wait, SQLOCK(sq)); in outer_enter()
5811 count = sq->sq_count; in outer_enter()
5812 SQ_PUTLOCKS_ENTER(sq); in outer_enter()
5813 SUM_SQ_PUTCOUNTS(sq, count); in outer_enter()
5815 SQ_PUTLOCKS_EXIT(sq); in outer_enter()
5816 mutex_exit(SQLOCK(sq)); in outer_enter()
5842 syncq_t *sq; in outer_exit() local
5873 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) in outer_exit()
5874 drain_needed += dropsq(sq, SQ_WRITER); in outer_exit()
5880 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) in outer_exit()
5881 emptysq(sq); in outer_exit()
5904 outer_insert(syncq_t *outer, syncq_t *sq) in outer_insert() argument
5908 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && in outer_insert()
5909 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ in outer_insert()
5916 mutex_enter(SQLOCK(sq)); in outer_insert()
5917 sq->sq_outer = outer; in outer_insert()
5918 outer->sq_onext->sq_oprev = sq; in outer_insert()
5919 sq->sq_onext = outer->sq_onext; in outer_insert()
5920 outer->sq_onext = sq; in outer_insert()
5921 sq->sq_oprev = outer; in outer_insert()
5922 mutex_exit(SQLOCK(sq)); in outer_insert()
5935 outer_remove(syncq_t *outer, syncq_t *sq) in outer_remove() argument
5939 ASSERT(sq->sq_outer == outer); in outer_remove()
5946 mutex_enter(SQLOCK(sq)); in outer_remove()
5947 sq->sq_outer = NULL; in outer_remove()
5948 sq->sq_onext->sq_oprev = sq->sq_oprev; in outer_remove()
5949 sq->sq_oprev->sq_onext = sq->sq_onext; in outer_remove()
5950 sq->sq_oprev = sq->sq_onext = NULL; in outer_remove()
5951 mutex_exit(SQLOCK(sq)); in outer_remove()
6004 syncq_t *osq, *sq, *outer; in qwriter_outer() local
6071 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { in qwriter_outer()
6073 uint_t maxcnt = (sq == osq) ? 1 : 0; in qwriter_outer()
6075 mutex_enter(SQLOCK(sq)); in qwriter_outer()
6076 count = sq->sq_count; in qwriter_outer()
6077 SQ_PUTLOCKS_ENTER(sq); in qwriter_outer()
6078 SUM_SQ_PUTCOUNTS(sq, count); in qwriter_outer()
6079 if (sq->sq_count > maxcnt) in qwriter_outer()
6081 sq->sq_flags |= SQ_WRITER; in qwriter_outer()
6082 SQ_PUTLOCKS_EXIT(sq); in qwriter_outer()
6083 mutex_exit(SQLOCK(sq)); in qwriter_outer()
6197 sq_run_events(syncq_t *sq) in sq_run_events() argument
6201 uint16_t flags = sq->sq_flags; in sq_run_events()
6204 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sq_run_events()
6205 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in sq_run_events()
6206 sq->sq_oprev == NULL) || in sq_run_events()
6207 (sq->sq_outer != NULL && sq->sq_onext != NULL && in sq_run_events()
6208 sq->sq_oprev != NULL)); in sq_run_events()
6211 ASSERT(sq->sq_count == 1); in sq_run_events()
6224 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { in sq_run_events()
6225 ASSERT(bp->b_queue->q_syncq == sq); in sq_run_events()
6226 ASSERT(sq->sq_flags & SQ_EVENTS); in sq_run_events()
6236 ASSERT(sq->sq_evhead == bp); in sq_run_events()
6237 sq->sq_evhead = bp->b_next; in sq_run_events()
6241 ASSERT(sq->sq_evtail == bp); in sq_run_events()
6242 sq->sq_evtail = NULL; in sq_run_events()
6243 sq->sq_flags &= ~SQ_EVENTS; in sq_run_events()
6248 mutex_exit(SQLOCK(sq)); in sq_run_events()
6252 mutex_enter(SQLOCK(sq)); in sq_run_events()
6256 flags = sq->sq_flags; in sq_run_events()
6259 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); in sq_run_events()
6260 ASSERT(!(sq->sq_flags & SQ_EVENTS)); in sq_run_events()
6264 cv_broadcast(&sq->sq_wait); in sq_run_events()
6268 cv_broadcast(&sq->sq_exitwait); in sq_run_events()
6270 sq->sq_flags = flags; in sq_run_events()
6281 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) in sqfill_events() argument
6285 ASSERT(MUTEX_HELD(SQLOCK(sq))); in sqfill_events()
6295 if (sq->sq_evhead == NULL) { in sqfill_events()
6296 sq->sq_evhead = sq->sq_evtail = mp; in sqfill_events()
6297 sq->sq_flags |= SQ_EVENTS; in sqfill_events()
6299 ASSERT(sq->sq_evtail != NULL); in sqfill_events()
6300 ASSERT(sq->sq_evtail->b_next == NULL); in sqfill_events()
6301 ASSERT(sq->sq_flags & SQ_EVENTS); in sqfill_events()
6302 sq->sq_evtail->b_next = mp; in sqfill_events()
6303 sq->sq_evtail = mp; in sqfill_events()
6316 count = sq->sq_count; in sqfill_events()
6317 SQ_PUTLOCKS_ENTER(sq); in sqfill_events()
6318 SUM_SQ_PUTCOUNTS(sq, count); in sqfill_events()
6332 SQ_PUTLOCKS_EXIT(sq); in sqfill_events()
6333 mutex_exit(SQLOCK(sq)); in sqfill_events()
6339 ASSERT((sq->sq_flags & SQ_EXCL) == 0); in sqfill_events()
6340 sq->sq_flags |= SQ_EXCL; in sqfill_events()
6341 ASSERT(sq->sq_count == 0); in sqfill_events()
6342 sq->sq_count++; in sqfill_events()
6343 SQ_PUTLOCKS_EXIT(sq); in sqfill_events()
6346 sq_run_events(sq); in sqfill_events()
6351 sq->sq_count--; in sqfill_events()
6358 sq->sq_flags &= ~SQ_EXCL; in sqfill_events()
6363 ASSERT(!(sq->sq_flags & SQ_EXCL)); in sqfill_events()
6372 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) in sqfill_events()
6373 drain_syncq(sq); in sqfill_events()
6375 mutex_exit(SQLOCK(sq)); in sqfill_events()
6395 drain_syncq(syncq_t *sq) in drain_syncq() argument
6399 uint16_t type = sq->sq_type; in drain_syncq()
6400 uint16_t flags = sq->sq_flags; in drain_syncq()
6401 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; in drain_syncq()
6404 "drain_syncq start:%p", sq); in drain_syncq()
6405 ASSERT(MUTEX_HELD(SQLOCK(sq))); in drain_syncq()
6406 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in drain_syncq()
6407 sq->sq_oprev == NULL) || in drain_syncq()
6408 (sq->sq_outer != NULL && sq->sq_onext != NULL && in drain_syncq()
6409 sq->sq_oprev != NULL)); in drain_syncq()
6415 sq->sq_svcflags &= ~SQ_SERVICE; in drain_syncq()
6423 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6424 sq->sq_servcount--; in drain_syncq()
6426 mutex_exit(SQLOCK(sq)); in drain_syncq()
6439 cv_broadcast(&sq->sq_wait); in drain_syncq()
6443 cv_broadcast(&sq->sq_exitwait); in drain_syncq()
6445 sq->sq_flags = flags; in drain_syncq()
6447 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6448 sq->sq_servcount--; in drain_syncq()
6450 mutex_exit(SQLOCK(sq)); in drain_syncq()
6461 type = sq->sq_type; in drain_syncq()
6463 if (sq->sq_count > 1) { in drain_syncq()
6465 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6466 sq->sq_servcount--; in drain_syncq()
6468 mutex_exit(SQLOCK(sq)); in drain_syncq()
6471 sq->sq_flags |= SQ_EXCL; in drain_syncq()
6485 sq->sq_count++; in drain_syncq()
6486 ASSERT(sq->sq_count != 0); /* wraparound */ in drain_syncq()
6488 while ((flags = sq->sq_flags) & SQ_QUEUED) { in drain_syncq()
6502 if (sq->sq_evhead != NULL) { in drain_syncq()
6503 ASSERT(sq->sq_flags & SQ_EVENTS); in drain_syncq()
6505 count = sq->sq_count; in drain_syncq()
6506 SQ_PUTLOCKS_ENTER(sq); in drain_syncq()
6507 SUM_SQ_PUTCOUNTS(sq, count); in drain_syncq()
6509 SQ_PUTLOCKS_EXIT(sq); in drain_syncq()
6514 sq->sq_flags = flags | SQ_EXCL; in drain_syncq()
6515 SQ_PUTLOCKS_EXIT(sq); in drain_syncq()
6520 sq_run_events(sq); in drain_syncq()
6528 ASSERT(sq->sq_flags & SQ_EXCL); in drain_syncq()
6529 sq->sq_flags &= ~SQ_EXCL; in drain_syncq()
6536 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); in drain_syncq()
6540 ASSERT(sq->sq_evhead == NULL); in drain_syncq()
6541 ASSERT(!(sq->sq_flags & SQ_EVENTS)); in drain_syncq()
6554 for (qp = sq->sq_head; in drain_syncq()
6574 mutex_exit(SQLOCK(sq)); in drain_syncq()
6576 qdrain_syncq(sq, qp); in drain_syncq()
6577 mutex_enter(SQLOCK(sq)); in drain_syncq()
6588 ASSERT(MUTEX_HELD(SQLOCK(sq))); in drain_syncq()
6589 flags = sq->sq_flags; in drain_syncq()
6604 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || in drain_syncq()
6605 (type & SQ_CI) || sq->sq_head->q_draining); in drain_syncq()
6615 cv_broadcast(&sq->sq_wait); in drain_syncq()
6619 cv_broadcast(&sq->sq_exitwait); in drain_syncq()
6621 sq->sq_flags = flags; in drain_syncq()
6623 ASSERT(sq->sq_count != 0); in drain_syncq()
6625 sq->sq_count--; in drain_syncq()
6628 ASSERT(sq->sq_servcount != 0); in drain_syncq()
6629 sq->sq_servcount--; in drain_syncq()
6632 mutex_exit(SQLOCK(sq)); in drain_syncq()
6635 "drain_syncq end:%p", sq); in drain_syncq()
6670 qdrain_syncq(syncq_t *sq, queue_t *q) in qdrain_syncq() argument
6678 "drain_syncq start:%p", sq); in qdrain_syncq()
6679 ASSERT(q->q_syncq == sq); in qdrain_syncq()
6681 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qdrain_syncq()
6687 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); in qdrain_syncq()
6688 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); in qdrain_syncq()
6689 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); in qdrain_syncq()
6693 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in qdrain_syncq()
6694 sq->sq_oprev == NULL) || in qdrain_syncq()
6695 (sq->sq_outer != NULL && sq->sq_onext != NULL && in qdrain_syncq()
6696 sq->sq_oprev != NULL)); in qdrain_syncq()
6698 count = sq->sq_count; in qdrain_syncq()
6706 SUM_SQ_PUTCOUNTS(sq, count); in qdrain_syncq()
6752 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { in qdrain_syncq()
6764 ASSERT(bp->b_queue->q_syncq == sq); in qdrain_syncq()
6819 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { in qdrain_syncq()
6820 mutex_enter(SQLOCK(sq)); in qdrain_syncq()
6821 sq->sq_flags &= ~SQ_EXCL; in qdrain_syncq()
6822 mutex_exit(SQLOCK(sq)); in qdrain_syncq()
6832 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); in qdrain_syncq()
6835 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qdrain_syncq()
6840 mutex_enter(SQLOCK(sq)); in qdrain_syncq()
6842 SQRM_Q(sq, q); in qdrain_syncq()
6843 mutex_exit(SQLOCK(sq)); in qdrain_syncq()
6864 "drain_syncq end:%p", sq); in qdrain_syncq()
6883 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) in qfill_syncq() argument
6885 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); in qfill_syncq()
6887 ASSERT(sq->sq_count > 0); in qfill_syncq()
6888 ASSERT(q->q_syncq == sq); in qfill_syncq()
6889 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && in qfill_syncq()
6890 sq->sq_oprev == NULL) || in qfill_syncq()
6891 (sq->sq_outer != NULL && sq->sq_onext != NULL && in qfill_syncq()
6892 sq->sq_oprev != NULL)); in qfill_syncq()
6910 ASSERT(q->q_syncq == sq); in qfill_syncq()
6917 mutex_enter(SQLOCK(sq)); in qfill_syncq()
6927 SQPUT_Q(sq, q); in qfill_syncq()
6934 ASSERT(sq->sq_tail != NULL); in qfill_syncq()
6935 if (sq->sq_tail == sq->sq_head) { in qfill_syncq()
6942 ASSERT(sq->sq_flags & SQ_QUEUED); in qfill_syncq()
6951 sq->sq_count--; in qfill_syncq()
6953 putnext_tail(sq, q, 0); in qfill_syncq()
6976 flush_syncq(syncq_t *sq, queue_t *qp) in flush_syncq() argument
6982 mutex_enter(SQLOCK(sq)); in flush_syncq()
6989 if (qp != NULL && sq->sq_evhead != NULL) { in flush_syncq()
6990 ASSERT(sq->sq_flags & SQ_EVENTS); in flush_syncq()
6993 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { in flush_syncq()
7003 if (bp == sq->sq_evtail) { in flush_syncq()
7005 sq->sq_evtail = mp_prev; in flush_syncq()
7008 sq->sq_evhead = mp_next; in flush_syncq()
7009 if (sq->sq_evhead == NULL) in flush_syncq()
7010 sq->sq_flags &= ~SQ_EVENTS; in flush_syncq()
7025 q = sq->sq_head; in flush_syncq()
7027 ASSERT(q->q_syncq == sq); in flush_syncq()
7041 mp_head->b_queue->q_syncq == sq); in flush_syncq()
7056 SQRM_Q(sq, q); in flush_syncq()
7065 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { in flush_syncq()
7066 sq->sq_flags &= ~SQ_WANTWAKEUP; in flush_syncq()
7067 cv_broadcast(&sq->sq_wait); in flush_syncq()
7070 mutex_exit(SQLOCK(sq)); in flush_syncq()
7082 mutex_enter(SQLOCK(sq)); in flush_syncq()
7087 q = sq->sq_head; in flush_syncq()
7092 ASSERT(MUTEX_HELD(SQLOCK(sq))); in flush_syncq()
7095 if (sq->sq_flags & SQ_WANTWAKEUP) { in flush_syncq()
7096 sq->sq_flags &= ~SQ_WANTWAKEUP; in flush_syncq()
7097 cv_broadcast(&sq->sq_wait); in flush_syncq()
7100 mutex_exit(SQLOCK(sq)); in flush_syncq()
7120 syncq_t *sq; in propagate_syncq() local
7131 sq = qp->q_syncq; in propagate_syncq()
7132 ASSERT(MUTEX_HELD(SQLOCK(sq))); in propagate_syncq()
7134 SQ_PUTLOCKS_HELD(sq); in propagate_syncq()
7140 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); in propagate_syncq()
7161 SQRM_Q(sq, qp); in propagate_syncq()
7225 if (sq->sq_evhead != NULL) { in propagate_syncq()
7226 ASSERT(sq->sq_flags & SQ_EVENTS); in propagate_syncq()
7228 for (bp = sq->sq_evhead; bp != NULL; bp = next) { in propagate_syncq()
7238 if (bp == sq->sq_evtail) { in propagate_syncq()
7240 sq->sq_evtail = prev; in propagate_syncq()
7243 sq->sq_evhead = next; in propagate_syncq()
7244 if (sq->sq_evhead == NULL) in propagate_syncq()
7245 sq->sq_flags &= ~SQ_EVENTS; in propagate_syncq()
7254 flags = sq->sq_flags; in propagate_syncq()
7259 cv_broadcast(&sq->sq_wait); in propagate_syncq()
7261 sq->sq_flags = flags; in propagate_syncq()
7279 syncq_t *sq = q->q_syncq; in qwriter_inner() local
7282 mutex_enter(SQLOCK(sq)); in qwriter_inner()
7283 count = sq->sq_count; in qwriter_inner()
7284 SQ_PUTLOCKS_ENTER(sq); in qwriter_inner()
7285 SUM_SQ_PUTCOUNTS(sq, count); in qwriter_inner()
7287 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); in qwriter_inner()
7295 sq->sq_flags |= SQ_EXCL; in qwriter_inner()
7296 SQ_PUTLOCKS_EXIT(sq); in qwriter_inner()
7297 mutex_exit(SQLOCK(sq)); in qwriter_inner()
7309 SQ_PUTLOCKS_EXIT(sq); in qwriter_inner()
7310 sqfill_events(sq, q, mp, func); in qwriter_inner()
7323 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) in callbparams_alloc() argument
7344 cbp->cbp_sq = sq; in callbparams_alloc()
7347 mutex_enter(SQLOCK(sq)); in callbparams_alloc()
7348 cbp->cbp_next = sq->sq_callbpend; in callbparams_alloc()
7349 sq->sq_callbpend = cbp; in callbparams_alloc()
7354 callbparams_free(syncq_t *sq, callbparams_t *cbp) in callbparams_free() argument
7358 ASSERT(MUTEX_HELD(SQLOCK(sq))); in callbparams_free()
7360 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { in callbparams_free()
7372 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) in callbparams_free_id() argument
7376 ASSERT(MUTEX_HELD(SQLOCK(sq))); in callbparams_free_id()
7378 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { in callbparams_free_id()
7399 syncq_t *sq; in qcallbwrapper() local
7404 sq = cbp->cbp_sq; in qcallbwrapper()
7405 mutex_enter(SQLOCK(sq)); in qcallbwrapper()
7406 type = sq->sq_type; in qcallbwrapper()
7408 count = sq->sq_count; in qcallbwrapper()
7409 SQ_PUTLOCKS_ENTER(sq); in qcallbwrapper()
7410 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); in qcallbwrapper()
7411 SUM_SQ_PUTCOUNTS(sq, count); in qcallbwrapper()
7412 sq->sq_needexcl++; in qcallbwrapper()
7413 ASSERT(sq->sq_needexcl != 0); /* wraparound */ in qcallbwrapper()
7419 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { in qcallbwrapper()
7420 if ((sq->sq_callbflags & cbp->cbp_flags) && in qcallbwrapper()
7421 (sq->sq_cancelid == cbp->cbp_id)) { in qcallbwrapper()
7423 sq->sq_callbflags |= SQ_CALLB_BYPASSED; in qcallbwrapper()
7424 callbparams_free(sq, cbp); in qcallbwrapper()
7426 ASSERT(sq->sq_needexcl > 0); in qcallbwrapper()
7427 sq->sq_needexcl--; in qcallbwrapper()
7428 if (sq->sq_needexcl == 0) { in qcallbwrapper()
7429 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in qcallbwrapper()
7431 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7433 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7436 sq->sq_flags |= SQ_WANTWAKEUP; in qcallbwrapper()
7438 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7440 cv_wait(&sq->sq_wait, SQLOCK(sq)); in qcallbwrapper()
7442 count = sq->sq_count; in qcallbwrapper()
7443 SQ_PUTLOCKS_ENTER(sq); in qcallbwrapper()
7444 SUM_SQ_PUTCOUNTS(sq, count); in qcallbwrapper()
7448 sq->sq_count++; in qcallbwrapper()
7449 ASSERT(sq->sq_count != 0); /* Wraparound */ in qcallbwrapper()
7452 sq->sq_flags |= SQ_EXCL; in qcallbwrapper()
7453 ASSERT(sq->sq_needexcl > 0); in qcallbwrapper()
7454 sq->sq_needexcl--; in qcallbwrapper()
7455 if (sq->sq_needexcl == 0) { in qcallbwrapper()
7456 SQ_PUTCOUNT_SETFAST_LOCKED(sq); in qcallbwrapper()
7458 SQ_PUTLOCKS_EXIT(sq); in qcallbwrapper()
7461 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7469 mutex_enter(SQLOCK(sq)); in qcallbwrapper()
7470 callbparams_free(sq, cbp); in qcallbwrapper()
7471 mutex_exit(SQLOCK(sq)); in qcallbwrapper()
7472 leavesq(sq, SQ_CALLBACK); in qcallbwrapper()
7494 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) in putnext_tail() argument
7496 uint16_t flags = sq->sq_flags; in putnext_tail()
7498 ASSERT(MUTEX_HELD(SQLOCK(sq))); in putnext_tail()
7507 cv_broadcast(&sq->sq_wait); in putnext_tail()
7511 cv_broadcast(&sq->sq_exitwait); in putnext_tail()
7513 sq->sq_flags = flags; in putnext_tail()
7523 (sq->sq_svcflags & SQ_DISABLED)) { in putnext_tail()
7525 drain_syncq(sq); in putnext_tail()
7528 sqenable(sq); in putnext_tail()
7532 mutex_exit(SQLOCK(sq)); in putnext_tail()
7534 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); in putnext_tail()
7844 syncq_t *sq; in pass_wput() local
7846 sq = _RD(q)->q_syncq; in pass_wput()
7847 if (sq->sq_flags & SQ_BLOCKED) in pass_wput()
7848 unblocksq(sq, SQ_BLOCKED, 0); in pass_wput()
8314 syncq_service(syncq_t *sq) in syncq_service() argument
8317 mutex_enter(SQLOCK(sq)); in syncq_service()
8318 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); in syncq_service()
8319 ASSERT(sq->sq_servcount != 0); in syncq_service()
8320 ASSERT(sq->sq_next == NULL); in syncq_service()
8323 if (sq->sq_svcflags & SQ_BGTHREAD) in syncq_service()
8324 sq->sq_svcflags &= ~SQ_BGTHREAD; in syncq_service()
8327 sq->sq_svcflags |= SQ_SERVICE; in syncq_service()
8328 drain_syncq(sq); in syncq_service()