xref: /illumos-gate/usr/src/uts/common/os/strsubr.c (revision 9b664393)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
22 /*	  All Rights Reserved	*/
23 
24 
25 /*
26  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
27  * Use is subject to license terms.
28  * Copyright (c) 2016 by Delphix. All rights reserved.
29  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
30  * Copyright 2018 Joyent, Inc.
31  * Copyright 2022 Garrett D'Amore
32  */
33 
34 #include <sys/types.h>
35 #include <sys/sysmacros.h>
36 #include <sys/param.h>
37 #include <sys/errno.h>
38 #include <sys/signal.h>
39 #include <sys/proc.h>
40 #include <sys/conf.h>
41 #include <sys/cred.h>
42 #include <sys/user.h>
43 #include <sys/vnode.h>
44 #include <sys/file.h>
45 #include <sys/session.h>
46 #include <sys/stream.h>
47 #include <sys/strsubr.h>
48 #include <sys/stropts.h>
49 #include <sys/poll.h>
50 #include <sys/systm.h>
51 #include <sys/cpuvar.h>
52 #include <sys/uio.h>
53 #include <sys/cmn_err.h>
54 #include <sys/priocntl.h>
55 #include <sys/procset.h>
56 #include <sys/vmem.h>
57 #include <sys/bitmap.h>
58 #include <sys/kmem.h>
59 #include <sys/siginfo.h>
60 #include <sys/vtrace.h>
61 #include <sys/callb.h>
62 #include <sys/debug.h>
63 #include <sys/modctl.h>
64 #include <sys/vmsystm.h>
65 #include <vm/page.h>
66 #include <sys/atomic.h>
67 #include <sys/suntpi.h>
68 #include <sys/strlog.h>
69 #include <sys/promif.h>
70 #include <sys/project.h>
71 #include <sys/vm.h>
72 #include <sys/taskq.h>
73 #include <sys/sunddi.h>
74 #include <sys/sunldi_impl.h>
75 #include <sys/strsun.h>
76 #include <sys/isa_defs.h>
77 #include <sys/pattr.h>
78 #include <sys/strft.h>
79 #include <sys/fs/snode.h>
80 #include <sys/zone.h>
81 #include <sys/open.h>
82 #include <sys/sunldi.h>
83 #include <sys/sad.h>
84 #include <sys/netstack.h>
85 
86 #define	O_SAMESTR(q)	(((q)->q_next) && \
87 	(((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
88 
89 /*
90  * WARNING:
91  * The variables and routines in this file are private, belonging
92  * to the STREAMS subsystem. These should not be used by modules
93  * or drivers. Compatibility will not be guaranteed.
94  */
95 
96 /*
97  * Id value used to distinguish between different multiplexor links.
98  */
99 static int32_t lnk_id = 0;
100 
101 #define	STREAMS_LOPRI MINCLSYSPRI
102 static pri_t streams_lopri = STREAMS_LOPRI;
103 
104 #define	STRSTAT(x)	(str_statistics.x.value.ui64++)
105 typedef struct str_stat {
106 	kstat_named_t	sqenables;
107 	kstat_named_t	stenables;
108 	kstat_named_t	syncqservice;
109 	kstat_named_t	freebs;
110 	kstat_named_t	qwr_outer;
111 	kstat_named_t	rservice;
112 	kstat_named_t	strwaits;
113 	kstat_named_t	taskqfails;
114 	kstat_named_t	bufcalls;
115 	kstat_named_t	qhelps;
116 	kstat_named_t	qremoved;
117 	kstat_named_t	sqremoved;
118 	kstat_named_t	bcwaits;
119 	kstat_named_t	sqtoomany;
120 } str_stat_t;
121 
122 static str_stat_t str_statistics = {
123 	{ "sqenables",		KSTAT_DATA_UINT64 },
124 	{ "stenables",		KSTAT_DATA_UINT64 },
125 	{ "syncqservice",	KSTAT_DATA_UINT64 },
126 	{ "freebs",		KSTAT_DATA_UINT64 },
127 	{ "qwr_outer",		KSTAT_DATA_UINT64 },
128 	{ "rservice",		KSTAT_DATA_UINT64 },
129 	{ "strwaits",		KSTAT_DATA_UINT64 },
130 	{ "taskqfails",		KSTAT_DATA_UINT64 },
131 	{ "bufcalls",		KSTAT_DATA_UINT64 },
132 	{ "qhelps",		KSTAT_DATA_UINT64 },
133 	{ "qremoved",		KSTAT_DATA_UINT64 },
134 	{ "sqremoved",		KSTAT_DATA_UINT64 },
135 	{ "bcwaits",		KSTAT_DATA_UINT64 },
136 	{ "sqtoomany",		KSTAT_DATA_UINT64 },
137 };
138 
139 static kstat_t *str_kstat;
140 
141 /*
142  * qrunflag was used previously to control background scheduling of queues. It
143  * is not used anymore, but kept here in case some module still wants to access
144  * it via qready() and setqsched macros.
145  */
146 char qrunflag;			/*  Unused */
147 
148 /*
149  * Most of the streams scheduling is done via task queues. Task queues may fail
150  * for non-sleep dispatches, so there are two backup threads servicing failed
151  * requests for queues and syncqs. Both of these threads also service failed
152  * dispatches freebs requests. Queues are put in the list specified by `qhead'
153  * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
154  * requests are put into `freebs_list' which has no tail pointer. All three
155  * lists are protected by a single `service_queue' lock and use
156  * `services_to_run' condition variable for signaling background threads. Use of
157  * a single lock should not be a problem because it is only used under heavy
158  * loads when task queues start to fail and at that time it may be a good idea
159  * to throttle scheduling requests.
160  *
161  * NOTE: queues and syncqs should be scheduled by two separate threads because
162  * queue servicing may be blocked waiting for a syncq which may be also
163  * scheduled for background execution. This may create a deadlock when only one
164  * thread is used for both.
165  */
166 
167 static taskq_t *streams_taskq;		/* Used for most STREAMS scheduling */
168 
169 static kmutex_t service_queue;		/* protects all of servicing vars */
170 static kcondvar_t services_to_run;	/* wake up background service thread */
171 static kcondvar_t syncqs_to_run;	/* wake up background service thread */
172 
173 /*
174  * List of queues scheduled for background processing due to lack of resources
175  * in the task queues. Protected by service_queue lock;
176  */
177 static struct queue *qhead;
178 static struct queue *qtail;
179 
180 /*
181  * Same list for syncqs
182  */
183 static syncq_t *sqhead;
184 static syncq_t *sqtail;
185 
186 static mblk_t *freebs_list;	/* list of buffers to free */
187 
188 /*
189  * Backup threads for servicing queues and syncqs
190  */
191 kthread_t *streams_qbkgrnd_thread;
192 kthread_t *streams_sqbkgrnd_thread;
193 
194 /*
195  * Bufcalls related variables.
196  */
197 struct bclist	strbcalls;	/* list of waiting bufcalls */
198 kmutex_t	strbcall_lock;	/* protects bufcall list (strbcalls) */
199 kcondvar_t	strbcall_cv;	/* Signaling when a bufcall is added */
200 kmutex_t	bcall_monitor;	/* sleep/wakeup style monitor */
201 kcondvar_t	bcall_cv;	/* wait 'till executing bufcall completes */
202 kthread_t	*bc_bkgrnd_thread; /* Thread to service bufcall requests */
203 
204 kmutex_t	strresources;	/* protects global resources */
205 kmutex_t	muxifier;	/* single-threads multiplexor creation */
206 
207 static void	*str_stack_init(netstackid_t stackid, netstack_t *ns);
208 static void	str_stack_shutdown(netstackid_t stackid, void *arg);
209 static void	str_stack_fini(netstackid_t stackid, void *arg);
210 
211 /*
212  * run_queues is no longer used, but is kept in case some 3rd party
213  * module/driver decides to use it.
214  */
215 int run_queues = 0;
216 
217 /*
218  * sq_max_size is the depth of the syncq (in number of messages) before
219  * qfill_syncq() starts QFULL'ing destination queues. As its primary
220  * consumer - IP is no longer D_MTPERMOD, but there may be other
221  * modules/drivers depend on this syncq flow control, we prefer to
222  * choose a large number as the default value. For potential
223  * performance gain, this value is tunable in /etc/system.
224  */
225 int sq_max_size = 10000;
226 
227 /*
228  * The number of ciputctrl structures per syncq and stream we create when
229  * needed.
230  */
231 int n_ciputctrl;
232 int max_n_ciputctrl = 16;
233 /*
234  * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
235  */
236 int min_n_ciputctrl = 2;
237 
238 /*
239  * Per-driver/module syncqs
240  * ========================
241  *
242  * For drivers/modules that use PERMOD or outer syncqs we keep a list of
243  * perdm structures, new entries being added (and new syncqs allocated) when
244  * setq() encounters a module/driver with a streamtab that it hasn't seen
245  * before.
246  * The reason for this mechanism is that some modules and drivers share a
247  * common streamtab and it is necessary for those modules and drivers to also
248  * share a common PERMOD syncq.
249  *
250  * perdm_list --> dm_str == streamtab_1
251  *                dm_sq == syncq_1
252  *                dm_ref
253  *                dm_next --> dm_str == streamtab_2
254  *                            dm_sq == syncq_2
255  *                            dm_ref
256  *                            dm_next --> ... NULL
257  *
258  * The dm_ref field is incremented for each new driver/module that takes
259  * a reference to the perdm structure and hence shares the syncq.
260  * References are held in the fmodsw_impl_t structure for each STREAMS module
261  * or the dev_impl array (indexed by device major number) for each driver.
262  *
263  * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
264  *		     ^                 ^ ^               ^
265  *                   |  ______________/  |               |
266  *                   | /                 |               |
267  * dev_impl:     ...|x|y|...          module A	      module B
268  *
269  * When a module/driver is unloaded the reference count is decremented and,
270  * when it falls to zero, the perdm structure is removed from the list and
271  * the syncq is freed (see rele_dm()).
272  */
273 perdm_t *perdm_list = NULL;
274 static krwlock_t perdm_rwlock;
275 cdevsw_impl_t *devimpl;
276 
277 extern struct qinit strdata;
278 extern struct qinit stwdata;
279 
280 static void runservice(queue_t *);
281 static void streams_bufcall_service(void);
282 static void streams_qbkgrnd_service(void);
283 static void streams_sqbkgrnd_service(void);
284 static syncq_t *new_syncq(void);
285 static void free_syncq(syncq_t *);
286 static void outer_insert(syncq_t *, syncq_t *);
287 static void outer_remove(syncq_t *, syncq_t *);
288 static void write_now(syncq_t *);
289 static void clr_qfull(queue_t *);
290 static void runbufcalls(void);
291 static void sqenable(syncq_t *);
292 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
293 static void wait_q_syncq(queue_t *);
294 static void backenable_insertedq(queue_t *);
295 
296 static void queue_service(queue_t *);
297 static void stream_service(stdata_t *);
298 static void syncq_service(syncq_t *);
299 static void qwriter_outer_service(syncq_t *);
300 static void mblk_free(mblk_t *);
301 #ifdef DEBUG
302 static int qprocsareon(queue_t *);
303 #endif
304 
305 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
306 static void reset_nfsrv_ptr(queue_t *, queue_t *);
307 void set_qfull(queue_t *);
308 
309 static void sq_run_events(syncq_t *);
310 static int propagate_syncq(queue_t *);
311 
312 static void	blocksq(syncq_t *, ushort_t, int);
313 static void	unblocksq(syncq_t *, ushort_t, int);
314 static int	dropsq(syncq_t *, uint16_t);
315 static void	emptysq(syncq_t *);
316 static sqlist_t *sqlist_alloc(struct stdata *, int);
317 static void	sqlist_free(sqlist_t *);
318 static sqlist_t	*sqlist_build(queue_t *, struct stdata *, boolean_t);
319 static void	sqlist_insert(sqlist_t *, syncq_t *);
320 static void	sqlist_insertall(sqlist_t *, queue_t *);
321 
322 static void	strsetuio(stdata_t *);
323 
324 struct kmem_cache *stream_head_cache;
325 struct kmem_cache *queue_cache;
326 struct kmem_cache *syncq_cache;
327 struct kmem_cache *qband_cache;
328 struct kmem_cache *linkinfo_cache;
329 struct kmem_cache *ciputctrl_cache = NULL;
330 
331 static linkinfo_t *linkinfo_list;
332 
333 /* Global esballoc throttling queue */
334 static esb_queue_t system_esbq;
335 
336 /* Array of esballoc throttling queues, of length esbq_nelem */
337 static esb_queue_t *volatile system_esbq_array;
338 static int esbq_nelem;
339 static kmutex_t esbq_lock;
340 static int esbq_log2_cpus_per_q = 0;
341 
342 /* Scale the system_esbq length by setting number of CPUs per queue. */
343 uint_t esbq_cpus_per_q = 1;
344 
345 /*
346  * esballoc tunable parameters.
347  */
348 int		esbq_max_qlen = 0x16;	/* throttled queue length */
349 clock_t		esbq_timeout = 0x8;	/* timeout to process esb queue */
350 
351 /*
352  * Routines to handle esballoc queueing.
353  */
354 static void esballoc_process_queue(esb_queue_t *);
355 static void esballoc_enqueue_mblk(mblk_t *);
356 static void esballoc_timer(void *);
357 static void esballoc_set_timer(esb_queue_t *, clock_t);
358 static void esballoc_mblk_free(mblk_t *);
359 
360 /*
361  *  Qinit structure and Module_info structures
362  *	for passthru read and write queues
363  */
364 
365 static int pass_rput(queue_t *, mblk_t *);
366 static int pass_wput(queue_t *, mblk_t *);
367 static queue_t *link_addpassthru(stdata_t *);
368 static void link_rempassthru(queue_t *);
369 
370 struct  module_info passthru_info = {
371 	0,
372 	"passthru",
373 	0,
374 	INFPSZ,
375 	STRHIGH,
376 	STRLOW
377 };
378 
379 struct  qinit passthru_rinit = {
380 	pass_rput,
381 	NULL,
382 	NULL,
383 	NULL,
384 	NULL,
385 	&passthru_info,
386 	NULL
387 };
388 
389 struct  qinit passthru_winit = {
390 	pass_wput,
391 	NULL,
392 	NULL,
393 	NULL,
394 	NULL,
395 	&passthru_info,
396 	NULL
397 };
398 
399 /*
400  * Verify correctness of list head/tail pointers.
401  */
402 #define	LISTCHECK(head, tail, link) {				\
403 	EQUIV(head, tail);					\
404 	IMPLY(tail != NULL, tail->link == NULL);		\
405 }
406 
407 /*
408  * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
409  * using a `link' field.
410  */
411 #define	ENQUEUE(el, head, tail, link) {				\
412 	ASSERT(el->link == NULL);				\
413 	LISTCHECK(head, tail, link);				\
414 	if (head == NULL)					\
415 		head = el;					\
416 	else							\
417 		tail->link = el;				\
418 	tail = el;						\
419 }
420 
421 /*
422  * Dequeue the first element of the list denoted by `head' and `tail' pointers
423  * using a `link' field and put result into `el'.
424  */
425 #define	DQ(el, head, tail, link) {				\
426 	LISTCHECK(head, tail, link);				\
427 	el = head;						\
428 	if (head != NULL) {					\
429 		head = head->link;				\
430 		if (head == NULL)				\
431 			tail = NULL;				\
432 		el->link = NULL;				\
433 	}							\
434 }
435 
436 /*
437  * Remove `el' from the list using `chase' and `curr' pointers and return result
438  * in `succeed'.
439  */
440 #define	RMQ(el, head, tail, link, chase, curr, succeed) {	\
441 	LISTCHECK(head, tail, link);				\
442 	chase = NULL;						\
443 	succeed = 0;						\
444 	for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
445 		chase = curr;					\
446 	if (curr != NULL) {					\
447 		succeed = 1;					\
448 		ASSERT(curr == el);				\
449 		if (chase != NULL)				\
450 			chase->link = curr->link;		\
451 		else						\
452 			head = curr->link;			\
453 		curr->link = NULL;				\
454 		if (curr == tail)				\
455 			tail = chase;				\
456 	}							\
457 	LISTCHECK(head, tail, link);				\
458 }
459 
460 /* Handling of delayed messages on the inner syncq. */
461 
462 /*
463  * DEBUG versions should use function versions (to simplify tracing) and
464  * non-DEBUG kernels should use macro versions.
465  */
466 
467 /*
468  * Put a queue on the syncq list of queues.
469  * Assumes SQLOCK held.
470  */
471 #define	SQPUT_Q(sq, qp)							\
472 {									\
473 	ASSERT(MUTEX_HELD(SQLOCK(sq)));					\
474 	if (!(qp->q_sqflags & Q_SQQUEUED)) {				\
475 		/* The queue should not be linked anywhere */		\
476 		ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
477 		/* Head and tail may only be NULL simultaneously */	\
478 		EQUIV(sq->sq_head, sq->sq_tail);			\
479 		/* Queue may be only enqueued on its syncq */		\
480 		ASSERT(sq == qp->q_syncq);				\
481 		/* Check the correctness of SQ_MESSAGES flag */		\
482 		EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES));	\
483 		/* Sanity check first/last elements of the list */	\
484 		IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
485 		IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
486 		/*							\
487 		 * Sanity check of priority field: empty queue should	\
488 		 * have zero priority					\
489 		 * and nqueues equal to zero.				\
490 		 */							\
491 		IMPLY(sq->sq_head == NULL, sq->sq_pri == 0);		\
492 		/* Sanity check of sq_nqueues field */			\
493 		EQUIV(sq->sq_head, sq->sq_nqueues);			\
494 		if (sq->sq_head == NULL) {				\
495 			sq->sq_head = sq->sq_tail = qp;			\
496 			sq->sq_flags |= SQ_MESSAGES;			\
497 		} else if (qp->q_spri == 0) {				\
498 			qp->q_sqprev = sq->sq_tail;			\
499 			sq->sq_tail->q_sqnext = qp;			\
500 			sq->sq_tail = qp;				\
501 		} else {						\
502 			/*						\
503 			 * Put this queue in priority order: higher	\
504 			 * priority gets closer to the head.		\
505 			 */						\
506 			queue_t **qpp = &sq->sq_tail;			\
507 			queue_t *qnext = NULL;				\
508 									\
509 			while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
510 				qnext = *qpp;				\
511 				qpp = &(*qpp)->q_sqprev;		\
512 			}						\
513 			qp->q_sqnext = qnext;				\
514 			qp->q_sqprev = *qpp;				\
515 			if (*qpp != NULL) {				\
516 				(*qpp)->q_sqnext = qp;			\
517 			} else {					\
518 				sq->sq_head = qp;			\
519 				sq->sq_pri = sq->sq_head->q_spri;	\
520 			}						\
521 			*qpp = qp;					\
522 		}							\
523 		qp->q_sqflags |= Q_SQQUEUED;				\
524 		qp->q_sqtstamp = ddi_get_lbolt();			\
525 		sq->sq_nqueues++;					\
526 	}								\
527 }
528 
529 /*
530  * Remove a queue from the syncq list
531  * Assumes SQLOCK held.
532  */
533 #define	SQRM_Q(sq, qp)							\
534 	{								\
535 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
536 		ASSERT(qp->q_sqflags & Q_SQQUEUED);			\
537 		ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL);	\
538 		ASSERT((sq->sq_flags & SQ_MESSAGES) != 0);		\
539 		/* Check that the queue is actually in the list */	\
540 		ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp);	\
541 		ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp);	\
542 		ASSERT(sq->sq_nqueues != 0);				\
543 		if (qp->q_sqprev == NULL) {				\
544 			/* First queue on list, make head q_sqnext */	\
545 			sq->sq_head = qp->q_sqnext;			\
546 		} else {						\
547 			/* Make prev->next == next */			\
548 			qp->q_sqprev->q_sqnext = qp->q_sqnext;		\
549 		}							\
550 		if (qp->q_sqnext == NULL) {				\
551 			/* Last queue on list, make tail sqprev */	\
552 			sq->sq_tail = qp->q_sqprev;			\
553 		} else {						\
554 			/* Make next->prev == prev */			\
555 			qp->q_sqnext->q_sqprev = qp->q_sqprev;		\
556 		}							\
557 		/* clear out references on this queue */		\
558 		qp->q_sqprev = qp->q_sqnext = NULL;			\
559 		qp->q_sqflags &= ~Q_SQQUEUED;				\
560 		/* If there is nothing queued, clear SQ_MESSAGES */	\
561 		if (sq->sq_head != NULL) {				\
562 			sq->sq_pri = sq->sq_head->q_spri;		\
563 		} else	{						\
564 			sq->sq_flags &= ~SQ_MESSAGES;			\
565 			sq->sq_pri = 0;					\
566 		}							\
567 		sq->sq_nqueues--;					\
568 		ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL ||	\
569 		    (sq->sq_flags & SQ_QUEUED) == 0);			\
570 	}
571 
572 /* Hide the definition from the header file. */
573 #ifdef SQPUT_MP
574 #undef SQPUT_MP
575 #endif
576 
577 /*
578  * Put a message on the queue syncq.
579  * Assumes QLOCK held.
580  */
581 #define	SQPUT_MP(qp, mp)						\
582 	{								\
583 		ASSERT(MUTEX_HELD(QLOCK(qp)));				\
584 		ASSERT(qp->q_sqhead == NULL ||				\
585 		    (qp->q_sqtail != NULL &&				\
586 		    qp->q_sqtail->b_next == NULL));			\
587 		qp->q_syncqmsgs++;					\
588 		ASSERT(qp->q_syncqmsgs != 0);	/* Wraparound */	\
589 		if (qp->q_sqhead == NULL) {				\
590 			qp->q_sqhead = qp->q_sqtail = mp;		\
591 		} else {						\
592 			qp->q_sqtail->b_next = mp;			\
593 			qp->q_sqtail = mp;				\
594 		}							\
595 		ASSERT(qp->q_syncqmsgs > 0);				\
596 		set_qfull(qp);						\
597 	}
598 
599 #define	SQ_PUTCOUNT_SETFAST_LOCKED(sq) {				\
600 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
601 		if ((sq)->sq_ciputctrl != NULL) {			\
602 			int i;						\
603 			int nlocks = (sq)->sq_nciputctrl;		\
604 			ciputctrl_t *cip = (sq)->sq_ciputctrl;		\
605 			ASSERT((sq)->sq_type & SQ_CIPUT);		\
606 			for (i = 0; i <= nlocks; i++) {			\
607 				ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
608 				cip[i].ciputctrl_count |= SQ_FASTPUT;	\
609 			}						\
610 		}							\
611 	}
612 
613 
614 #define	SQ_PUTCOUNT_CLRFAST_LOCKED(sq) {				\
615 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
616 		if ((sq)->sq_ciputctrl != NULL) {			\
617 			int i;						\
618 			int nlocks = (sq)->sq_nciputctrl;		\
619 			ciputctrl_t *cip = (sq)->sq_ciputctrl;		\
620 			ASSERT((sq)->sq_type & SQ_CIPUT);		\
621 			for (i = 0; i <= nlocks; i++) {			\
622 				ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
623 				cip[i].ciputctrl_count &= ~SQ_FASTPUT;	\
624 			}						\
625 		}							\
626 	}
627 
628 /*
629  * Run service procedures for all queues in the stream head.
630  */
631 #define	STR_SERVICE(stp, q) {						\
632 	ASSERT(MUTEX_HELD(&stp->sd_qlock));				\
633 	while (stp->sd_qhead != NULL) {					\
634 		DQ(q, stp->sd_qhead, stp->sd_qtail, q_link);		\
635 		ASSERT(stp->sd_nqueues > 0);				\
636 		stp->sd_nqueues--;					\
637 		ASSERT(!(q->q_flag & QINSERVICE));			\
638 		mutex_exit(&stp->sd_qlock);				\
639 		queue_service(q);					\
640 		mutex_enter(&stp->sd_qlock);				\
641 	}								\
642 	ASSERT(stp->sd_nqueues == 0);					\
643 	ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL));	\
644 }
645 
646 /*
647  * Constructor/destructor routines for the stream head cache
648  */
649 /* ARGSUSED */
650 static int
stream_head_constructor(void * buf,void * cdrarg,int kmflags)651 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
652 {
653 	stdata_t *stp = buf;
654 
655 	mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
656 	mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
657 	mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
658 	cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
659 	cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
660 	cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
661 	cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
662 	cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
663 	stp->sd_wrq = NULL;
664 
665 	return (0);
666 }
667 
668 /* ARGSUSED */
669 static void
stream_head_destructor(void * buf,void * cdrarg)670 stream_head_destructor(void *buf, void *cdrarg)
671 {
672 	stdata_t *stp = buf;
673 
674 	mutex_destroy(&stp->sd_lock);
675 	mutex_destroy(&stp->sd_reflock);
676 	mutex_destroy(&stp->sd_qlock);
677 	cv_destroy(&stp->sd_monitor);
678 	cv_destroy(&stp->sd_iocmonitor);
679 	cv_destroy(&stp->sd_refmonitor);
680 	cv_destroy(&stp->sd_qcv);
681 	cv_destroy(&stp->sd_zcopy_wait);
682 }
683 
684 /*
685  * Constructor/destructor routines for the queue cache
686  */
687 /* ARGSUSED */
688 static int
queue_constructor(void * buf,void * cdrarg,int kmflags)689 queue_constructor(void *buf, void *cdrarg, int kmflags)
690 {
691 	queinfo_t *qip = buf;
692 	queue_t *qp = &qip->qu_rqueue;
693 	queue_t *wqp = &qip->qu_wqueue;
694 	syncq_t	*sq = &qip->qu_syncq;
695 
696 	qp->q_first = NULL;
697 	qp->q_link = NULL;
698 	qp->q_count = 0;
699 	qp->q_mblkcnt = 0;
700 	qp->q_sqhead = NULL;
701 	qp->q_sqtail = NULL;
702 	qp->q_sqnext = NULL;
703 	qp->q_sqprev = NULL;
704 	qp->q_sqflags = 0;
705 	qp->q_rwcnt = 0;
706 	qp->q_spri = 0;
707 
708 	mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
709 	cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
710 
711 	wqp->q_first = NULL;
712 	wqp->q_link = NULL;
713 	wqp->q_count = 0;
714 	wqp->q_mblkcnt = 0;
715 	wqp->q_sqhead = NULL;
716 	wqp->q_sqtail = NULL;
717 	wqp->q_sqnext = NULL;
718 	wqp->q_sqprev = NULL;
719 	wqp->q_sqflags = 0;
720 	wqp->q_rwcnt = 0;
721 	wqp->q_spri = 0;
722 
723 	mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
724 	cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
725 
726 	sq->sq_head = NULL;
727 	sq->sq_tail = NULL;
728 	sq->sq_evhead = NULL;
729 	sq->sq_evtail = NULL;
730 	sq->sq_callbpend = NULL;
731 	sq->sq_outer = NULL;
732 	sq->sq_onext = NULL;
733 	sq->sq_oprev = NULL;
734 	sq->sq_next = NULL;
735 	sq->sq_svcflags = 0;
736 	sq->sq_servcount = 0;
737 	sq->sq_needexcl = 0;
738 	sq->sq_nqueues = 0;
739 	sq->sq_pri = 0;
740 
741 	mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
742 	cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
743 	cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
744 
745 	return (0);
746 }
747 
748 /* ARGSUSED */
749 static void
queue_destructor(void * buf,void * cdrarg)750 queue_destructor(void *buf, void *cdrarg)
751 {
752 	queinfo_t *qip = buf;
753 	queue_t *qp = &qip->qu_rqueue;
754 	queue_t *wqp = &qip->qu_wqueue;
755 	syncq_t	*sq = &qip->qu_syncq;
756 
757 	ASSERT(qp->q_sqhead == NULL);
758 	ASSERT(wqp->q_sqhead == NULL);
759 	ASSERT(qp->q_sqnext == NULL);
760 	ASSERT(wqp->q_sqnext == NULL);
761 	ASSERT(qp->q_rwcnt == 0);
762 	ASSERT(wqp->q_rwcnt == 0);
763 
764 	mutex_destroy(&qp->q_lock);
765 	cv_destroy(&qp->q_wait);
766 
767 	mutex_destroy(&wqp->q_lock);
768 	cv_destroy(&wqp->q_wait);
769 
770 	mutex_destroy(&sq->sq_lock);
771 	cv_destroy(&sq->sq_wait);
772 	cv_destroy(&sq->sq_exitwait);
773 }
774 
775 /*
776  * Constructor/destructor routines for the syncq cache
777  */
778 /* ARGSUSED */
779 static int
syncq_constructor(void * buf,void * cdrarg,int kmflags)780 syncq_constructor(void *buf, void *cdrarg, int kmflags)
781 {
782 	syncq_t	*sq = buf;
783 
784 	bzero(buf, sizeof (syncq_t));
785 
786 	mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
787 	cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
788 	cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
789 
790 	return (0);
791 }
792 
793 /* ARGSUSED */
794 static void
syncq_destructor(void * buf,void * cdrarg)795 syncq_destructor(void *buf, void *cdrarg)
796 {
797 	syncq_t	*sq = buf;
798 
799 	ASSERT(sq->sq_head == NULL);
800 	ASSERT(sq->sq_tail == NULL);
801 	ASSERT(sq->sq_evhead == NULL);
802 	ASSERT(sq->sq_evtail == NULL);
803 	ASSERT(sq->sq_callbpend == NULL);
804 	ASSERT(sq->sq_callbflags == 0);
805 	ASSERT(sq->sq_outer == NULL);
806 	ASSERT(sq->sq_onext == NULL);
807 	ASSERT(sq->sq_oprev == NULL);
808 	ASSERT(sq->sq_next == NULL);
809 	ASSERT(sq->sq_needexcl == 0);
810 	ASSERT(sq->sq_svcflags == 0);
811 	ASSERT(sq->sq_servcount == 0);
812 	ASSERT(sq->sq_nqueues == 0);
813 	ASSERT(sq->sq_pri == 0);
814 	ASSERT(sq->sq_count == 0);
815 	ASSERT(sq->sq_rmqcount == 0);
816 	ASSERT(sq->sq_cancelid == 0);
817 	ASSERT(sq->sq_ciputctrl == NULL);
818 	ASSERT(sq->sq_nciputctrl == 0);
819 	ASSERT(sq->sq_type == 0);
820 	ASSERT(sq->sq_flags == 0);
821 
822 	mutex_destroy(&sq->sq_lock);
823 	cv_destroy(&sq->sq_wait);
824 	cv_destroy(&sq->sq_exitwait);
825 }
826 
827 /* ARGSUSED */
828 static int
ciputctrl_constructor(void * buf,void * cdrarg,int kmflags)829 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
830 {
831 	ciputctrl_t *cip = buf;
832 	int i;
833 
834 	for (i = 0; i < n_ciputctrl; i++) {
835 		cip[i].ciputctrl_count = SQ_FASTPUT;
836 		mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
837 	}
838 
839 	return (0);
840 }
841 
842 /* ARGSUSED */
843 static void
ciputctrl_destructor(void * buf,void * cdrarg)844 ciputctrl_destructor(void *buf, void *cdrarg)
845 {
846 	ciputctrl_t *cip = buf;
847 	int i;
848 
849 	for (i = 0; i < n_ciputctrl; i++) {
850 		ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
851 		mutex_destroy(&cip[i].ciputctrl_lock);
852 	}
853 }
854 
855 /*
856  * Init routine run from main at boot time.
857  */
858 void
strinit(void)859 strinit(void)
860 {
861 	int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
862 
863 	stream_head_cache = kmem_cache_create("stream_head_cache",
864 	    sizeof (stdata_t), 0,
865 	    stream_head_constructor, stream_head_destructor, NULL,
866 	    NULL, NULL, 0);
867 
868 	queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
869 	    queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
870 
871 	syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
872 	    syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
873 
874 	qband_cache = kmem_cache_create("qband_cache",
875 	    sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
876 
877 	linkinfo_cache = kmem_cache_create("linkinfo_cache",
878 	    sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
879 
880 	n_ciputctrl = ncpus;
881 	n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
882 	ASSERT(n_ciputctrl >= 1);
883 	n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
884 	if (n_ciputctrl >= min_n_ciputctrl) {
885 		ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
886 		    sizeof (ciputctrl_t) * n_ciputctrl,
887 		    sizeof (ciputctrl_t), ciputctrl_constructor,
888 		    ciputctrl_destructor, NULL, NULL, NULL, 0);
889 	}
890 
891 	streams_taskq = system_taskq;
892 
893 	if (streams_taskq == NULL)
894 		panic("strinit: no memory for streams taskq!");
895 
896 	bc_bkgrnd_thread = thread_create(NULL, 0,
897 	    streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
898 
899 	streams_qbkgrnd_thread = thread_create(NULL, 0,
900 	    streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
901 
902 	streams_sqbkgrnd_thread = thread_create(NULL, 0,
903 	    streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
904 
905 	/*
906 	 * Create STREAMS kstats.
907 	 */
908 	str_kstat = kstat_create("streams", 0, "strstat",
909 	    "net", KSTAT_TYPE_NAMED,
910 	    sizeof (str_statistics) / sizeof (kstat_named_t),
911 	    KSTAT_FLAG_VIRTUAL);
912 
913 	if (str_kstat != NULL) {
914 		str_kstat->ks_data = &str_statistics;
915 		kstat_install(str_kstat);
916 	}
917 
918 	/*
919 	 * TPI support routine initialisation.
920 	 */
921 	tpi_init();
922 
923 	/*
924 	 * Handle to have autopush and persistent link information per
925 	 * zone.
926 	 * Note: uses shutdown hook instead of destroy hook so that the
927 	 * persistent links can be torn down before the destroy hooks
928 	 * in the TCP/IP stack are called.
929 	 */
930 	netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
931 	    str_stack_fini);
932 }
933 
934 void
str_sendsig(vnode_t * vp,int event,uchar_t band,int error)935 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
936 {
937 	struct stdata *stp;
938 
939 	ASSERT(vp->v_stream);
940 	stp = vp->v_stream;
941 	/* Have to hold sd_lock to prevent siglist from changing */
942 	mutex_enter(&stp->sd_lock);
943 	if (stp->sd_sigflags & event)
944 		strsendsig(stp->sd_siglist, event, band, error);
945 	mutex_exit(&stp->sd_lock);
946 }
947 
948 /*
949  * Send the "sevent" set of signals to a process.
950  * This might send more than one signal if the process is registered
951  * for multiple events. The caller should pass in an sevent that only
952  * includes the events for which the process has registered.
953  */
954 static void
dosendsig(proc_t * proc,int events,int sevent,k_siginfo_t * info,uchar_t band,int error)955 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
956     uchar_t band, int error)
957 {
958 	ASSERT(MUTEX_HELD(&proc->p_lock));
959 
960 	info->si_band = 0;
961 	info->si_errno = 0;
962 
963 	if (sevent & S_ERROR) {
964 		sevent &= ~S_ERROR;
965 		info->si_code = POLL_ERR;
966 		info->si_errno = error;
967 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
968 		    "strsendsig:proc %p info %p", proc, info);
969 		sigaddq(proc, NULL, info, KM_NOSLEEP);
970 		info->si_errno = 0;
971 	}
972 	if (sevent & S_HANGUP) {
973 		sevent &= ~S_HANGUP;
974 		info->si_code = POLL_HUP;
975 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
976 		    "strsendsig:proc %p info %p", proc, info);
977 		sigaddq(proc, NULL, info, KM_NOSLEEP);
978 	}
979 	if (sevent & S_HIPRI) {
980 		sevent &= ~S_HIPRI;
981 		info->si_code = POLL_PRI;
982 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
983 		    "strsendsig:proc %p info %p", proc, info);
984 		sigaddq(proc, NULL, info, KM_NOSLEEP);
985 	}
986 	if (sevent & S_RDBAND) {
987 		sevent &= ~S_RDBAND;
988 		if (events & S_BANDURG)
989 			sigtoproc(proc, NULL, SIGURG);
990 		else
991 			sigtoproc(proc, NULL, SIGPOLL);
992 	}
993 	if (sevent & S_WRBAND) {
994 		sevent &= ~S_WRBAND;
995 		sigtoproc(proc, NULL, SIGPOLL);
996 	}
997 	if (sevent & S_INPUT) {
998 		sevent &= ~S_INPUT;
999 		info->si_code = POLL_IN;
1000 		info->si_band = band;
1001 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1002 		    "strsendsig:proc %p info %p", proc, info);
1003 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1004 		info->si_band = 0;
1005 	}
1006 	if (sevent & S_OUTPUT) {
1007 		sevent &= ~S_OUTPUT;
1008 		info->si_code = POLL_OUT;
1009 		info->si_band = band;
1010 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1011 		    "strsendsig:proc %p info %p", proc, info);
1012 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1013 		info->si_band = 0;
1014 	}
1015 	if (sevent & S_MSG) {
1016 		sevent &= ~S_MSG;
1017 		info->si_code = POLL_MSG;
1018 		info->si_band = band;
1019 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1020 		    "strsendsig:proc %p info %p", proc, info);
1021 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1022 		info->si_band = 0;
1023 	}
1024 	if (sevent & S_RDNORM) {
1025 		sevent &= ~S_RDNORM;
1026 		sigtoproc(proc, NULL, SIGPOLL);
1027 	}
1028 	if (sevent != 0) {
1029 		panic("strsendsig: unknown event(s) %x", sevent);
1030 	}
1031 }
1032 
1033 /*
1034  * Send SIGPOLL/SIGURG signal to all processes and process groups
1035  * registered on the given signal list that want a signal for at
1036  * least one of the specified events.
1037  *
1038  * Must be called with exclusive access to siglist (caller holding sd_lock).
1039  *
1040  * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1041  * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1042  * while it is in the siglist.
1043  *
1044  * For performance reasons (MP scalability) the code drops pidlock
1045  * when sending signals to a single process.
1046  * When sending to a process group the code holds
1047  * pidlock to prevent the membership in the process group from changing
1048  * while walking the p_pglink list.
1049  */
1050 void
strsendsig(strsig_t * siglist,int event,uchar_t band,int error)1051 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1052 {
1053 	strsig_t *ssp;
1054 	k_siginfo_t info;
1055 	struct pid *pidp;
1056 	proc_t  *proc;
1057 
1058 	info.si_signo = SIGPOLL;
1059 	info.si_errno = 0;
1060 	for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1061 		int sevent;
1062 
1063 		sevent = ssp->ss_events & event;
1064 		if (sevent == 0)
1065 			continue;
1066 
1067 		if ((pidp = ssp->ss_pidp) == NULL) {
1068 			/* pid was released but still on event list */
1069 			continue;
1070 		}
1071 
1072 
1073 		if (ssp->ss_pid > 0) {
1074 			/*
1075 			 * XXX This unfortunately still generates
1076 			 * a signal when a fd is closed but
1077 			 * the proc is active.
1078 			 */
1079 			ASSERT(ssp->ss_pid == pidp->pid_id);
1080 
1081 			mutex_enter(&pidlock);
1082 			proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1083 			if (proc == NULL) {
1084 				mutex_exit(&pidlock);
1085 				continue;
1086 			}
1087 			mutex_enter(&proc->p_lock);
1088 			mutex_exit(&pidlock);
1089 			dosendsig(proc, ssp->ss_events, sevent, &info,
1090 			    band, error);
1091 			mutex_exit(&proc->p_lock);
1092 		} else {
1093 			/*
1094 			 * Send to process group. Hold pidlock across
1095 			 * calls to dosendsig().
1096 			 */
1097 			pid_t pgrp = -ssp->ss_pid;
1098 
1099 			mutex_enter(&pidlock);
1100 			proc = pgfind_zone(pgrp, ALL_ZONES);
1101 			while (proc != NULL) {
1102 				mutex_enter(&proc->p_lock);
1103 				dosendsig(proc, ssp->ss_events, sevent,
1104 				    &info, band, error);
1105 				mutex_exit(&proc->p_lock);
1106 				proc = proc->p_pglink;
1107 			}
1108 			mutex_exit(&pidlock);
1109 		}
1110 	}
1111 }
1112 
1113 /*
1114  * Attach a stream device or module.
1115  * qp is a read queue; the new queue goes in so its next
1116  * read ptr is the argument, and the write queue corresponding
1117  * to the argument points to this queue. Return 0 on success,
1118  * or a non-zero errno on failure.
1119  */
1120 int
qattach(queue_t * qp,dev_t * devp,int oflag,cred_t * crp,fmodsw_impl_t * fp,boolean_t is_insert)1121 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1122     boolean_t is_insert)
1123 {
1124 	major_t			major;
1125 	cdevsw_impl_t		*dp;
1126 	struct streamtab	*str;
1127 	queue_t			*rq;
1128 	queue_t			*wrq;
1129 	uint32_t		qflag;
1130 	uint32_t		sqtype;
1131 	perdm_t			*dmp;
1132 	int			error;
1133 	int			sflag;
1134 
1135 	rq = allocq();
1136 	wrq = _WR(rq);
1137 	STREAM(rq) = STREAM(wrq) = STREAM(qp);
1138 
1139 	if (fp != NULL) {
1140 		str = fp->f_str;
1141 		qflag = fp->f_qflag;
1142 		sqtype = fp->f_sqtype;
1143 		dmp = fp->f_dmp;
1144 		IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1145 		sflag = MODOPEN;
1146 
1147 		/*
1148 		 * stash away a pointer to the module structure so we can
1149 		 * unref it in qdetach.
1150 		 */
1151 		rq->q_fp = fp;
1152 	} else {
1153 		ASSERT(!is_insert);
1154 
1155 		major = getmajor(*devp);
1156 		dp = &devimpl[major];
1157 
1158 		str = dp->d_str;
1159 		ASSERT(str == STREAMSTAB(major));
1160 
1161 		qflag = dp->d_qflag;
1162 		ASSERT(qflag & QISDRV);
1163 		sqtype = dp->d_sqtype;
1164 
1165 		/* create perdm_t if needed */
1166 		if (NEED_DM(dp->d_dmp, qflag))
1167 			dp->d_dmp = hold_dm(str, qflag, sqtype);
1168 
1169 		dmp = dp->d_dmp;
1170 		sflag = 0;
1171 	}
1172 
1173 	TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1174 	    "qattach:qflag == %X(%X)", qflag, *devp);
1175 
1176 	/* setq might sleep in allocator - avoid holding locks. */
1177 	setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1178 
1179 	/*
1180 	 * Before calling the module's open routine, set up the q_next
1181 	 * pointer for inserting a module in the middle of a stream.
1182 	 *
1183 	 * Note that we can always set _QINSERTING and set up q_next
1184 	 * pointer for both inserting and pushing a module.  Then there
1185 	 * is no need for the is_insert parameter.  In insertq(), called
1186 	 * by qprocson(), assume that q_next of the new module always points
1187 	 * to the correct queue and use it for insertion.  Everything should
1188 	 * work out fine.  But in the first release of _I_INSERT, we
1189 	 * distinguish between inserting and pushing to make sure that
1190 	 * pushing a module follows the same code path as before.
1191 	 */
1192 	if (is_insert) {
1193 		rq->q_flag |= _QINSERTING;
1194 		rq->q_next = qp;
1195 	}
1196 
1197 	/*
1198 	 * If there is an outer perimeter get exclusive access during
1199 	 * the open procedure.  Bump up the reference count on the queue.
1200 	 */
1201 	entersq(rq->q_syncq, SQ_OPENCLOSE);
1202 	error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1203 	if (error != 0)
1204 		goto failed;
1205 	leavesq(rq->q_syncq, SQ_OPENCLOSE);
1206 	ASSERT(qprocsareon(rq));
1207 	return (0);
1208 
1209 failed:
1210 	rq->q_flag &= ~_QINSERTING;
1211 	if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1212 		qprocsoff(rq);
1213 	leavesq(rq->q_syncq, SQ_OPENCLOSE);
1214 	rq->q_next = wrq->q_next = NULL;
1215 	qdetach(rq, 0, 0, crp, B_FALSE);
1216 	return (error);
1217 }
1218 
1219 /*
1220  * Handle second open of stream. For modules, set the
1221  * last argument to MODOPEN and do not pass any open flags.
1222  * Ignore dummydev since this is not the first open.
1223  */
1224 int
qreopen(queue_t * qp,dev_t * devp,int flag,cred_t * crp)1225 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1226 {
1227 	int	error;
1228 	dev_t dummydev;
1229 	queue_t *wqp = _WR(qp);
1230 
1231 	ASSERT(qp->q_flag & QREADR);
1232 	entersq(qp->q_syncq, SQ_OPENCLOSE);
1233 
1234 	dummydev = *devp;
1235 	if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1236 	    (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1237 		leavesq(qp->q_syncq, SQ_OPENCLOSE);
1238 		mutex_enter(&STREAM(qp)->sd_lock);
1239 		qp->q_stream->sd_flag |= STREOPENFAIL;
1240 		mutex_exit(&STREAM(qp)->sd_lock);
1241 		return (error);
1242 	}
1243 	leavesq(qp->q_syncq, SQ_OPENCLOSE);
1244 
1245 	/*
1246 	 * successful open should have done qprocson()
1247 	 */
1248 	ASSERT(qprocsareon(_RD(qp)));
1249 	return (0);
1250 }
1251 
1252 /*
1253  * Detach a stream module or device.
1254  * If clmode == 1 then the module or driver was opened and its
1255  * close routine must be called. If clmode == 0, the module
1256  * or driver was never opened or the open failed, and so its close
1257  * should not be called.
1258  */
1259 void
qdetach(queue_t * qp,int clmode,int flag,cred_t * crp,boolean_t is_remove)1260 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1261 {
1262 	queue_t *wqp = _WR(qp);
1263 	ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1264 
1265 	if (STREAM_NEEDSERVICE(STREAM(qp)))
1266 		stream_runservice(STREAM(qp));
1267 
1268 	if (clmode) {
1269 		/*
1270 		 * Make sure that all the messages on the write side syncq are
1271 		 * processed and nothing is left. Since we are closing, no new
1272 		 * messages may appear there.
1273 		 */
1274 		wait_q_syncq(wqp);
1275 
1276 		entersq(qp->q_syncq, SQ_OPENCLOSE);
1277 		if (is_remove) {
1278 			mutex_enter(QLOCK(qp));
1279 			qp->q_flag |= _QREMOVING;
1280 			mutex_exit(QLOCK(qp));
1281 		}
1282 		(*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1283 		/*
1284 		 * Check that qprocsoff() was actually called.
1285 		 */
1286 		ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1287 
1288 		leavesq(qp->q_syncq, SQ_OPENCLOSE);
1289 	} else {
1290 		disable_svc(qp);
1291 	}
1292 
1293 	/*
1294 	 * Allow any threads blocked in entersq to proceed and discover
1295 	 * the QWCLOSE is set.
1296 	 * Note: This assumes that all users of entersq check QWCLOSE.
1297 	 * Currently runservice is the only entersq that can happen
1298 	 * after removeq has finished.
1299 	 * Removeq will have discarded all messages destined to the closing
1300 	 * pair of queues from the syncq.
1301 	 * NOTE: Calling a function inside an assert is unconventional.
1302 	 * However, it does not cause any problem since flush_syncq() does
1303 	 * not change any state except when it returns non-zero i.e.
1304 	 * when the assert will trigger.
1305 	 */
1306 	ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1307 	ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1308 	ASSERT((qp->q_flag & QPERMOD) ||
1309 	    ((qp->q_syncq->sq_head == NULL) &&
1310 	    (wqp->q_syncq->sq_head == NULL)));
1311 
1312 	/* release any fmodsw_impl_t structure held on behalf of the queue */
1313 	ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1314 	if (qp->q_fp != NULL)
1315 		fmodsw_rele(qp->q_fp);
1316 
1317 	/* freeq removes us from the outer perimeter if any */
1318 	freeq(qp);
1319 }
1320 
1321 /* Prevent service procedures from being called */
1322 void
disable_svc(queue_t * qp)1323 disable_svc(queue_t *qp)
1324 {
1325 	queue_t *wqp = _WR(qp);
1326 
1327 	ASSERT(qp->q_flag & QREADR);
1328 	mutex_enter(QLOCK(qp));
1329 	qp->q_flag |= QWCLOSE;
1330 	mutex_exit(QLOCK(qp));
1331 	mutex_enter(QLOCK(wqp));
1332 	wqp->q_flag |= QWCLOSE;
1333 	mutex_exit(QLOCK(wqp));
1334 }
1335 
1336 /* Allow service procedures to be called again */
1337 void
enable_svc(queue_t * qp)1338 enable_svc(queue_t *qp)
1339 {
1340 	queue_t *wqp = _WR(qp);
1341 
1342 	ASSERT(qp->q_flag & QREADR);
1343 	mutex_enter(QLOCK(qp));
1344 	qp->q_flag &= ~QWCLOSE;
1345 	mutex_exit(QLOCK(qp));
1346 	mutex_enter(QLOCK(wqp));
1347 	wqp->q_flag &= ~QWCLOSE;
1348 	mutex_exit(QLOCK(wqp));
1349 }
1350 
1351 /*
1352  * Remove queue from qhead/qtail if it is enabled.
1353  * Only reset QENAB if the queue was removed from the runlist.
1354  * A queue goes through 3 stages:
1355  *	It is on the service list and QENAB is set.
1356  *	It is removed from the service list but QENAB is still set.
1357  *	QENAB gets changed to QINSERVICE.
1358  *	QINSERVICE is reset (when the service procedure is done)
1359  * Thus we can not reset QENAB unless we actually removed it from the service
1360  * queue.
1361  */
1362 void
remove_runlist(queue_t * qp)1363 remove_runlist(queue_t *qp)
1364 {
1365 	if (qp->q_flag & QENAB && qhead != NULL) {
1366 		queue_t *q_chase;
1367 		queue_t *q_curr;
1368 		int removed;
1369 
1370 		mutex_enter(&service_queue);
1371 		RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1372 		mutex_exit(&service_queue);
1373 		if (removed) {
1374 			STRSTAT(qremoved);
1375 			qp->q_flag &= ~QENAB;
1376 		}
1377 	}
1378 }
1379 
1380 
1381 /*
1382  * Wait for any pending service processing to complete.
1383  * The removal of queues from the runlist is not atomic with the
1384  * clearing of the QENABLED flag and setting the INSERVICE flag.
1385  * consequently it is possible for remove_runlist in strclose
1386  * to not find the queue on the runlist but for it to be QENABLED
1387  * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1388  * as well as INSERVICE.
1389  */
1390 void
wait_svc(queue_t * qp)1391 wait_svc(queue_t *qp)
1392 {
1393 	queue_t *wqp = _WR(qp);
1394 
1395 	ASSERT(qp->q_flag & QREADR);
1396 
1397 	/*
1398 	 * Try to remove queues from qhead/qtail list.
1399 	 */
1400 	if (qhead != NULL) {
1401 		remove_runlist(qp);
1402 		remove_runlist(wqp);
1403 	}
1404 	/*
1405 	 * Wait till the syncqs associated with the queue disappear from the
1406 	 * background processing list.
1407 	 * This only needs to be done for non-PERMOD perimeters since
1408 	 * for PERMOD perimeters the syncq may be shared and will only be freed
1409 	 * when the last module/driver is unloaded.
1410 	 * If for PERMOD perimeters queue was on the syncq list, removeq()
1411 	 * should call propagate_syncq() or drain_syncq() for it. Both of these
1412 	 * functions remove the queue from its syncq list, so sqthread will not
1413 	 * try to access the queue.
1414 	 */
1415 	if (!(qp->q_flag & QPERMOD)) {
1416 		syncq_t *rsq = qp->q_syncq;
1417 		syncq_t *wsq = wqp->q_syncq;
1418 
1419 		/*
1420 		 * Disable rsq and wsq and wait for any background processing of
1421 		 * syncq to complete.
1422 		 */
1423 		wait_sq_svc(rsq);
1424 		if (wsq != rsq)
1425 			wait_sq_svc(wsq);
1426 	}
1427 
1428 	mutex_enter(QLOCK(qp));
1429 	while (qp->q_flag & (QINSERVICE|QENAB))
1430 		cv_wait(&qp->q_wait, QLOCK(qp));
1431 	mutex_exit(QLOCK(qp));
1432 	mutex_enter(QLOCK(wqp));
1433 	while (wqp->q_flag & (QINSERVICE|QENAB))
1434 		cv_wait(&wqp->q_wait, QLOCK(wqp));
1435 	mutex_exit(QLOCK(wqp));
1436 }
1437 
1438 /*
1439  * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1440  * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1441  * also be set, and is passed through to allocb_cred_wait().
1442  *
1443  * Returns errno on failure, zero on success.
1444  */
1445 int
putiocd(mblk_t * bp,char * arg,int flag,cred_t * cr)1446 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1447 {
1448 	mblk_t *tmp;
1449 	ssize_t  count;
1450 	int error = 0;
1451 
1452 	ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1453 	    (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1454 
1455 	if (bp->b_datap->db_type == M_IOCTL) {
1456 		count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1457 	} else {
1458 		ASSERT(bp->b_datap->db_type == M_COPYIN);
1459 		count = ((struct copyreq *)bp->b_rptr)->cq_size;
1460 	}
1461 	/*
1462 	 * strdoioctl validates ioc_count, so if this assert fails it
1463 	 * cannot be due to user error.
1464 	 */
1465 	ASSERT(count >= 0);
1466 
1467 	if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1468 	    curproc->p_pid)) == NULL) {
1469 		return (error);
1470 	}
1471 	error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1472 	if (error != 0) {
1473 		freeb(tmp);
1474 		return (error);
1475 	}
1476 	DB_CPID(tmp) = curproc->p_pid;
1477 	tmp->b_wptr += count;
1478 	bp->b_cont = tmp;
1479 
1480 	return (0);
1481 }
1482 
1483 /*
1484  * Copy ioctl data to user-land. Return non-zero errno on failure,
1485  * 0 for success.
1486  */
1487 int
getiocd(mblk_t * bp,char * arg,int copymode)1488 getiocd(mblk_t *bp, char *arg, int copymode)
1489 {
1490 	ssize_t count;
1491 	size_t  n;
1492 	int	error;
1493 
1494 	if (bp->b_datap->db_type == M_IOCACK)
1495 		count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1496 	else {
1497 		ASSERT(bp->b_datap->db_type == M_COPYOUT);
1498 		count = ((struct copyreq *)bp->b_rptr)->cq_size;
1499 	}
1500 	ASSERT(count >= 0);
1501 
1502 	for (bp = bp->b_cont; bp && count;
1503 	    count -= n, bp = bp->b_cont, arg += n) {
1504 		n = MIN(count, bp->b_wptr - bp->b_rptr);
1505 		error = strcopyout(bp->b_rptr, arg, n, copymode);
1506 		if (error)
1507 			return (error);
1508 	}
1509 	ASSERT(count == 0);
1510 	return (0);
1511 }
1512 
1513 /*
1514  * Allocate a linkinfo entry given the write queue of the
1515  * bottom module of the top stream and the write queue of the
1516  * stream head of the bottom stream.
1517  */
1518 linkinfo_t *
alloclink(queue_t * qup,queue_t * qdown,file_t * fpdown)1519 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1520 {
1521 	linkinfo_t *linkp;
1522 
1523 	linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1524 
1525 	linkp->li_lblk.l_qtop = qup;
1526 	linkp->li_lblk.l_qbot = qdown;
1527 	linkp->li_fpdown = fpdown;
1528 
1529 	mutex_enter(&strresources);
1530 	linkp->li_next = linkinfo_list;
1531 	linkp->li_prev = NULL;
1532 	if (linkp->li_next)
1533 		linkp->li_next->li_prev = linkp;
1534 	linkinfo_list = linkp;
1535 	linkp->li_lblk.l_index = ++lnk_id;
1536 	ASSERT(lnk_id != 0);	/* this should never wrap in practice */
1537 	mutex_exit(&strresources);
1538 
1539 	return (linkp);
1540 }
1541 
1542 /*
1543  * Free a linkinfo entry.
1544  */
1545 void
lbfree(linkinfo_t * linkp)1546 lbfree(linkinfo_t *linkp)
1547 {
1548 	mutex_enter(&strresources);
1549 	if (linkp->li_next)
1550 		linkp->li_next->li_prev = linkp->li_prev;
1551 	if (linkp->li_prev)
1552 		linkp->li_prev->li_next = linkp->li_next;
1553 	else
1554 		linkinfo_list = linkp->li_next;
1555 	mutex_exit(&strresources);
1556 
1557 	kmem_cache_free(linkinfo_cache, linkp);
1558 }
1559 
1560 /*
1561  * Check for a potential linking cycle.
1562  * Return 1 if a link will result in a cycle,
1563  * and 0 otherwise.
1564  */
1565 int
linkcycle(stdata_t * upstp,stdata_t * lostp,str_stack_t * ss)1566 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1567 {
1568 	struct mux_node *np;
1569 	struct mux_edge *ep;
1570 	int i;
1571 	major_t lomaj;
1572 	major_t upmaj;
1573 	/*
1574 	 * if the lower stream is a pipe/FIFO, return, since link
1575 	 * cycles can not happen on pipes/FIFOs
1576 	 */
1577 	if (lostp->sd_vnode->v_type == VFIFO)
1578 		return (0);
1579 
1580 	for (i = 0; i < ss->ss_devcnt; i++) {
1581 		np = &ss->ss_mux_nodes[i];
1582 		MUX_CLEAR(np);
1583 	}
1584 	lomaj = getmajor(lostp->sd_vnode->v_rdev);
1585 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
1586 	np = &ss->ss_mux_nodes[lomaj];
1587 	for (;;) {
1588 		if (!MUX_DIDVISIT(np)) {
1589 			if (np->mn_imaj == upmaj)
1590 				return (1);
1591 			if (np->mn_outp == NULL) {
1592 				MUX_VISIT(np);
1593 				if (np->mn_originp == NULL)
1594 					return (0);
1595 				np = np->mn_originp;
1596 				continue;
1597 			}
1598 			MUX_VISIT(np);
1599 			np->mn_startp = np->mn_outp;
1600 		} else {
1601 			if (np->mn_startp == NULL) {
1602 				if (np->mn_originp == NULL)
1603 					return (0);
1604 				else {
1605 					np = np->mn_originp;
1606 					continue;
1607 				}
1608 			}
1609 			/*
1610 			 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1611 			 * ignore the edge and move on. ep->me_nodep gets
1612 			 * set to NULL in mux_addedge() if it is a FIFO.
1613 			 *
1614 			 */
1615 			ep = np->mn_startp;
1616 			np->mn_startp = ep->me_nextp;
1617 			if (ep->me_nodep == NULL)
1618 				continue;
1619 			ep->me_nodep->mn_originp = np;
1620 			np = ep->me_nodep;
1621 		}
1622 	}
1623 }
1624 
1625 /*
1626  * Find linkinfo entry corresponding to the parameters.
1627  */
1628 linkinfo_t *
findlinks(stdata_t * stp,int index,int type,str_stack_t * ss)1629 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1630 {
1631 	linkinfo_t *linkp;
1632 	struct mux_edge *mep;
1633 	struct mux_node *mnp;
1634 	queue_t *qup;
1635 
1636 	mutex_enter(&strresources);
1637 	if ((type & LINKTYPEMASK) == LINKNORMAL) {
1638 		qup = getendq(stp->sd_wrq);
1639 		for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1640 			if ((qup == linkp->li_lblk.l_qtop) &&
1641 			    (!index || (index == linkp->li_lblk.l_index))) {
1642 				mutex_exit(&strresources);
1643 				return (linkp);
1644 			}
1645 		}
1646 	} else {
1647 		ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1648 		mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1649 		mep = mnp->mn_outp;
1650 		while (mep) {
1651 			if ((index == 0) || (index == mep->me_muxid))
1652 				break;
1653 			mep = mep->me_nextp;
1654 		}
1655 		if (!mep) {
1656 			mutex_exit(&strresources);
1657 			return (NULL);
1658 		}
1659 		for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1660 			if ((!linkp->li_lblk.l_qtop) &&
1661 			    (mep->me_muxid == linkp->li_lblk.l_index)) {
1662 				mutex_exit(&strresources);
1663 				return (linkp);
1664 			}
1665 		}
1666 	}
1667 	mutex_exit(&strresources);
1668 	return (NULL);
1669 }
1670 
1671 /*
1672  * Given a queue ptr, follow the chain of q_next pointers until you reach the
1673  * last queue on the chain and return it.
1674  */
1675 queue_t *
getendq(queue_t * q)1676 getendq(queue_t *q)
1677 {
1678 	ASSERT(q != NULL);
1679 	while (_SAMESTR(q))
1680 		q = q->q_next;
1681 	return (q);
1682 }
1683 
1684 /*
1685  * Wait for the syncq count to drop to zero.
1686  * sq could be either outer or inner.
1687  */
1688 
1689 static void
wait_syncq(syncq_t * sq)1690 wait_syncq(syncq_t *sq)
1691 {
1692 	uint16_t count;
1693 
1694 	mutex_enter(SQLOCK(sq));
1695 	count = sq->sq_count;
1696 	SQ_PUTLOCKS_ENTER(sq);
1697 	SUM_SQ_PUTCOUNTS(sq, count);
1698 	while (count != 0) {
1699 		sq->sq_flags |= SQ_WANTWAKEUP;
1700 		SQ_PUTLOCKS_EXIT(sq);
1701 		cv_wait(&sq->sq_wait, SQLOCK(sq));
1702 		count = sq->sq_count;
1703 		SQ_PUTLOCKS_ENTER(sq);
1704 		SUM_SQ_PUTCOUNTS(sq, count);
1705 	}
1706 	SQ_PUTLOCKS_EXIT(sq);
1707 	mutex_exit(SQLOCK(sq));
1708 }
1709 
1710 /*
1711  * Wait while there are any messages for the queue in its syncq.
1712  */
1713 static void
wait_q_syncq(queue_t * q)1714 wait_q_syncq(queue_t *q)
1715 {
1716 	if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1717 		syncq_t *sq = q->q_syncq;
1718 
1719 		mutex_enter(SQLOCK(sq));
1720 		while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1721 			sq->sq_flags |= SQ_WANTWAKEUP;
1722 			cv_wait(&sq->sq_wait, SQLOCK(sq));
1723 		}
1724 		mutex_exit(SQLOCK(sq));
1725 	}
1726 }
1727 
1728 
1729 int
mlink_file(vnode_t * vp,int cmd,struct file * fpdown,cred_t * crp,int * rvalp,int lhlink)1730 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1731     int lhlink)
1732 {
1733 	struct stdata *stp;
1734 	struct strioctl strioc;
1735 	struct linkinfo *linkp;
1736 	struct stdata *stpdown;
1737 	struct streamtab *str;
1738 	queue_t *passq;
1739 	syncq_t *passyncq;
1740 	queue_t *rq;
1741 	cdevsw_impl_t *dp;
1742 	uint32_t qflag;
1743 	uint32_t sqtype;
1744 	perdm_t *dmp;
1745 	int error = 0;
1746 	netstack_t *ns;
1747 	str_stack_t *ss;
1748 
1749 	stp = vp->v_stream;
1750 	TRACE_1(TR_FAC_STREAMS_FR,
1751 	    TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1752 	/*
1753 	 * Test for invalid upper stream
1754 	 */
1755 	if (stp->sd_flag & STRHUP) {
1756 		return (ENXIO);
1757 	}
1758 	if (vp->v_type == VFIFO) {
1759 		return (EINVAL);
1760 	}
1761 	if (stp->sd_strtab == NULL) {
1762 		return (EINVAL);
1763 	}
1764 	if (!stp->sd_strtab->st_muxwinit) {
1765 		return (EINVAL);
1766 	}
1767 	if (fpdown == NULL) {
1768 		return (EBADF);
1769 	}
1770 	ns = netstack_find_by_cred(crp);
1771 	ASSERT(ns != NULL);
1772 	ss = ns->netstack_str;
1773 	ASSERT(ss != NULL);
1774 
1775 	if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1776 		netstack_rele(ss->ss_netstack);
1777 		return (EINVAL);
1778 	}
1779 	mutex_enter(&muxifier);
1780 	if (stp->sd_flag & STPLEX) {
1781 		mutex_exit(&muxifier);
1782 		netstack_rele(ss->ss_netstack);
1783 		return (ENXIO);
1784 	}
1785 
1786 	/*
1787 	 * Test for invalid lower stream.
1788 	 * The check for the v_type != VFIFO and having a major
1789 	 * number not >= devcnt is done to avoid problems with
1790 	 * adding mux_node entry past the end of mux_nodes[].
1791 	 * For FIFO's we don't add an entry so this isn't a
1792 	 * problem.
1793 	 */
1794 	if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1795 	    (stpdown == stp) || (stpdown->sd_flag &
1796 	    (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1797 	    ((stpdown->sd_vnode->v_type != VFIFO) &&
1798 	    (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1799 	    linkcycle(stp, stpdown, ss)) {
1800 		mutex_exit(&muxifier);
1801 		netstack_rele(ss->ss_netstack);
1802 		return (EINVAL);
1803 	}
1804 	TRACE_1(TR_FAC_STREAMS_FR,
1805 	    TR_STPDOWN, "stpdown:%p", stpdown);
1806 	rq = getendq(stp->sd_wrq);
1807 	if (cmd == I_PLINK)
1808 		rq = NULL;
1809 
1810 	linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1811 
1812 	strioc.ic_cmd = cmd;
1813 	strioc.ic_timout = INFTIM;
1814 	strioc.ic_len = sizeof (struct linkblk);
1815 	strioc.ic_dp = (char *)&linkp->li_lblk;
1816 
1817 	/*
1818 	 * STRPLUMB protects plumbing changes and should be set before
1819 	 * link_addpassthru()/link_rempassthru() are called, so it is set here
1820 	 * and cleared in the end of mlink when passthru queue is removed.
1821 	 * Setting of STRPLUMB prevents reopens of the stream while passthru
1822 	 * queue is in-place (it is not a proper module and doesn't have open
1823 	 * entry point).
1824 	 *
1825 	 * STPLEX prevents any threads from entering the stream from above. It
1826 	 * can't be set before the call to link_addpassthru() because putnext
1827 	 * from below may cause stream head I/O routines to be called and these
1828 	 * routines assert that STPLEX is not set. After link_addpassthru()
1829 	 * nothing may come from below since the pass queue syncq is blocked.
1830 	 * Note also that STPLEX should be cleared before the call to
1831 	 * link_rempassthru() since when messages start flowing to the stream
1832 	 * head (e.g. because of message propagation from the pass queue) stream
1833 	 * head I/O routines may be called with STPLEX flag set.
1834 	 *
1835 	 * When STPLEX is set, nothing may come into the stream from above and
1836 	 * it is safe to do a setq which will change stream head. So, the
1837 	 * correct sequence of actions is:
1838 	 *
1839 	 * 1) Set STRPLUMB
1840 	 * 2) Call link_addpassthru()
1841 	 * 3) Set STPLEX
1842 	 * 4) Call setq and update the stream state
1843 	 * 5) Clear STPLEX
1844 	 * 6) Call link_rempassthru()
1845 	 * 7) Clear STRPLUMB
1846 	 *
1847 	 * The same sequence applies to munlink() code.
1848 	 */
1849 	mutex_enter(&stpdown->sd_lock);
1850 	stpdown->sd_flag |= STRPLUMB;
1851 	mutex_exit(&stpdown->sd_lock);
1852 	/*
1853 	 * Add passthru queue below lower mux. This will block
1854 	 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1855 	 */
1856 	passq = link_addpassthru(stpdown);
1857 
1858 	mutex_enter(&stpdown->sd_lock);
1859 	stpdown->sd_flag |= STPLEX;
1860 	mutex_exit(&stpdown->sd_lock);
1861 
1862 	rq = _RD(stpdown->sd_wrq);
1863 	/*
1864 	 * There may be messages in the streamhead's syncq due to messages
1865 	 * that arrived before link_addpassthru() was done. To avoid
1866 	 * background processing of the syncq happening simultaneous with
1867 	 * setq processing, we disable the streamhead syncq and wait until
1868 	 * existing background thread finishes working on it.
1869 	 */
1870 	wait_sq_svc(rq->q_syncq);
1871 	passyncq = passq->q_syncq;
1872 	if (!(passyncq->sq_flags & SQ_BLOCKED))
1873 		blocksq(passyncq, SQ_BLOCKED, 0);
1874 
1875 	ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1876 	ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1877 	rq->q_ptr = _WR(rq)->q_ptr = NULL;
1878 
1879 	/* setq might sleep in allocator - avoid holding locks. */
1880 	/* Note: we are holding muxifier here. */
1881 
1882 	str = stp->sd_strtab;
1883 	dp = &devimpl[getmajor(vp->v_rdev)];
1884 	ASSERT(dp->d_str == str);
1885 
1886 	qflag = dp->d_qflag;
1887 	sqtype = dp->d_sqtype;
1888 
1889 	/* create perdm_t if needed */
1890 	if (NEED_DM(dp->d_dmp, qflag))
1891 		dp->d_dmp = hold_dm(str, qflag, sqtype);
1892 
1893 	dmp = dp->d_dmp;
1894 
1895 	setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1896 	    B_TRUE);
1897 
1898 	/*
1899 	 * XXX Remove any "odd" messages from the queue.
1900 	 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1901 	 */
1902 	error = strdoioctl(stp, &strioc, FNATIVE,
1903 	    K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1904 	if (error != 0)
1905 		goto cleanup;
1906 
1907 	mutex_enter(&fpdown->f_tlock);
1908 	fpdown->f_count++;
1909 	mutex_exit(&fpdown->f_tlock);
1910 
1911 	/*
1912 	 * if we've made it here the linkage is all set up so we should also
1913 	 * set up the layered driver linkages
1914 	 */
1915 
1916 	ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1917 	if (cmd == I_LINK) {
1918 		error = ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1919 	} else {
1920 		error = ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1921 	}
1922 
1923 	if (error != 0) {
1924 		mutex_enter(&fpdown->f_tlock);
1925 		fpdown->f_count--;
1926 		mutex_exit(&fpdown->f_tlock);
1927 		goto cleanup;
1928 	}
1929 
1930 	link_rempassthru(passq);
1931 
1932 	mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1933 
1934 	/*
1935 	 * Mark the upper stream as having dependent links
1936 	 * so that strclose can clean it up.
1937 	 */
1938 	if (cmd == I_LINK) {
1939 		mutex_enter(&stp->sd_lock);
1940 		stp->sd_flag |= STRHASLINKS;
1941 		mutex_exit(&stp->sd_lock);
1942 	}
1943 	/*
1944 	 * Wake up any other processes that may have been
1945 	 * waiting on the lower stream. These will all
1946 	 * error out.
1947 	 */
1948 	mutex_enter(&stpdown->sd_lock);
1949 	/* The passthru module is removed so we may release STRPLUMB */
1950 	stpdown->sd_flag &= ~STRPLUMB;
1951 	cv_broadcast(&rq->q_wait);
1952 	cv_broadcast(&_WR(rq)->q_wait);
1953 	cv_broadcast(&stpdown->sd_monitor);
1954 	mutex_exit(&stpdown->sd_lock);
1955 	mutex_exit(&muxifier);
1956 	*rvalp = linkp->li_lblk.l_index;
1957 	netstack_rele(ss->ss_netstack);
1958 	return (0);
1959 
1960 cleanup:
1961 	lbfree(linkp);
1962 
1963 	if (!(passyncq->sq_flags & SQ_BLOCKED))
1964 		blocksq(passyncq, SQ_BLOCKED, 0);
1965 	/*
1966 	 * Restore the stream head queue and then remove
1967 	 * the passq. Turn off STPLEX before we turn on
1968 	 * the stream by removing the passq.
1969 	 */
1970 	rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1971 	setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1972 	    B_TRUE);
1973 
1974 	mutex_enter(&stpdown->sd_lock);
1975 	stpdown->sd_flag &= ~STPLEX;
1976 	mutex_exit(&stpdown->sd_lock);
1977 
1978 	link_rempassthru(passq);
1979 
1980 	mutex_enter(&stpdown->sd_lock);
1981 	stpdown->sd_flag &= ~STRPLUMB;
1982 	/* Wakeup anyone waiting for STRPLUMB to clear. */
1983 	cv_broadcast(&stpdown->sd_monitor);
1984 	mutex_exit(&stpdown->sd_lock);
1985 
1986 	mutex_exit(&muxifier);
1987 	netstack_rele(ss->ss_netstack);
1988 	return (error);
1989 }
1990 
1991 int
mlink(vnode_t * vp,int cmd,int arg,cred_t * crp,int * rvalp,int lhlink)1992 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1993 {
1994 	int		ret;
1995 	struct file	*fpdown;
1996 
1997 	fpdown = getf(arg);
1998 	ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1999 	if (fpdown != NULL)
2000 		releasef(arg);
2001 	return (ret);
2002 }
2003 
2004 /*
2005  * Unlink a multiplexor link. Stp is the controlling stream for the
2006  * link, and linkp points to the link's entry in the linkinfo list.
2007  * The muxifier lock must be held on entry and is dropped on exit.
2008  *
2009  * NOTE : Currently it is assumed that mux would process all the messages
2010  * sitting on it's queue before ACKing the UNLINK. It is the responsibility
2011  * of the mux to handle all the messages that arrive before UNLINK.
2012  * If the mux has to send down messages on its lower stream before
2013  * ACKing I_UNLINK, then it *should* know to handle messages even
2014  * after the UNLINK is acked (actually it should be able to handle till we
2015  * re-block the read side of the pass queue here). If the mux does not
2016  * open up the lower stream, any messages that arrive during UNLINK
2017  * will be put in the stream head. In the case of lower stream opening
2018  * up, some messages might land in the stream head depending on when
2019  * the message arrived and when the read side of the pass queue was
2020  * re-blocked.
2021  */
2022 int
munlink(stdata_t * stp,linkinfo_t * linkp,int flag,cred_t * crp,int * rvalp,str_stack_t * ss)2023 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2024     str_stack_t *ss)
2025 {
2026 	struct strioctl strioc;
2027 	struct stdata *stpdown;
2028 	queue_t *rq, *wrq;
2029 	queue_t	*passq;
2030 	syncq_t *passyncq;
2031 	int error = 0;
2032 	file_t *fpdown;
2033 
2034 	ASSERT(MUTEX_HELD(&muxifier));
2035 
2036 	stpdown = linkp->li_fpdown->f_vnode->v_stream;
2037 
2038 	/*
2039 	 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2040 	 */
2041 	mutex_enter(&stpdown->sd_lock);
2042 	stpdown->sd_flag |= STRPLUMB;
2043 	mutex_exit(&stpdown->sd_lock);
2044 
2045 	/*
2046 	 * Add passthru queue below lower mux. This will block
2047 	 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2048 	 */
2049 	passq = link_addpassthru(stpdown);
2050 
2051 	if ((flag & LINKTYPEMASK) == LINKNORMAL)
2052 		strioc.ic_cmd = I_UNLINK;
2053 	else
2054 		strioc.ic_cmd = I_PUNLINK;
2055 	strioc.ic_timout = INFTIM;
2056 	strioc.ic_len = sizeof (struct linkblk);
2057 	strioc.ic_dp = (char *)&linkp->li_lblk;
2058 
2059 	error = strdoioctl(stp, &strioc, FNATIVE,
2060 	    K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2061 
2062 	/*
2063 	 * If there was an error and this is not called via strclose,
2064 	 * return to the user. Otherwise, pretend there was no error
2065 	 * and close the link.
2066 	 */
2067 	if (error) {
2068 		if (flag & LINKCLOSE) {
2069 			cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2070 			    "unlink ioctl, closing anyway (%d)\n", error);
2071 		} else {
2072 			link_rempassthru(passq);
2073 			mutex_enter(&stpdown->sd_lock);
2074 			stpdown->sd_flag &= ~STRPLUMB;
2075 			cv_broadcast(&stpdown->sd_monitor);
2076 			mutex_exit(&stpdown->sd_lock);
2077 			mutex_exit(&muxifier);
2078 			return (error);
2079 		}
2080 	}
2081 
2082 	mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2083 	fpdown = linkp->li_fpdown;
2084 	lbfree(linkp);
2085 
2086 	/*
2087 	 * We go ahead and drop muxifier here--it's a nasty global lock that
2088 	 * can slow others down. It's okay to since attempts to mlink() this
2089 	 * stream will be stopped because STPLEX is still set in the stdata
2090 	 * structure, and munlink() is stopped because mux_rmvedge() and
2091 	 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2092 	 * respectively.  Note that we defer the closef() of fpdown until
2093 	 * after we drop muxifier since strclose() can call munlinkall().
2094 	 */
2095 	mutex_exit(&muxifier);
2096 
2097 	wrq = stpdown->sd_wrq;
2098 	rq = _RD(wrq);
2099 
2100 	/*
2101 	 * Get rid of outstanding service procedure runs, before we make
2102 	 * it a stream head, since a stream head doesn't have any service
2103 	 * procedure.
2104 	 */
2105 	disable_svc(rq);
2106 	wait_svc(rq);
2107 
2108 	/*
2109 	 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2110 	 * is queued up to be finished. mux should take care that nothing is
2111 	 * send down to this queue. We should do it now as we're going to block
2112 	 * passyncq if it was unblocked.
2113 	 */
2114 	if (wrq->q_flag & QPERMOD) {
2115 		syncq_t	*sq = wrq->q_syncq;
2116 
2117 		mutex_enter(SQLOCK(sq));
2118 		while (wrq->q_sqflags & Q_SQQUEUED) {
2119 			sq->sq_flags |= SQ_WANTWAKEUP;
2120 			cv_wait(&sq->sq_wait, SQLOCK(sq));
2121 		}
2122 		mutex_exit(SQLOCK(sq));
2123 	}
2124 	passyncq = passq->q_syncq;
2125 	if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2126 
2127 		syncq_t *sq, *outer;
2128 
2129 		/*
2130 		 * Messages could be flowing from underneath. We will
2131 		 * block the read side of the passq. This would be
2132 		 * sufficient for QPAIR and QPERQ muxes to ensure
2133 		 * that no data is flowing up into this queue
2134 		 * and hence no thread active in this instance of
2135 		 * lower mux. But for QPERMOD and QMTOUTPERIM there
2136 		 * could be messages on the inner and outer/inner
2137 		 * syncqs respectively. We will wait for them to drain.
2138 		 * Because passq is blocked messages end up in the syncq
2139 		 * And qfill_syncq could possibly end up setting QFULL
2140 		 * which will access the rq->q_flag. Hence, we have to
2141 		 * acquire the QLOCK in setq.
2142 		 *
2143 		 * XXX Messages can also flow from top into this
2144 		 * queue though the unlink is over (Ex. some instance
2145 		 * in putnext() called from top that has still not
2146 		 * accessed this queue. And also putq(lowerq) ?).
2147 		 * Solution : How about blocking the l_qtop queue ?
2148 		 * Do we really care about such pure D_MP muxes ?
2149 		 */
2150 
2151 		blocksq(passyncq, SQ_BLOCKED, 0);
2152 
2153 		sq = rq->q_syncq;
2154 		if ((outer = sq->sq_outer) != NULL) {
2155 
2156 			/*
2157 			 * We have to just wait for the outer sq_count
2158 			 * drop to zero. As this does not prevent new
2159 			 * messages to enter the outer perimeter, this
2160 			 * is subject to starvation.
2161 			 *
2162 			 * NOTE :Because of blocksq above, messages could
2163 			 * be in the inner syncq only because of some
2164 			 * thread holding the outer perimeter exclusively.
2165 			 * Hence it would be sufficient to wait for the
2166 			 * exclusive holder of the outer perimeter to drain
2167 			 * the inner and outer syncqs. But we will not depend
2168 			 * on this feature and hence check the inner syncqs
2169 			 * separately.
2170 			 */
2171 			wait_syncq(outer);
2172 		}
2173 
2174 
2175 		/*
2176 		 * There could be messages destined for
2177 		 * this queue. Let the exclusive holder
2178 		 * drain it.
2179 		 */
2180 
2181 		wait_syncq(sq);
2182 		ASSERT((rq->q_flag & QPERMOD) ||
2183 		    ((rq->q_syncq->sq_head == NULL) &&
2184 		    (_WR(rq)->q_syncq->sq_head == NULL)));
2185 	}
2186 
2187 	/*
2188 	 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2189 	 * case as we don't disable its syncq or remove it off the syncq
2190 	 * service list.
2191 	 */
2192 	if (rq->q_flag & QPERMOD) {
2193 		syncq_t	*sq = rq->q_syncq;
2194 
2195 		mutex_enter(SQLOCK(sq));
2196 		while (rq->q_sqflags & Q_SQQUEUED) {
2197 			sq->sq_flags |= SQ_WANTWAKEUP;
2198 			cv_wait(&sq->sq_wait, SQLOCK(sq));
2199 		}
2200 		mutex_exit(SQLOCK(sq));
2201 	}
2202 
2203 	/*
2204 	 * flush_syncq changes states only when there are some messages to
2205 	 * free, i.e. when it returns non-zero value to return.
2206 	 */
2207 	ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2208 	ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2209 
2210 	/*
2211 	 * Nobody else should know about this queue now.
2212 	 * If the mux did not process the messages before
2213 	 * acking the I_UNLINK, free them now.
2214 	 */
2215 
2216 	flushq(rq, FLUSHALL);
2217 	flushq(_WR(rq), FLUSHALL);
2218 
2219 	/*
2220 	 * Convert the mux lower queue into a stream head queue.
2221 	 * Turn off STPLEX before we turn on the stream by removing the passq.
2222 	 */
2223 	rq->q_ptr = wrq->q_ptr = stpdown;
2224 	setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2225 
2226 	ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2227 	ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2228 
2229 	enable_svc(rq);
2230 
2231 	/*
2232 	 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2233 	 * needs to be set to prevent reopen() of the stream - such reopen may
2234 	 * try to call non-existent pass queue open routine and panic.
2235 	 */
2236 	mutex_enter(&stpdown->sd_lock);
2237 	stpdown->sd_flag &= ~STPLEX;
2238 	mutex_exit(&stpdown->sd_lock);
2239 
2240 	ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2241 	    ((flag & LINKTYPEMASK) == LINKPERSIST));
2242 
2243 	/* clean up the layered driver linkages */
2244 	if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2245 		VERIFY0(ldi_munlink_fp(stp, fpdown, LINKNORMAL));
2246 	} else {
2247 		VERIFY0(ldi_munlink_fp(stp, fpdown, LINKPERSIST));
2248 	}
2249 
2250 	link_rempassthru(passq);
2251 
2252 	/*
2253 	 * Now all plumbing changes are finished and STRPLUMB is no
2254 	 * longer needed.
2255 	 */
2256 	mutex_enter(&stpdown->sd_lock);
2257 	stpdown->sd_flag &= ~STRPLUMB;
2258 	cv_broadcast(&stpdown->sd_monitor);
2259 	mutex_exit(&stpdown->sd_lock);
2260 
2261 	(void) closef(fpdown);
2262 	return (0);
2263 }
2264 
2265 /*
2266  * Unlink all multiplexor links for which stp is the controlling stream.
2267  * Return 0, or a non-zero errno on failure.
2268  */
2269 int
munlinkall(stdata_t * stp,int flag,cred_t * crp,int * rvalp,str_stack_t * ss)2270 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2271 {
2272 	linkinfo_t *linkp;
2273 	int error = 0;
2274 
2275 	mutex_enter(&muxifier);
2276 	while (linkp = findlinks(stp, 0, flag, ss)) {
2277 		/*
2278 		 * munlink() releases the muxifier lock.
2279 		 */
2280 		if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2281 			return (error);
2282 		mutex_enter(&muxifier);
2283 	}
2284 	mutex_exit(&muxifier);
2285 	return (0);
2286 }
2287 
2288 /*
2289  * A multiplexor link has been made. Add an
2290  * edge to the directed graph.
2291  */
2292 void
mux_addedge(stdata_t * upstp,stdata_t * lostp,int muxid,str_stack_t * ss)2293 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2294 {
2295 	struct mux_node *np;
2296 	struct mux_edge *ep;
2297 	major_t upmaj;
2298 	major_t lomaj;
2299 
2300 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
2301 	lomaj = getmajor(lostp->sd_vnode->v_rdev);
2302 	np = &ss->ss_mux_nodes[upmaj];
2303 	if (np->mn_outp) {
2304 		ep = np->mn_outp;
2305 		while (ep->me_nextp)
2306 			ep = ep->me_nextp;
2307 		ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2308 		ep = ep->me_nextp;
2309 	} else {
2310 		np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2311 		ep = np->mn_outp;
2312 	}
2313 	ep->me_nextp = NULL;
2314 	ep->me_muxid = muxid;
2315 	/*
2316 	 * Save the dev_t for the purposes of str_stack_shutdown.
2317 	 * str_stack_shutdown assumes that the device allows reopen, since
2318 	 * this dev_t is the one after any cloning by xx_open().
2319 	 * Would prefer finding the dev_t from before any cloning,
2320 	 * but specfs doesn't retain that.
2321 	 */
2322 	ep->me_dev = upstp->sd_vnode->v_rdev;
2323 	if (lostp->sd_vnode->v_type == VFIFO)
2324 		ep->me_nodep = NULL;
2325 	else
2326 		ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2327 }
2328 
2329 /*
2330  * A multiplexor link has been removed. Remove the
2331  * edge in the directed graph.
2332  */
2333 void
mux_rmvedge(stdata_t * upstp,int muxid,str_stack_t * ss)2334 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2335 {
2336 	struct mux_node *np;
2337 	struct mux_edge *ep;
2338 	struct mux_edge *pep = NULL;
2339 	major_t upmaj;
2340 
2341 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
2342 	np = &ss->ss_mux_nodes[upmaj];
2343 	ASSERT(np->mn_outp != NULL);
2344 	ep = np->mn_outp;
2345 	while (ep) {
2346 		if (ep->me_muxid == muxid) {
2347 			if (pep)
2348 				pep->me_nextp = ep->me_nextp;
2349 			else
2350 				np->mn_outp = ep->me_nextp;
2351 			kmem_free(ep, sizeof (struct mux_edge));
2352 			return;
2353 		}
2354 		pep = ep;
2355 		ep = ep->me_nextp;
2356 	}
2357 	ASSERT(0);	/* should not reach here */
2358 }
2359 
2360 /*
2361  * Translate the device flags (from conf.h) to the corresponding
2362  * qflag and sq_flag (type) values.
2363  */
2364 int
devflg_to_qflag(struct streamtab * stp,uint32_t devflag,uint32_t * qflagp,uint32_t * sqtypep)2365 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2366     uint32_t *sqtypep)
2367 {
2368 	uint32_t qflag = 0;
2369 	uint32_t sqtype = 0;
2370 
2371 	if (devflag & _D_OLD)
2372 		goto bad;
2373 
2374 	/* Inner perimeter presence and scope */
2375 	switch (devflag & D_MTINNER_MASK) {
2376 	case D_MP:
2377 		qflag |= QMTSAFE;
2378 		sqtype |= SQ_CI;
2379 		break;
2380 	case D_MTPERQ|D_MP:
2381 		qflag |= QPERQ;
2382 		break;
2383 	case D_MTQPAIR|D_MP:
2384 		qflag |= QPAIR;
2385 		break;
2386 	case D_MTPERMOD|D_MP:
2387 		qflag |= QPERMOD;
2388 		break;
2389 	default:
2390 		goto bad;
2391 	}
2392 
2393 	/* Outer perimeter */
2394 	if (devflag & D_MTOUTPERIM) {
2395 		switch (devflag & D_MTINNER_MASK) {
2396 		case D_MP:
2397 		case D_MTPERQ|D_MP:
2398 		case D_MTQPAIR|D_MP:
2399 			break;
2400 		default:
2401 			goto bad;
2402 		}
2403 		qflag |= QMTOUTPERIM;
2404 	}
2405 
2406 	/* Inner perimeter modifiers */
2407 	if (devflag & D_MTINNER_MOD) {
2408 		switch (devflag & D_MTINNER_MASK) {
2409 		case D_MP:
2410 			goto bad;
2411 		default:
2412 			break;
2413 		}
2414 		if (devflag & D_MTPUTSHARED)
2415 			sqtype |= SQ_CIPUT;
2416 		if (devflag & _D_MTOCSHARED) {
2417 			/*
2418 			 * The code in putnext assumes that it has the
2419 			 * highest concurrency by not checking sq_count.
2420 			 * Thus _D_MTOCSHARED can only be supported when
2421 			 * D_MTPUTSHARED is set.
2422 			 */
2423 			if (!(devflag & D_MTPUTSHARED))
2424 				goto bad;
2425 			sqtype |= SQ_CIOC;
2426 		}
2427 		if (devflag & _D_MTCBSHARED) {
2428 			/*
2429 			 * The code in putnext assumes that it has the
2430 			 * highest concurrency by not checking sq_count.
2431 			 * Thus _D_MTCBSHARED can only be supported when
2432 			 * D_MTPUTSHARED is set.
2433 			 */
2434 			if (!(devflag & D_MTPUTSHARED))
2435 				goto bad;
2436 			sqtype |= SQ_CICB;
2437 		}
2438 		if (devflag & _D_MTSVCSHARED) {
2439 			/*
2440 			 * The code in putnext assumes that it has the
2441 			 * highest concurrency by not checking sq_count.
2442 			 * Thus _D_MTSVCSHARED can only be supported when
2443 			 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2444 			 * supported only for QPERMOD.
2445 			 */
2446 			if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2447 				goto bad;
2448 			sqtype |= SQ_CISVC;
2449 		}
2450 	}
2451 
2452 	/* Default outer perimeter concurrency */
2453 	sqtype |= SQ_CO;
2454 
2455 	/* Outer perimeter modifiers */
2456 	if (devflag & D_MTOCEXCL) {
2457 		if (!(devflag & D_MTOUTPERIM)) {
2458 			/* No outer perimeter */
2459 			goto bad;
2460 		}
2461 		sqtype &= ~SQ_COOC;
2462 	}
2463 
2464 	/* Synchronous Streams extended qinit structure */
2465 	if (devflag & D_SYNCSTR)
2466 		qflag |= QSYNCSTR;
2467 
2468 	/*
2469 	 * Private flag used by a transport module to indicate
2470 	 * to sockfs that it supports direct-access mode without
2471 	 * having to go through STREAMS.
2472 	 */
2473 	if (devflag & _D_DIRECT) {
2474 		/* Reject unless the module is fully-MT (no perimeter) */
2475 		if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2476 			goto bad;
2477 		qflag |= _QDIRECT;
2478 	}
2479 
2480 	/*
2481 	 * Private flag used to indicate that a streams module should only
2482 	 * be pushed once. The TTY streams modules have this flag since if
2483 	 * libc believes itself to be an xpg4 process then it will
2484 	 * automatically and unconditionally push them when a PTS device is
2485 	 * opened. If an application is not aware of this then without this
2486 	 * flag we would end up with duplicate modules.
2487 	 */
2488 	if (devflag & _D_SINGLE_INSTANCE)
2489 		qflag |= _QSINGLE_INSTANCE;
2490 
2491 	*qflagp = qflag;
2492 	*sqtypep = sqtype;
2493 	return (0);
2494 
2495 bad:
2496 	cmn_err(CE_WARN,
2497 	    "stropen: bad MT flags (0x%x) in driver '%s'",
2498 	    (int)(qflag & D_MTSAFETY_MASK),
2499 	    stp->st_rdinit->qi_minfo->mi_idname);
2500 
2501 	return (EINVAL);
2502 }
2503 
2504 /*
2505  * Set the interface values for a pair of queues (qinit structure,
2506  * packet sizes, water marks).
2507  * setq assumes that the caller does not have a claim (entersq or claimq)
2508  * on the queue.
2509  */
2510 void
setq(queue_t * rq,struct qinit * rinit,struct qinit * winit,perdm_t * dmp,uint32_t qflag,uint32_t sqtype,boolean_t lock_needed)2511 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2512     perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2513 {
2514 	queue_t *wq;
2515 	syncq_t	*sq, *outer;
2516 
2517 	ASSERT(rq->q_flag & QREADR);
2518 	ASSERT((qflag & QMT_TYPEMASK) != 0);
2519 	IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2520 
2521 	wq = _WR(rq);
2522 	rq->q_qinfo = rinit;
2523 	rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2524 	rq->q_lowat = rinit->qi_minfo->mi_lowat;
2525 	rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2526 	rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2527 	wq->q_qinfo = winit;
2528 	wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2529 	wq->q_lowat = winit->qi_minfo->mi_lowat;
2530 	wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2531 	wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2532 
2533 	/* Remove old syncqs */
2534 	sq = rq->q_syncq;
2535 	outer = sq->sq_outer;
2536 	if (outer != NULL) {
2537 		ASSERT(wq->q_syncq->sq_outer == outer);
2538 		outer_remove(outer, rq->q_syncq);
2539 		if (wq->q_syncq != rq->q_syncq)
2540 			outer_remove(outer, wq->q_syncq);
2541 	}
2542 	ASSERT(sq->sq_outer == NULL);
2543 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2544 
2545 	if (sq != SQ(rq)) {
2546 		if (!(rq->q_flag & QPERMOD))
2547 			free_syncq(sq);
2548 		if (wq->q_syncq == rq->q_syncq)
2549 			wq->q_syncq = NULL;
2550 		rq->q_syncq = NULL;
2551 	}
2552 	if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2553 	    wq->q_syncq != SQ(rq)) {
2554 		free_syncq(wq->q_syncq);
2555 		wq->q_syncq = NULL;
2556 	}
2557 	ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2558 	    rq->q_syncq->sq_tail == NULL));
2559 	ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2560 	    wq->q_syncq->sq_tail == NULL));
2561 
2562 	if (!(rq->q_flag & QPERMOD) &&
2563 	    rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2564 		ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2565 		SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2566 		    rq->q_syncq->sq_nciputctrl, 0);
2567 		ASSERT(ciputctrl_cache != NULL);
2568 		kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2569 		rq->q_syncq->sq_ciputctrl = NULL;
2570 		rq->q_syncq->sq_nciputctrl = 0;
2571 	}
2572 
2573 	if (!(wq->q_flag & QPERMOD) &&
2574 	    wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2575 		ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2576 		SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2577 		    wq->q_syncq->sq_nciputctrl, 0);
2578 		ASSERT(ciputctrl_cache != NULL);
2579 		kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2580 		wq->q_syncq->sq_ciputctrl = NULL;
2581 		wq->q_syncq->sq_nciputctrl = 0;
2582 	}
2583 
2584 	sq = SQ(rq);
2585 	ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2586 	ASSERT(sq->sq_outer == NULL);
2587 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2588 
2589 	/*
2590 	 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2591 	 * bits in sq_flag based on the sqtype.
2592 	 */
2593 	ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2594 
2595 	rq->q_syncq = wq->q_syncq = sq;
2596 	sq->sq_type = sqtype;
2597 	sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2598 
2599 	/*
2600 	 *  We are making sq_svcflags zero,
2601 	 *  resetting SQ_DISABLED in case it was set by
2602 	 *  wait_svc() in the munlink path.
2603 	 *
2604 	 */
2605 	ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2606 	sq->sq_svcflags = 0;
2607 
2608 	/*
2609 	 * We need to acquire the lock here for the mlink and munlink case,
2610 	 * where canputnext, backenable, etc can access the q_flag.
2611 	 */
2612 	if (lock_needed) {
2613 		mutex_enter(QLOCK(rq));
2614 		rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2615 		mutex_exit(QLOCK(rq));
2616 		mutex_enter(QLOCK(wq));
2617 		wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2618 		mutex_exit(QLOCK(wq));
2619 	} else {
2620 		rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2621 		wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2622 	}
2623 
2624 	if (qflag & QPERQ) {
2625 		/* Allocate a separate syncq for the write side */
2626 		sq = new_syncq();
2627 		sq->sq_type = rq->q_syncq->sq_type;
2628 		sq->sq_flags = rq->q_syncq->sq_flags;
2629 		ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2630 		    sq->sq_oprev == NULL);
2631 		wq->q_syncq = sq;
2632 	}
2633 	if (qflag & QPERMOD) {
2634 		sq = dmp->dm_sq;
2635 
2636 		/*
2637 		 * Assert that we do have an inner perimeter syncq and that it
2638 		 * does not have an outer perimeter associated with it.
2639 		 */
2640 		ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2641 		    sq->sq_oprev == NULL);
2642 		rq->q_syncq = wq->q_syncq = sq;
2643 	}
2644 	if (qflag & QMTOUTPERIM) {
2645 		outer = dmp->dm_sq;
2646 
2647 		ASSERT(outer->sq_outer == NULL);
2648 		outer_insert(outer, rq->q_syncq);
2649 		if (wq->q_syncq != rq->q_syncq)
2650 			outer_insert(outer, wq->q_syncq);
2651 	}
2652 	ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2653 	    (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2654 	ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2655 	    (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2656 	ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2657 
2658 	/*
2659 	 * Initialize struio() types.
2660 	 */
2661 	rq->q_struiot =
2662 	    (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2663 	wq->q_struiot =
2664 	    (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2665 }
2666 
2667 perdm_t *
hold_dm(struct streamtab * str,uint32_t qflag,uint32_t sqtype)2668 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2669 {
2670 	syncq_t	*sq;
2671 	perdm_t	**pp;
2672 	perdm_t	*p;
2673 	perdm_t	*dmp;
2674 
2675 	ASSERT(str != NULL);
2676 	ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2677 
2678 	rw_enter(&perdm_rwlock, RW_READER);
2679 	for (p = perdm_list; p != NULL; p = p->dm_next) {
2680 		if (p->dm_str == str) {	/* found one */
2681 			atomic_inc_32(&(p->dm_ref));
2682 			rw_exit(&perdm_rwlock);
2683 			return (p);
2684 		}
2685 	}
2686 	rw_exit(&perdm_rwlock);
2687 
2688 	sq = new_syncq();
2689 	if (qflag & QPERMOD) {
2690 		sq->sq_type = sqtype | SQ_PERMOD;
2691 		sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2692 	} else {
2693 		ASSERT(qflag & QMTOUTPERIM);
2694 		sq->sq_onext = sq->sq_oprev = sq;
2695 	}
2696 
2697 	dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2698 	dmp->dm_sq = sq;
2699 	dmp->dm_str = str;
2700 	dmp->dm_ref = 1;
2701 	dmp->dm_next = NULL;
2702 
2703 	rw_enter(&perdm_rwlock, RW_WRITER);
2704 	for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2705 		if (p->dm_str == str) {	/* already present */
2706 			p->dm_ref++;
2707 			rw_exit(&perdm_rwlock);
2708 			free_syncq(sq);
2709 			kmem_free(dmp, sizeof (perdm_t));
2710 			return (p);
2711 		}
2712 	}
2713 
2714 	*pp = dmp;
2715 	rw_exit(&perdm_rwlock);
2716 	return (dmp);
2717 }
2718 
2719 void
rele_dm(perdm_t * dmp)2720 rele_dm(perdm_t *dmp)
2721 {
2722 	perdm_t **pp;
2723 	perdm_t *p;
2724 
2725 	rw_enter(&perdm_rwlock, RW_WRITER);
2726 	ASSERT(dmp->dm_ref > 0);
2727 
2728 	if (--dmp->dm_ref > 0) {
2729 		rw_exit(&perdm_rwlock);
2730 		return;
2731 	}
2732 
2733 	for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2734 		if (p == dmp)
2735 			break;
2736 	ASSERT(p == dmp);
2737 	*pp = p->dm_next;
2738 	rw_exit(&perdm_rwlock);
2739 
2740 	/*
2741 	 * Wait for any background processing that relies on the
2742 	 * syncq to complete before it is freed.
2743 	 */
2744 	wait_sq_svc(p->dm_sq);
2745 	free_syncq(p->dm_sq);
2746 	kmem_free(p, sizeof (perdm_t));
2747 }
2748 
2749 /*
2750  * Make a protocol message given control and data buffers.
2751  * n.b., this can block; be careful of what locks you hold when calling it.
2752  *
2753  * If sd_maxblk is less than *iosize this routine can fail part way through
2754  * (due to an allocation failure). In this case on return *iosize will contain
2755  * the amount that was consumed. Otherwise *iosize will not be modified
2756  * i.e. it will contain the amount that was consumed.
2757  */
2758 int
strmakemsg(struct strbuf * mctl,ssize_t * iosize,struct uio * uiop,stdata_t * stp,int32_t flag,mblk_t ** mpp)2759 strmakemsg(
2760 	struct strbuf *mctl,
2761 	ssize_t *iosize,
2762 	struct uio *uiop,
2763 	stdata_t *stp,
2764 	int32_t flag,
2765 	mblk_t **mpp)
2766 {
2767 	mblk_t *mpctl = NULL;
2768 	mblk_t *mpdata = NULL;
2769 	int error;
2770 
2771 	ASSERT(uiop != NULL);
2772 
2773 	*mpp = NULL;
2774 	/* Create control part, if any */
2775 	if ((mctl != NULL) && (mctl->len >= 0)) {
2776 		error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2777 		if (error)
2778 			return (error);
2779 	}
2780 	/* Create data part, if any */
2781 	if (*iosize >= 0) {
2782 		error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2783 		if (error) {
2784 			freemsg(mpctl);
2785 			return (error);
2786 		}
2787 	}
2788 	if (mpctl != NULL) {
2789 		if (mpdata != NULL)
2790 			linkb(mpctl, mpdata);
2791 		*mpp = mpctl;
2792 	} else {
2793 		*mpp = mpdata;
2794 	}
2795 	return (0);
2796 }
2797 
2798 /*
2799  * Make the control part of a protocol message given a control buffer.
2800  * n.b., this can block; be careful of what locks you hold when calling it.
2801  */
2802 int
strmakectl(struct strbuf * mctl,int32_t flag,int32_t fflag,mblk_t ** mpp)2803 strmakectl(
2804 	struct strbuf *mctl,
2805 	int32_t flag,
2806 	int32_t fflag,
2807 	mblk_t **mpp)
2808 {
2809 	mblk_t *bp = NULL;
2810 	unsigned char msgtype;
2811 	int error = 0;
2812 	cred_t *cr = CRED();
2813 
2814 	/* We do not support interrupt threads using the stream head to send */
2815 	ASSERT(cr != NULL);
2816 
2817 	*mpp = NULL;
2818 	/*
2819 	 * Create control part of message, if any.
2820 	 */
2821 	if ((mctl != NULL) && (mctl->len >= 0)) {
2822 		caddr_t base;
2823 		int ctlcount;
2824 		int allocsz;
2825 
2826 		if (flag & RS_HIPRI)
2827 			msgtype = M_PCPROTO;
2828 		else
2829 			msgtype = M_PROTO;
2830 
2831 		ctlcount = mctl->len;
2832 		base = mctl->buf;
2833 
2834 		/*
2835 		 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2836 		 * blocks by increasing the size to something more usable.
2837 		 */
2838 		allocsz = MAX(ctlcount, 64);
2839 
2840 		/*
2841 		 * Range checking has already been done; simply try
2842 		 * to allocate a message block for the ctl part.
2843 		 */
2844 		while ((bp = allocb_cred(allocsz, cr,
2845 		    curproc->p_pid)) == NULL) {
2846 			if (fflag & (FNDELAY|FNONBLOCK))
2847 				return (EAGAIN);
2848 			if (error = strwaitbuf(allocsz, BPRI_MED))
2849 				return (error);
2850 		}
2851 
2852 		bp->b_datap->db_type = msgtype;
2853 		if (copyin(base, bp->b_wptr, ctlcount)) {
2854 			freeb(bp);
2855 			return (EFAULT);
2856 		}
2857 		bp->b_wptr += ctlcount;
2858 	}
2859 	*mpp = bp;
2860 	return (0);
2861 }
2862 
2863 /*
2864  * Make a protocol message given data buffers.
2865  * n.b., this can block; be careful of what locks you hold when calling it.
2866  *
2867  * If sd_maxblk is less than *iosize this routine can fail part way through
2868  * (due to an allocation failure). In this case on return *iosize will contain
2869  * the amount that was consumed. Otherwise *iosize will not be modified
2870  * i.e. it will contain the amount that was consumed.
2871  */
2872 int
strmakedata(ssize_t * iosize,struct uio * uiop,stdata_t * stp,int32_t flag,mblk_t ** mpp)2873 strmakedata(
2874 	ssize_t   *iosize,
2875 	struct uio *uiop,
2876 	stdata_t *stp,
2877 	int32_t flag,
2878 	mblk_t **mpp)
2879 {
2880 	mblk_t *mp = NULL;
2881 	mblk_t *bp;
2882 	int wroff = (int)stp->sd_wroff;
2883 	int tail_len = (int)stp->sd_tail;
2884 	int extra = wroff + tail_len;
2885 	int error = 0;
2886 	ssize_t maxblk;
2887 	ssize_t count = *iosize;
2888 	cred_t *cr;
2889 
2890 	*mpp = NULL;
2891 	if (count < 0)
2892 		return (0);
2893 
2894 	/* We do not support interrupt threads using the stream head to send */
2895 	cr = CRED();
2896 	ASSERT(cr != NULL);
2897 
2898 	maxblk = stp->sd_maxblk;
2899 	if (maxblk == INFPSZ)
2900 		maxblk = count;
2901 
2902 	/*
2903 	 * Create data part of message, if any.
2904 	 */
2905 	do {
2906 		ssize_t size;
2907 		dblk_t  *dp;
2908 
2909 		ASSERT(uiop);
2910 
2911 		size = MIN(count, maxblk);
2912 
2913 		while ((bp = allocb_cred(size + extra, cr,
2914 		    curproc->p_pid)) == NULL) {
2915 			error = EAGAIN;
2916 			if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2917 			    (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2918 				if (count == *iosize) {
2919 					freemsg(mp);
2920 					return (error);
2921 				} else {
2922 					*iosize -= count;
2923 					*mpp = mp;
2924 					return (0);
2925 				}
2926 			}
2927 		}
2928 		dp = bp->b_datap;
2929 		dp->db_cpid = curproc->p_pid;
2930 		ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2931 		bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2932 
2933 		if (flag & STRUIO_POSTPONE) {
2934 			/*
2935 			 * Setup the stream uio portion of the
2936 			 * dblk for subsequent use by struioget().
2937 			 */
2938 			dp->db_struioflag = STRUIO_SPEC;
2939 			dp->db_cksumstart = 0;
2940 			dp->db_cksumstuff = 0;
2941 			dp->db_cksumend = size;
2942 			*(long long *)dp->db_struioun.data = 0ll;
2943 			bp->b_wptr += size;
2944 		} else {
2945 			if (stp->sd_copyflag & STRCOPYCACHED)
2946 				uiop->uio_extflg |= UIO_COPY_CACHED;
2947 
2948 			if (size != 0) {
2949 				error = uiomove(bp->b_wptr, size, UIO_WRITE,
2950 				    uiop);
2951 				if (error != 0) {
2952 					freeb(bp);
2953 					freemsg(mp);
2954 					return (error);
2955 				}
2956 			}
2957 			bp->b_wptr += size;
2958 
2959 			if (stp->sd_wputdatafunc != NULL) {
2960 				mblk_t *newbp;
2961 
2962 				newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2963 				    bp, NULL, NULL, NULL, NULL);
2964 				if (newbp == NULL) {
2965 					freeb(bp);
2966 					freemsg(mp);
2967 					return (ECOMM);
2968 				}
2969 				bp = newbp;
2970 			}
2971 		}
2972 
2973 		count -= size;
2974 
2975 		if (mp == NULL)
2976 			mp = bp;
2977 		else
2978 			linkb(mp, bp);
2979 	} while (count > 0);
2980 
2981 	*mpp = mp;
2982 	return (0);
2983 }
2984 
2985 /*
2986  * Wait for a buffer to become available. Return non-zero errno
2987  * if not able to wait, 0 if buffer is probably there.
2988  */
2989 int
strwaitbuf(size_t size,int pri)2990 strwaitbuf(size_t size, int pri)
2991 {
2992 	bufcall_id_t id;
2993 
2994 	mutex_enter(&bcall_monitor);
2995 	if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2996 	    &ttoproc(curthread)->p_flag_cv)) == 0) {
2997 		mutex_exit(&bcall_monitor);
2998 		return (ENOSR);
2999 	}
3000 	if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
3001 		unbufcall(id);
3002 		mutex_exit(&bcall_monitor);
3003 		return (EINTR);
3004 	}
3005 	unbufcall(id);
3006 	mutex_exit(&bcall_monitor);
3007 	return (0);
3008 }
3009 
3010 /*
3011  * This function waits for a read or write event to happen on a stream.
3012  * fmode can specify FNDELAY and/or FNONBLOCK.
3013  * The timeout is in ms with -1 meaning infinite.
3014  * The flag values work as follows:
3015  *	READWAIT	Check for read side errors, send M_READ
3016  *	GETWAIT		Check for read side errors, no M_READ
3017  *	WRITEWAIT	Check for write side errors.
3018  *	NOINTR		Do not return error if nonblocking or timeout.
3019  *	STR_NOERROR	Ignore all errors except STPLEX.
3020  *	STR_NOSIG	Ignore/hold signals during the duration of the call.
3021  *	STR_PEEK	Pass through the strgeterr().
3022  */
3023 int
strwaitq(stdata_t * stp,int flag,ssize_t count,int fmode,clock_t timout,int * done)3024 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3025     int *done)
3026 {
3027 	int slpflg, errs;
3028 	int error;
3029 	kcondvar_t *sleepon;
3030 	mblk_t *mp;
3031 	ssize_t *rd_count;
3032 	clock_t rval;
3033 
3034 	ASSERT(MUTEX_HELD(&stp->sd_lock));
3035 	if ((flag & READWAIT) || (flag & GETWAIT)) {
3036 		slpflg = RSLEEP;
3037 		sleepon = &_RD(stp->sd_wrq)->q_wait;
3038 		errs = STRDERR|STPLEX;
3039 	} else {
3040 		slpflg = WSLEEP;
3041 		sleepon = &stp->sd_wrq->q_wait;
3042 		errs = STWRERR|STRHUP|STPLEX;
3043 	}
3044 	if (flag & STR_NOERROR)
3045 		errs = STPLEX;
3046 
3047 	if (stp->sd_wakeq & slpflg) {
3048 		/*
3049 		 * A strwakeq() is pending, no need to sleep.
3050 		 */
3051 		stp->sd_wakeq &= ~slpflg;
3052 		*done = 0;
3053 		return (0);
3054 	}
3055 
3056 	if (stp->sd_flag & errs) {
3057 		/*
3058 		 * Check for errors before going to sleep since the
3059 		 * caller might not have checked this while holding
3060 		 * sd_lock.
3061 		 */
3062 		error = strgeterr(stp, errs, (flag & STR_PEEK));
3063 		if (error != 0) {
3064 			*done = 1;
3065 			return (error);
3066 		}
3067 	}
3068 
3069 	/*
3070 	 * If any module downstream has requested read notification
3071 	 * by setting SNDMREAD flag using M_SETOPTS, send a message
3072 	 * down stream.
3073 	 */
3074 	if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3075 		mutex_exit(&stp->sd_lock);
3076 		if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3077 		    (flag & STR_NOSIG), &error))) {
3078 			mutex_enter(&stp->sd_lock);
3079 			*done = 1;
3080 			return (error);
3081 		}
3082 		mp->b_datap->db_type = M_READ;
3083 		rd_count = (ssize_t *)mp->b_wptr;
3084 		*rd_count = count;
3085 		mp->b_wptr += sizeof (ssize_t);
3086 		/*
3087 		 * Send the number of bytes requested by the
3088 		 * read as the argument to M_READ.
3089 		 */
3090 		stream_willservice(stp);
3091 		putnext(stp->sd_wrq, mp);
3092 		stream_runservice(stp);
3093 		mutex_enter(&stp->sd_lock);
3094 
3095 		/*
3096 		 * If any data arrived due to inline processing
3097 		 * of putnext(), don't sleep.
3098 		 */
3099 		if (_RD(stp->sd_wrq)->q_first != NULL) {
3100 			*done = 0;
3101 			return (0);
3102 		}
3103 	}
3104 
3105 	if (fmode & (FNDELAY|FNONBLOCK)) {
3106 		if (!(flag & NOINTR))
3107 			error = EAGAIN;
3108 		else
3109 			error = 0;
3110 		*done = 1;
3111 		return (error);
3112 	}
3113 
3114 	stp->sd_flag |= slpflg;
3115 	TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3116 	    "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3117 	    stp, flag, count, fmode, done);
3118 
3119 	rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3120 	if (rval > 0) {
3121 		/* EMPTY */
3122 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3123 		    "strwaitq awakes(2):%X, %X, %X, %X, %X",
3124 		    stp, flag, count, fmode, done);
3125 	} else if (rval == 0) {
3126 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3127 		    "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3128 		    stp, flag, count, fmode, done);
3129 		stp->sd_flag &= ~slpflg;
3130 		cv_broadcast(sleepon);
3131 		if (!(flag & NOINTR))
3132 			error = EINTR;
3133 		else
3134 			error = 0;
3135 		*done = 1;
3136 		return (error);
3137 	} else {
3138 		/* timeout */
3139 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3140 		    "strwaitq timeout:%p, %X, %lX, %X, %p",
3141 		    stp, flag, count, fmode, done);
3142 		*done = 1;
3143 		if (!(flag & NOINTR))
3144 			return (ETIME);
3145 		else
3146 			return (0);
3147 	}
3148 	/*
3149 	 * If the caller implements delayed errors (i.e. queued after data)
3150 	 * we can not check for errors here since data as well as an
3151 	 * error might have arrived at the stream head. We return to
3152 	 * have the caller check the read queue before checking for errors.
3153 	 */
3154 	if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3155 		error = strgeterr(stp, errs, (flag & STR_PEEK));
3156 		if (error != 0) {
3157 			*done = 1;
3158 			return (error);
3159 		}
3160 	}
3161 	*done = 0;
3162 	return (0);
3163 }
3164 
3165 /*
3166  * Perform job control discipline access checks.
3167  * Return 0 for success and the errno for failure.
3168  */
3169 
3170 #define	cantsend(p, t, sig) \
3171 	(sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3172 
3173 int
straccess(struct stdata * stp,enum jcaccess mode)3174 straccess(struct stdata *stp, enum jcaccess mode)
3175 {
3176 	extern kcondvar_t lbolt_cv;	/* XXX: should be in a header file */
3177 	kthread_t *t = curthread;
3178 	proc_t *p = ttoproc(t);
3179 	sess_t *sp;
3180 
3181 	ASSERT(mutex_owned(&stp->sd_lock));
3182 
3183 	if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3184 		return (0);
3185 
3186 	mutex_enter(&p->p_lock);		/* protects p_pgidp */
3187 
3188 	for (;;) {
3189 		mutex_enter(&p->p_splock);	/* protects p->p_sessp */
3190 		sp = p->p_sessp;
3191 		mutex_enter(&sp->s_lock);	/* protects sp->* */
3192 
3193 		/*
3194 		 * If this is not the calling process's controlling terminal
3195 		 * or if the calling process is already in the foreground
3196 		 * then allow access.
3197 		 */
3198 		if (sp->s_dev != stp->sd_vnode->v_rdev ||
3199 		    p->p_pgidp == stp->sd_pgidp) {
3200 			mutex_exit(&sp->s_lock);
3201 			mutex_exit(&p->p_splock);
3202 			mutex_exit(&p->p_lock);
3203 			return (0);
3204 		}
3205 
3206 		/*
3207 		 * Check to see if controlling terminal has been deallocated.
3208 		 */
3209 		if (sp->s_vp == NULL) {
3210 			if (!cantsend(p, t, SIGHUP))
3211 				sigtoproc(p, t, SIGHUP);
3212 			mutex_exit(&sp->s_lock);
3213 			mutex_exit(&p->p_splock);
3214 			mutex_exit(&p->p_lock);
3215 			return (EIO);
3216 		}
3217 
3218 		mutex_exit(&sp->s_lock);
3219 		mutex_exit(&p->p_splock);
3220 
3221 		if (mode == JCGETP) {
3222 			mutex_exit(&p->p_lock);
3223 			return (0);
3224 		}
3225 
3226 		if (mode == JCREAD) {
3227 			if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3228 				mutex_exit(&p->p_lock);
3229 				return (EIO);
3230 			}
3231 			mutex_exit(&p->p_lock);
3232 			mutex_exit(&stp->sd_lock);
3233 			pgsignal(p->p_pgidp, SIGTTIN);
3234 			mutex_enter(&stp->sd_lock);
3235 			mutex_enter(&p->p_lock);
3236 		} else {  /* mode == JCWRITE or JCSETP */
3237 			if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3238 			    cantsend(p, t, SIGTTOU)) {
3239 				mutex_exit(&p->p_lock);
3240 				return (0);
3241 			}
3242 			if (p->p_detached) {
3243 				mutex_exit(&p->p_lock);
3244 				return (EIO);
3245 			}
3246 			mutex_exit(&p->p_lock);
3247 			mutex_exit(&stp->sd_lock);
3248 			pgsignal(p->p_pgidp, SIGTTOU);
3249 			mutex_enter(&stp->sd_lock);
3250 			mutex_enter(&p->p_lock);
3251 		}
3252 
3253 		/*
3254 		 * We call cv_wait_sig_swap() to cause the appropriate
3255 		 * action for the jobcontrol signal to take place.
3256 		 * If the signal is being caught, we will take the
3257 		 * EINTR error return.  Otherwise, the default action
3258 		 * of causing the process to stop will take place.
3259 		 * In this case, we rely on the periodic cv_broadcast() on
3260 		 * &lbolt_cv to wake us up to loop around and test again.
3261 		 * We can't get here if the signal is ignored or
3262 		 * if the current thread is blocking the signal.
3263 		 */
3264 		mutex_exit(&stp->sd_lock);
3265 		if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3266 			mutex_exit(&p->p_lock);
3267 			mutex_enter(&stp->sd_lock);
3268 			return (EINTR);
3269 		}
3270 		mutex_exit(&p->p_lock);
3271 		mutex_enter(&stp->sd_lock);
3272 		mutex_enter(&p->p_lock);
3273 	}
3274 }
3275 
3276 /*
3277  * Return size of message of block type (bp->b_datap->db_type)
3278  */
3279 size_t
xmsgsize(mblk_t * bp)3280 xmsgsize(mblk_t *bp)
3281 {
3282 	unsigned char type;
3283 	size_t count = 0;
3284 
3285 	type = bp->b_datap->db_type;
3286 
3287 	for (; bp; bp = bp->b_cont) {
3288 		if (type != bp->b_datap->db_type)
3289 			break;
3290 		ASSERT(bp->b_wptr >= bp->b_rptr);
3291 		count += bp->b_wptr - bp->b_rptr;
3292 	}
3293 	return (count);
3294 }
3295 
3296 /*
3297  * Allocate a stream head.
3298  */
3299 struct stdata *
shalloc(queue_t * qp)3300 shalloc(queue_t *qp)
3301 {
3302 	stdata_t *stp;
3303 
3304 	stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3305 
3306 	stp->sd_wrq = _WR(qp);
3307 	stp->sd_strtab = NULL;
3308 	stp->sd_iocid = 0;
3309 	stp->sd_mate = NULL;
3310 	stp->sd_freezer = NULL;
3311 	stp->sd_refcnt = 0;
3312 	stp->sd_wakeq = 0;
3313 	stp->sd_anchor = 0;
3314 	stp->sd_struiowrq = NULL;
3315 	stp->sd_struiordq = NULL;
3316 	stp->sd_struiodnak = 0;
3317 	stp->sd_struionak = NULL;
3318 	stp->sd_t_audit_data = NULL;
3319 	stp->sd_rput_opt = 0;
3320 	stp->sd_wput_opt = 0;
3321 	stp->sd_read_opt = 0;
3322 	stp->sd_rprotofunc = strrput_proto;
3323 	stp->sd_rmiscfunc = strrput_misc;
3324 	stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3325 	stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3326 	stp->sd_ciputctrl = NULL;
3327 	stp->sd_nciputctrl = 0;
3328 	stp->sd_qhead = NULL;
3329 	stp->sd_qtail = NULL;
3330 	stp->sd_servid = NULL;
3331 	stp->sd_nqueues = 0;
3332 	stp->sd_svcflags = 0;
3333 	stp->sd_copyflag = 0;
3334 
3335 	return (stp);
3336 }
3337 
3338 /*
3339  * Free a stream head.
3340  */
3341 void
shfree(stdata_t * stp)3342 shfree(stdata_t *stp)
3343 {
3344 	ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3345 
3346 	stp->sd_wrq = NULL;
3347 
3348 	mutex_enter(&stp->sd_qlock);
3349 	while (stp->sd_svcflags & STRS_SCHEDULED) {
3350 		STRSTAT(strwaits);
3351 		cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3352 	}
3353 	mutex_exit(&stp->sd_qlock);
3354 
3355 	if (stp->sd_ciputctrl != NULL) {
3356 		ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3357 		SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3358 		    stp->sd_nciputctrl, 0);
3359 		ASSERT(ciputctrl_cache != NULL);
3360 		kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3361 		stp->sd_ciputctrl = NULL;
3362 		stp->sd_nciputctrl = 0;
3363 	}
3364 	ASSERT(stp->sd_qhead == NULL);
3365 	ASSERT(stp->sd_qtail == NULL);
3366 	ASSERT(stp->sd_nqueues == 0);
3367 	kmem_cache_free(stream_head_cache, stp);
3368 }
3369 
3370 /*
3371  * Allocate a pair of queues and a syncq for the pair
3372  */
3373 queue_t *
allocq(void)3374 allocq(void)
3375 {
3376 	queinfo_t *qip;
3377 	queue_t *qp, *wqp;
3378 	syncq_t	*sq;
3379 
3380 	qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3381 
3382 	qp = &qip->qu_rqueue;
3383 	wqp = &qip->qu_wqueue;
3384 	sq = &qip->qu_syncq;
3385 
3386 	qp->q_last	= NULL;
3387 	qp->q_next	= NULL;
3388 	qp->q_ptr	= NULL;
3389 	qp->q_flag	= QUSE | QREADR;
3390 	qp->q_bandp	= NULL;
3391 	qp->q_stream	= NULL;
3392 	qp->q_syncq	= sq;
3393 	qp->q_nband	= 0;
3394 	qp->q_nfsrv	= NULL;
3395 	qp->q_draining	= 0;
3396 	qp->q_syncqmsgs	= 0;
3397 	qp->q_spri	= 0;
3398 	qp->q_qtstamp	= 0;
3399 	qp->q_sqtstamp	= 0;
3400 	qp->q_fp	= NULL;
3401 
3402 	wqp->q_last	= NULL;
3403 	wqp->q_next	= NULL;
3404 	wqp->q_ptr	= NULL;
3405 	wqp->q_flag	= QUSE;
3406 	wqp->q_bandp	= NULL;
3407 	wqp->q_stream	= NULL;
3408 	wqp->q_syncq	= sq;
3409 	wqp->q_nband	= 0;
3410 	wqp->q_nfsrv	= NULL;
3411 	wqp->q_draining	= 0;
3412 	wqp->q_syncqmsgs = 0;
3413 	wqp->q_qtstamp	= 0;
3414 	wqp->q_sqtstamp	= 0;
3415 	wqp->q_spri	= 0;
3416 
3417 	sq->sq_count	= 0;
3418 	sq->sq_rmqcount	= 0;
3419 	sq->sq_flags	= 0;
3420 	sq->sq_type	= 0;
3421 	sq->sq_callbflags = 0;
3422 	sq->sq_cancelid	= 0;
3423 	sq->sq_ciputctrl = NULL;
3424 	sq->sq_nciputctrl = 0;
3425 	sq->sq_needexcl = 0;
3426 	sq->sq_svcflags = 0;
3427 
3428 	return (qp);
3429 }
3430 
3431 /*
3432  * Free a pair of queues and the "attached" syncq.
3433  * Discard any messages left on the syncq(s), remove the syncq(s) from the
3434  * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3435  */
3436 void
freeq(queue_t * qp)3437 freeq(queue_t *qp)
3438 {
3439 	qband_t *qbp, *nqbp;
3440 	syncq_t *sq, *outer;
3441 	queue_t *wqp = _WR(qp);
3442 
3443 	ASSERT(qp->q_flag & QREADR);
3444 
3445 	/*
3446 	 * If a previously dispatched taskq job is scheduled to run
3447 	 * sync_service() or a service routine is scheduled for the
3448 	 * queues about to be freed, wait here until all service is
3449 	 * done on the queue and all associated queues and syncqs.
3450 	 */
3451 	wait_svc(qp);
3452 
3453 	(void) flush_syncq(qp->q_syncq, qp);
3454 	(void) flush_syncq(wqp->q_syncq, wqp);
3455 	ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3456 
3457 	/*
3458 	 * Flush the queues before q_next is set to NULL This is needed
3459 	 * in order to backenable any downstream queue before we go away.
3460 	 * Note: we are already removed from the stream so that the
3461 	 * backenabling will not cause any messages to be delivered to our
3462 	 * put procedures.
3463 	 */
3464 	flushq(qp, FLUSHALL);
3465 	flushq(wqp, FLUSHALL);
3466 
3467 	/* Tidy up - removeq only does a half-remove from stream */
3468 	qp->q_next = wqp->q_next = NULL;
3469 	ASSERT(!(qp->q_flag & QENAB));
3470 	ASSERT(!(wqp->q_flag & QENAB));
3471 
3472 	outer = qp->q_syncq->sq_outer;
3473 	if (outer != NULL) {
3474 		outer_remove(outer, qp->q_syncq);
3475 		if (wqp->q_syncq != qp->q_syncq)
3476 			outer_remove(outer, wqp->q_syncq);
3477 	}
3478 	/*
3479 	 * Free any syncqs that are outside what allocq returned.
3480 	 */
3481 	if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3482 		free_syncq(qp->q_syncq);
3483 	if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3484 		free_syncq(wqp->q_syncq);
3485 
3486 	ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3487 	ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3488 	ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3489 	ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3490 	sq = SQ(qp);
3491 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3492 	ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3493 	ASSERT(sq->sq_outer == NULL);
3494 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3495 	ASSERT(sq->sq_callbpend == NULL);
3496 	ASSERT(sq->sq_needexcl == 0);
3497 
3498 	if (sq->sq_ciputctrl != NULL) {
3499 		ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3500 		SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3501 		    sq->sq_nciputctrl, 0);
3502 		ASSERT(ciputctrl_cache != NULL);
3503 		kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3504 		sq->sq_ciputctrl = NULL;
3505 		sq->sq_nciputctrl = 0;
3506 	}
3507 
3508 	ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3509 	ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3510 	ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3511 
3512 	qp->q_flag &= ~QUSE;
3513 	wqp->q_flag &= ~QUSE;
3514 
3515 	/* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3516 	/* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3517 
3518 	qbp = qp->q_bandp;
3519 	while (qbp) {
3520 		nqbp = qbp->qb_next;
3521 		freeband(qbp);
3522 		qbp = nqbp;
3523 	}
3524 	qbp = wqp->q_bandp;
3525 	while (qbp) {
3526 		nqbp = qbp->qb_next;
3527 		freeband(qbp);
3528 		qbp = nqbp;
3529 	}
3530 	kmem_cache_free(queue_cache, qp);
3531 }
3532 
3533 /*
3534  * Allocate a qband structure.
3535  */
3536 qband_t *
allocband(void)3537 allocband(void)
3538 {
3539 	qband_t *qbp;
3540 
3541 	qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3542 	if (qbp == NULL)
3543 		return (NULL);
3544 
3545 	qbp->qb_next	= NULL;
3546 	qbp->qb_count	= 0;
3547 	qbp->qb_mblkcnt	= 0;
3548 	qbp->qb_first	= NULL;
3549 	qbp->qb_last	= NULL;
3550 	qbp->qb_flag	= 0;
3551 
3552 	return (qbp);
3553 }
3554 
3555 /*
3556  * Free a qband structure.
3557  */
3558 void
freeband(qband_t * qbp)3559 freeband(qband_t *qbp)
3560 {
3561 	kmem_cache_free(qband_cache, qbp);
3562 }
3563 
3564 /*
3565  * Just like putnextctl(9F), except that allocb_wait() is used.
3566  *
3567  * Consolidation Private, and of course only callable from the stream head or
3568  * routines that may block.
3569  */
3570 int
putnextctl_wait(queue_t * q,int type)3571 putnextctl_wait(queue_t *q, int type)
3572 {
3573 	mblk_t *bp;
3574 	int error;
3575 
3576 	if ((datamsg(type) && (type != M_DELAY)) ||
3577 	    (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3578 		return (0);
3579 
3580 	bp->b_datap->db_type = (unsigned char)type;
3581 	putnext(q, bp);
3582 	return (1);
3583 }
3584 
3585 /*
3586  * Run any possible bufcalls.
3587  */
3588 void
runbufcalls(void)3589 runbufcalls(void)
3590 {
3591 	strbufcall_t *bcp;
3592 
3593 	mutex_enter(&bcall_monitor);
3594 	mutex_enter(&strbcall_lock);
3595 
3596 	if (strbcalls.bc_head) {
3597 		size_t count;
3598 		int nevent;
3599 
3600 		/*
3601 		 * count how many events are on the list
3602 		 * now so we can check to avoid looping
3603 		 * in low memory situations
3604 		 */
3605 		nevent = 0;
3606 		for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3607 			nevent++;
3608 
3609 		/*
3610 		 * get estimate of available memory from kmem_avail().
3611 		 * awake all bufcall functions waiting for
3612 		 * memory whose request could be satisfied
3613 		 * by 'count' memory and let 'em fight for it.
3614 		 */
3615 		count = kmem_avail();
3616 		while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3617 			STRSTAT(bufcalls);
3618 			--nevent;
3619 			if (bcp->bc_size <= count) {
3620 				bcp->bc_executor = curthread;
3621 				mutex_exit(&strbcall_lock);
3622 				(*bcp->bc_func)(bcp->bc_arg);
3623 				mutex_enter(&strbcall_lock);
3624 				bcp->bc_executor = NULL;
3625 				cv_broadcast(&bcall_cv);
3626 				strbcalls.bc_head = bcp->bc_next;
3627 				kmem_free(bcp, sizeof (strbufcall_t));
3628 			} else {
3629 				/*
3630 				 * too big, try again later - note
3631 				 * that nevent was decremented above
3632 				 * so we won't retry this one on this
3633 				 * iteration of the loop
3634 				 */
3635 				if (bcp->bc_next != NULL) {
3636 					strbcalls.bc_head = bcp->bc_next;
3637 					bcp->bc_next = NULL;
3638 					strbcalls.bc_tail->bc_next = bcp;
3639 					strbcalls.bc_tail = bcp;
3640 				}
3641 			}
3642 		}
3643 		if (strbcalls.bc_head == NULL)
3644 			strbcalls.bc_tail = NULL;
3645 	}
3646 
3647 	mutex_exit(&strbcall_lock);
3648 	mutex_exit(&bcall_monitor);
3649 }
3650 
3651 
3652 /*
3653  * Actually run queue's service routine.
3654  */
3655 static void
runservice(queue_t * q)3656 runservice(queue_t *q)
3657 {
3658 	qband_t *qbp;
3659 
3660 	ASSERT(q->q_qinfo->qi_srvp);
3661 again:
3662 	entersq(q->q_syncq, SQ_SVC);
3663 	TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3664 	    "runservice starts:%p", q);
3665 
3666 	if (!(q->q_flag & QWCLOSE))
3667 		(*q->q_qinfo->qi_srvp)(q);
3668 
3669 	TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3670 	    "runservice ends:(%p)", q);
3671 
3672 	leavesq(q->q_syncq, SQ_SVC);
3673 
3674 	mutex_enter(QLOCK(q));
3675 	if (q->q_flag & QENAB) {
3676 		q->q_flag &= ~QENAB;
3677 		mutex_exit(QLOCK(q));
3678 		goto again;
3679 	}
3680 	q->q_flag &= ~QINSERVICE;
3681 	q->q_flag &= ~QBACK;
3682 	for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3683 		qbp->qb_flag &= ~QB_BACK;
3684 	/*
3685 	 * Wakeup thread waiting for the service procedure
3686 	 * to be run (strclose and qdetach).
3687 	 */
3688 	cv_broadcast(&q->q_wait);
3689 
3690 	mutex_exit(QLOCK(q));
3691 }
3692 
3693 /*
3694  * Background processing of bufcalls.
3695  */
3696 void
streams_bufcall_service(void)3697 streams_bufcall_service(void)
3698 {
3699 	callb_cpr_t	cprinfo;
3700 
3701 	CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3702 	    "streams_bufcall_service");
3703 
3704 	mutex_enter(&strbcall_lock);
3705 
3706 	for (;;) {
3707 		if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3708 			mutex_exit(&strbcall_lock);
3709 			runbufcalls();
3710 			mutex_enter(&strbcall_lock);
3711 		}
3712 		if (strbcalls.bc_head != NULL) {
3713 			STRSTAT(bcwaits);
3714 			/* Wait for memory to become available */
3715 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3716 			(void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3717 			    SEC_TO_TICK(60), TR_CLOCK_TICK);
3718 			CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3719 		}
3720 
3721 		/* Wait for new work to arrive */
3722 		if (strbcalls.bc_head == NULL) {
3723 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3724 			cv_wait(&strbcall_cv, &strbcall_lock);
3725 			CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3726 		}
3727 	}
3728 }
3729 
3730 /*
3731  * Background processing of streams background tasks which failed
3732  * taskq_dispatch.
3733  */
3734 static void
streams_qbkgrnd_service(void)3735 streams_qbkgrnd_service(void)
3736 {
3737 	callb_cpr_t cprinfo;
3738 	queue_t *q;
3739 
3740 	CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3741 	    "streams_bkgrnd_service");
3742 
3743 	mutex_enter(&service_queue);
3744 
3745 	for (;;) {
3746 		/*
3747 		 * Wait for work to arrive.
3748 		 */
3749 		while ((freebs_list == NULL) && (qhead == NULL)) {
3750 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3751 			cv_wait(&services_to_run, &service_queue);
3752 			CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3753 		}
3754 		/*
3755 		 * Handle all pending freebs requests to free memory.
3756 		 */
3757 		while (freebs_list != NULL) {
3758 			mblk_t *mp = freebs_list;
3759 			freebs_list = mp->b_next;
3760 			mutex_exit(&service_queue);
3761 			mblk_free(mp);
3762 			mutex_enter(&service_queue);
3763 		}
3764 		/*
3765 		 * Run pending queues.
3766 		 */
3767 		while (qhead != NULL) {
3768 			DQ(q, qhead, qtail, q_link);
3769 			ASSERT(q != NULL);
3770 			mutex_exit(&service_queue);
3771 			queue_service(q);
3772 			mutex_enter(&service_queue);
3773 		}
3774 		ASSERT(qhead == NULL && qtail == NULL);
3775 	}
3776 }
3777 
3778 /*
3779  * Background processing of streams background tasks which failed
3780  * taskq_dispatch.
3781  */
3782 static void
streams_sqbkgrnd_service(void)3783 streams_sqbkgrnd_service(void)
3784 {
3785 	callb_cpr_t cprinfo;
3786 	syncq_t *sq;
3787 
3788 	CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3789 	    "streams_sqbkgrnd_service");
3790 
3791 	mutex_enter(&service_queue);
3792 
3793 	for (;;) {
3794 		/*
3795 		 * Wait for work to arrive.
3796 		 */
3797 		while (sqhead == NULL) {
3798 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3799 			cv_wait(&syncqs_to_run, &service_queue);
3800 			CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3801 		}
3802 
3803 		/*
3804 		 * Run pending syncqs.
3805 		 */
3806 		while (sqhead != NULL) {
3807 			DQ(sq, sqhead, sqtail, sq_next);
3808 			ASSERT(sq != NULL);
3809 			ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3810 			mutex_exit(&service_queue);
3811 			syncq_service(sq);
3812 			mutex_enter(&service_queue);
3813 		}
3814 	}
3815 }
3816 
3817 /*
3818  * Disable the syncq and wait for background syncq processing to complete.
3819  * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3820  * list.
3821  */
3822 void
wait_sq_svc(syncq_t * sq)3823 wait_sq_svc(syncq_t *sq)
3824 {
3825 	mutex_enter(SQLOCK(sq));
3826 	sq->sq_svcflags |= SQ_DISABLED;
3827 	if (sq->sq_svcflags & SQ_BGTHREAD) {
3828 		syncq_t *sq_chase;
3829 		syncq_t *sq_curr;
3830 		int removed;
3831 
3832 		ASSERT(sq->sq_servcount == 1);
3833 		mutex_enter(&service_queue);
3834 		RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3835 		mutex_exit(&service_queue);
3836 		if (removed) {
3837 			sq->sq_svcflags &= ~SQ_BGTHREAD;
3838 			sq->sq_servcount = 0;
3839 			STRSTAT(sqremoved);
3840 			goto done;
3841 		}
3842 	}
3843 	while (sq->sq_servcount != 0) {
3844 		sq->sq_flags |= SQ_WANTWAKEUP;
3845 		cv_wait(&sq->sq_wait, SQLOCK(sq));
3846 	}
3847 done:
3848 	mutex_exit(SQLOCK(sq));
3849 }
3850 
3851 /*
3852  * Put a syncq on the list of syncq's to be serviced by the sqthread.
3853  * Add the argument to the end of the sqhead list and set the flag
3854  * indicating this syncq has been enabled.  If it has already been
3855  * enabled, don't do anything.
3856  * This routine assumes that SQLOCK is held.
3857  * NOTE that the lock order is to have the SQLOCK first,
3858  * so if the service_syncq lock is held, we need to release it
3859  * before acquiring the SQLOCK (mostly relevant for the background
3860  * thread, and this seems to be common among the STREAMS global locks).
3861  * Note that the sq_svcflags are protected by the SQLOCK.
3862  */
3863 void
sqenable(syncq_t * sq)3864 sqenable(syncq_t *sq)
3865 {
3866 	/*
3867 	 * This is probably not important except for where I believe it
3868 	 * is being called.  At that point, it should be held (and it
3869 	 * is a pain to release it just for this routine, so don't do
3870 	 * it).
3871 	 */
3872 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
3873 
3874 	IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3875 	IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3876 
3877 	/*
3878 	 * Do not put on list if background thread is scheduled or
3879 	 * syncq is disabled.
3880 	 */
3881 	if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3882 		return;
3883 
3884 	/*
3885 	 * Check whether we should enable sq at all.
3886 	 * Non PERMOD syncqs may be drained by at most one thread.
3887 	 * PERMOD syncqs may be drained by several threads but we limit the
3888 	 * total amount to the lesser of
3889 	 *	Number of queues on the squeue and
3890 	 *	Number of CPUs.
3891 	 */
3892 	if (sq->sq_servcount != 0) {
3893 		if (((sq->sq_type & SQ_PERMOD) == 0) ||
3894 		    (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3895 			STRSTAT(sqtoomany);
3896 			return;
3897 		}
3898 	}
3899 
3900 	sq->sq_tstamp = ddi_get_lbolt();
3901 	STRSTAT(sqenables);
3902 
3903 	/* Attempt a taskq dispatch */
3904 	sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3905 	    (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3906 	if (sq->sq_servid != NULL) {
3907 		sq->sq_servcount++;
3908 		return;
3909 	}
3910 
3911 	/*
3912 	 * This taskq dispatch failed, but a previous one may have succeeded.
3913 	 * Don't try to schedule on the background thread whilst there is
3914 	 * outstanding taskq processing.
3915 	 */
3916 	if (sq->sq_servcount != 0)
3917 		return;
3918 
3919 	/*
3920 	 * System is low on resources and can't perform a non-sleeping
3921 	 * dispatch. Schedule the syncq for a background thread and mark the
3922 	 * syncq to avoid any further taskq dispatch attempts.
3923 	 */
3924 	mutex_enter(&service_queue);
3925 	STRSTAT(taskqfails);
3926 	ENQUEUE(sq, sqhead, sqtail, sq_next);
3927 	sq->sq_svcflags |= SQ_BGTHREAD;
3928 	sq->sq_servcount = 1;
3929 	cv_signal(&syncqs_to_run);
3930 	mutex_exit(&service_queue);
3931 }
3932 
3933 /*
3934  * Note: fifo_close() depends on the mblk_t on the queue being freed
3935  * asynchronously. The asynchronous freeing of messages breaks the
3936  * recursive call chain of fifo_close() while there are I_SENDFD type of
3937  * messages referring to other file pointers on the queue. Then when
3938  * closing pipes it can avoid stack overflow in case of daisy-chained
3939  * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3940  * share the same fifolock_t).
3941  *
3942  * No need to kpreempt_disable to access cpu_seqid.  If we migrate and
3943  * the esb queue does not match the new CPU, that is OK.
3944  */
3945 void
freebs_enqueue(mblk_t * mp,dblk_t * dbp)3946 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3947 {
3948 	int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3949 	esb_queue_t *eqp;
3950 
3951 	ASSERT(dbp->db_mblk == mp);
3952 	ASSERT(qindex < esbq_nelem);
3953 
3954 	eqp = system_esbq_array;
3955 	if (eqp != NULL) {
3956 		eqp += qindex;
3957 	} else {
3958 		mutex_enter(&esbq_lock);
3959 		if (kmem_ready && system_esbq_array == NULL)
3960 			system_esbq_array = (esb_queue_t *)kmem_zalloc(
3961 			    esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3962 		mutex_exit(&esbq_lock);
3963 		eqp = system_esbq_array;
3964 		if (eqp != NULL)
3965 			eqp += qindex;
3966 		else
3967 			eqp = &system_esbq;
3968 	}
3969 
3970 	/*
3971 	 * Check data sanity. The dblock should have non-empty free function.
3972 	 * It is better to panic here then later when the dblock is freed
3973 	 * asynchronously when the context is lost.
3974 	 */
3975 	if (dbp->db_frtnp->free_func == NULL) {
3976 		panic("freebs_enqueue: dblock %p has a NULL free callback",
3977 		    (void *)dbp);
3978 	}
3979 
3980 	mutex_enter(&eqp->eq_lock);
3981 	/* queue the new mblk on the esballoc queue */
3982 	if (eqp->eq_head == NULL) {
3983 		eqp->eq_head = eqp->eq_tail = mp;
3984 	} else {
3985 		eqp->eq_tail->b_next = mp;
3986 		eqp->eq_tail = mp;
3987 	}
3988 	eqp->eq_len++;
3989 
3990 	/* If we're the first thread to reach the threshold, process */
3991 	if (eqp->eq_len >= esbq_max_qlen &&
3992 	    !(eqp->eq_flags & ESBQ_PROCESSING))
3993 		esballoc_process_queue(eqp);
3994 
3995 	esballoc_set_timer(eqp, esbq_timeout);
3996 	mutex_exit(&eqp->eq_lock);
3997 }
3998 
3999 static void
esballoc_process_queue(esb_queue_t * eqp)4000 esballoc_process_queue(esb_queue_t *eqp)
4001 {
4002 	mblk_t	*mp;
4003 
4004 	ASSERT(MUTEX_HELD(&eqp->eq_lock));
4005 
4006 	eqp->eq_flags |= ESBQ_PROCESSING;
4007 
4008 	do {
4009 		/*
4010 		 * Detach the message chain for processing.
4011 		 */
4012 		mp = eqp->eq_head;
4013 		eqp->eq_tail->b_next = NULL;
4014 		eqp->eq_head = eqp->eq_tail = NULL;
4015 		eqp->eq_len = 0;
4016 		mutex_exit(&eqp->eq_lock);
4017 
4018 		/*
4019 		 * Process the message chain.
4020 		 */
4021 		esballoc_enqueue_mblk(mp);
4022 		mutex_enter(&eqp->eq_lock);
4023 	} while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
4024 
4025 	eqp->eq_flags &= ~ESBQ_PROCESSING;
4026 }
4027 
4028 /*
4029  * taskq callback routine to free esballoced mblk's
4030  */
4031 static void
esballoc_mblk_free(mblk_t * mp)4032 esballoc_mblk_free(mblk_t *mp)
4033 {
4034 	mblk_t	*nextmp;
4035 
4036 	for (; mp != NULL; mp = nextmp) {
4037 		nextmp = mp->b_next;
4038 		mp->b_next = NULL;
4039 		mblk_free(mp);
4040 	}
4041 }
4042 
4043 static void
esballoc_enqueue_mblk(mblk_t * mp)4044 esballoc_enqueue_mblk(mblk_t *mp)
4045 {
4046 
4047 	if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4048 	    TQ_NOSLEEP) == TASKQID_INVALID) {
4049 		mblk_t *first_mp = mp;
4050 		/*
4051 		 * System is low on resources and can't perform a non-sleeping
4052 		 * dispatch. Schedule for a background thread.
4053 		 */
4054 		mutex_enter(&service_queue);
4055 		STRSTAT(taskqfails);
4056 
4057 		while (mp->b_next != NULL)
4058 			mp = mp->b_next;
4059 
4060 		mp->b_next = freebs_list;
4061 		freebs_list = first_mp;
4062 		cv_signal(&services_to_run);
4063 		mutex_exit(&service_queue);
4064 	}
4065 }
4066 
4067 static void
esballoc_timer(void * arg)4068 esballoc_timer(void *arg)
4069 {
4070 	esb_queue_t *eqp = arg;
4071 
4072 	mutex_enter(&eqp->eq_lock);
4073 	eqp->eq_flags &= ~ESBQ_TIMER;
4074 
4075 	if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4076 	    eqp->eq_len > 0)
4077 		esballoc_process_queue(eqp);
4078 
4079 	esballoc_set_timer(eqp, esbq_timeout);
4080 	mutex_exit(&eqp->eq_lock);
4081 }
4082 
4083 static void
esballoc_set_timer(esb_queue_t * eqp,clock_t eq_timeout)4084 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4085 {
4086 	ASSERT(MUTEX_HELD(&eqp->eq_lock));
4087 
4088 	if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4089 		(void) timeout(esballoc_timer, eqp, eq_timeout);
4090 		eqp->eq_flags |= ESBQ_TIMER;
4091 	}
4092 }
4093 
4094 /*
4095  * Setup esbq array length based upon NCPU scaled by CPUs per
4096  * queue. Use static system_esbq until kmem_ready and we can
4097  * create an array in freebs_enqueue().
4098  */
4099 void
esballoc_queue_init(void)4100 esballoc_queue_init(void)
4101 {
4102 	esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4103 	esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4104 	esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4105 	system_esbq.eq_len = 0;
4106 	system_esbq.eq_head = system_esbq.eq_tail = NULL;
4107 	system_esbq.eq_flags = 0;
4108 }
4109 
4110 /*
4111  * Set the QBACK or QB_BACK flag in the given queue for
4112  * the given priority band.
4113  */
4114 void
setqback(queue_t * q,unsigned char pri)4115 setqback(queue_t *q, unsigned char pri)
4116 {
4117 	int i;
4118 	qband_t *qbp;
4119 	qband_t **qbpp;
4120 
4121 	ASSERT(MUTEX_HELD(QLOCK(q)));
4122 	if (pri != 0) {
4123 		if (pri > q->q_nband) {
4124 			qbpp = &q->q_bandp;
4125 			while (*qbpp)
4126 				qbpp = &(*qbpp)->qb_next;
4127 			while (pri > q->q_nband) {
4128 				if ((*qbpp = allocband()) == NULL) {
4129 					cmn_err(CE_WARN,
4130 					    "setqback: can't allocate qband\n");
4131 					return;
4132 				}
4133 				(*qbpp)->qb_hiwat = q->q_hiwat;
4134 				(*qbpp)->qb_lowat = q->q_lowat;
4135 				q->q_nband++;
4136 				qbpp = &(*qbpp)->qb_next;
4137 			}
4138 		}
4139 		qbp = q->q_bandp;
4140 		i = pri;
4141 		while (--i)
4142 			qbp = qbp->qb_next;
4143 		qbp->qb_flag |= QB_BACK;
4144 	} else {
4145 		q->q_flag |= QBACK;
4146 	}
4147 }
4148 
4149 int
strcopyin(void * from,void * to,size_t len,int copyflag)4150 strcopyin(void *from, void *to, size_t len, int copyflag)
4151 {
4152 	if (copyflag & U_TO_K) {
4153 		ASSERT((copyflag & K_TO_K) == 0);
4154 		if (copyin(from, to, len))
4155 			return (EFAULT);
4156 	} else {
4157 		ASSERT(copyflag & K_TO_K);
4158 		bcopy(from, to, len);
4159 	}
4160 	return (0);
4161 }
4162 
4163 int
strcopyout(void * from,void * to,size_t len,int copyflag)4164 strcopyout(void *from, void *to, size_t len, int copyflag)
4165 {
4166 	if (copyflag & U_TO_K) {
4167 		if (copyout(from, to, len))
4168 			return (EFAULT);
4169 	} else {
4170 		ASSERT(copyflag & K_TO_K);
4171 		bcopy(from, to, len);
4172 	}
4173 	return (0);
4174 }
4175 
4176 /*
4177  * strsignal_nolock() posts a signal to the process(es) at the stream head.
4178  * It assumes that the stream head lock is already held, whereas strsignal()
4179  * acquires the lock first.  This routine was created because a few callers
4180  * release the stream head lock before calling only to re-acquire it after
4181  * it returns.
4182  */
4183 void
strsignal_nolock(stdata_t * stp,int sig,uchar_t band)4184 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4185 {
4186 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4187 	switch (sig) {
4188 	case SIGPOLL:
4189 		if (stp->sd_sigflags & S_MSG)
4190 			strsendsig(stp->sd_siglist, S_MSG, band, 0);
4191 		break;
4192 	default:
4193 		if (stp->sd_pgidp)
4194 			pgsignal(stp->sd_pgidp, sig);
4195 		break;
4196 	}
4197 }
4198 
4199 void
strsignal(stdata_t * stp,int sig,int32_t band)4200 strsignal(stdata_t *stp, int sig, int32_t band)
4201 {
4202 	TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4203 	    "strsignal:%p, %X, %X", stp, sig, band);
4204 
4205 	mutex_enter(&stp->sd_lock);
4206 	switch (sig) {
4207 	case SIGPOLL:
4208 		if (stp->sd_sigflags & S_MSG)
4209 			strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4210 		break;
4211 
4212 	default:
4213 		if (stp->sd_pgidp) {
4214 			pgsignal(stp->sd_pgidp, sig);
4215 		}
4216 		break;
4217 	}
4218 	mutex_exit(&stp->sd_lock);
4219 }
4220 
4221 void
strhup(stdata_t * stp)4222 strhup(stdata_t *stp)
4223 {
4224 	ASSERT(mutex_owned(&stp->sd_lock));
4225 	pollwakeup(&stp->sd_pollist, POLLHUP);
4226 	if (stp->sd_sigflags & S_HANGUP)
4227 		strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4228 }
4229 
4230 /*
4231  * Backenable the first queue upstream from `q' with a service procedure.
4232  */
4233 void
backenable(queue_t * q,uchar_t pri)4234 backenable(queue_t *q, uchar_t pri)
4235 {
4236 	queue_t	*nq;
4237 
4238 	/*
4239 	 * Our presence might not prevent other modules in our own
4240 	 * stream from popping/pushing since the caller of getq might not
4241 	 * have a claim on the queue (some drivers do a getq on somebody
4242 	 * else's queue - they know that the queue itself is not going away
4243 	 * but the framework has to guarantee q_next in that stream).
4244 	 */
4245 	claimstr(q);
4246 
4247 	/* Find nearest back queue with service proc */
4248 	for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4249 		ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4250 	}
4251 
4252 	if (nq) {
4253 		kthread_t *freezer;
4254 		/*
4255 		 * backenable can be called either with no locks held
4256 		 * or with the stream frozen (the latter occurs when a module
4257 		 * calls rmvq with the stream frozen). If the stream is frozen
4258 		 * by the caller the caller will hold all qlocks in the stream.
4259 		 * Note that a frozen stream doesn't freeze a mated stream,
4260 		 * so we explicitly check for that.
4261 		 */
4262 		freezer = STREAM(q)->sd_freezer;
4263 		if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4264 			mutex_enter(QLOCK(nq));
4265 		}
4266 #ifdef DEBUG
4267 		else {
4268 			ASSERT(frozenstr(q));
4269 			ASSERT(MUTEX_HELD(QLOCK(q)));
4270 			ASSERT(MUTEX_HELD(QLOCK(nq)));
4271 		}
4272 #endif
4273 		setqback(nq, pri);
4274 		qenable_locked(nq);
4275 		if (freezer != curthread || STREAM(q) != STREAM(nq))
4276 			mutex_exit(QLOCK(nq));
4277 	}
4278 	releasestr(q);
4279 }
4280 
4281 /*
4282  * Return the appropriate errno when one of flags_to_check is set
4283  * in sd_flags. Uses the exported error routines if they are set.
4284  * Will return 0 if non error is set (or if the exported error routines
4285  * do not return an error).
4286  *
4287  * If there is both a read and write error to check, we prefer the read error.
4288  * Also, give preference to recorded errno's over the error functions.
4289  * The flags that are handled are:
4290  *	STPLEX		return EINVAL
4291  *	STRDERR		return sd_rerror (and clear if STRDERRNONPERSIST)
4292  *	STWRERR		return sd_werror (and clear if STWRERRNONPERSIST)
4293  *	STRHUP		return sd_werror
4294  *
4295  * If the caller indicates that the operation is a peek, a nonpersistent error
4296  * is not cleared.
4297  */
4298 int
strgeterr(stdata_t * stp,int32_t flags_to_check,int ispeek)4299 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4300 {
4301 	int32_t sd_flag = stp->sd_flag & flags_to_check;
4302 	int error = 0;
4303 
4304 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4305 	ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4306 	if (sd_flag & STPLEX)
4307 		error = EINVAL;
4308 	else if (sd_flag & STRDERR) {
4309 		error = stp->sd_rerror;
4310 		if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4311 			/*
4312 			 * Read errors are non-persistent i.e. discarded once
4313 			 * returned to a non-peeking caller,
4314 			 */
4315 			stp->sd_rerror = 0;
4316 			stp->sd_flag &= ~STRDERR;
4317 		}
4318 		if (error == 0 && stp->sd_rderrfunc != NULL) {
4319 			int clearerr = 0;
4320 
4321 			error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4322 			    &clearerr);
4323 			if (clearerr) {
4324 				stp->sd_flag &= ~STRDERR;
4325 				stp->sd_rderrfunc = NULL;
4326 			}
4327 		}
4328 	} else if (sd_flag & STWRERR) {
4329 		error = stp->sd_werror;
4330 		if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4331 			/*
4332 			 * Write errors are non-persistent i.e. discarded once
4333 			 * returned to a non-peeking caller,
4334 			 */
4335 			stp->sd_werror = 0;
4336 			stp->sd_flag &= ~STWRERR;
4337 		}
4338 		if (error == 0 && stp->sd_wrerrfunc != NULL) {
4339 			int clearerr = 0;
4340 
4341 			error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4342 			    &clearerr);
4343 			if (clearerr) {
4344 				stp->sd_flag &= ~STWRERR;
4345 				stp->sd_wrerrfunc = NULL;
4346 			}
4347 		}
4348 	} else if (sd_flag & STRHUP) {
4349 		/* sd_werror set when STRHUP */
4350 		error = stp->sd_werror;
4351 	}
4352 	return (error);
4353 }
4354 
4355 
4356 /*
4357  * Single-thread open/close/push/pop
4358  * for twisted streams also
4359  */
4360 int
strstartplumb(stdata_t * stp,int flag,int cmd)4361 strstartplumb(stdata_t *stp, int flag, int cmd)
4362 {
4363 	int waited = 1;
4364 	int error = 0;
4365 
4366 	if (STRMATED(stp)) {
4367 		struct stdata *stmatep = stp->sd_mate;
4368 
4369 		STRLOCKMATES(stp);
4370 		while (waited) {
4371 			waited = 0;
4372 			while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4373 				if ((cmd == I_POP) &&
4374 				    (flag & (FNDELAY|FNONBLOCK))) {
4375 					STRUNLOCKMATES(stp);
4376 					return (EAGAIN);
4377 				}
4378 				waited = 1;
4379 				mutex_exit(&stp->sd_lock);
4380 				if (!cv_wait_sig(&stmatep->sd_monitor,
4381 				    &stmatep->sd_lock)) {
4382 					mutex_exit(&stmatep->sd_lock);
4383 					return (EINTR);
4384 				}
4385 				mutex_exit(&stmatep->sd_lock);
4386 				STRLOCKMATES(stp);
4387 			}
4388 			while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4389 				if ((cmd == I_POP) &&
4390 				    (flag & (FNDELAY|FNONBLOCK))) {
4391 					STRUNLOCKMATES(stp);
4392 					return (EAGAIN);
4393 				}
4394 				waited = 1;
4395 				mutex_exit(&stmatep->sd_lock);
4396 				if (!cv_wait_sig(&stp->sd_monitor,
4397 				    &stp->sd_lock)) {
4398 					mutex_exit(&stp->sd_lock);
4399 					return (EINTR);
4400 				}
4401 				mutex_exit(&stp->sd_lock);
4402 				STRLOCKMATES(stp);
4403 			}
4404 			if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4405 				error = strgeterr(stp,
4406 				    STRDERR|STWRERR|STRHUP|STPLEX, 0);
4407 				if (error != 0) {
4408 					STRUNLOCKMATES(stp);
4409 					return (error);
4410 				}
4411 			}
4412 		}
4413 		stp->sd_flag |= STRPLUMB;
4414 		STRUNLOCKMATES(stp);
4415 	} else {
4416 		mutex_enter(&stp->sd_lock);
4417 		while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4418 			if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4419 			    (flag & (FNDELAY|FNONBLOCK))) {
4420 				mutex_exit(&stp->sd_lock);
4421 				return (EAGAIN);
4422 			}
4423 			if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4424 				mutex_exit(&stp->sd_lock);
4425 				return (EINTR);
4426 			}
4427 			if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4428 				error = strgeterr(stp,
4429 				    STRDERR|STWRERR|STRHUP|STPLEX, 0);
4430 				if (error != 0) {
4431 					mutex_exit(&stp->sd_lock);
4432 					return (error);
4433 				}
4434 			}
4435 		}
4436 		stp->sd_flag |= STRPLUMB;
4437 		mutex_exit(&stp->sd_lock);
4438 	}
4439 	return (0);
4440 }
4441 
4442 /*
4443  * Complete the plumbing operation associated with stream `stp'.
4444  */
4445 void
strendplumb(stdata_t * stp)4446 strendplumb(stdata_t *stp)
4447 {
4448 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4449 	ASSERT(stp->sd_flag & STRPLUMB);
4450 	stp->sd_flag &= ~STRPLUMB;
4451 	cv_broadcast(&stp->sd_monitor);
4452 }
4453 
4454 /*
4455  * This describes how the STREAMS framework handles synchronization
4456  * during open/push and close/pop.
4457  * The key interfaces for open and close are qprocson and qprocsoff,
4458  * respectively. While the close case in general is harder both open
4459  * have close have significant similarities.
4460  *
4461  * During close the STREAMS framework has to both ensure that there
4462  * are no stale references to the queue pair (and syncq) that
4463  * are being closed and also provide the guarantees that are documented
4464  * in qprocsoff(9F).
4465  * If there are stale references to the queue that is closing it can
4466  * result in kernel memory corruption or kernel panics.
4467  *
4468  * Note that is it up to the module/driver to ensure that it itself
4469  * does not have any stale references to the closing queues once its close
4470  * routine returns. This includes:
4471  *  - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4472  *    associated with the queues. For timeout and bufcall callbacks the
4473  *    module/driver also has to ensure (or wait for) any callbacks that
4474  *    are in progress.
4475  *  - If the module/driver is using esballoc it has to ensure that any
4476  *    esballoc free functions do not refer to a queue that has closed.
4477  *    (Note that in general the close routine can not wait for the esballoc'ed
4478  *    messages to be freed since that can cause a deadlock.)
4479  *  - Cancelling any interrupts that refer to the closing queues and
4480  *    also ensuring that there are no interrupts in progress that will
4481  *    refer to the closing queues once the close routine returns.
4482  *  - For multiplexors removing any driver global state that refers to
4483  *    the closing queue and also ensuring that there are no threads in
4484  *    the multiplexor that has picked up a queue pointer but not yet
4485  *    finished using it.
4486  *
4487  * In addition, a driver/module can only reference the q_next pointer
4488  * in its open, close, put, or service procedures or in a
4489  * qtimeout/qbufcall callback procedure executing "on" the correct
4490  * stream. Thus it can not reference the q_next pointer in an interrupt
4491  * routine or a timeout, bufcall or esballoc callback routine. Likewise
4492  * it can not reference q_next of a different queue e.g. in a mux that
4493  * passes messages from one queues put/service procedure to another queue.
4494  * In all the cases when the driver/module can not access the q_next
4495  * field it must use the *next* versions e.g. canputnext instead of
4496  * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4497  *
4498  *
4499  * Assuming that the driver/module conforms to the above constraints
4500  * the STREAMS framework has to avoid stale references to q_next for all
4501  * the framework internal cases which include (but are not limited to):
4502  *  - Threads in canput/canputnext/backenable and elsewhere that are
4503  *    walking q_next.
4504  *  - Messages on a syncq that have a reference to the queue through b_queue.
4505  *  - Messages on an outer perimeter (syncq) that have a reference to the
4506  *    queue through b_queue.
4507  *  - Threads that use q_nfsrv (e.g. canput) to find a queue.
4508  *    Note that only canput and bcanput use q_nfsrv without any locking.
4509  *
4510  * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4511  * after qprocsoff returns, the framework has to ensure that no threads can
4512  * enter the put or service routines for the closing read or write-side queue.
4513  * In addition to preventing "direct" entry into the put procedures
4514  * the framework also has to prevent messages being drained from
4515  * the syncq or the outer perimeter.
4516  * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4517  * mechanism to prevent qwriter(PERIM_OUTER) from running after
4518  * qprocsoff has returned.
4519  * Note that if a module/driver uses put(9F) on one of its own queues
4520  * it is up to the module/driver to ensure that the put() doesn't
4521  * get called when the queue is closing.
4522  *
4523  *
4524  * The framework aspects of the above "contract" is implemented by
4525  * qprocsoff, removeq, and strlock:
4526  *  - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4527  *    entering the service procedures.
4528  *  - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4529  *    canputnext, backenable etc from dereferencing the q_next that will
4530  *    soon change.
4531  *  - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4532  *    or other q_next walker that uses claimstr/releasestr to finish.
4533  *  - optionally for every syncq in the stream strlock acquires all the
4534  *    sq_lock's and waits for all sq_counts to drop to a value that indicates
4535  *    that no thread executes in the put or service procedures and that no
4536  *    thread is draining into the module/driver. This ensures that no
4537  *    open, close, put, service, or qtimeout/qbufcall callback procedure is
4538  *    currently executing hence no such thread can end up with the old stale
4539  *    q_next value and no canput/backenable can have the old stale
4540  *    q_nfsrv/q_next.
4541  *  - qdetach (wait_svc) makes sure that any scheduled or running threads
4542  *    have either finished or observed the QWCLOSE flag and gone away.
4543  */
4544 
4545 
4546 /*
4547  * Get all the locks necessary to change q_next.
4548  *
4549  * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4550  * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4551  * the only threads inside the syncq are threads currently calling removeq().
4552  * Since threads calling removeq() are in the process of removing their queues
4553  * from the stream, we do not need to worry about them accessing a stale q_next
4554  * pointer and thus we do not need to wait for them to exit (in fact, waiting
4555  * for them can cause deadlock).
4556  *
4557  * This routine is subject to starvation since it does not set any flag to
4558  * prevent threads from entering a module in the stream (i.e. sq_count can
4559  * increase on some syncq while it is waiting on some other syncq).
4560  *
4561  * Assumes that only one thread attempts to call strlock for a given
4562  * stream. If this is not the case the two threads would deadlock.
4563  * This assumption is guaranteed since strlock is only called by insertq
4564  * and removeq and streams plumbing changes are single-threaded for
4565  * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4566  *
4567  * For pipes, it is not difficult to atomically designate a pair of streams
4568  * to be mated. Once mated atomically by the framework the twisted pair remain
4569  * configured that way until dismantled atomically by the framework.
4570  * When plumbing takes place on a twisted stream it is necessary to ensure that
4571  * this operation is done exclusively on the twisted stream since two such
4572  * operations, each initiated on different ends of the pipe will deadlock
4573  * waiting for each other to complete.
4574  *
4575  * On entry, no locks should be held.
4576  * The locks acquired and held by strlock depends on a few factors.
4577  * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4578  *   and held on exit and all sq_count are at an acceptable level.
4579  * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4580  *   sd_refcnt being zero.
4581  */
4582 
4583 static void
strlock(struct stdata * stp,sqlist_t * sqlist)4584 strlock(struct stdata *stp, sqlist_t *sqlist)
4585 {
4586 	syncql_t *sql, *sql2;
4587 retry:
4588 	/*
4589 	 * Wait for any claimstr to go away.
4590 	 */
4591 	if (STRMATED(stp)) {
4592 		struct stdata *stp1, *stp2;
4593 
4594 		STRLOCKMATES(stp);
4595 		/*
4596 		 * Note that the selection of locking order is not
4597 		 * important, just that they are always acquired in
4598 		 * the same order.  To assure this, we choose this
4599 		 * order based on the value of the pointer, and since
4600 		 * the pointer will not change for the life of this
4601 		 * pair, we will always grab the locks in the same
4602 		 * order (and hence, prevent deadlocks).
4603 		 */
4604 		if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4605 			stp1 = stp;
4606 			stp2 = stp->sd_mate;
4607 		} else {
4608 			stp2 = stp;
4609 			stp1 = stp->sd_mate;
4610 		}
4611 		mutex_enter(&stp1->sd_reflock);
4612 		if (stp1->sd_refcnt > 0) {
4613 			STRUNLOCKMATES(stp);
4614 			cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4615 			mutex_exit(&stp1->sd_reflock);
4616 			goto retry;
4617 		}
4618 		mutex_enter(&stp2->sd_reflock);
4619 		if (stp2->sd_refcnt > 0) {
4620 			STRUNLOCKMATES(stp);
4621 			mutex_exit(&stp1->sd_reflock);
4622 			cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4623 			mutex_exit(&stp2->sd_reflock);
4624 			goto retry;
4625 		}
4626 		STREAM_PUTLOCKS_ENTER(stp1);
4627 		STREAM_PUTLOCKS_ENTER(stp2);
4628 	} else {
4629 		mutex_enter(&stp->sd_lock);
4630 		mutex_enter(&stp->sd_reflock);
4631 		while (stp->sd_refcnt > 0) {
4632 			mutex_exit(&stp->sd_lock);
4633 			cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4634 			if (mutex_tryenter(&stp->sd_lock) == 0) {
4635 				mutex_exit(&stp->sd_reflock);
4636 				mutex_enter(&stp->sd_lock);
4637 				mutex_enter(&stp->sd_reflock);
4638 			}
4639 		}
4640 		STREAM_PUTLOCKS_ENTER(stp);
4641 	}
4642 
4643 	if (sqlist == NULL)
4644 		return;
4645 
4646 	for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4647 		syncq_t *sq = sql->sql_sq;
4648 		uint16_t count;
4649 
4650 		mutex_enter(SQLOCK(sq));
4651 		count = sq->sq_count;
4652 		ASSERT(sq->sq_rmqcount <= count);
4653 		SQ_PUTLOCKS_ENTER(sq);
4654 		SUM_SQ_PUTCOUNTS(sq, count);
4655 		if (count == sq->sq_rmqcount)
4656 			continue;
4657 
4658 		/* Failed - drop all locks that we have acquired so far */
4659 		if (STRMATED(stp)) {
4660 			STREAM_PUTLOCKS_EXIT(stp);
4661 			STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4662 			STRUNLOCKMATES(stp);
4663 			mutex_exit(&stp->sd_reflock);
4664 			mutex_exit(&stp->sd_mate->sd_reflock);
4665 		} else {
4666 			STREAM_PUTLOCKS_EXIT(stp);
4667 			mutex_exit(&stp->sd_lock);
4668 			mutex_exit(&stp->sd_reflock);
4669 		}
4670 		for (sql2 = sqlist->sqlist_head; sql2 != sql;
4671 		    sql2 = sql2->sql_next) {
4672 			SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4673 			mutex_exit(SQLOCK(sql2->sql_sq));
4674 		}
4675 
4676 		/*
4677 		 * The wait loop below may starve when there are many threads
4678 		 * claiming the syncq. This is especially a problem with permod
4679 		 * syncqs (IP). To lessen the impact of the problem we increment
4680 		 * sq_needexcl and clear fastbits so that putnexts will slow
4681 		 * down and call sqenable instead of draining right away.
4682 		 */
4683 		sq->sq_needexcl++;
4684 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4685 		while (count > sq->sq_rmqcount) {
4686 			sq->sq_flags |= SQ_WANTWAKEUP;
4687 			SQ_PUTLOCKS_EXIT(sq);
4688 			cv_wait(&sq->sq_wait, SQLOCK(sq));
4689 			count = sq->sq_count;
4690 			SQ_PUTLOCKS_ENTER(sq);
4691 			SUM_SQ_PUTCOUNTS(sq, count);
4692 		}
4693 		sq->sq_needexcl--;
4694 		if (sq->sq_needexcl == 0)
4695 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4696 		SQ_PUTLOCKS_EXIT(sq);
4697 		ASSERT(count == sq->sq_rmqcount);
4698 		mutex_exit(SQLOCK(sq));
4699 		goto retry;
4700 	}
4701 }
4702 
4703 /*
4704  * Drop all the locks that strlock acquired.
4705  */
4706 static void
strunlock(struct stdata * stp,sqlist_t * sqlist)4707 strunlock(struct stdata *stp, sqlist_t *sqlist)
4708 {
4709 	syncql_t *sql;
4710 
4711 	if (STRMATED(stp)) {
4712 		STREAM_PUTLOCKS_EXIT(stp);
4713 		STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4714 		STRUNLOCKMATES(stp);
4715 		mutex_exit(&stp->sd_reflock);
4716 		mutex_exit(&stp->sd_mate->sd_reflock);
4717 	} else {
4718 		STREAM_PUTLOCKS_EXIT(stp);
4719 		mutex_exit(&stp->sd_lock);
4720 		mutex_exit(&stp->sd_reflock);
4721 	}
4722 
4723 	if (sqlist == NULL)
4724 		return;
4725 
4726 	for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4727 		SQ_PUTLOCKS_EXIT(sql->sql_sq);
4728 		mutex_exit(SQLOCK(sql->sql_sq));
4729 	}
4730 }
4731 
4732 /*
4733  * When the module has service procedure, we need check if the next
4734  * module which has service procedure is in flow control to trigger
4735  * the backenable.
4736  */
4737 static void
backenable_insertedq(queue_t * q)4738 backenable_insertedq(queue_t *q)
4739 {
4740 	qband_t	*qbp;
4741 
4742 	claimstr(q);
4743 	if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4744 		if (q->q_next->q_nfsrv->q_flag & QWANTW)
4745 			backenable(q, 0);
4746 
4747 		qbp = q->q_next->q_nfsrv->q_bandp;
4748 		for (; qbp != NULL; qbp = qbp->qb_next)
4749 			if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4750 				backenable(q, qbp->qb_first->b_band);
4751 	}
4752 	releasestr(q);
4753 }
4754 
4755 /*
4756  * Given two read queues, insert a new single one after another.
4757  *
4758  * This routine acquires all the necessary locks in order to change
4759  * q_next and related pointer using strlock().
4760  * It depends on the stream head ensuring that there are no concurrent
4761  * insertq or removeq on the same stream. The stream head ensures this
4762  * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4763  *
4764  * Note that no syncq locks are held during the q_next change. This is
4765  * applied to all streams since, unlike removeq, there is no problem of stale
4766  * pointers when adding a module to the stream. Thus drivers/modules that do a
4767  * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4768  * applied this optimization to all streams.
4769  */
4770 void
insertq(struct stdata * stp,queue_t * new)4771 insertq(struct stdata *stp, queue_t *new)
4772 {
4773 	queue_t	*after;
4774 	queue_t *wafter;
4775 	queue_t *wnew = _WR(new);
4776 	boolean_t have_fifo = B_FALSE;
4777 
4778 	if (new->q_flag & _QINSERTING) {
4779 		ASSERT(stp->sd_vnode->v_type != VFIFO);
4780 		after = new->q_next;
4781 		wafter = _WR(new->q_next);
4782 	} else {
4783 		after = _RD(stp->sd_wrq);
4784 		wafter = stp->sd_wrq;
4785 	}
4786 
4787 	TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4788 	    "insertq:%p, %p", after, new);
4789 	ASSERT(after->q_flag & QREADR);
4790 	ASSERT(new->q_flag & QREADR);
4791 
4792 	strlock(stp, NULL);
4793 
4794 	/* Do we have a FIFO? */
4795 	if (wafter->q_next == after) {
4796 		have_fifo = B_TRUE;
4797 		wnew->q_next = new;
4798 	} else {
4799 		wnew->q_next = wafter->q_next;
4800 	}
4801 	new->q_next = after;
4802 
4803 	set_nfsrv_ptr(new, wnew, after, wafter);
4804 	/*
4805 	 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4806 	 * so only reset this flag after calling it.
4807 	 */
4808 	new->q_flag &= ~_QINSERTING;
4809 
4810 	if (have_fifo) {
4811 		wafter->q_next = wnew;
4812 	} else {
4813 		if (wafter->q_next)
4814 			_OTHERQ(wafter->q_next)->q_next = new;
4815 		wafter->q_next = wnew;
4816 	}
4817 
4818 	set_qend(new);
4819 	/* The QEND flag might have to be updated for the upstream guy */
4820 	set_qend(after);
4821 
4822 	ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4823 	ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4824 	ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4825 	ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4826 	strsetuio(stp);
4827 
4828 	/*
4829 	 * If this was a module insertion, bump the push count.
4830 	 */
4831 	if (!(new->q_flag & QISDRV))
4832 		stp->sd_pushcnt++;
4833 
4834 	strunlock(stp, NULL);
4835 
4836 	/* check if the write Q needs backenable */
4837 	backenable_insertedq(wnew);
4838 
4839 	/* check if the read Q needs backenable */
4840 	backenable_insertedq(new);
4841 }
4842 
4843 /*
4844  * Given a read queue, unlink it from any neighbors.
4845  *
4846  * This routine acquires all the necessary locks in order to
4847  * change q_next and related pointers and also guard against
4848  * stale references (e.g. through q_next) to the queue that
4849  * is being removed. It also plays part of the role in ensuring
4850  * that the module's/driver's put procedure doesn't get called
4851  * after qprocsoff returns.
4852  *
4853  * Removeq depends on the stream head ensuring that there are
4854  * no concurrent insertq or removeq on the same stream. The
4855  * stream head ensures this using the flags STWOPEN, STRCLOSE and
4856  * STRPLUMB.
4857  *
4858  * The set of locks needed to remove the queue is different in
4859  * different cases:
4860  *
4861  * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4862  * waiting for the syncq reference count to drop to 0 indicating that no
4863  * non-close threads are present anywhere in the stream. This ensures that any
4864  * module/driver can reference q_next in its open, close, put, or service
4865  * procedures.
4866  *
4867  * The sq_rmqcount counter tracks the number of threads inside removeq().
4868  * strlock() ensures that there is either no threads executing inside perimeter
4869  * or there is only a thread calling qprocsoff().
4870  *
4871  * strlock() compares the value of sq_count with the number of threads inside
4872  * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4873  * any threads waiting in strlock() when the sq_rmqcount increases.
4874  */
4875 
4876 void
removeq(queue_t * qp)4877 removeq(queue_t *qp)
4878 {
4879 	queue_t *wqp = _WR(qp);
4880 	struct stdata *stp = STREAM(qp);
4881 	sqlist_t *sqlist = NULL;
4882 	boolean_t isdriver;
4883 	int moved;
4884 	syncq_t *sq = qp->q_syncq;
4885 	syncq_t *wsq = wqp->q_syncq;
4886 
4887 	ASSERT(stp);
4888 
4889 	TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4890 	    "removeq:%p %p", qp, wqp);
4891 	ASSERT(qp->q_flag&QREADR);
4892 
4893 	/*
4894 	 * For queues using Synchronous streams, we must wait for all threads in
4895 	 * rwnext() to drain out before proceeding.
4896 	 */
4897 	if (qp->q_flag & QSYNCSTR) {
4898 		/* First, we need wakeup any threads blocked in rwnext() */
4899 		mutex_enter(SQLOCK(sq));
4900 		if (sq->sq_flags & SQ_WANTWAKEUP) {
4901 			sq->sq_flags &= ~SQ_WANTWAKEUP;
4902 			cv_broadcast(&sq->sq_wait);
4903 		}
4904 		mutex_exit(SQLOCK(sq));
4905 
4906 		if (wsq != sq) {
4907 			mutex_enter(SQLOCK(wsq));
4908 			if (wsq->sq_flags & SQ_WANTWAKEUP) {
4909 				wsq->sq_flags &= ~SQ_WANTWAKEUP;
4910 				cv_broadcast(&wsq->sq_wait);
4911 			}
4912 			mutex_exit(SQLOCK(wsq));
4913 		}
4914 
4915 		mutex_enter(QLOCK(qp));
4916 		while (qp->q_rwcnt > 0) {
4917 			qp->q_flag |= QWANTRMQSYNC;
4918 			cv_wait(&qp->q_wait, QLOCK(qp));
4919 		}
4920 		mutex_exit(QLOCK(qp));
4921 
4922 		mutex_enter(QLOCK(wqp));
4923 		while (wqp->q_rwcnt > 0) {
4924 			wqp->q_flag |= QWANTRMQSYNC;
4925 			cv_wait(&wqp->q_wait, QLOCK(wqp));
4926 		}
4927 		mutex_exit(QLOCK(wqp));
4928 	}
4929 
4930 	mutex_enter(SQLOCK(sq));
4931 	sq->sq_rmqcount++;
4932 	if (sq->sq_flags & SQ_WANTWAKEUP) {
4933 		sq->sq_flags &= ~SQ_WANTWAKEUP;
4934 		cv_broadcast(&sq->sq_wait);
4935 	}
4936 	mutex_exit(SQLOCK(sq));
4937 
4938 	isdriver = (qp->q_flag & QISDRV);
4939 
4940 	sqlist = sqlist_build(qp, stp, STRMATED(stp));
4941 	strlock(stp, sqlist);
4942 
4943 	reset_nfsrv_ptr(qp, wqp);
4944 
4945 	ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4946 	ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4947 	/* Do we have a FIFO? */
4948 	if (wqp->q_next == qp) {
4949 		stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4950 	} else {
4951 		if (wqp->q_next)
4952 			backq(qp)->q_next = qp->q_next;
4953 		if (qp->q_next)
4954 			backq(wqp)->q_next = wqp->q_next;
4955 	}
4956 
4957 	/* The QEND flag might have to be updated for the upstream guy */
4958 	if (qp->q_next)
4959 		set_qend(qp->q_next);
4960 
4961 	ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4962 	ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4963 
4964 	/*
4965 	 * Move any messages destined for the put procedures to the next
4966 	 * syncq in line. Otherwise free them.
4967 	 */
4968 	moved = 0;
4969 	/*
4970 	 * Quick check to see whether there are any messages or events.
4971 	 */
4972 	if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4973 		moved += propagate_syncq(qp);
4974 	if (wqp->q_syncqmsgs != 0 ||
4975 	    (wqp->q_syncq->sq_flags & SQ_EVENTS))
4976 		moved += propagate_syncq(wqp);
4977 
4978 	strsetuio(stp);
4979 
4980 	/*
4981 	 * If this was a module removal, decrement the push count.
4982 	 */
4983 	if (!isdriver)
4984 		stp->sd_pushcnt--;
4985 
4986 	strunlock(stp, sqlist);
4987 	sqlist_free(sqlist);
4988 
4989 	/*
4990 	 * Make sure any messages that were propagated are drained.
4991 	 * Also clear any QFULL bit caused by messages that were propagated.
4992 	 */
4993 
4994 	if (qp->q_next != NULL) {
4995 		clr_qfull(qp);
4996 		/*
4997 		 * For the driver calling qprocsoff, propagate_syncq
4998 		 * frees all the messages instead of putting it in
4999 		 * the stream head
5000 		 */
5001 		if (!isdriver && (moved > 0))
5002 			emptysq(qp->q_next->q_syncq);
5003 	}
5004 	if (wqp->q_next != NULL) {
5005 		clr_qfull(wqp);
5006 		/*
5007 		 * We come here for any pop of a module except for the
5008 		 * case of driver being removed. We don't call emptysq
5009 		 * if we did not move any messages. This will avoid holding
5010 		 * PERMOD syncq locks in emptysq
5011 		 */
5012 		if (moved > 0)
5013 			emptysq(wqp->q_next->q_syncq);
5014 	}
5015 
5016 	mutex_enter(SQLOCK(sq));
5017 	sq->sq_rmqcount--;
5018 	mutex_exit(SQLOCK(sq));
5019 }
5020 
5021 /*
5022  * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
5023  * SQ_WRITER) on a syncq.
5024  * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5025  * sync queue and waits until sq_count reaches maxcnt.
5026  *
5027  * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5028  * does not care about putnext threads that are in the middle of calling put
5029  * entry points.
5030  *
5031  * This routine is used for both inner and outer syncqs.
5032  */
5033 static void
blocksq(syncq_t * sq,ushort_t flag,int maxcnt)5034 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5035 {
5036 	uint16_t count = 0;
5037 
5038 	mutex_enter(SQLOCK(sq));
5039 	/*
5040 	 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5041 	 * SQ_FROZEN will be set if there is a frozen stream that has a
5042 	 * queue which also refers to this "shared" syncq.
5043 	 * SQ_BLOCKED will be set if there is "off" queue which also
5044 	 * refers to this "shared" syncq.
5045 	 */
5046 	if (maxcnt != -1) {
5047 		count = sq->sq_count;
5048 		SQ_PUTLOCKS_ENTER(sq);
5049 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5050 		SUM_SQ_PUTCOUNTS(sq, count);
5051 	}
5052 	sq->sq_needexcl++;
5053 	ASSERT(sq->sq_needexcl != 0);	/* wraparound */
5054 
5055 	while ((sq->sq_flags & flag) ||
5056 	    (maxcnt != -1 && count > (unsigned)maxcnt)) {
5057 		sq->sq_flags |= SQ_WANTWAKEUP;
5058 		if (maxcnt != -1) {
5059 			SQ_PUTLOCKS_EXIT(sq);
5060 		}
5061 		cv_wait(&sq->sq_wait, SQLOCK(sq));
5062 		if (maxcnt != -1) {
5063 			count = sq->sq_count;
5064 			SQ_PUTLOCKS_ENTER(sq);
5065 			SUM_SQ_PUTCOUNTS(sq, count);
5066 		}
5067 	}
5068 	sq->sq_needexcl--;
5069 	sq->sq_flags |= flag;
5070 	ASSERT(maxcnt == -1 || count == maxcnt);
5071 	if (maxcnt != -1) {
5072 		if (sq->sq_needexcl == 0) {
5073 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5074 		}
5075 		SQ_PUTLOCKS_EXIT(sq);
5076 	} else if (sq->sq_needexcl == 0) {
5077 		SQ_PUTCOUNT_SETFAST(sq);
5078 	}
5079 
5080 	mutex_exit(SQLOCK(sq));
5081 }
5082 
5083 /*
5084  * Reset a flag that was set with blocksq.
5085  *
5086  * Can not use this routine to reset SQ_WRITER.
5087  *
5088  * If "isouter" is set then the syncq is assumed to be an outer perimeter
5089  * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5090  * to handle the queued qwriter operations.
5091  *
5092  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5093  * sq_putlocks are used.
5094  */
5095 static void
unblocksq(syncq_t * sq,uint16_t resetflag,int isouter)5096 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5097 {
5098 	uint16_t flags;
5099 
5100 	mutex_enter(SQLOCK(sq));
5101 	ASSERT(resetflag != SQ_WRITER);
5102 	ASSERT(sq->sq_flags & resetflag);
5103 	flags = sq->sq_flags & ~resetflag;
5104 	sq->sq_flags = flags;
5105 	if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5106 		if (flags & SQ_WANTWAKEUP) {
5107 			flags &= ~SQ_WANTWAKEUP;
5108 			cv_broadcast(&sq->sq_wait);
5109 		}
5110 		sq->sq_flags = flags;
5111 		if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5112 			if (!isouter) {
5113 				/* drain_syncq drops SQLOCK */
5114 				drain_syncq(sq);
5115 				return;
5116 			}
5117 		}
5118 	}
5119 	mutex_exit(SQLOCK(sq));
5120 }
5121 
5122 /*
5123  * Reset a flag that was set with blocksq.
5124  * Does not drain the syncq. Use emptysq() for that.
5125  * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5126  *
5127  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5128  * sq_putlocks are used.
5129  */
5130 static int
dropsq(syncq_t * sq,uint16_t resetflag)5131 dropsq(syncq_t *sq, uint16_t resetflag)
5132 {
5133 	uint16_t flags;
5134 
5135 	mutex_enter(SQLOCK(sq));
5136 	ASSERT(sq->sq_flags & resetflag);
5137 	flags = sq->sq_flags & ~resetflag;
5138 	if (flags & SQ_WANTWAKEUP) {
5139 		flags &= ~SQ_WANTWAKEUP;
5140 		cv_broadcast(&sq->sq_wait);
5141 	}
5142 	sq->sq_flags = flags;
5143 	mutex_exit(SQLOCK(sq));
5144 	if (flags & SQ_QUEUED)
5145 		return (1);
5146 	return (0);
5147 }
5148 
5149 /*
5150  * Empty all the messages on a syncq.
5151  *
5152  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5153  * sq_putlocks are used.
5154  */
5155 static void
emptysq(syncq_t * sq)5156 emptysq(syncq_t *sq)
5157 {
5158 	uint16_t flags;
5159 
5160 	mutex_enter(SQLOCK(sq));
5161 	flags = sq->sq_flags;
5162 	if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5163 		/*
5164 		 * To prevent potential recursive invocation of drain_syncq we
5165 		 * do not call drain_syncq if count is non-zero.
5166 		 */
5167 		if (sq->sq_count == 0) {
5168 			/* drain_syncq() drops SQLOCK */
5169 			drain_syncq(sq);
5170 			return;
5171 		} else
5172 			sqenable(sq);
5173 	}
5174 	mutex_exit(SQLOCK(sq));
5175 }
5176 
5177 /*
5178  * Ordered insert while removing duplicates.
5179  */
5180 static void
sqlist_insert(sqlist_t * sqlist,syncq_t * sqp)5181 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5182 {
5183 	syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5184 
5185 	prev_sqlpp = &sqlist->sqlist_head;
5186 	while ((sqlp = *prev_sqlpp) != NULL) {
5187 		if (sqlp->sql_sq >= sqp) {
5188 			if (sqlp->sql_sq == sqp)	/* duplicate */
5189 				return;
5190 			break;
5191 		}
5192 		prev_sqlpp = &sqlp->sql_next;
5193 	}
5194 	new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5195 	ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5196 	new_sqlp->sql_next = sqlp;
5197 	new_sqlp->sql_sq = sqp;
5198 	*prev_sqlpp = new_sqlp;
5199 }
5200 
5201 /*
5202  * Walk the write side queues until we hit either the driver
5203  * or a twist in the stream (_SAMESTR will return false in both
5204  * these cases) then turn around and walk the read side queues
5205  * back up to the stream head.
5206  */
5207 static void
sqlist_insertall(sqlist_t * sqlist,queue_t * q)5208 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5209 {
5210 	while (q != NULL) {
5211 		sqlist_insert(sqlist, q->q_syncq);
5212 
5213 		if (_SAMESTR(q))
5214 			q = q->q_next;
5215 		else if (!(q->q_flag & QREADR))
5216 			q = _RD(q);
5217 		else
5218 			q = NULL;
5219 	}
5220 }
5221 
5222 /*
5223  * Allocate and build a list of all syncqs in a stream and the syncq(s)
5224  * associated with the "q" parameter. The resulting list is sorted in a
5225  * canonical order and is free of duplicates.
5226  * Assumes the passed queue is a _RD(q).
5227  */
5228 static sqlist_t *
sqlist_build(queue_t * q,struct stdata * stp,boolean_t do_twist)5229 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5230 {
5231 	sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5232 
5233 	/*
5234 	 * start with the current queue/qpair
5235 	 */
5236 	ASSERT(q->q_flag & QREADR);
5237 
5238 	sqlist_insert(sqlist, q->q_syncq);
5239 	sqlist_insert(sqlist, _WR(q)->q_syncq);
5240 
5241 	sqlist_insertall(sqlist, stp->sd_wrq);
5242 	if (do_twist)
5243 		sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5244 
5245 	return (sqlist);
5246 }
5247 
5248 static sqlist_t *
sqlist_alloc(struct stdata * stp,int kmflag)5249 sqlist_alloc(struct stdata *stp, int kmflag)
5250 {
5251 	size_t sqlist_size;
5252 	sqlist_t *sqlist;
5253 
5254 	/*
5255 	 * Allocate 2 syncql_t's for each pushed module. Note that
5256 	 * the sqlist_t structure already has 4 syncql_t's built in:
5257 	 * 2 for the stream head, and 2 for the driver/other stream head.
5258 	 */
5259 	sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5260 	    sizeof (sqlist_t);
5261 	if (STRMATED(stp))
5262 		sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5263 	sqlist = kmem_alloc(sqlist_size, kmflag);
5264 
5265 	sqlist->sqlist_head = NULL;
5266 	sqlist->sqlist_size = sqlist_size;
5267 	sqlist->sqlist_index = 0;
5268 
5269 	return (sqlist);
5270 }
5271 
5272 /*
5273  * Free the list created by sqlist_alloc()
5274  */
5275 static void
sqlist_free(sqlist_t * sqlist)5276 sqlist_free(sqlist_t *sqlist)
5277 {
5278 	kmem_free(sqlist, sqlist->sqlist_size);
5279 }
5280 
5281 /*
5282  * Prevent any new entries into any syncq in this stream.
5283  * Used by freezestr.
5284  */
5285 void
strblock(queue_t * q)5286 strblock(queue_t *q)
5287 {
5288 	struct stdata	*stp;
5289 	syncql_t	*sql;
5290 	sqlist_t	*sqlist;
5291 
5292 	q = _RD(q);
5293 
5294 	stp = STREAM(q);
5295 	ASSERT(stp != NULL);
5296 
5297 	/*
5298 	 * Get a sorted list with all the duplicates removed containing
5299 	 * all the syncqs referenced by this stream.
5300 	 */
5301 	sqlist = sqlist_build(q, stp, B_FALSE);
5302 	for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5303 		blocksq(sql->sql_sq, SQ_FROZEN, -1);
5304 	sqlist_free(sqlist);
5305 }
5306 
5307 /*
5308  * Release the block on new entries into this stream
5309  */
5310 void
strunblock(queue_t * q)5311 strunblock(queue_t *q)
5312 {
5313 	struct stdata	*stp;
5314 	syncql_t	*sql;
5315 	sqlist_t	*sqlist;
5316 	int		drain_needed;
5317 
5318 	q = _RD(q);
5319 
5320 	/*
5321 	 * Get a sorted list with all the duplicates removed containing
5322 	 * all the syncqs referenced by this stream.
5323 	 * Have to drop the SQ_FROZEN flag on all the syncqs before
5324 	 * starting to drain them; otherwise the draining might
5325 	 * cause a freezestr in some module on the stream (which
5326 	 * would deadlock).
5327 	 */
5328 	stp = STREAM(q);
5329 	ASSERT(stp != NULL);
5330 	sqlist = sqlist_build(q, stp, B_FALSE);
5331 	drain_needed = 0;
5332 	for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5333 		drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5334 	if (drain_needed) {
5335 		for (sql = sqlist->sqlist_head; sql != NULL;
5336 		    sql = sql->sql_next)
5337 			emptysq(sql->sql_sq);
5338 	}
5339 	sqlist_free(sqlist);
5340 }
5341 
5342 #ifdef DEBUG
5343 static int
qprocsareon(queue_t * rq)5344 qprocsareon(queue_t *rq)
5345 {
5346 	if (rq->q_next == NULL)
5347 		return (0);
5348 	return (_WR(rq->q_next)->q_next == _WR(rq));
5349 }
5350 
5351 int
qclaimed(queue_t * q)5352 qclaimed(queue_t *q)
5353 {
5354 	uint_t count;
5355 
5356 	count = q->q_syncq->sq_count;
5357 	SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5358 	return (count != 0);
5359 }
5360 
5361 /*
5362  * Check if anyone has frozen this stream with freezestr
5363  */
5364 int
frozenstr(queue_t * q)5365 frozenstr(queue_t *q)
5366 {
5367 	return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5368 }
5369 #endif /* DEBUG */
5370 
5371 /*
5372  * Enter a queue.
5373  * Obsoleted interface. Should not be used.
5374  */
5375 void
enterq(queue_t * q)5376 enterq(queue_t *q)
5377 {
5378 	entersq(q->q_syncq, SQ_CALLBACK);
5379 }
5380 
5381 void
leaveq(queue_t * q)5382 leaveq(queue_t *q)
5383 {
5384 	leavesq(q->q_syncq, SQ_CALLBACK);
5385 }
5386 
5387 /*
5388  * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5389  * to check.
5390  * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5391  * calls and the running of open, close and service procedures.
5392  *
5393  * If c_inner bit is set no need to grab sq_putlocks since we don't care
5394  * if other threads have entered or are entering put entry point.
5395  *
5396  * If c_inner bit is set it might have been possible to use
5397  * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5398  * open/close path for IP) but since the count may need to be decremented in
5399  * qwait() we wouldn't know which counter to decrement. Currently counter is
5400  * selected by current cpu_seqid and current CPU can change at any moment. XXX
5401  * in the future we might use curthread id bits to select the counter and this
5402  * would stay constant across routine calls.
5403  */
5404 void
entersq(syncq_t * sq,int entrypoint)5405 entersq(syncq_t *sq, int entrypoint)
5406 {
5407 	uint16_t	count = 0;
5408 	uint16_t	flags;
5409 	uint16_t	waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5410 	uint16_t	type;
5411 	uint_t		c_inner = entrypoint & SQ_CI;
5412 	uint_t		c_outer = entrypoint & SQ_CO;
5413 
5414 	/*
5415 	 * Increment ref count to keep closes out of this queue.
5416 	 */
5417 	ASSERT(sq);
5418 	ASSERT(c_inner && c_outer);
5419 	mutex_enter(SQLOCK(sq));
5420 	flags = sq->sq_flags;
5421 	type = sq->sq_type;
5422 	if (!(type & c_inner)) {
5423 		/* Make sure all putcounts now use slowlock. */
5424 		count = sq->sq_count;
5425 		SQ_PUTLOCKS_ENTER(sq);
5426 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5427 		SUM_SQ_PUTCOUNTS(sq, count);
5428 		sq->sq_needexcl++;
5429 		ASSERT(sq->sq_needexcl != 0);	/* wraparound */
5430 		waitflags |= SQ_MESSAGES;
5431 	}
5432 	/*
5433 	 * Wait until we can enter the inner perimeter.
5434 	 * If we want exclusive access we wait until sq_count is 0.
5435 	 * We have to do this before entering the outer perimeter in order
5436 	 * to preserve put/close message ordering.
5437 	 */
5438 	while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5439 		sq->sq_flags = flags | SQ_WANTWAKEUP;
5440 		if (!(type & c_inner)) {
5441 			SQ_PUTLOCKS_EXIT(sq);
5442 		}
5443 		cv_wait(&sq->sq_wait, SQLOCK(sq));
5444 		if (!(type & c_inner)) {
5445 			count = sq->sq_count;
5446 			SQ_PUTLOCKS_ENTER(sq);
5447 			SUM_SQ_PUTCOUNTS(sq, count);
5448 		}
5449 		flags = sq->sq_flags;
5450 	}
5451 
5452 	if (!(type & c_inner)) {
5453 		ASSERT(sq->sq_needexcl > 0);
5454 		sq->sq_needexcl--;
5455 		if (sq->sq_needexcl == 0) {
5456 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5457 		}
5458 	}
5459 
5460 	/* Check if we need to enter the outer perimeter */
5461 	if (!(type & c_outer)) {
5462 		/*
5463 		 * We have to enter the outer perimeter exclusively before
5464 		 * we can increment sq_count to avoid deadlock. This implies
5465 		 * that we have to re-check sq_flags and sq_count.
5466 		 *
5467 		 * is it possible to have c_inner set when c_outer is not set?
5468 		 */
5469 		if (!(type & c_inner)) {
5470 			SQ_PUTLOCKS_EXIT(sq);
5471 		}
5472 		mutex_exit(SQLOCK(sq));
5473 		outer_enter(sq->sq_outer, SQ_GOAWAY);
5474 		mutex_enter(SQLOCK(sq));
5475 		flags = sq->sq_flags;
5476 		/*
5477 		 * there should be no need to recheck sq_putcounts
5478 		 * because outer_enter() has already waited for them to clear
5479 		 * after setting SQ_WRITER.
5480 		 */
5481 		count = sq->sq_count;
5482 #ifdef DEBUG
5483 		/*
5484 		 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5485 		 * of doing an ASSERT internally. Others should do
5486 		 * something like
5487 		 *	 ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5488 		 * without the need to #ifdef DEBUG it.
5489 		 */
5490 		SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5491 #endif
5492 		while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5493 		    (!(type & c_inner) && count != 0)) {
5494 			sq->sq_flags = flags | SQ_WANTWAKEUP;
5495 			cv_wait(&sq->sq_wait, SQLOCK(sq));
5496 			count = sq->sq_count;
5497 			flags = sq->sq_flags;
5498 		}
5499 	}
5500 
5501 	sq->sq_count++;
5502 	ASSERT(sq->sq_count != 0);	/* Wraparound */
5503 	if (!(type & c_inner)) {
5504 		/* Exclusive entry */
5505 		ASSERT(sq->sq_count == 1);
5506 		sq->sq_flags |= SQ_EXCL;
5507 		if (type & c_outer) {
5508 			SQ_PUTLOCKS_EXIT(sq);
5509 		}
5510 	}
5511 	mutex_exit(SQLOCK(sq));
5512 }
5513 
5514 /*
5515  * Leave a syncq. Announce to framework that closes may proceed.
5516  * c_inner and c_outer specify which concurrency bits to check.
5517  *
5518  * Must never be called from driver or module put entry point.
5519  *
5520  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5521  * sq_putlocks are used.
5522  */
5523 void
leavesq(syncq_t * sq,int entrypoint)5524 leavesq(syncq_t *sq, int entrypoint)
5525 {
5526 	uint16_t	flags;
5527 	uint16_t	type;
5528 	uint_t		c_outer = entrypoint & SQ_CO;
5529 #ifdef DEBUG
5530 	uint_t		c_inner = entrypoint & SQ_CI;
5531 #endif
5532 
5533 	/*
5534 	 * Decrement ref count, drain the syncq if possible, and wake up
5535 	 * any waiting close.
5536 	 */
5537 	ASSERT(sq);
5538 	ASSERT(c_inner && c_outer);
5539 	mutex_enter(SQLOCK(sq));
5540 	flags = sq->sq_flags;
5541 	type = sq->sq_type;
5542 	if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5543 
5544 		if (flags & SQ_WANTWAKEUP) {
5545 			flags &= ~SQ_WANTWAKEUP;
5546 			cv_broadcast(&sq->sq_wait);
5547 		}
5548 		if (flags & SQ_WANTEXWAKEUP) {
5549 			flags &= ~SQ_WANTEXWAKEUP;
5550 			cv_broadcast(&sq->sq_exitwait);
5551 		}
5552 
5553 		if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5554 			/*
5555 			 * The syncq needs to be drained. "Exit" the syncq
5556 			 * before calling drain_syncq.
5557 			 */
5558 			ASSERT(sq->sq_count != 0);
5559 			sq->sq_count--;
5560 			ASSERT((flags & SQ_EXCL) || (type & c_inner));
5561 			sq->sq_flags = flags & ~SQ_EXCL;
5562 			drain_syncq(sq);
5563 			ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5564 			/* Check if we need to exit the outer perimeter */
5565 			/* XXX will this ever be true? */
5566 			if (!(type & c_outer))
5567 				outer_exit(sq->sq_outer);
5568 			return;
5569 		}
5570 	}
5571 	ASSERT(sq->sq_count != 0);
5572 	sq->sq_count--;
5573 	ASSERT((flags & SQ_EXCL) || (type & c_inner));
5574 	sq->sq_flags = flags & ~SQ_EXCL;
5575 	mutex_exit(SQLOCK(sq));
5576 
5577 	/* Check if we need to exit the outer perimeter */
5578 	if (!(sq->sq_type & c_outer))
5579 		outer_exit(sq->sq_outer);
5580 }
5581 
5582 /*
5583  * Prevent q_next from changing in this stream by incrementing sq_count.
5584  *
5585  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5586  * sq_putlocks are used.
5587  */
5588 void
claimq(queue_t * qp)5589 claimq(queue_t *qp)
5590 {
5591 	syncq_t	*sq = qp->q_syncq;
5592 
5593 	mutex_enter(SQLOCK(sq));
5594 	sq->sq_count++;
5595 	ASSERT(sq->sq_count != 0);	/* Wraparound */
5596 	mutex_exit(SQLOCK(sq));
5597 }
5598 
5599 /*
5600  * Undo claimq.
5601  *
5602  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5603  * sq_putlocks are used.
5604  */
5605 void
releaseq(queue_t * qp)5606 releaseq(queue_t *qp)
5607 {
5608 	syncq_t	*sq = qp->q_syncq;
5609 	uint16_t flags;
5610 
5611 	mutex_enter(SQLOCK(sq));
5612 	ASSERT(sq->sq_count > 0);
5613 	sq->sq_count--;
5614 
5615 	flags = sq->sq_flags;
5616 	if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5617 		if (flags & SQ_WANTWAKEUP) {
5618 			flags &= ~SQ_WANTWAKEUP;
5619 			cv_broadcast(&sq->sq_wait);
5620 		}
5621 		sq->sq_flags = flags;
5622 		if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5623 			/*
5624 			 * To prevent potential recursive invocation of
5625 			 * drain_syncq we do not call drain_syncq if count is
5626 			 * non-zero.
5627 			 */
5628 			if (sq->sq_count == 0) {
5629 				drain_syncq(sq);
5630 				return;
5631 			} else
5632 				sqenable(sq);
5633 		}
5634 	}
5635 	mutex_exit(SQLOCK(sq));
5636 }
5637 
5638 /*
5639  * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5640  */
5641 void
claimstr(queue_t * qp)5642 claimstr(queue_t *qp)
5643 {
5644 	struct stdata *stp = STREAM(qp);
5645 
5646 	mutex_enter(&stp->sd_reflock);
5647 	stp->sd_refcnt++;
5648 	ASSERT(stp->sd_refcnt != 0);	/* Wraparound */
5649 	mutex_exit(&stp->sd_reflock);
5650 }
5651 
5652 /*
5653  * Undo claimstr.
5654  */
5655 void
releasestr(queue_t * qp)5656 releasestr(queue_t *qp)
5657 {
5658 	struct stdata *stp = STREAM(qp);
5659 
5660 	mutex_enter(&stp->sd_reflock);
5661 	ASSERT(stp->sd_refcnt != 0);
5662 	if (--stp->sd_refcnt == 0)
5663 		cv_broadcast(&stp->sd_refmonitor);
5664 	mutex_exit(&stp->sd_reflock);
5665 }
5666 
5667 static syncq_t *
new_syncq(void)5668 new_syncq(void)
5669 {
5670 	return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5671 }
5672 
5673 static void
free_syncq(syncq_t * sq)5674 free_syncq(syncq_t *sq)
5675 {
5676 	ASSERT(sq->sq_head == NULL);
5677 	ASSERT(sq->sq_outer == NULL);
5678 	ASSERT(sq->sq_callbpend == NULL);
5679 	ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5680 	    (sq->sq_onext == sq && sq->sq_oprev == sq));
5681 
5682 	if (sq->sq_ciputctrl != NULL) {
5683 		ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5684 		SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5685 		    sq->sq_nciputctrl, 0);
5686 		ASSERT(ciputctrl_cache != NULL);
5687 		kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5688 	}
5689 
5690 	sq->sq_tail = NULL;
5691 	sq->sq_evhead = NULL;
5692 	sq->sq_evtail = NULL;
5693 	sq->sq_ciputctrl = NULL;
5694 	sq->sq_nciputctrl = 0;
5695 	sq->sq_count = 0;
5696 	sq->sq_rmqcount = 0;
5697 	sq->sq_callbflags = 0;
5698 	sq->sq_cancelid = 0;
5699 	sq->sq_next = NULL;
5700 	sq->sq_needexcl = 0;
5701 	sq->sq_svcflags = 0;
5702 	sq->sq_nqueues = 0;
5703 	sq->sq_pri = 0;
5704 	sq->sq_onext = NULL;
5705 	sq->sq_oprev = NULL;
5706 	sq->sq_flags = 0;
5707 	sq->sq_type = 0;
5708 	sq->sq_servcount = 0;
5709 
5710 	kmem_cache_free(syncq_cache, sq);
5711 }
5712 
5713 /* Outer perimeter code */
5714 
5715 /*
5716  * The outer syncq uses the fields and flags in the syncq slightly
5717  * differently from the inner syncqs.
5718  *	sq_count	Incremented when there are pending or running
5719  *			writers at the outer perimeter to prevent the set of
5720  *			inner syncqs that belong to the outer perimeter from
5721  *			changing.
5722  *	sq_head/tail	List of deferred qwriter(OUTER) operations.
5723  *
5724  *	SQ_BLOCKED	Set to prevent traversing of sq_next,sq_prev while
5725  *			inner syncqs are added to or removed from the
5726  *			outer perimeter.
5727  *	SQ_QUEUED	sq_head/tail has messages or events queued.
5728  *
5729  *	SQ_WRITER	A thread is currently traversing all the inner syncqs
5730  *			setting the SQ_WRITER flag.
5731  */
5732 
5733 /*
5734  * Get write access at the outer perimeter.
5735  * Note that read access is done by entersq, putnext, and put by simply
5736  * incrementing sq_count in the inner syncq.
5737  *
5738  * Waits until "flags" is no longer set in the outer to prevent multiple
5739  * threads from having write access at the same time. SQ_WRITER has to be part
5740  * of "flags".
5741  *
5742  * Increases sq_count on the outer syncq to keep away outer_insert/remove
5743  * until the outer_exit is finished.
5744  *
5745  * outer_enter is vulnerable to starvation since it does not prevent new
5746  * threads from entering the inner syncqs while it is waiting for sq_count to
5747  * go to zero.
5748  */
5749 void
outer_enter(syncq_t * outer,uint16_t flags)5750 outer_enter(syncq_t *outer, uint16_t flags)
5751 {
5752 	syncq_t	*sq;
5753 	int	wait_needed;
5754 	uint16_t	count;
5755 
5756 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5757 	    outer->sq_oprev != NULL);
5758 	ASSERT(flags & SQ_WRITER);
5759 
5760 retry:
5761 	mutex_enter(SQLOCK(outer));
5762 	while (outer->sq_flags & flags) {
5763 		outer->sq_flags |= SQ_WANTWAKEUP;
5764 		cv_wait(&outer->sq_wait, SQLOCK(outer));
5765 	}
5766 
5767 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5768 	outer->sq_flags |= SQ_WRITER;
5769 	outer->sq_count++;
5770 	ASSERT(outer->sq_count != 0);	/* wraparound */
5771 	wait_needed = 0;
5772 	/*
5773 	 * Set SQ_WRITER on all the inner syncqs while holding
5774 	 * the SQLOCK on the outer syncq. This ensures that the changing
5775 	 * of SQ_WRITER is atomic under the outer SQLOCK.
5776 	 */
5777 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5778 		mutex_enter(SQLOCK(sq));
5779 		count = sq->sq_count;
5780 		SQ_PUTLOCKS_ENTER(sq);
5781 		sq->sq_flags |= SQ_WRITER;
5782 		SUM_SQ_PUTCOUNTS(sq, count);
5783 		if (count != 0)
5784 			wait_needed = 1;
5785 		SQ_PUTLOCKS_EXIT(sq);
5786 		mutex_exit(SQLOCK(sq));
5787 	}
5788 	mutex_exit(SQLOCK(outer));
5789 
5790 	/*
5791 	 * Get everybody out of the syncqs sequentially.
5792 	 * Note that we don't actually need to acquire the PUTLOCKS, since
5793 	 * we have already cleared the fastbit, and set QWRITER.  By
5794 	 * definition, the count can not increase since putnext will
5795 	 * take the slowlock path (and the purpose of acquiring the
5796 	 * putlocks was to make sure it didn't increase while we were
5797 	 * waiting).
5798 	 *
5799 	 * Note that we still acquire the PUTLOCKS to be safe.
5800 	 */
5801 	if (wait_needed) {
5802 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5803 			mutex_enter(SQLOCK(sq));
5804 			count = sq->sq_count;
5805 			SQ_PUTLOCKS_ENTER(sq);
5806 			SUM_SQ_PUTCOUNTS(sq, count);
5807 			while (count != 0) {
5808 				sq->sq_flags |= SQ_WANTWAKEUP;
5809 				SQ_PUTLOCKS_EXIT(sq);
5810 				cv_wait(&sq->sq_wait, SQLOCK(sq));
5811 				count = sq->sq_count;
5812 				SQ_PUTLOCKS_ENTER(sq);
5813 				SUM_SQ_PUTCOUNTS(sq, count);
5814 			}
5815 			SQ_PUTLOCKS_EXIT(sq);
5816 			mutex_exit(SQLOCK(sq));
5817 		}
5818 		/*
5819 		 * Verify that none of the flags got set while we
5820 		 * were waiting for the sq_counts to drop.
5821 		 * If this happens we exit and retry entering the
5822 		 * outer perimeter.
5823 		 */
5824 		mutex_enter(SQLOCK(outer));
5825 		if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5826 			mutex_exit(SQLOCK(outer));
5827 			outer_exit(outer);
5828 			goto retry;
5829 		}
5830 		mutex_exit(SQLOCK(outer));
5831 	}
5832 }
5833 
5834 /*
5835  * Drop the write access at the outer perimeter.
5836  * Read access is dropped implicitly (by putnext, put, and leavesq) by
5837  * decrementing sq_count.
5838  */
5839 void
outer_exit(syncq_t * outer)5840 outer_exit(syncq_t *outer)
5841 {
5842 	syncq_t	*sq;
5843 	int	 drain_needed;
5844 	uint16_t flags;
5845 
5846 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5847 	    outer->sq_oprev != NULL);
5848 	ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5849 
5850 	/*
5851 	 * Atomically (from the perspective of threads calling become_writer)
5852 	 * drop the write access at the outer perimeter by holding
5853 	 * SQLOCK(outer) across all the dropsq calls and the resetting of
5854 	 * SQ_WRITER.
5855 	 * This defines a locking order between the outer perimeter
5856 	 * SQLOCK and the inner perimeter SQLOCKs.
5857 	 */
5858 	mutex_enter(SQLOCK(outer));
5859 	flags = outer->sq_flags;
5860 	ASSERT(outer->sq_flags & SQ_WRITER);
5861 	if (flags & SQ_QUEUED) {
5862 		write_now(outer);
5863 		flags = outer->sq_flags;
5864 	}
5865 
5866 	/*
5867 	 * sq_onext is stable since sq_count has not yet been decreased.
5868 	 * Reset the SQ_WRITER flags in all syncqs.
5869 	 * After dropping SQ_WRITER on the outer syncq we empty all the
5870 	 * inner syncqs.
5871 	 */
5872 	drain_needed = 0;
5873 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5874 		drain_needed += dropsq(sq, SQ_WRITER);
5875 	ASSERT(!(outer->sq_flags & SQ_QUEUED));
5876 	flags &= ~SQ_WRITER;
5877 	if (drain_needed) {
5878 		outer->sq_flags = flags;
5879 		mutex_exit(SQLOCK(outer));
5880 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5881 			emptysq(sq);
5882 		mutex_enter(SQLOCK(outer));
5883 		flags = outer->sq_flags;
5884 	}
5885 	if (flags & SQ_WANTWAKEUP) {
5886 		flags &= ~SQ_WANTWAKEUP;
5887 		cv_broadcast(&outer->sq_wait);
5888 	}
5889 	outer->sq_flags = flags;
5890 	ASSERT(outer->sq_count > 0);
5891 	outer->sq_count--;
5892 	mutex_exit(SQLOCK(outer));
5893 }
5894 
5895 /*
5896  * Add another syncq to an outer perimeter.
5897  * Block out all other access to the outer perimeter while it is being
5898  * changed using blocksq.
5899  * Assumes that the caller has *not* done an outer_enter.
5900  *
5901  * Vulnerable to starvation in blocksq.
5902  */
5903 static void
outer_insert(syncq_t * outer,syncq_t * sq)5904 outer_insert(syncq_t *outer, syncq_t *sq)
5905 {
5906 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5907 	    outer->sq_oprev != NULL);
5908 	ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5909 	    sq->sq_oprev == NULL);	/* Can't be in an outer perimeter */
5910 
5911 	/* Get exclusive access to the outer perimeter list */
5912 	blocksq(outer, SQ_BLOCKED, 0);
5913 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5914 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5915 
5916 	mutex_enter(SQLOCK(sq));
5917 	sq->sq_outer = outer;
5918 	outer->sq_onext->sq_oprev = sq;
5919 	sq->sq_onext = outer->sq_onext;
5920 	outer->sq_onext = sq;
5921 	sq->sq_oprev = outer;
5922 	mutex_exit(SQLOCK(sq));
5923 	unblocksq(outer, SQ_BLOCKED, 1);
5924 }
5925 
5926 /*
5927  * Remove a syncq from an outer perimeter.
5928  * Block out all other access to the outer perimeter while it is being
5929  * changed using blocksq.
5930  * Assumes that the caller has *not* done an outer_enter.
5931  *
5932  * Vulnerable to starvation in blocksq.
5933  */
5934 static void
outer_remove(syncq_t * outer,syncq_t * sq)5935 outer_remove(syncq_t *outer, syncq_t *sq)
5936 {
5937 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5938 	    outer->sq_oprev != NULL);
5939 	ASSERT(sq->sq_outer == outer);
5940 
5941 	/* Get exclusive access to the outer perimeter list */
5942 	blocksq(outer, SQ_BLOCKED, 0);
5943 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5944 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5945 
5946 	mutex_enter(SQLOCK(sq));
5947 	sq->sq_outer = NULL;
5948 	sq->sq_onext->sq_oprev = sq->sq_oprev;
5949 	sq->sq_oprev->sq_onext = sq->sq_onext;
5950 	sq->sq_oprev = sq->sq_onext = NULL;
5951 	mutex_exit(SQLOCK(sq));
5952 	unblocksq(outer, SQ_BLOCKED, 1);
5953 }
5954 
5955 /*
5956  * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5957  * If this is the first callback for this outer perimeter then add
5958  * this outer perimeter to the list of outer perimeters that
5959  * the qwriter_outer_thread will process.
5960  *
5961  * Increments sq_count in the outer syncq to prevent the membership
5962  * of the outer perimeter (in terms of inner syncqs) to change while
5963  * the callback is pending.
5964  */
5965 static void
queue_writer(syncq_t * outer,void (* func)(),queue_t * q,mblk_t * mp)5966 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5967 {
5968 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
5969 
5970 	mp->b_prev = (mblk_t *)func;
5971 	mp->b_queue = q;
5972 	mp->b_next = NULL;
5973 	outer->sq_count++;	/* Decremented when dequeued */
5974 	ASSERT(outer->sq_count != 0);	/* Wraparound */
5975 	if (outer->sq_evhead == NULL) {
5976 		/* First message. */
5977 		outer->sq_evhead = outer->sq_evtail = mp;
5978 		outer->sq_flags |= SQ_EVENTS;
5979 		mutex_exit(SQLOCK(outer));
5980 		STRSTAT(qwr_outer);
5981 		(void) taskq_dispatch(streams_taskq,
5982 		    (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5983 	} else {
5984 		ASSERT(outer->sq_flags & SQ_EVENTS);
5985 		outer->sq_evtail->b_next = mp;
5986 		outer->sq_evtail = mp;
5987 		mutex_exit(SQLOCK(outer));
5988 	}
5989 }
5990 
5991 /*
5992  * Try and upgrade to write access at the outer perimeter. If this can
5993  * not be done without blocking then queue the callback to be done
5994  * by the qwriter_outer_thread.
5995  *
5996  * This routine can only be called from put or service procedures plus
5997  * asynchronous callback routines that have properly entered the queue (with
5998  * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5999  * associated with q.
6000  */
6001 void
qwriter_outer(queue_t * q,mblk_t * mp,void (* func)())6002 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
6003 {
6004 	syncq_t	*osq, *sq, *outer;
6005 	int	failed;
6006 	uint16_t flags;
6007 
6008 	osq = q->q_syncq;
6009 	outer = osq->sq_outer;
6010 	if (outer == NULL)
6011 		panic("qwriter(PERIM_OUTER): no outer perimeter");
6012 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6013 	    outer->sq_oprev != NULL);
6014 
6015 	mutex_enter(SQLOCK(outer));
6016 	flags = outer->sq_flags;
6017 	/*
6018 	 * If some thread is traversing sq_next, or if we are blocked by
6019 	 * outer_insert or outer_remove, or if the we already have queued
6020 	 * callbacks, then queue this callback for later processing.
6021 	 *
6022 	 * Also queue the qwriter for an interrupt thread in order
6023 	 * to reduce the time spent running at high IPL.
6024 	 * to identify there are events.
6025 	 */
6026 	if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6027 		/*
6028 		 * Queue the become_writer request.
6029 		 * The queueing is atomic under SQLOCK(outer) in order
6030 		 * to synchronize with outer_exit.
6031 		 * queue_writer will drop the outer SQLOCK
6032 		 */
6033 		if (flags & SQ_BLOCKED) {
6034 			/* Must set SQ_WRITER on inner perimeter */
6035 			mutex_enter(SQLOCK(osq));
6036 			osq->sq_flags |= SQ_WRITER;
6037 			mutex_exit(SQLOCK(osq));
6038 		} else {
6039 			if (!(flags & SQ_WRITER)) {
6040 				/*
6041 				 * The outer could have been SQ_BLOCKED thus
6042 				 * SQ_WRITER might not be set on the inner.
6043 				 */
6044 				mutex_enter(SQLOCK(osq));
6045 				osq->sq_flags |= SQ_WRITER;
6046 				mutex_exit(SQLOCK(osq));
6047 			}
6048 			ASSERT(osq->sq_flags & SQ_WRITER);
6049 		}
6050 		queue_writer(outer, func, q, mp);
6051 		return;
6052 	}
6053 	/*
6054 	 * We are half-way to exclusive access to the outer perimeter.
6055 	 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6056 	 * while the inner syncqs are traversed.
6057 	 */
6058 	outer->sq_count++;
6059 	ASSERT(outer->sq_count != 0);	/* wraparound */
6060 	flags |= SQ_WRITER;
6061 	/*
6062 	 * Check if we can run the function immediately. Mark all
6063 	 * syncqs with the writer flag to prevent new entries into
6064 	 * put and service procedures.
6065 	 *
6066 	 * Set SQ_WRITER on all the inner syncqs while holding
6067 	 * the SQLOCK on the outer syncq. This ensures that the changing
6068 	 * of SQ_WRITER is atomic under the outer SQLOCK.
6069 	 */
6070 	failed = 0;
6071 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6072 		uint16_t count;
6073 		uint_t	maxcnt = (sq == osq) ? 1 : 0;
6074 
6075 		mutex_enter(SQLOCK(sq));
6076 		count = sq->sq_count;
6077 		SQ_PUTLOCKS_ENTER(sq);
6078 		SUM_SQ_PUTCOUNTS(sq, count);
6079 		if (sq->sq_count > maxcnt)
6080 			failed = 1;
6081 		sq->sq_flags |= SQ_WRITER;
6082 		SQ_PUTLOCKS_EXIT(sq);
6083 		mutex_exit(SQLOCK(sq));
6084 	}
6085 	if (failed) {
6086 		/*
6087 		 * Some other thread has a read claim on the outer perimeter.
6088 		 * Queue the callback for deferred processing.
6089 		 *
6090 		 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6091 		 * so that other qwriter(OUTER) calls will queue their
6092 		 * callbacks as well. queue_writer increments sq_count so we
6093 		 * decrement to compensate for the our increment.
6094 		 *
6095 		 * Dropping SQ_WRITER enables the writer thread to work
6096 		 * on this outer perimeter.
6097 		 */
6098 		outer->sq_flags = flags;
6099 		queue_writer(outer, func, q, mp);
6100 		/* queue_writer dropper the lock */
6101 		mutex_enter(SQLOCK(outer));
6102 		ASSERT(outer->sq_count > 0);
6103 		outer->sq_count--;
6104 		ASSERT(outer->sq_flags & SQ_WRITER);
6105 		flags = outer->sq_flags;
6106 		flags &= ~SQ_WRITER;
6107 		if (flags & SQ_WANTWAKEUP) {
6108 			flags &= ~SQ_WANTWAKEUP;
6109 			cv_broadcast(&outer->sq_wait);
6110 		}
6111 		outer->sq_flags = flags;
6112 		mutex_exit(SQLOCK(outer));
6113 		return;
6114 	} else {
6115 		outer->sq_flags = flags;
6116 		mutex_exit(SQLOCK(outer));
6117 	}
6118 
6119 	/* Can run it immediately */
6120 	(*func)(q, mp);
6121 
6122 	outer_exit(outer);
6123 }
6124 
6125 /*
6126  * Dequeue all writer callbacks from the outer perimeter and run them.
6127  */
6128 static void
write_now(syncq_t * outer)6129 write_now(syncq_t *outer)
6130 {
6131 	mblk_t		*mp;
6132 	queue_t		*q;
6133 	void	(*func)();
6134 
6135 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6136 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6137 	    outer->sq_oprev != NULL);
6138 	while ((mp = outer->sq_evhead) != NULL) {
6139 		/*
6140 		 * queues cannot be placed on the queuelist on the outer
6141 		 * perimeter.
6142 		 */
6143 		ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6144 		ASSERT((outer->sq_flags & SQ_EVENTS));
6145 
6146 		outer->sq_evhead = mp->b_next;
6147 		if (outer->sq_evhead == NULL) {
6148 			outer->sq_evtail = NULL;
6149 			outer->sq_flags &= ~SQ_EVENTS;
6150 		}
6151 		ASSERT(outer->sq_count != 0);
6152 		outer->sq_count--;	/* Incremented when enqueued. */
6153 		mutex_exit(SQLOCK(outer));
6154 		/*
6155 		 * Drop the message if the queue is closing.
6156 		 * Make sure that the queue is "claimed" when the callback
6157 		 * is run in order to satisfy various ASSERTs.
6158 		 */
6159 		q = mp->b_queue;
6160 		func = (void (*)())mp->b_prev;
6161 		ASSERT(func != NULL);
6162 		mp->b_next = mp->b_prev = NULL;
6163 		if (q->q_flag & QWCLOSE) {
6164 			freemsg(mp);
6165 		} else {
6166 			claimq(q);
6167 			(*func)(q, mp);
6168 			releaseq(q);
6169 		}
6170 		mutex_enter(SQLOCK(outer));
6171 	}
6172 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6173 }
6174 
6175 /*
6176  * The list of messages on the inner syncq is effectively hashed
6177  * by destination queue.  These destination queues are doubly
6178  * linked lists (hopefully) in priority order.  Messages are then
6179  * put on the queue referenced by the q_sqhead/q_sqtail elements.
6180  * Additional messages are linked together by the b_next/b_prev
6181  * elements in the mblk, with (similar to putq()) the first message
6182  * having a NULL b_prev and the last message having a NULL b_next.
6183  *
6184  * Events, such as qwriter callbacks, are put onto a list in FIFO
6185  * order referenced by sq_evhead, and sq_evtail.  This is a singly
6186  * linked list, and messages here MUST be processed in the order queued.
6187  */
6188 
6189 /*
6190  * Run the events on the syncq event list (sq_evhead).
6191  * Assumes there is only one claim on the syncq, it is
6192  * already exclusive (SQ_EXCL set), and the SQLOCK held.
6193  * Messages here are processed in order, with the SQ_EXCL bit
6194  * held all the way through till the last message is processed.
6195  */
6196 void
sq_run_events(syncq_t * sq)6197 sq_run_events(syncq_t *sq)
6198 {
6199 	mblk_t		*bp;
6200 	queue_t		*qp;
6201 	uint16_t	flags = sq->sq_flags;
6202 	void		(*func)();
6203 
6204 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6205 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6206 	    sq->sq_oprev == NULL) ||
6207 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6208 	    sq->sq_oprev != NULL));
6209 
6210 	ASSERT(flags & SQ_EXCL);
6211 	ASSERT(sq->sq_count == 1);
6212 
6213 	/*
6214 	 * We need to process all of the events on this list.  It
6215 	 * is possible that new events will be added while we are
6216 	 * away processing a callback, so on every loop, we start
6217 	 * back at the beginning of the list.
6218 	 */
6219 	/*
6220 	 * We have to reaccess sq_evhead since there is a
6221 	 * possibility of a new entry while we were running
6222 	 * the callback.
6223 	 */
6224 	for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6225 		ASSERT(bp->b_queue->q_syncq == sq);
6226 		ASSERT(sq->sq_flags & SQ_EVENTS);
6227 
6228 		qp = bp->b_queue;
6229 		func = (void (*)())bp->b_prev;
6230 		ASSERT(func != NULL);
6231 
6232 		/*
6233 		 * Messages from the event queue must be taken off in
6234 		 * FIFO order.
6235 		 */
6236 		ASSERT(sq->sq_evhead == bp);
6237 		sq->sq_evhead = bp->b_next;
6238 
6239 		if (bp->b_next == NULL) {
6240 			/* Deleting last */
6241 			ASSERT(sq->sq_evtail == bp);
6242 			sq->sq_evtail = NULL;
6243 			sq->sq_flags &= ~SQ_EVENTS;
6244 		}
6245 		bp->b_prev = bp->b_next = NULL;
6246 		ASSERT(bp->b_datap->db_ref != 0);
6247 
6248 		mutex_exit(SQLOCK(sq));
6249 
6250 		(*func)(qp, bp);
6251 
6252 		mutex_enter(SQLOCK(sq));
6253 		/*
6254 		 * re-read the flags, since they could have changed.
6255 		 */
6256 		flags = sq->sq_flags;
6257 		ASSERT(flags & SQ_EXCL);
6258 	}
6259 	ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6260 	ASSERT(!(sq->sq_flags & SQ_EVENTS));
6261 
6262 	if (flags & SQ_WANTWAKEUP) {
6263 		flags &= ~SQ_WANTWAKEUP;
6264 		cv_broadcast(&sq->sq_wait);
6265 	}
6266 	if (flags & SQ_WANTEXWAKEUP) {
6267 		flags &= ~SQ_WANTEXWAKEUP;
6268 		cv_broadcast(&sq->sq_exitwait);
6269 	}
6270 	sq->sq_flags = flags;
6271 }
6272 
6273 /*
6274  * Put messages on the event list.
6275  * If we can go exclusive now, do so and process the event list, otherwise
6276  * let the last claim service this list (or wake the sqthread).
6277  * This procedure assumes SQLOCK is held.  To run the event list, it
6278  * must be called with no claims.
6279  */
6280 static void
sqfill_events(syncq_t * sq,queue_t * q,mblk_t * mp,void (* func)())6281 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6282 {
6283 	uint16_t count;
6284 
6285 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6286 	ASSERT(func != NULL);
6287 
6288 	/*
6289 	 * This is a callback.  Add it to the list of callbacks
6290 	 * and see about upgrading.
6291 	 */
6292 	mp->b_prev = (mblk_t *)func;
6293 	mp->b_queue = q;
6294 	mp->b_next = NULL;
6295 	if (sq->sq_evhead == NULL) {
6296 		sq->sq_evhead = sq->sq_evtail = mp;
6297 		sq->sq_flags |= SQ_EVENTS;
6298 	} else {
6299 		ASSERT(sq->sq_evtail != NULL);
6300 		ASSERT(sq->sq_evtail->b_next == NULL);
6301 		ASSERT(sq->sq_flags & SQ_EVENTS);
6302 		sq->sq_evtail->b_next = mp;
6303 		sq->sq_evtail = mp;
6304 	}
6305 	/*
6306 	 * We have set SQ_EVENTS, so threads will have to
6307 	 * unwind out of the perimeter, and new entries will
6308 	 * not grab a putlock.  But we still need to know
6309 	 * how many threads have already made a claim to the
6310 	 * syncq, so grab the putlocks, and sum the counts.
6311 	 * If there are no claims on the syncq, we can upgrade
6312 	 * to exclusive, and run the event list.
6313 	 * NOTE: We hold the SQLOCK, so we can just grab the
6314 	 * putlocks.
6315 	 */
6316 	count = sq->sq_count;
6317 	SQ_PUTLOCKS_ENTER(sq);
6318 	SUM_SQ_PUTCOUNTS(sq, count);
6319 	/*
6320 	 * We have no claim, so we need to check if there
6321 	 * are no others, then we can upgrade.
6322 	 */
6323 	/*
6324 	 * There are currently no claims on
6325 	 * the syncq by this thread (at least on this entry). The thread who has
6326 	 * the claim should drain syncq.
6327 	 */
6328 	if (count > 0) {
6329 		/*
6330 		 * Can't upgrade - other threads inside.
6331 		 */
6332 		SQ_PUTLOCKS_EXIT(sq);
6333 		mutex_exit(SQLOCK(sq));
6334 		return;
6335 	}
6336 	/*
6337 	 * Need to set SQ_EXCL and make a claim on the syncq.
6338 	 */
6339 	ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6340 	sq->sq_flags |= SQ_EXCL;
6341 	ASSERT(sq->sq_count == 0);
6342 	sq->sq_count++;
6343 	SQ_PUTLOCKS_EXIT(sq);
6344 
6345 	/* Process the events list */
6346 	sq_run_events(sq);
6347 
6348 	/*
6349 	 * Release our claim...
6350 	 */
6351 	sq->sq_count--;
6352 
6353 	/*
6354 	 * And release SQ_EXCL.
6355 	 * We don't need to acquire the putlocks to release
6356 	 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6357 	 */
6358 	sq->sq_flags &= ~SQ_EXCL;
6359 
6360 	/*
6361 	 * sq_run_events should have released SQ_EXCL
6362 	 */
6363 	ASSERT(!(sq->sq_flags & SQ_EXCL));
6364 
6365 	/*
6366 	 * If anything happened while we were running the
6367 	 * events (or was there before), we need to process
6368 	 * them now.  We shouldn't be exclusive sine we
6369 	 * released the perimeter above (plus, we asserted
6370 	 * for it).
6371 	 */
6372 	if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6373 		drain_syncq(sq);
6374 	else
6375 		mutex_exit(SQLOCK(sq));
6376 }
6377 
6378 /*
6379  * Perform delayed processing. The caller has to make sure that it is safe
6380  * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6381  * set).
6382  *
6383  * Assume that the caller has NO claims on the syncq.  However, a claim
6384  * on the syncq does not indicate that a thread is draining the syncq.
6385  * There may be more claims on the syncq than there are threads draining
6386  * (i.e.  #_threads_draining <= sq_count)
6387  *
6388  * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6389  * in order to preserve qwriter(OUTER) ordering constraints.
6390  *
6391  * sq_putcount only needs to be checked when dispatching the queued
6392  * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6393  */
6394 void
drain_syncq(syncq_t * sq)6395 drain_syncq(syncq_t *sq)
6396 {
6397 	queue_t		*qp;
6398 	uint16_t	count;
6399 	uint16_t	type = sq->sq_type;
6400 	uint16_t	flags = sq->sq_flags;
6401 	boolean_t	bg_service = sq->sq_svcflags & SQ_SERVICE;
6402 
6403 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6404 	    "drain_syncq start:%p", sq);
6405 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6406 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6407 	    sq->sq_oprev == NULL) ||
6408 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6409 	    sq->sq_oprev != NULL));
6410 
6411 	/*
6412 	 * Drop SQ_SERVICE flag.
6413 	 */
6414 	if (bg_service)
6415 		sq->sq_svcflags &= ~SQ_SERVICE;
6416 
6417 	/*
6418 	 * If SQ_EXCL is set, someone else is processing this syncq - let them
6419 	 * finish the job.
6420 	 */
6421 	if (flags & SQ_EXCL) {
6422 		if (bg_service) {
6423 			ASSERT(sq->sq_servcount != 0);
6424 			sq->sq_servcount--;
6425 		}
6426 		mutex_exit(SQLOCK(sq));
6427 		return;
6428 	}
6429 
6430 	/*
6431 	 * This routine can be called by a background thread if
6432 	 * it was scheduled by a hi-priority thread.  SO, if there are
6433 	 * NOT messages queued, return (remember, we have the SQLOCK,
6434 	 * and it cannot change until we release it). Wakeup any waiters also.
6435 	 */
6436 	if (!(flags & SQ_QUEUED)) {
6437 		if (flags & SQ_WANTWAKEUP) {
6438 			flags &= ~SQ_WANTWAKEUP;
6439 			cv_broadcast(&sq->sq_wait);
6440 		}
6441 		if (flags & SQ_WANTEXWAKEUP) {
6442 			flags &= ~SQ_WANTEXWAKEUP;
6443 			cv_broadcast(&sq->sq_exitwait);
6444 		}
6445 		sq->sq_flags = flags;
6446 		if (bg_service) {
6447 			ASSERT(sq->sq_servcount != 0);
6448 			sq->sq_servcount--;
6449 		}
6450 		mutex_exit(SQLOCK(sq));
6451 		return;
6452 	}
6453 
6454 	/*
6455 	 * If this is not a concurrent put perimeter, we need to
6456 	 * become exclusive to drain.  Also, if not CIPUT, we would
6457 	 * not have acquired a putlock, so we don't need to check
6458 	 * the putcounts.  If not entering with a claim, we test
6459 	 * for sq_count == 0.
6460 	 */
6461 	type = sq->sq_type;
6462 	if (!(type & SQ_CIPUT)) {
6463 		if (sq->sq_count > 1) {
6464 			if (bg_service) {
6465 				ASSERT(sq->sq_servcount != 0);
6466 				sq->sq_servcount--;
6467 			}
6468 			mutex_exit(SQLOCK(sq));
6469 			return;
6470 		}
6471 		sq->sq_flags |= SQ_EXCL;
6472 	}
6473 
6474 	/*
6475 	 * This is where we make a claim to the syncq.
6476 	 * This can either be done by incrementing a putlock, or
6477 	 * the sq_count.  But since we already have the SQLOCK
6478 	 * here, we just bump the sq_count.
6479 	 *
6480 	 * Note that after we make a claim, we need to let the code
6481 	 * fall through to the end of this routine to clean itself
6482 	 * up.  A return in the while loop will put the syncq in a
6483 	 * very bad state.
6484 	 */
6485 	sq->sq_count++;
6486 	ASSERT(sq->sq_count != 0);	/* wraparound */
6487 
6488 	while ((flags = sq->sq_flags) & SQ_QUEUED) {
6489 		/*
6490 		 * If we are told to stayaway or went exclusive,
6491 		 * we are done.
6492 		 */
6493 		if (flags & (SQ_STAYAWAY)) {
6494 			break;
6495 		}
6496 
6497 		/*
6498 		 * If there are events to run, do so.
6499 		 * We have one claim to the syncq, so if there are
6500 		 * more than one, other threads are running.
6501 		 */
6502 		if (sq->sq_evhead != NULL) {
6503 			ASSERT(sq->sq_flags & SQ_EVENTS);
6504 
6505 			count = sq->sq_count;
6506 			SQ_PUTLOCKS_ENTER(sq);
6507 			SUM_SQ_PUTCOUNTS(sq, count);
6508 			if (count > 1) {
6509 				SQ_PUTLOCKS_EXIT(sq);
6510 				/* Can't upgrade - other threads inside */
6511 				break;
6512 			}
6513 			ASSERT((flags & SQ_EXCL) == 0);
6514 			sq->sq_flags = flags | SQ_EXCL;
6515 			SQ_PUTLOCKS_EXIT(sq);
6516 			/*
6517 			 * we have the only claim, run the events,
6518 			 * sq_run_events will clear the SQ_EXCL flag.
6519 			 */
6520 			sq_run_events(sq);
6521 
6522 			/*
6523 			 * If this is a CIPUT perimeter, we need
6524 			 * to drop the SQ_EXCL flag so we can properly
6525 			 * continue draining the syncq.
6526 			 */
6527 			if (type & SQ_CIPUT) {
6528 				ASSERT(sq->sq_flags & SQ_EXCL);
6529 				sq->sq_flags &= ~SQ_EXCL;
6530 			}
6531 
6532 			/*
6533 			 * And go back to the beginning just in case
6534 			 * anything changed while we were away.
6535 			 */
6536 			ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6537 			continue;
6538 		}
6539 
6540 		ASSERT(sq->sq_evhead == NULL);
6541 		ASSERT(!(sq->sq_flags & SQ_EVENTS));
6542 
6543 		/*
6544 		 * Find the queue that is not draining.
6545 		 *
6546 		 * q_draining is protected by QLOCK which we do not hold.
6547 		 * But if it was set, then a thread was draining, and if it gets
6548 		 * cleared, then it was because the thread has successfully
6549 		 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6550 		 * state to happen, a thread needs the SQLOCK which we hold, and
6551 		 * if there was such a flag, we would have already seen it.
6552 		 */
6553 
6554 		for (qp = sq->sq_head;
6555 		    qp != NULL && (qp->q_draining ||
6556 		    (qp->q_sqflags & Q_SQDRAINING));
6557 		    qp = qp->q_sqnext)
6558 			;
6559 
6560 		if (qp == NULL)
6561 			break;
6562 
6563 		/*
6564 		 * We have a queue to work on, and we hold the
6565 		 * SQLOCK and one claim, call qdrain_syncq.
6566 		 * This means we need to release the SQLOCK and
6567 		 * acquire the QLOCK (OK since we have a claim).
6568 		 * Note that qdrain_syncq will actually dequeue
6569 		 * this queue from the sq_head list when it is
6570 		 * convinced all the work is done and release
6571 		 * the QLOCK before returning.
6572 		 */
6573 		qp->q_sqflags |= Q_SQDRAINING;
6574 		mutex_exit(SQLOCK(sq));
6575 		mutex_enter(QLOCK(qp));
6576 		qdrain_syncq(sq, qp);
6577 		mutex_enter(SQLOCK(sq));
6578 
6579 		/* The queue is drained */
6580 		ASSERT(qp->q_sqflags & Q_SQDRAINING);
6581 		qp->q_sqflags &= ~Q_SQDRAINING;
6582 		/*
6583 		 * NOTE: After this point qp should not be used since it may be
6584 		 * closed.
6585 		 */
6586 	}
6587 
6588 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6589 	flags = sq->sq_flags;
6590 
6591 	/*
6592 	 * sq->sq_head cannot change because we hold the
6593 	 * sqlock. However, a thread CAN decide that it is no longer
6594 	 * going to drain that queue.  However, this should be due to
6595 	 * a GOAWAY state, and we should see that here.
6596 	 *
6597 	 * This loop is not very efficient. One solution may be adding a second
6598 	 * pointer to the "draining" queue, but it is difficult to do when
6599 	 * queues are inserted in the middle due to priority ordering. Another
6600 	 * possibility is to yank the queue out of the sq list and put it onto
6601 	 * the "draining list" and then put it back if it can't be drained.
6602 	 */
6603 
6604 	ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6605 	    (type & SQ_CI) || sq->sq_head->q_draining);
6606 
6607 	/* Drop SQ_EXCL for non-CIPUT perimeters */
6608 	if (!(type & SQ_CIPUT))
6609 		flags &= ~SQ_EXCL;
6610 	ASSERT((flags & SQ_EXCL) == 0);
6611 
6612 	/* Wake up any waiters. */
6613 	if (flags & SQ_WANTWAKEUP) {
6614 		flags &= ~SQ_WANTWAKEUP;
6615 		cv_broadcast(&sq->sq_wait);
6616 	}
6617 	if (flags & SQ_WANTEXWAKEUP) {
6618 		flags &= ~SQ_WANTEXWAKEUP;
6619 		cv_broadcast(&sq->sq_exitwait);
6620 	}
6621 	sq->sq_flags = flags;
6622 
6623 	ASSERT(sq->sq_count != 0);
6624 	/* Release our claim. */
6625 	sq->sq_count--;
6626 
6627 	if (bg_service) {
6628 		ASSERT(sq->sq_servcount != 0);
6629 		sq->sq_servcount--;
6630 	}
6631 
6632 	mutex_exit(SQLOCK(sq));
6633 
6634 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6635 	    "drain_syncq end:%p", sq);
6636 }
6637 
6638 
6639 /*
6640  *
6641  * qdrain_syncq can be called (currently) from only one of two places:
6642  *	drain_syncq
6643  *	putnext  (or some variation of it).
6644  * and eventually
6645  *	qwait(_sig)
6646  *
6647  * If called from drain_syncq, we found it in the list of queues needing
6648  * service, so there is work to be done (or it wouldn't be in the list).
6649  *
6650  * If called from some putnext variation, it was because the
6651  * perimeter is open, but messages are blocking a putnext and
6652  * there is not a thread working on it.  Now a thread could start
6653  * working on it while we are getting ready to do so ourself, but
6654  * the thread would set the q_draining flag, and we can spin out.
6655  *
6656  * As for qwait(_sig), I think I shall let it continue to call
6657  * drain_syncq directly (after all, it will get here eventually).
6658  *
6659  * qdrain_syncq has to terminate when:
6660  * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6661  * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6662  *
6663  * ASSUMES:
6664  *	One claim
6665  *	QLOCK held
6666  *	SQLOCK not held
6667  *	Will release QLOCK before returning
6668  */
6669 void
qdrain_syncq(syncq_t * sq,queue_t * q)6670 qdrain_syncq(syncq_t *sq, queue_t *q)
6671 {
6672 	mblk_t		*bp;
6673 #ifdef DEBUG
6674 	uint16_t	count;
6675 #endif
6676 
6677 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6678 	    "drain_syncq start:%p", sq);
6679 	ASSERT(q->q_syncq == sq);
6680 	ASSERT(MUTEX_HELD(QLOCK(q)));
6681 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6682 	/*
6683 	 * For non-CIPUT perimeters, we should be called with the exclusive bit
6684 	 * set already. For CIPUT perimeters, we will be doing a concurrent
6685 	 * drain, so it better not be set.
6686 	 */
6687 	ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6688 	ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6689 	ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6690 	/*
6691 	 * All outer pointers are set, or none of them are
6692 	 */
6693 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6694 	    sq->sq_oprev == NULL) ||
6695 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6696 	    sq->sq_oprev != NULL));
6697 #ifdef DEBUG
6698 	count = sq->sq_count;
6699 	/*
6700 	 * This is OK without the putlocks, because we have one
6701 	 * claim either from the sq_count, or a putcount.  We could
6702 	 * get an erroneous value from other counts, but ours won't
6703 	 * change, so one way or another, we will have at least a
6704 	 * value of one.
6705 	 */
6706 	SUM_SQ_PUTCOUNTS(sq, count);
6707 	ASSERT(count >= 1);
6708 #endif /* DEBUG */
6709 
6710 	/*
6711 	 * The first thing to do is find out if a thread is already draining
6712 	 * this queue. If so, we are done, just return.
6713 	 */
6714 	if (q->q_draining) {
6715 		mutex_exit(QLOCK(q));
6716 		return;
6717 	}
6718 
6719 	/*
6720 	 * If the perimeter is exclusive, there is nothing we can do right now,
6721 	 * go away. Note that there is nothing to prevent this case from
6722 	 * changing right after this check, but the spin-out will catch it.
6723 	 */
6724 
6725 	/* Tell other threads that we are draining this queue */
6726 	q->q_draining = 1;	/* Protected by QLOCK */
6727 
6728 	/*
6729 	 * If there is nothing to do, clear QFULL as necessary. This caters for
6730 	 * the case where an empty queue was enqueued onto the syncq.
6731 	 */
6732 	if (q->q_sqhead == NULL) {
6733 		ASSERT(q->q_syncqmsgs == 0);
6734 		mutex_exit(QLOCK(q));
6735 		clr_qfull(q);
6736 		mutex_enter(QLOCK(q));
6737 	}
6738 
6739 	/*
6740 	 * Note that q_sqhead must be re-checked here in case another message
6741 	 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6742 	 */
6743 	for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6744 		/*
6745 		 * Because we can enter this routine just because a putnext is
6746 		 * blocked, we need to spin out if the perimeter wants to go
6747 		 * exclusive as well as just blocked. We need to spin out also
6748 		 * if events are queued on the syncq.
6749 		 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6750 		 * set it, and it can't become exclusive while we hold a claim.
6751 		 */
6752 		if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6753 			break;
6754 		}
6755 
6756 #ifdef DEBUG
6757 		/*
6758 		 * Since we are in qdrain_syncq, we already know the queue,
6759 		 * but for sanity, we want to check this against the qp that
6760 		 * was passed in by bp->b_queue.
6761 		 */
6762 
6763 		ASSERT(bp->b_queue == q);
6764 		ASSERT(bp->b_queue->q_syncq == sq);
6765 		bp->b_queue = NULL;
6766 
6767 		/*
6768 		 * We would have the following check in the DEBUG code:
6769 		 *
6770 		 * if (bp->b_prev != NULL)  {
6771 		 *	ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6772 		 * }
6773 		 *
6774 		 * This can't be done, however, since IP modifies qinfo
6775 		 * structure at run-time (switching between IPv4 qinfo and IPv6
6776 		 * qinfo), invalidating the check.
6777 		 * So the assignment to func is left here, but the ASSERT itself
6778 		 * is removed until the whole issue is resolved.
6779 		 */
6780 #endif
6781 		ASSERT(q->q_sqhead == bp);
6782 		q->q_sqhead = bp->b_next;
6783 		bp->b_prev = bp->b_next = NULL;
6784 		ASSERT(q->q_syncqmsgs > 0);
6785 		mutex_exit(QLOCK(q));
6786 
6787 		ASSERT(bp->b_datap->db_ref != 0);
6788 
6789 		(void) (*q->q_qinfo->qi_putp)(q, bp);
6790 
6791 		mutex_enter(QLOCK(q));
6792 
6793 		/*
6794 		 * q_syncqmsgs should only be decremented after executing the
6795 		 * put procedure to avoid message re-ordering. This is due to an
6796 		 * optimisation in putnext() which can call the put procedure
6797 		 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6798 		 * being set).
6799 		 *
6800 		 * We also need to clear QFULL in the next service procedure
6801 		 * queue if this is the last message destined for that queue.
6802 		 *
6803 		 * It would make better sense to have some sort of tunable for
6804 		 * the low water mark, but these semantics are not yet defined.
6805 		 * So, alas, we use a constant.
6806 		 */
6807 		if (--q->q_syncqmsgs == 0) {
6808 			mutex_exit(QLOCK(q));
6809 			clr_qfull(q);
6810 			mutex_enter(QLOCK(q));
6811 		}
6812 
6813 		/*
6814 		 * Always clear SQ_EXCL when CIPUT in order to handle
6815 		 * qwriter(INNER). The putp() can call qwriter and get exclusive
6816 		 * access IFF this is the only claim. So, we need to test for
6817 		 * this possibility, acquire the mutex and clear the bit.
6818 		 */
6819 		if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6820 			mutex_enter(SQLOCK(sq));
6821 			sq->sq_flags &= ~SQ_EXCL;
6822 			mutex_exit(SQLOCK(sq));
6823 		}
6824 	}
6825 
6826 	/*
6827 	 * We should either have no messages on this queue, or we were told to
6828 	 * goaway by a waiter (which we will wake up at the end of this
6829 	 * function).
6830 	 */
6831 	ASSERT((q->q_sqhead == NULL) ||
6832 	    (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6833 
6834 	ASSERT(MUTEX_HELD(QLOCK(q)));
6835 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6836 
6837 	/* Remove the q from the syncq list if all the messages are drained. */
6838 	if (q->q_sqhead == NULL) {
6839 		ASSERT(q->q_syncqmsgs == 0);
6840 		mutex_enter(SQLOCK(sq));
6841 		if (q->q_sqflags & Q_SQQUEUED)
6842 			SQRM_Q(sq, q);
6843 		mutex_exit(SQLOCK(sq));
6844 		/*
6845 		 * Since the queue is removed from the list, reset its priority.
6846 		 */
6847 		q->q_spri = 0;
6848 	}
6849 
6850 	/*
6851 	 * Remember, the q_draining flag is used to let another thread know
6852 	 * that there is a thread currently draining the messages for a queue.
6853 	 * Since we are now done with this queue (even if there may be messages
6854 	 * still there), we need to clear this flag so some thread will work on
6855 	 * it if needed.
6856 	 */
6857 	ASSERT(q->q_draining);
6858 	q->q_draining = 0;
6859 
6860 	/* Called with a claim, so OK to drop all locks. */
6861 	mutex_exit(QLOCK(q));
6862 
6863 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6864 	    "drain_syncq end:%p", sq);
6865 }
6866 /* END OF QDRAIN_SYNCQ  */
6867 
6868 
6869 /*
6870  * This is the mate to qdrain_syncq, except that it is putting the message onto
6871  * the queue instead of draining. Since the message is destined for the queue
6872  * that is selected, there is no need to identify the function because the
6873  * message is intended for the put routine for the queue. For debug kernels,
6874  * this routine will do it anyway just in case.
6875  *
6876  * After the message is enqueued on the syncq, it calls putnext_tail()
6877  * which will schedule a background thread to actually process the message.
6878  *
6879  * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6880  * SQLOCK(sq) and QLOCK(q) are not held.
6881  */
6882 void
qfill_syncq(syncq_t * sq,queue_t * q,mblk_t * mp)6883 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6884 {
6885 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6886 	ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6887 	ASSERT(sq->sq_count > 0);
6888 	ASSERT(q->q_syncq == sq);
6889 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6890 	    sq->sq_oprev == NULL) ||
6891 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6892 	    sq->sq_oprev != NULL));
6893 
6894 	mutex_enter(QLOCK(q));
6895 
6896 #ifdef DEBUG
6897 	/*
6898 	 * This is used for debug in the qfill_syncq/qdrain_syncq case
6899 	 * to trace the queue that the message is intended for.  Note
6900 	 * that the original use was to identify the queue and function
6901 	 * to call on the drain.  In the new syncq, we have the context
6902 	 * of the queue that we are draining, so call it's putproc and
6903 	 * don't rely on the saved values.  But for debug this is still
6904 	 * useful information.
6905 	 */
6906 	mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6907 	mp->b_queue = q;
6908 	mp->b_next = NULL;
6909 #endif
6910 	ASSERT(q->q_syncq == sq);
6911 	/*
6912 	 * Enqueue the message on the list.
6913 	 * SQPUT_MP() accesses q_syncqmsgs.  We are already holding QLOCK to
6914 	 * protect it.  So it's ok to acquire SQLOCK after SQPUT_MP().
6915 	 */
6916 	SQPUT_MP(q, mp);
6917 	mutex_enter(SQLOCK(sq));
6918 
6919 	/*
6920 	 * And queue on syncq for scheduling, if not already queued.
6921 	 * Note that we need the SQLOCK for this, and for testing flags
6922 	 * at the end to see if we will drain.  So grab it now, and
6923 	 * release it before we call qdrain_syncq or return.
6924 	 */
6925 	if (!(q->q_sqflags & Q_SQQUEUED)) {
6926 		q->q_spri = curthread->t_pri;
6927 		SQPUT_Q(sq, q);
6928 	}
6929 #ifdef DEBUG
6930 	else {
6931 		/*
6932 		 * All of these conditions MUST be true!
6933 		 */
6934 		ASSERT(sq->sq_tail != NULL);
6935 		if (sq->sq_tail == sq->sq_head) {
6936 			ASSERT((q->q_sqprev == NULL) &&
6937 			    (q->q_sqnext == NULL));
6938 		} else {
6939 			ASSERT((q->q_sqprev != NULL) ||
6940 			    (q->q_sqnext != NULL));
6941 		}
6942 		ASSERT(sq->sq_flags & SQ_QUEUED);
6943 		ASSERT(q->q_syncqmsgs != 0);
6944 		ASSERT(q->q_sqflags & Q_SQQUEUED);
6945 	}
6946 #endif
6947 	mutex_exit(QLOCK(q));
6948 	/*
6949 	 * SQLOCK is still held, so sq_count can be safely decremented.
6950 	 */
6951 	sq->sq_count--;
6952 
6953 	putnext_tail(sq, q, 0);
6954 	/* Should not reference sq or q after this point. */
6955 }
6956 
6957 /*  End of qfill_syncq  */
6958 
6959 /*
6960  * Remove all messages from a syncq (if qp is NULL) or remove all messages
6961  * that would be put into qp by drain_syncq.
6962  * Used when deleting the syncq (qp == NULL) or when detaching
6963  * a queue (qp != NULL).
6964  * Return non-zero if one or more messages were freed.
6965  *
6966  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6967  * sq_putlocks are used.
6968  *
6969  * NOTE: This function assumes that it is called from the close() context and
6970  * that all the queues in the syncq are going away. For this reason it doesn't
6971  * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6972  * currently valid, but it is useful to rethink this function to behave properly
6973  * in other cases.
6974  */
6975 int
flush_syncq(syncq_t * sq,queue_t * qp)6976 flush_syncq(syncq_t *sq, queue_t *qp)
6977 {
6978 	mblk_t		*bp, *mp_head, *mp_next, *mp_prev;
6979 	queue_t		*q;
6980 	int		ret = 0;
6981 
6982 	mutex_enter(SQLOCK(sq));
6983 
6984 	/*
6985 	 * Before we leave, we need to make sure there are no
6986 	 * events listed for this queue.  All events for this queue
6987 	 * will just be freed.
6988 	 */
6989 	if (qp != NULL && sq->sq_evhead != NULL) {
6990 		ASSERT(sq->sq_flags & SQ_EVENTS);
6991 
6992 		mp_prev = NULL;
6993 		for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6994 			mp_next = bp->b_next;
6995 			if (bp->b_queue == qp) {
6996 				/* Delete this message */
6997 				if (mp_prev != NULL) {
6998 					mp_prev->b_next = mp_next;
6999 					/*
7000 					 * Update sq_evtail if the last element
7001 					 * is removed.
7002 					 */
7003 					if (bp == sq->sq_evtail) {
7004 						ASSERT(mp_next == NULL);
7005 						sq->sq_evtail = mp_prev;
7006 					}
7007 				} else
7008 					sq->sq_evhead = mp_next;
7009 				if (sq->sq_evhead == NULL)
7010 					sq->sq_flags &= ~SQ_EVENTS;
7011 				bp->b_prev = bp->b_next = NULL;
7012 				freemsg(bp);
7013 				ret++;
7014 			} else {
7015 				mp_prev = bp;
7016 			}
7017 		}
7018 	}
7019 
7020 	/*
7021 	 * Walk sq_head and:
7022 	 *	- match qp if qp is set, remove it's messages
7023 	 *	- all if qp is not set
7024 	 */
7025 	q = sq->sq_head;
7026 	while (q != NULL) {
7027 		ASSERT(q->q_syncq == sq);
7028 		if ((qp == NULL) || (qp == q)) {
7029 			/*
7030 			 * Yank the messages as a list off the queue
7031 			 */
7032 			mp_head = q->q_sqhead;
7033 			/*
7034 			 * We do not have QLOCK(q) here (which is safe due to
7035 			 * assumptions mentioned above). To obtain the lock we
7036 			 * need to release SQLOCK which may allow lots of things
7037 			 * to change upon us. This place requires more analysis.
7038 			 */
7039 			q->q_sqhead = q->q_sqtail = NULL;
7040 			ASSERT(mp_head->b_queue &&
7041 			    mp_head->b_queue->q_syncq == sq);
7042 
7043 			/*
7044 			 * Free each of the messages.
7045 			 */
7046 			for (bp = mp_head; bp != NULL; bp = mp_next) {
7047 				mp_next = bp->b_next;
7048 				bp->b_prev = bp->b_next = NULL;
7049 				freemsg(bp);
7050 				ret++;
7051 			}
7052 			/*
7053 			 * Now remove the queue from the syncq.
7054 			 */
7055 			ASSERT(q->q_sqflags & Q_SQQUEUED);
7056 			SQRM_Q(sq, q);
7057 			q->q_spri = 0;
7058 			q->q_syncqmsgs = 0;
7059 
7060 			/*
7061 			 * If qp was specified, we are done with it and are
7062 			 * going to drop SQLOCK(sq) and return. We wakeup syncq
7063 			 * waiters while we still have the SQLOCK.
7064 			 */
7065 			if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7066 				sq->sq_flags &= ~SQ_WANTWAKEUP;
7067 				cv_broadcast(&sq->sq_wait);
7068 			}
7069 			/* Drop SQLOCK across clr_qfull */
7070 			mutex_exit(SQLOCK(sq));
7071 
7072 			/*
7073 			 * We avoid doing the test that drain_syncq does and
7074 			 * unconditionally clear qfull for every flushed
7075 			 * message. Since flush_syncq is only called during
7076 			 * close this should not be a problem.
7077 			 */
7078 			clr_qfull(q);
7079 			if (qp != NULL) {
7080 				return (ret);
7081 			} else {
7082 				mutex_enter(SQLOCK(sq));
7083 				/*
7084 				 * The head was removed by SQRM_Q above.
7085 				 * reread the new head and flush it.
7086 				 */
7087 				q = sq->sq_head;
7088 			}
7089 		} else {
7090 			q = q->q_sqnext;
7091 		}
7092 		ASSERT(MUTEX_HELD(SQLOCK(sq)));
7093 	}
7094 
7095 	if (sq->sq_flags & SQ_WANTWAKEUP) {
7096 		sq->sq_flags &= ~SQ_WANTWAKEUP;
7097 		cv_broadcast(&sq->sq_wait);
7098 	}
7099 
7100 	mutex_exit(SQLOCK(sq));
7101 	return (ret);
7102 }
7103 
7104 /*
7105  * Propagate all messages from a syncq to the next syncq that are associated
7106  * with the specified queue. If the queue is attached to a driver or if the
7107  * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7108  *
7109  * Assumes that the stream is strlock()'ed. We don't come here if there
7110  * are no messages to propagate.
7111  *
7112  * NOTE : If the queue is attached to a driver, all the messages are freed
7113  * as there is no point in propagating the messages from the driver syncq
7114  * to the closing stream head which will in turn get freed later.
7115  */
7116 static int
propagate_syncq(queue_t * qp)7117 propagate_syncq(queue_t *qp)
7118 {
7119 	mblk_t		*bp, *head, *tail, *prev, *next;
7120 	syncq_t		*sq;
7121 	queue_t		*nqp;
7122 	syncq_t		*nsq;
7123 	boolean_t	isdriver;
7124 	int		moved = 0;
7125 	uint16_t	flags;
7126 	pri_t		priority = curthread->t_pri;
7127 #ifdef DEBUG
7128 	void		(*func)();
7129 #endif
7130 
7131 	sq = qp->q_syncq;
7132 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7133 	/* debug macro */
7134 	SQ_PUTLOCKS_HELD(sq);
7135 	/*
7136 	 * As entersq() does not increment the sq_count for
7137 	 * the write side, check sq_count for non-QPERQ
7138 	 * perimeters alone.
7139 	 */
7140 	ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7141 
7142 	/*
7143 	 * propagate_syncq() can be called because of either messages on the
7144 	 * queue syncq or because on events on the queue syncq. Do actual
7145 	 * message propagations if there are any messages.
7146 	 */
7147 	if (qp->q_syncqmsgs) {
7148 		isdriver = (qp->q_flag & QISDRV);
7149 
7150 		if (!isdriver) {
7151 			nqp = qp->q_next;
7152 			nsq = nqp->q_syncq;
7153 			ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7154 			/* debug macro */
7155 			SQ_PUTLOCKS_HELD(nsq);
7156 #ifdef DEBUG
7157 			func = (void (*)())(uintptr_t)nqp->q_qinfo->qi_putp;
7158 #endif
7159 		}
7160 
7161 		SQRM_Q(sq, qp);
7162 		priority = MAX(qp->q_spri, priority);
7163 		qp->q_spri = 0;
7164 		head = qp->q_sqhead;
7165 		tail = qp->q_sqtail;
7166 		qp->q_sqhead = qp->q_sqtail = NULL;
7167 		qp->q_syncqmsgs = 0;
7168 
7169 		/*
7170 		 * Walk the list of messages, and free them if this is a driver,
7171 		 * otherwise reset the b_prev and b_queue value to the new putp.
7172 		 * Afterward, we will just add the head to the end of the next
7173 		 * syncq, and point the tail to the end of this one.
7174 		 */
7175 
7176 		for (bp = head; bp != NULL; bp = next) {
7177 			next = bp->b_next;
7178 			if (isdriver) {
7179 				bp->b_prev = bp->b_next = NULL;
7180 				freemsg(bp);
7181 				continue;
7182 			}
7183 			/* Change the q values for this message */
7184 			bp->b_queue = nqp;
7185 #ifdef DEBUG
7186 			bp->b_prev = (mblk_t *)func;
7187 #endif
7188 			moved++;
7189 		}
7190 		/*
7191 		 * Attach list of messages to the end of the new queue (if there
7192 		 * is a list of messages).
7193 		 */
7194 
7195 		if (!isdriver && head != NULL) {
7196 			ASSERT(tail != NULL);
7197 			if (nqp->q_sqhead == NULL) {
7198 				nqp->q_sqhead = head;
7199 			} else {
7200 				ASSERT(nqp->q_sqtail != NULL);
7201 				nqp->q_sqtail->b_next = head;
7202 			}
7203 			nqp->q_sqtail = tail;
7204 			/*
7205 			 * When messages are moved from high priority queue to
7206 			 * another queue, the destination queue priority is
7207 			 * upgraded.
7208 			 */
7209 
7210 			if (priority > nqp->q_spri)
7211 				nqp->q_spri = priority;
7212 
7213 			SQPUT_Q(nsq, nqp);
7214 
7215 			nqp->q_syncqmsgs += moved;
7216 			ASSERT(nqp->q_syncqmsgs != 0);
7217 		}
7218 	}
7219 
7220 	/*
7221 	 * Before we leave, we need to make sure there are no
7222 	 * events listed for this queue.  All events for this queue
7223 	 * will just be freed.
7224 	 */
7225 	if (sq->sq_evhead != NULL) {
7226 		ASSERT(sq->sq_flags & SQ_EVENTS);
7227 		prev = NULL;
7228 		for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7229 			next = bp->b_next;
7230 			if (bp->b_queue == qp) {
7231 				/* Delete this message */
7232 				if (prev != NULL) {
7233 					prev->b_next = next;
7234 					/*
7235 					 * Update sq_evtail if the last element
7236 					 * is removed.
7237 					 */
7238 					if (bp == sq->sq_evtail) {
7239 						ASSERT(next == NULL);
7240 						sq->sq_evtail = prev;
7241 					}
7242 				} else
7243 					sq->sq_evhead = next;
7244 				if (sq->sq_evhead == NULL)
7245 					sq->sq_flags &= ~SQ_EVENTS;
7246 				bp->b_prev = bp->b_next = NULL;
7247 				freemsg(bp);
7248 			} else {
7249 				prev = bp;
7250 			}
7251 		}
7252 	}
7253 
7254 	flags = sq->sq_flags;
7255 
7256 	/* Wake up any waiter before leaving. */
7257 	if (flags & SQ_WANTWAKEUP) {
7258 		flags &= ~SQ_WANTWAKEUP;
7259 		cv_broadcast(&sq->sq_wait);
7260 	}
7261 	sq->sq_flags = flags;
7262 
7263 	return (moved);
7264 }
7265 
7266 /*
7267  * Try and upgrade to exclusive access at the inner perimeter. If this can
7268  * not be done without blocking then request will be queued on the syncq
7269  * and drain_syncq will run it later.
7270  *
7271  * This routine can only be called from put or service procedures plus
7272  * asynchronous callback routines that have properly entered the queue (with
7273  * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7274  * associated with q.
7275  */
7276 void
qwriter_inner(queue_t * q,mblk_t * mp,void (* func)())7277 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7278 {
7279 	syncq_t	*sq = q->q_syncq;
7280 	uint16_t count;
7281 
7282 	mutex_enter(SQLOCK(sq));
7283 	count = sq->sq_count;
7284 	SQ_PUTLOCKS_ENTER(sq);
7285 	SUM_SQ_PUTCOUNTS(sq, count);
7286 	ASSERT(count >= 1);
7287 	ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7288 
7289 	if (count == 1) {
7290 		/*
7291 		 * Can upgrade. This case also handles nested qwriter calls
7292 		 * (when the qwriter callback function calls qwriter). In that
7293 		 * case SQ_EXCL is already set.
7294 		 */
7295 		sq->sq_flags |= SQ_EXCL;
7296 		SQ_PUTLOCKS_EXIT(sq);
7297 		mutex_exit(SQLOCK(sq));
7298 		(*func)(q, mp);
7299 		/*
7300 		 * Assumes that leavesq, putnext, and drain_syncq will reset
7301 		 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7302 		 * until putnext, leavesq, or drain_syncq drops it.
7303 		 * That way we handle nested qwriter(INNER) without dropping
7304 		 * SQ_EXCL until the outermost qwriter callback routine is
7305 		 * done.
7306 		 */
7307 		return;
7308 	}
7309 	SQ_PUTLOCKS_EXIT(sq);
7310 	sqfill_events(sq, q, mp, func);
7311 }
7312 
7313 /*
7314  * Synchronous callback support functions
7315  */
7316 
7317 /*
7318  * Allocate a callback parameter structure.
7319  * Assumes that caller initializes the flags and the id.
7320  * Acquires SQLOCK(sq) if non-NULL is returned.
7321  */
7322 callbparams_t *
callbparams_alloc(syncq_t * sq,void (* func)(void *),void * arg,int kmflags)7323 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7324 {
7325 	callbparams_t *cbp;
7326 	size_t size = sizeof (callbparams_t);
7327 
7328 	cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7329 
7330 	/*
7331 	 * Only try tryhard allocation if the caller is ready to panic.
7332 	 * Otherwise just fail.
7333 	 */
7334 	if (cbp == NULL) {
7335 		if (kmflags & KM_PANIC)
7336 			cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7337 			    &size, kmflags);
7338 		else
7339 			return (NULL);
7340 	}
7341 
7342 	ASSERT(size >= sizeof (callbparams_t));
7343 	cbp->cbp_size = size;
7344 	cbp->cbp_sq = sq;
7345 	cbp->cbp_func = func;
7346 	cbp->cbp_arg = arg;
7347 	mutex_enter(SQLOCK(sq));
7348 	cbp->cbp_next = sq->sq_callbpend;
7349 	sq->sq_callbpend = cbp;
7350 	return (cbp);
7351 }
7352 
7353 void
callbparams_free(syncq_t * sq,callbparams_t * cbp)7354 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7355 {
7356 	callbparams_t **pp, *p;
7357 
7358 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7359 
7360 	for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7361 		if (p == cbp) {
7362 			*pp = p->cbp_next;
7363 			kmem_free(p, p->cbp_size);
7364 			return;
7365 		}
7366 	}
7367 	(void) (STRLOG(0, 0, 0, SL_CONSOLE,
7368 	    "callbparams_free: not found\n"));
7369 }
7370 
7371 void
callbparams_free_id(syncq_t * sq,callbparams_id_t id,int32_t flag)7372 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7373 {
7374 	callbparams_t **pp, *p;
7375 
7376 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7377 
7378 	for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7379 		if (p->cbp_id == id && p->cbp_flags == flag) {
7380 			*pp = p->cbp_next;
7381 			kmem_free(p, p->cbp_size);
7382 			return;
7383 		}
7384 	}
7385 	(void) (STRLOG(0, 0, 0, SL_CONSOLE,
7386 	    "callbparams_free_id: not found\n"));
7387 }
7388 
7389 /*
7390  * Callback wrapper function used by once-only callbacks that can be
7391  * cancelled (qtimeout and qbufcall)
7392  * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7393  * cancelled by the qun* functions.
7394  */
7395 void
qcallbwrapper(void * arg)7396 qcallbwrapper(void *arg)
7397 {
7398 	callbparams_t *cbp = arg;
7399 	syncq_t	*sq;
7400 	uint16_t count = 0;
7401 	uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7402 	uint16_t type;
7403 
7404 	sq = cbp->cbp_sq;
7405 	mutex_enter(SQLOCK(sq));
7406 	type = sq->sq_type;
7407 	if (!(type & SQ_CICB)) {
7408 		count = sq->sq_count;
7409 		SQ_PUTLOCKS_ENTER(sq);
7410 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7411 		SUM_SQ_PUTCOUNTS(sq, count);
7412 		sq->sq_needexcl++;
7413 		ASSERT(sq->sq_needexcl != 0);	/* wraparound */
7414 		waitflags |= SQ_MESSAGES;
7415 	}
7416 	/* Can not handle exclusive entry at outer perimeter */
7417 	ASSERT(type & SQ_COCB);
7418 
7419 	while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7420 		if ((sq->sq_callbflags & cbp->cbp_flags) &&
7421 		    (sq->sq_cancelid == cbp->cbp_id)) {
7422 			/* timeout has been cancelled */
7423 			sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7424 			callbparams_free(sq, cbp);
7425 			if (!(type & SQ_CICB)) {
7426 				ASSERT(sq->sq_needexcl > 0);
7427 				sq->sq_needexcl--;
7428 				if (sq->sq_needexcl == 0) {
7429 					SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7430 				}
7431 				SQ_PUTLOCKS_EXIT(sq);
7432 			}
7433 			mutex_exit(SQLOCK(sq));
7434 			return;
7435 		}
7436 		sq->sq_flags |= SQ_WANTWAKEUP;
7437 		if (!(type & SQ_CICB)) {
7438 			SQ_PUTLOCKS_EXIT(sq);
7439 		}
7440 		cv_wait(&sq->sq_wait, SQLOCK(sq));
7441 		if (!(type & SQ_CICB)) {
7442 			count = sq->sq_count;
7443 			SQ_PUTLOCKS_ENTER(sq);
7444 			SUM_SQ_PUTCOUNTS(sq, count);
7445 		}
7446 	}
7447 
7448 	sq->sq_count++;
7449 	ASSERT(sq->sq_count != 0);	/* Wraparound */
7450 	if (!(type & SQ_CICB)) {
7451 		ASSERT(count == 0);
7452 		sq->sq_flags |= SQ_EXCL;
7453 		ASSERT(sq->sq_needexcl > 0);
7454 		sq->sq_needexcl--;
7455 		if (sq->sq_needexcl == 0) {
7456 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7457 		}
7458 		SQ_PUTLOCKS_EXIT(sq);
7459 	}
7460 
7461 	mutex_exit(SQLOCK(sq));
7462 
7463 	cbp->cbp_func(cbp->cbp_arg);
7464 
7465 	/*
7466 	 * We drop the lock only for leavesq to re-acquire it.
7467 	 * Possible optimization is inline of leavesq.
7468 	 */
7469 	mutex_enter(SQLOCK(sq));
7470 	callbparams_free(sq, cbp);
7471 	mutex_exit(SQLOCK(sq));
7472 	leavesq(sq, SQ_CALLBACK);
7473 }
7474 
7475 /*
7476  * No need to grab sq_putlocks here. See comment in strsubr.h that
7477  * explains when sq_putlocks are used.
7478  *
7479  * sq_count (or one of the sq_putcounts) has already been
7480  * decremented by the caller, and if SQ_QUEUED, we need to call
7481  * drain_syncq (the global syncq drain).
7482  * If putnext_tail is called with the SQ_EXCL bit set, we are in
7483  * one of two states, non-CIPUT perimeter, and we need to clear
7484  * it, or we went exclusive in the put procedure.  In any case,
7485  * we want to clear the bit now, and it is probably easier to do
7486  * this at the beginning of this function (remember, we hold
7487  * the SQLOCK).  Lastly, if there are other messages queued
7488  * on the syncq (and not for our destination), enable the syncq
7489  * for background work.
7490  */
7491 
7492 /* ARGSUSED */
7493 void
putnext_tail(syncq_t * sq,queue_t * qp,uint32_t passflags)7494 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7495 {
7496 	uint16_t	flags = sq->sq_flags;
7497 
7498 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7499 	ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7500 
7501 	/* Clear SQ_EXCL if set in passflags */
7502 	if (passflags & SQ_EXCL) {
7503 		flags &= ~SQ_EXCL;
7504 	}
7505 	if (flags & SQ_WANTWAKEUP) {
7506 		flags &= ~SQ_WANTWAKEUP;
7507 		cv_broadcast(&sq->sq_wait);
7508 	}
7509 	if (flags & SQ_WANTEXWAKEUP) {
7510 		flags &= ~SQ_WANTEXWAKEUP;
7511 		cv_broadcast(&sq->sq_exitwait);
7512 	}
7513 	sq->sq_flags = flags;
7514 
7515 	/*
7516 	 * We have cleared SQ_EXCL if we were asked to, and started
7517 	 * the wakeup process for waiters.  If there are no writers
7518 	 * then we need to drain the syncq if we were told to, or
7519 	 * enable the background thread to do it.
7520 	 */
7521 	if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7522 		if ((passflags & SQ_QUEUED) ||
7523 		    (sq->sq_svcflags & SQ_DISABLED)) {
7524 			/* drain_syncq will take care of events in the list */
7525 			drain_syncq(sq);
7526 			return;
7527 		} else if (flags & SQ_QUEUED) {
7528 			sqenable(sq);
7529 		}
7530 	}
7531 	/* Drop the SQLOCK on exit */
7532 	mutex_exit(SQLOCK(sq));
7533 	TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7534 	    "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7535 }
7536 
7537 void
set_qend(queue_t * q)7538 set_qend(queue_t *q)
7539 {
7540 	mutex_enter(QLOCK(q));
7541 	if (!O_SAMESTR(q))
7542 		q->q_flag |= QEND;
7543 	else
7544 		q->q_flag &= ~QEND;
7545 	mutex_exit(QLOCK(q));
7546 	q = _OTHERQ(q);
7547 	mutex_enter(QLOCK(q));
7548 	if (!O_SAMESTR(q))
7549 		q->q_flag |= QEND;
7550 	else
7551 		q->q_flag &= ~QEND;
7552 	mutex_exit(QLOCK(q));
7553 }
7554 
7555 /*
7556  * Set QFULL in next service procedure queue (that cares) if not already
7557  * set and if there are already more messages on the syncq than
7558  * sq_max_size.  If sq_max_size is 0, no flow control will be asserted on
7559  * any syncq.
7560  *
7561  * The fq here is the next queue with a service procedure.  This is where
7562  * we would fail canputnext, so this is where we need to set QFULL.
7563  * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7564  *
7565  * We already have QLOCK at this point. To avoid cross-locks with
7566  * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7567  * SQLOCK and sd_reflock, we need to drop respective locks first.
7568  */
7569 void
set_qfull(queue_t * q)7570 set_qfull(queue_t *q)
7571 {
7572 	queue_t		*fq = NULL;
7573 
7574 	ASSERT(MUTEX_HELD(QLOCK(q)));
7575 	if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7576 	    (q->q_syncqmsgs > sq_max_size)) {
7577 		if ((fq = q->q_nfsrv) == q) {
7578 			fq->q_flag |= QFULL;
7579 		} else {
7580 			mutex_exit(QLOCK(q));
7581 			mutex_enter(QLOCK(fq));
7582 			fq->q_flag |= QFULL;
7583 			mutex_exit(QLOCK(fq));
7584 			mutex_enter(QLOCK(q));
7585 		}
7586 	}
7587 }
7588 
7589 void
clr_qfull(queue_t * q)7590 clr_qfull(queue_t *q)
7591 {
7592 	queue_t	*oq = q;
7593 
7594 	q = q->q_nfsrv;
7595 	/* Fast check if there is any work to do before getting the lock. */
7596 	if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7597 		return;
7598 	}
7599 
7600 	/*
7601 	 * Do not reset QFULL (and backenable) if the q_count is the reason
7602 	 * for QFULL being set.
7603 	 */
7604 	mutex_enter(QLOCK(q));
7605 	/*
7606 	 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7607 	 * Hence clear the QFULL.
7608 	 * If both q_count and q_mblkcnt are less than the hiwat mark,
7609 	 * clear the QFULL.
7610 	 */
7611 	if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7612 	    (q->q_mblkcnt < q->q_hiwat))) {
7613 		q->q_flag &= ~QFULL;
7614 		/*
7615 		 * A little more confusing, how about this way:
7616 		 * if someone wants to write,
7617 		 * AND
7618 		 *    both counts are less than the lowat mark
7619 		 *    OR
7620 		 *    the lowat mark is zero
7621 		 * THEN
7622 		 * backenable
7623 		 */
7624 		if ((q->q_flag & QWANTW) &&
7625 		    (((q->q_count < q->q_lowat) &&
7626 		    (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7627 			q->q_flag &= ~QWANTW;
7628 			mutex_exit(QLOCK(q));
7629 			backenable(oq, 0);
7630 		} else
7631 			mutex_exit(QLOCK(q));
7632 	} else
7633 		mutex_exit(QLOCK(q));
7634 }
7635 
7636 /*
7637  * Set the forward service procedure pointer.
7638  *
7639  * Called at insert-time to cache a queue's next forward service procedure in
7640  * q_nfsrv; used by canput() and canputnext().  If the queue to be inserted
7641  * has a service procedure then q_nfsrv points to itself.  If the queue to be
7642  * inserted does not have a service procedure, then q_nfsrv points to the next
7643  * queue forward that has a service procedure.  If the queue is at the logical
7644  * end of the stream (driver for write side, stream head for the read side)
7645  * and does not have a service procedure, then q_nfsrv also points to itself.
7646  */
7647 void
set_nfsrv_ptr(queue_t * rnew,queue_t * wnew,queue_t * prev_rq,queue_t * prev_wq)7648 set_nfsrv_ptr(
7649 	queue_t  *rnew,		/* read queue pointer to new module */
7650 	queue_t  *wnew,		/* write queue pointer to new module */
7651 	queue_t  *prev_rq,	/* read queue pointer to the module above */
7652 	queue_t  *prev_wq)	/* write queue pointer to the module above */
7653 {
7654 	queue_t *qp;
7655 
7656 	if (prev_wq->q_next == NULL) {
7657 		/*
7658 		 * Insert the driver, initialize the driver and stream head.
7659 		 * In this case, prev_rq/prev_wq should be the stream head.
7660 		 * _I_INSERT does not allow inserting a driver.  Make sure
7661 		 * that it is not an insertion.
7662 		 */
7663 		ASSERT(!(rnew->q_flag & _QINSERTING));
7664 		wnew->q_nfsrv = wnew;
7665 		if (rnew->q_qinfo->qi_srvp)
7666 			rnew->q_nfsrv = rnew;
7667 		else
7668 			rnew->q_nfsrv = prev_rq;
7669 		prev_rq->q_nfsrv = prev_rq;
7670 		prev_wq->q_nfsrv = prev_wq;
7671 	} else {
7672 		/*
7673 		 * set up read side q_nfsrv pointer.  This MUST be done
7674 		 * before setting the write side, because the setting of
7675 		 * the write side for a fifo may depend on it.
7676 		 *
7677 		 * Suppose we have a fifo that only has pipemod pushed.
7678 		 * pipemod has no read or write service procedures, so
7679 		 * nfsrv for both pipemod queues points to prev_rq (the
7680 		 * stream read head).  Now push bufmod (which has only a
7681 		 * read service procedure).  Doing the write side first,
7682 		 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7683 		 * is WRONG; the next queue forward from wnew with a
7684 		 * service procedure will be rnew, not the stream read head.
7685 		 * Since the downstream queue (which in the case of a fifo
7686 		 * is the read queue rnew) can affect upstream queues, it
7687 		 * needs to be done first.  Setting up the read side first
7688 		 * sets nfsrv for both pipemod queues to rnew and then
7689 		 * when the write side is set up, wnew-q_nfsrv will also
7690 		 * point to rnew.
7691 		 */
7692 		if (rnew->q_qinfo->qi_srvp) {
7693 			/*
7694 			 * use _OTHERQ() because, if this is a pipe, next
7695 			 * module may have been pushed from other end and
7696 			 * q_next could be a read queue.
7697 			 */
7698 			qp = _OTHERQ(prev_wq->q_next);
7699 			while (qp && qp->q_nfsrv != qp) {
7700 				qp->q_nfsrv = rnew;
7701 				qp = backq(qp);
7702 			}
7703 			rnew->q_nfsrv = rnew;
7704 		} else
7705 			rnew->q_nfsrv = prev_rq->q_nfsrv;
7706 
7707 		/* set up write side q_nfsrv pointer */
7708 		if (wnew->q_qinfo->qi_srvp) {
7709 			wnew->q_nfsrv = wnew;
7710 
7711 			/*
7712 			 * For insertion, need to update nfsrv of the modules
7713 			 * above which do not have a service routine.
7714 			 */
7715 			if (rnew->q_flag & _QINSERTING) {
7716 				for (qp = prev_wq;
7717 				    qp != NULL && qp->q_nfsrv != qp;
7718 				    qp = backq(qp)) {
7719 					qp->q_nfsrv = wnew->q_nfsrv;
7720 				}
7721 			}
7722 		} else {
7723 			if (prev_wq->q_next == prev_rq)
7724 				/*
7725 				 * Since prev_wq/prev_rq are the middle of a
7726 				 * fifo, wnew/rnew will also be the middle of
7727 				 * a fifo and wnew's nfsrv is same as rnew's.
7728 				 */
7729 				wnew->q_nfsrv = rnew->q_nfsrv;
7730 			else
7731 				wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7732 		}
7733 	}
7734 }
7735 
7736 /*
7737  * Reset the forward service procedure pointer; called at remove-time.
7738  */
7739 void
reset_nfsrv_ptr(queue_t * rqp,queue_t * wqp)7740 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7741 {
7742 	queue_t *tmp_qp;
7743 
7744 	/* Reset the write side q_nfsrv pointer for _I_REMOVE */
7745 	if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7746 		for (tmp_qp = backq(wqp);
7747 		    tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7748 		    tmp_qp = backq(tmp_qp)) {
7749 			tmp_qp->q_nfsrv = wqp->q_nfsrv;
7750 		}
7751 	}
7752 
7753 	/* reset the read side q_nfsrv pointer */
7754 	if (rqp->q_qinfo->qi_srvp) {
7755 		if (wqp->q_next) {	/* non-driver case */
7756 			tmp_qp = _OTHERQ(wqp->q_next);
7757 			while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7758 				/* Note that rqp->q_next cannot be NULL */
7759 				ASSERT(rqp->q_next != NULL);
7760 				tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7761 				tmp_qp = backq(tmp_qp);
7762 			}
7763 		}
7764 	}
7765 }
7766 
7767 /*
7768  * This routine should be called after all stream geometry changes to update
7769  * the stream head cached struio() rd/wr queue pointers. Note must be called
7770  * with the streamlock()ed.
7771  *
7772  * Note: only enables Synchronous STREAMS for a side of a Stream which has
7773  *	 an explicit synchronous barrier module queue. That is, a queue that
7774  *	 has specified a struio() type.
7775  */
7776 static void
strsetuio(stdata_t * stp)7777 strsetuio(stdata_t *stp)
7778 {
7779 	queue_t *wrq;
7780 
7781 	if (stp->sd_flag & STPLEX) {
7782 		/*
7783 		 * Not streamhead, but a mux, so no Synchronous STREAMS.
7784 		 */
7785 		stp->sd_struiowrq = NULL;
7786 		stp->sd_struiordq = NULL;
7787 		return;
7788 	}
7789 	/*
7790 	 * Scan the write queue(s) while synchronous
7791 	 * until we find a qinfo uio type specified.
7792 	 */
7793 	wrq = stp->sd_wrq->q_next;
7794 	while (wrq) {
7795 		if (wrq->q_struiot == STRUIOT_NONE) {
7796 			wrq = 0;
7797 			break;
7798 		}
7799 		if (wrq->q_struiot != STRUIOT_DONTCARE)
7800 			break;
7801 		if (! _SAMESTR(wrq)) {
7802 			wrq = 0;
7803 			break;
7804 		}
7805 		wrq = wrq->q_next;
7806 	}
7807 	stp->sd_struiowrq = wrq;
7808 	/*
7809 	 * Scan the read queue(s) while synchronous
7810 	 * until we find a qinfo uio type specified.
7811 	 */
7812 	wrq = stp->sd_wrq->q_next;
7813 	while (wrq) {
7814 		if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7815 			wrq = 0;
7816 			break;
7817 		}
7818 		if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7819 			break;
7820 		if (! _SAMESTR(wrq)) {
7821 			wrq = 0;
7822 			break;
7823 		}
7824 		wrq = wrq->q_next;
7825 	}
7826 	stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7827 }
7828 
7829 static int
pass_rput(queue_t * q,mblk_t * mp)7830 pass_rput(queue_t *q, mblk_t *mp)
7831 {
7832 	putnext(q, mp);
7833 	return (0);
7834 }
7835 
7836 /*
7837  * pass_wput, unblocks the passthru queues, so that
7838  * messages can arrive at muxs lower read queue, before
7839  * I_LINK/I_UNLINK is acked/nacked.
7840  */
7841 static int
pass_wput(queue_t * q,mblk_t * mp)7842 pass_wput(queue_t *q, mblk_t *mp)
7843 {
7844 	syncq_t *sq;
7845 
7846 	sq = _RD(q)->q_syncq;
7847 	if (sq->sq_flags & SQ_BLOCKED)
7848 		unblocksq(sq, SQ_BLOCKED, 0);
7849 	putnext(q, mp);
7850 	return (0);
7851 }
7852 
7853 /*
7854  * Set up queues for the link/unlink.
7855  * Create a new queue and block it and then insert it
7856  * below the stream head on the lower stream.
7857  * This prevents any messages from arriving during the setq
7858  * as well as while the mux is processing the LINK/I_UNLINK.
7859  * The blocked passq is unblocked once the LINK/I_UNLINK has
7860  * been acked or nacked or if a message is generated and sent
7861  * down muxs write put procedure.
7862  * See pass_wput().
7863  *
7864  * After the new queue is inserted, all messages coming from below are
7865  * blocked. The call to strlock will ensure that all activity in the stream head
7866  * read queue syncq is stopped (sq_count drops to zero).
7867  */
7868 static queue_t *
link_addpassthru(stdata_t * stpdown)7869 link_addpassthru(stdata_t *stpdown)
7870 {
7871 	queue_t *passq;
7872 	sqlist_t sqlist;
7873 
7874 	passq = allocq();
7875 	STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7876 	/* setq might sleep in allocator - avoid holding locks. */
7877 	setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7878 	    SQ_CI|SQ_CO, B_FALSE);
7879 	claimq(passq);
7880 	blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7881 	insertq(STREAM(passq), passq);
7882 
7883 	/*
7884 	 * Use strlock() to wait for the stream head sq_count to drop to zero
7885 	 * since we are going to change q_ptr in the stream head.  Note that
7886 	 * insertq() doesn't wait for any syncq counts to drop to zero.
7887 	 */
7888 	sqlist.sqlist_head = NULL;
7889 	sqlist.sqlist_index = 0;
7890 	sqlist.sqlist_size = sizeof (sqlist_t);
7891 	sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7892 	strlock(stpdown, &sqlist);
7893 	strunlock(stpdown, &sqlist);
7894 
7895 	releaseq(passq);
7896 	return (passq);
7897 }
7898 
7899 /*
7900  * Let messages flow up into the mux by removing
7901  * the passq.
7902  */
7903 static void
link_rempassthru(queue_t * passq)7904 link_rempassthru(queue_t *passq)
7905 {
7906 	claimq(passq);
7907 	removeq(passq);
7908 	releaseq(passq);
7909 	freeq(passq);
7910 }
7911 
7912 /*
7913  * Wait for the condition variable pointed to by `cvp' to be signaled,
7914  * or for `tim' milliseconds to elapse, whichever comes first.  If `tim'
7915  * is negative, then there is no time limit.  If `nosigs' is non-zero,
7916  * then the wait will be non-interruptible.
7917  *
7918  * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7919  */
7920 clock_t
str_cv_wait(kcondvar_t * cvp,kmutex_t * mp,clock_t tim,int nosigs)7921 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7922 {
7923 	clock_t ret;
7924 
7925 	if (tim < 0) {
7926 		if (nosigs) {
7927 			cv_wait(cvp, mp);
7928 			ret = 1;
7929 		} else {
7930 			ret = cv_wait_sig(cvp, mp);
7931 		}
7932 	} else if (tim > 0) {
7933 		/*
7934 		 * convert milliseconds to clock ticks
7935 		 */
7936 		if (nosigs) {
7937 			ret = cv_reltimedwait(cvp, mp,
7938 			    MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7939 		} else {
7940 			ret = cv_reltimedwait_sig(cvp, mp,
7941 			    MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7942 		}
7943 	} else {
7944 		ret = -1;
7945 	}
7946 	return (ret);
7947 }
7948 
7949 /*
7950  * Wait until the stream head can determine if it is at the mark but
7951  * don't wait forever to prevent a race condition between the "mark" state
7952  * in the stream head and any mark state in the caller/user of this routine.
7953  *
7954  * This is used by sockets and for a socket it would be incorrect
7955  * to return a failure for SIOCATMARK when there is no data in the receive
7956  * queue and the marked urgent data is traveling up the stream.
7957  *
7958  * This routine waits until the mark is known by waiting for one of these
7959  * three events:
7960  *	The stream head read queue becoming non-empty (including an EOF).
7961  *	The STRATMARK flag being set (due to a MSGMARKNEXT message).
7962  *	The STRNOTATMARK flag being set (which indicates that the transport
7963  *	has sent a MSGNOTMARKNEXT message to indicate that it is not at
7964  *	the mark).
7965  *
7966  * The routine returns 1 if the stream is at the mark; 0 if it can
7967  * be determined that the stream is not at the mark.
7968  * If the wait times out and it can't determine
7969  * whether or not the stream might be at the mark the routine will return -1.
7970  *
7971  * Note: This routine should only be used when a mark is pending i.e.,
7972  * in the socket case the SIGURG has been posted.
7973  * Note2: This can not wakeup just because synchronous streams indicate
7974  * that data is available since it is not possible to use the synchronous
7975  * streams interfaces to determine the b_flag value for the data queued below
7976  * the stream head.
7977  */
7978 int
strwaitmark(vnode_t * vp)7979 strwaitmark(vnode_t *vp)
7980 {
7981 	struct stdata *stp = vp->v_stream;
7982 	queue_t *rq = _RD(stp->sd_wrq);
7983 	int mark;
7984 
7985 	mutex_enter(&stp->sd_lock);
7986 	while (rq->q_first == NULL &&
7987 	    !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7988 		stp->sd_flag |= RSLEEP;
7989 
7990 		/* Wait for 100 milliseconds for any state change. */
7991 		if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7992 			mutex_exit(&stp->sd_lock);
7993 			return (-1);
7994 		}
7995 	}
7996 	if (stp->sd_flag & STRATMARK)
7997 		mark = 1;
7998 	else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7999 		mark = 1;
8000 	else
8001 		mark = 0;
8002 
8003 	mutex_exit(&stp->sd_lock);
8004 	return (mark);
8005 }
8006 
8007 /*
8008  * Set a read side error. If persist is set change the socket error
8009  * to persistent. If errfunc is set install the function as the exported
8010  * error handler.
8011  */
8012 void
strsetrerror(vnode_t * vp,int error,int persist,errfunc_t errfunc)8013 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8014 {
8015 	struct stdata *stp = vp->v_stream;
8016 
8017 	mutex_enter(&stp->sd_lock);
8018 	stp->sd_rerror = error;
8019 	if (error == 0 && errfunc == NULL)
8020 		stp->sd_flag &= ~STRDERR;
8021 	else
8022 		stp->sd_flag |= STRDERR;
8023 	if (persist) {
8024 		stp->sd_flag &= ~STRDERRNONPERSIST;
8025 	} else {
8026 		stp->sd_flag |= STRDERRNONPERSIST;
8027 	}
8028 	stp->sd_rderrfunc = errfunc;
8029 	if (error != 0 || errfunc != NULL) {
8030 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);	/* readers */
8031 		cv_broadcast(&stp->sd_wrq->q_wait);		/* writers */
8032 		cv_broadcast(&stp->sd_monitor);			/* ioctllers */
8033 
8034 		mutex_exit(&stp->sd_lock);
8035 		pollwakeup(&stp->sd_pollist, POLLERR);
8036 		mutex_enter(&stp->sd_lock);
8037 
8038 		if (stp->sd_sigflags & S_ERROR)
8039 			strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8040 	}
8041 	mutex_exit(&stp->sd_lock);
8042 }
8043 
8044 /*
8045  * Set a write side error. If persist is set change the socket error
8046  * to persistent.
8047  */
8048 void
strsetwerror(vnode_t * vp,int error,int persist,errfunc_t errfunc)8049 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8050 {
8051 	struct stdata *stp = vp->v_stream;
8052 
8053 	mutex_enter(&stp->sd_lock);
8054 	stp->sd_werror = error;
8055 	if (error == 0 && errfunc == NULL)
8056 		stp->sd_flag &= ~STWRERR;
8057 	else
8058 		stp->sd_flag |= STWRERR;
8059 	if (persist) {
8060 		stp->sd_flag &= ~STWRERRNONPERSIST;
8061 	} else {
8062 		stp->sd_flag |= STWRERRNONPERSIST;
8063 	}
8064 	stp->sd_wrerrfunc = errfunc;
8065 	if (error != 0 || errfunc != NULL) {
8066 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);	/* readers */
8067 		cv_broadcast(&stp->sd_wrq->q_wait);		/* writers */
8068 		cv_broadcast(&stp->sd_monitor);			/* ioctllers */
8069 
8070 		mutex_exit(&stp->sd_lock);
8071 		pollwakeup(&stp->sd_pollist, POLLERR);
8072 		mutex_enter(&stp->sd_lock);
8073 
8074 		if (stp->sd_sigflags & S_ERROR)
8075 			strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8076 	}
8077 	mutex_exit(&stp->sd_lock);
8078 }
8079 
8080 /*
8081  * Make the stream return 0 (EOF) when all data has been read.
8082  * No effect on write side.
8083  */
8084 void
strseteof(vnode_t * vp,int eof)8085 strseteof(vnode_t *vp, int eof)
8086 {
8087 	struct stdata *stp = vp->v_stream;
8088 
8089 	mutex_enter(&stp->sd_lock);
8090 	if (!eof) {
8091 		stp->sd_flag &= ~STREOF;
8092 		mutex_exit(&stp->sd_lock);
8093 		return;
8094 	}
8095 	stp->sd_flag |= STREOF;
8096 	if (stp->sd_flag & RSLEEP) {
8097 		stp->sd_flag &= ~RSLEEP;
8098 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8099 	}
8100 
8101 	mutex_exit(&stp->sd_lock);
8102 	pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8103 	mutex_enter(&stp->sd_lock);
8104 
8105 	if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8106 		strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8107 	mutex_exit(&stp->sd_lock);
8108 }
8109 
8110 void
strflushrq(vnode_t * vp,int flag)8111 strflushrq(vnode_t *vp, int flag)
8112 {
8113 	struct stdata *stp = vp->v_stream;
8114 
8115 	mutex_enter(&stp->sd_lock);
8116 	flushq(_RD(stp->sd_wrq), flag);
8117 	mutex_exit(&stp->sd_lock);
8118 }
8119 
8120 void
strsetrputhooks(vnode_t * vp,uint_t flags,msgfunc_t protofunc,msgfunc_t miscfunc)8121 strsetrputhooks(vnode_t *vp, uint_t flags,
8122     msgfunc_t protofunc, msgfunc_t miscfunc)
8123 {
8124 	struct stdata *stp = vp->v_stream;
8125 
8126 	mutex_enter(&stp->sd_lock);
8127 
8128 	if (protofunc == NULL)
8129 		stp->sd_rprotofunc = strrput_proto;
8130 	else
8131 		stp->sd_rprotofunc = protofunc;
8132 
8133 	if (miscfunc == NULL)
8134 		stp->sd_rmiscfunc = strrput_misc;
8135 	else
8136 		stp->sd_rmiscfunc = miscfunc;
8137 
8138 	if (flags & SH_CONSOL_DATA)
8139 		stp->sd_rput_opt |= SR_CONSOL_DATA;
8140 	else
8141 		stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8142 
8143 	if (flags & SH_SIGALLDATA)
8144 		stp->sd_rput_opt |= SR_SIGALLDATA;
8145 	else
8146 		stp->sd_rput_opt &= ~SR_SIGALLDATA;
8147 
8148 	if (flags & SH_IGN_ZEROLEN)
8149 		stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8150 	else
8151 		stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8152 
8153 	mutex_exit(&stp->sd_lock);
8154 }
8155 
8156 void
strsetwputhooks(vnode_t * vp,uint_t flags,clock_t closetime)8157 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8158 {
8159 	struct stdata *stp = vp->v_stream;
8160 
8161 	mutex_enter(&stp->sd_lock);
8162 	stp->sd_closetime = closetime;
8163 
8164 	if (flags & SH_SIGPIPE)
8165 		stp->sd_wput_opt |= SW_SIGPIPE;
8166 	else
8167 		stp->sd_wput_opt &= ~SW_SIGPIPE;
8168 	if (flags & SH_RECHECK_ERR)
8169 		stp->sd_wput_opt |= SW_RECHECK_ERR;
8170 	else
8171 		stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8172 
8173 	mutex_exit(&stp->sd_lock);
8174 }
8175 
8176 void
strsetrwputdatahooks(vnode_t * vp,msgfunc_t rdatafunc,msgfunc_t wdatafunc)8177 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8178 {
8179 	struct stdata *stp = vp->v_stream;
8180 
8181 	mutex_enter(&stp->sd_lock);
8182 
8183 	stp->sd_rputdatafunc = rdatafunc;
8184 	stp->sd_wputdatafunc = wdatafunc;
8185 
8186 	mutex_exit(&stp->sd_lock);
8187 }
8188 
8189 /* Used within framework when the queue is already locked */
8190 void
qenable_locked(queue_t * q)8191 qenable_locked(queue_t *q)
8192 {
8193 	stdata_t *stp = STREAM(q);
8194 
8195 	ASSERT(MUTEX_HELD(QLOCK(q)));
8196 
8197 	if (!q->q_qinfo->qi_srvp)
8198 		return;
8199 
8200 	/*
8201 	 * Do not place on run queue if already enabled or closing.
8202 	 */
8203 	if (q->q_flag & (QWCLOSE|QENAB))
8204 		return;
8205 
8206 	/*
8207 	 * mark queue enabled and place on run list if it is not already being
8208 	 * serviced. If it is serviced, the runservice() function will detect
8209 	 * that QENAB is set and call service procedure before clearing
8210 	 * QINSERVICE flag.
8211 	 */
8212 	q->q_flag |= QENAB;
8213 	if (q->q_flag & QINSERVICE)
8214 		return;
8215 
8216 	/* Record the time of qenable */
8217 	q->q_qtstamp = ddi_get_lbolt();
8218 
8219 	/*
8220 	 * Put the queue in the stp list and schedule it for background
8221 	 * processing if it is not already scheduled or if stream head does not
8222 	 * intent to process it in the foreground later by setting
8223 	 * STRS_WILLSERVICE flag.
8224 	 */
8225 	mutex_enter(&stp->sd_qlock);
8226 	/*
8227 	 * If there are already something on the list, stp flags should show
8228 	 * intention to drain it.
8229 	 */
8230 	IMPLY(STREAM_NEEDSERVICE(stp),
8231 	    (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8232 
8233 	ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8234 	stp->sd_nqueues++;
8235 
8236 	/*
8237 	 * If no one will drain this stream we are the first producer and
8238 	 * need to schedule it for background thread.
8239 	 */
8240 	if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8241 		/*
8242 		 * No one will service this stream later, so we have to
8243 		 * schedule it now.
8244 		 */
8245 		STRSTAT(stenables);
8246 		stp->sd_svcflags |= STRS_SCHEDULED;
8247 		stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8248 		    (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8249 
8250 		if (stp->sd_servid == NULL) {
8251 			/*
8252 			 * Task queue failed so fail over to the backup
8253 			 * servicing thread.
8254 			 */
8255 			STRSTAT(taskqfails);
8256 			/*
8257 			 * It is safe to clear STRS_SCHEDULED flag because it
8258 			 * was set by this thread above.
8259 			 */
8260 			stp->sd_svcflags &= ~STRS_SCHEDULED;
8261 
8262 			/*
8263 			 * Failover scheduling is protected by service_queue
8264 			 * lock.
8265 			 */
8266 			mutex_enter(&service_queue);
8267 			ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8268 			ASSERT(q->q_link == NULL);
8269 			/*
8270 			 * Append the queue to qhead/qtail list.
8271 			 */
8272 			if (qhead == NULL)
8273 				qhead = q;
8274 			else
8275 				qtail->q_link = q;
8276 			qtail = q;
8277 			/*
8278 			 * Clear stp queue list.
8279 			 */
8280 			stp->sd_qhead = stp->sd_qtail = NULL;
8281 			stp->sd_nqueues = 0;
8282 			/*
8283 			 * Wakeup background queue processing thread.
8284 			 */
8285 			cv_signal(&services_to_run);
8286 			mutex_exit(&service_queue);
8287 		}
8288 	}
8289 	mutex_exit(&stp->sd_qlock);
8290 }
8291 
8292 static void
queue_service(queue_t * q)8293 queue_service(queue_t *q)
8294 {
8295 	/*
8296 	 * The queue in the list should have
8297 	 * QENAB flag set and should not have
8298 	 * QINSERVICE flag set. QINSERVICE is
8299 	 * set when the queue is dequeued and
8300 	 * qenable_locked doesn't enqueue a
8301 	 * queue with QINSERVICE set.
8302 	 */
8303 
8304 	ASSERT(!(q->q_flag & QINSERVICE));
8305 	ASSERT((q->q_flag & QENAB));
8306 	mutex_enter(QLOCK(q));
8307 	q->q_flag &= ~QENAB;
8308 	q->q_flag |= QINSERVICE;
8309 	mutex_exit(QLOCK(q));
8310 	runservice(q);
8311 }
8312 
8313 static void
syncq_service(syncq_t * sq)8314 syncq_service(syncq_t *sq)
8315 {
8316 	STRSTAT(syncqservice);
8317 	mutex_enter(SQLOCK(sq));
8318 	ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8319 	ASSERT(sq->sq_servcount != 0);
8320 	ASSERT(sq->sq_next == NULL);
8321 
8322 	/* if we came here from the background thread, clear the flag */
8323 	if (sq->sq_svcflags & SQ_BGTHREAD)
8324 		sq->sq_svcflags &= ~SQ_BGTHREAD;
8325 
8326 	/* let drain_syncq know that it's being called in the background */
8327 	sq->sq_svcflags |= SQ_SERVICE;
8328 	drain_syncq(sq);
8329 }
8330 
8331 static void
qwriter_outer_service(syncq_t * outer)8332 qwriter_outer_service(syncq_t *outer)
8333 {
8334 	/*
8335 	 * Note that SQ_WRITER is used on the outer perimeter
8336 	 * to signal that a qwriter(OUTER) is either investigating
8337 	 * running or that it is actually running a function.
8338 	 */
8339 	outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8340 
8341 	/*
8342 	 * All inner syncq are empty and have SQ_WRITER set
8343 	 * to block entering the outer perimeter.
8344 	 *
8345 	 * We do not need to explicitly call write_now since
8346 	 * outer_exit does it for us.
8347 	 */
8348 	outer_exit(outer);
8349 }
8350 
8351 static void
mblk_free(mblk_t * mp)8352 mblk_free(mblk_t *mp)
8353 {
8354 	dblk_t *dbp = mp->b_datap;
8355 	frtn_t *frp = dbp->db_frtnp;
8356 
8357 	mp->b_next = NULL;
8358 	if (dbp->db_fthdr != NULL)
8359 		str_ftfree(dbp);
8360 
8361 	ASSERT(dbp->db_fthdr == NULL);
8362 	frp->free_func(frp->free_arg);
8363 	ASSERT(dbp->db_mblk == mp);
8364 
8365 	if (dbp->db_credp != NULL) {
8366 		crfree(dbp->db_credp);
8367 		dbp->db_credp = NULL;
8368 	}
8369 	dbp->db_cpid = -1;
8370 	dbp->db_struioflag = 0;
8371 	dbp->db_struioun.cksum.flags = 0;
8372 
8373 	kmem_cache_free(dbp->db_cache, dbp);
8374 }
8375 
8376 /*
8377  * Background processing of the stream queue list.
8378  */
8379 static void
stream_service(stdata_t * stp)8380 stream_service(stdata_t *stp)
8381 {
8382 	queue_t *q;
8383 
8384 	mutex_enter(&stp->sd_qlock);
8385 
8386 	STR_SERVICE(stp, q);
8387 
8388 	stp->sd_svcflags &= ~STRS_SCHEDULED;
8389 	stp->sd_servid = NULL;
8390 	cv_signal(&stp->sd_qcv);
8391 	mutex_exit(&stp->sd_qlock);
8392 }
8393 
8394 /*
8395  * Foreground processing of the stream queue list.
8396  */
8397 void
stream_runservice(stdata_t * stp)8398 stream_runservice(stdata_t *stp)
8399 {
8400 	queue_t *q;
8401 
8402 	mutex_enter(&stp->sd_qlock);
8403 	STRSTAT(rservice);
8404 	/*
8405 	 * We are going to drain this stream queue list, so qenable_locked will
8406 	 * not schedule it until we finish.
8407 	 */
8408 	stp->sd_svcflags |= STRS_WILLSERVICE;
8409 
8410 	STR_SERVICE(stp, q);
8411 
8412 	stp->sd_svcflags &= ~STRS_WILLSERVICE;
8413 	mutex_exit(&stp->sd_qlock);
8414 	/*
8415 	 * Help backup background thread to drain the qhead/qtail list.
8416 	 */
8417 	while (qhead != NULL) {
8418 		STRSTAT(qhelps);
8419 		mutex_enter(&service_queue);
8420 		DQ(q, qhead, qtail, q_link);
8421 		mutex_exit(&service_queue);
8422 		if (q != NULL)
8423 			queue_service(q);
8424 	}
8425 }
8426 
8427 void
stream_willservice(stdata_t * stp)8428 stream_willservice(stdata_t *stp)
8429 {
8430 	mutex_enter(&stp->sd_qlock);
8431 	stp->sd_svcflags |= STRS_WILLSERVICE;
8432 	mutex_exit(&stp->sd_qlock);
8433 }
8434 
8435 /*
8436  * Replace the cred currently in the mblk with a different one.
8437  * Also update db_cpid.
8438  */
8439 void
mblk_setcred(mblk_t * mp,cred_t * cr,pid_t cpid)8440 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8441 {
8442 	dblk_t *dbp = mp->b_datap;
8443 	cred_t *ocr = dbp->db_credp;
8444 
8445 	ASSERT(cr != NULL);
8446 
8447 	if (cr != ocr) {
8448 		crhold(dbp->db_credp = cr);
8449 		if (ocr != NULL)
8450 			crfree(ocr);
8451 	}
8452 	/* Don't overwrite with NOPID */
8453 	if (cpid != NOPID)
8454 		dbp->db_cpid = cpid;
8455 }
8456 
8457 /*
8458  * If the src message has a cred, then replace the cred currently in the mblk
8459  * with it.
8460  * Also update db_cpid.
8461  */
8462 void
mblk_copycred(mblk_t * mp,const mblk_t * src)8463 mblk_copycred(mblk_t *mp, const mblk_t *src)
8464 {
8465 	dblk_t *dbp = mp->b_datap;
8466 	cred_t *cr, *ocr;
8467 	pid_t cpid;
8468 
8469 	cr = msg_getcred(src, &cpid);
8470 	if (cr == NULL)
8471 		return;
8472 
8473 	ocr = dbp->db_credp;
8474 	if (cr != ocr) {
8475 		crhold(dbp->db_credp = cr);
8476 		if (ocr != NULL)
8477 			crfree(ocr);
8478 	}
8479 	/* Don't overwrite with NOPID */
8480 	if (cpid != NOPID)
8481 		dbp->db_cpid = cpid;
8482 }
8483 
8484 void
lso_info_set(mblk_t * mp,uint32_t mss,uint32_t flags)8485 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8486 {
8487 	ASSERT(DB_TYPE(mp) == M_DATA);
8488 	ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8489 
8490 	/* Set the flags */
8491 	DB_LSOFLAGS(mp) |= flags;
8492 	DB_LSOMSS(mp) = mss;
8493 }
8494 
8495 void
lso_info_cleanup(mblk_t * mp)8496 lso_info_cleanup(mblk_t *mp)
8497 {
8498 	ASSERT(DB_TYPE(mp) == M_DATA);
8499 
8500 	/* Clear the flags */
8501 	DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8502 	DB_LSOMSS(mp) = 0;
8503 }
8504 
8505 /*
8506  * Checksum buffer *bp for len bytes with psum partial checksum,
8507  * or 0 if none, and return the 16 bit partial checksum.
8508  */
8509 unsigned
bcksum(uchar_t * bp,int len,unsigned int psum)8510 bcksum(uchar_t *bp, int len, unsigned int psum)
8511 {
8512 	int odd = len & 1;
8513 	extern unsigned int ip_ocsum();
8514 
8515 	if (((intptr_t)bp & 1) == 0 && !odd) {
8516 		/*
8517 		 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8518 		 */
8519 		return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8520 	}
8521 	if (((intptr_t)bp & 1) != 0) {
8522 		/*
8523 		 * Bp isn't 16 bit aligned.
8524 		 */
8525 		unsigned int tsum;
8526 
8527 #ifdef _LITTLE_ENDIAN
8528 		psum += *bp;
8529 #else
8530 		psum += *bp << 8;
8531 #endif
8532 		len--;
8533 		bp++;
8534 		tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8535 		psum += (tsum << 8) & 0xffff | (tsum >> 8);
8536 		if (len & 1) {
8537 			bp += len - 1;
8538 #ifdef _LITTLE_ENDIAN
8539 			psum += *bp << 8;
8540 #else
8541 			psum += *bp;
8542 #endif
8543 		}
8544 	} else {
8545 		/*
8546 		 * Bp is 16 bit aligned.
8547 		 */
8548 		psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8549 		if (odd) {
8550 			bp += len - 1;
8551 #ifdef _LITTLE_ENDIAN
8552 			psum += *bp;
8553 #else
8554 			psum += *bp << 8;
8555 #endif
8556 		}
8557 	}
8558 	/*
8559 	 * Normalize psum to 16 bits before returning the new partial
8560 	 * checksum. The max psum value before normalization is 0x3FDFE.
8561 	 */
8562 	return ((psum >> 16) + (psum & 0xFFFF));
8563 }
8564 
8565 void
freemsgchain(mblk_t * mp)8566 freemsgchain(mblk_t *mp)
8567 {
8568 	mblk_t	*next;
8569 
8570 	while (mp != NULL) {
8571 		next = mp->b_next;
8572 		mp->b_next = NULL;
8573 
8574 		freemsg(mp);
8575 		mp = next;
8576 	}
8577 }
8578 
8579 mblk_t *
copymsgchain(mblk_t * mp)8580 copymsgchain(mblk_t *mp)
8581 {
8582 	mblk_t	*nmp = NULL;
8583 	mblk_t	**nmpp = &nmp;
8584 
8585 	for (; mp != NULL; mp = mp->b_next) {
8586 		if ((*nmpp = copymsg(mp)) == NULL) {
8587 			freemsgchain(nmp);
8588 			return (NULL);
8589 		}
8590 
8591 		nmpp = &((*nmpp)->b_next);
8592 	}
8593 
8594 	return (nmp);
8595 }
8596 
8597 /* NOTE: Do not add code after this point. */
8598 #undef QLOCK
8599 
8600 /*
8601  * Replacement for QLOCK macro for those that can't use it.
8602  */
8603 kmutex_t *
QLOCK(queue_t * q)8604 QLOCK(queue_t *q)
8605 {
8606 	return (&(q)->q_lock);
8607 }
8608 
8609 /*
8610  * Dummy runqueues/queuerun functions functions for backwards compatibility.
8611  */
8612 #undef runqueues
8613 void
runqueues(void)8614 runqueues(void)
8615 {
8616 }
8617 
8618 #undef queuerun
8619 void
queuerun(void)8620 queuerun(void)
8621 {
8622 }
8623 
8624 /*
8625  * Initialize the STR stack instance, which tracks autopush and persistent
8626  * links.
8627  */
8628 /* ARGSUSED */
8629 static void *
str_stack_init(netstackid_t stackid,netstack_t * ns)8630 str_stack_init(netstackid_t stackid, netstack_t *ns)
8631 {
8632 	str_stack_t	*ss;
8633 	int i;
8634 
8635 	ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8636 	ss->ss_netstack = ns;
8637 
8638 	/*
8639 	 * set up autopush
8640 	 */
8641 	sad_initspace(ss);
8642 
8643 	/*
8644 	 * set up mux_node structures.
8645 	 */
8646 	ss->ss_devcnt = devcnt;	/* In case it should change before free */
8647 	ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8648 	    ss->ss_devcnt), KM_SLEEP);
8649 	for (i = 0; i < ss->ss_devcnt; i++)
8650 		ss->ss_mux_nodes[i].mn_imaj = i;
8651 	return (ss);
8652 }
8653 
8654 /*
8655  * Note: run at zone shutdown and not destroy so that the PLINKs are
8656  * gone by the time other cleanup happens from the destroy callbacks.
8657  */
8658 static void
str_stack_shutdown(netstackid_t stackid,void * arg)8659 str_stack_shutdown(netstackid_t stackid, void *arg)
8660 {
8661 	str_stack_t *ss = (str_stack_t *)arg;
8662 	int i;
8663 	cred_t *cr;
8664 
8665 	cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8666 	ASSERT(cr != NULL);
8667 
8668 	/* Undo all the I_PLINKs for this zone */
8669 	for (i = 0; i < ss->ss_devcnt; i++) {
8670 		struct mux_edge		*ep;
8671 		ldi_handle_t		lh;
8672 		ldi_ident_t		li;
8673 		int			ret;
8674 		int			rval;
8675 		dev_t			rdev;
8676 
8677 		ep = ss->ss_mux_nodes[i].mn_outp;
8678 		if (ep == NULL)
8679 			continue;
8680 		ret = ldi_ident_from_major((major_t)i, &li);
8681 		if (ret != 0) {
8682 			continue;
8683 		}
8684 		rdev = ep->me_dev;
8685 		ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8686 		    cr, &lh, li);
8687 		if (ret != 0) {
8688 			ldi_ident_release(li);
8689 			continue;
8690 		}
8691 
8692 		ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8693 		    cr, &rval);
8694 		if (ret) {
8695 			(void) ldi_close(lh, FREAD|FWRITE, cr);
8696 			ldi_ident_release(li);
8697 			continue;
8698 		}
8699 		(void) ldi_close(lh, FREAD|FWRITE, cr);
8700 
8701 		/* Close layered handles */
8702 		ldi_ident_release(li);
8703 	}
8704 	crfree(cr);
8705 
8706 	sad_freespace(ss);
8707 
8708 	kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8709 	ss->ss_mux_nodes = NULL;
8710 }
8711 
8712 /*
8713  * Free the structure; str_stack_shutdown did the other cleanup work.
8714  */
8715 /* ARGSUSED */
8716 static void
str_stack_fini(netstackid_t stackid,void * arg)8717 str_stack_fini(netstackid_t stackid, void *arg)
8718 {
8719 	str_stack_t	*ss = (str_stack_t *)arg;
8720 
8721 	kmem_free(ss, sizeof (*ss));
8722 }
8723