1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
27  * Copyright 2019 Joyent, Inc.
28  */
29 
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/signal.h>
33 #include <sys/cmn_err.h>
34 
35 #include <sys/stropts.h>
36 #include <sys/socket.h>
37 #include <sys/socketvar.h>
38 #include <sys/sockio.h>
39 #include <sys/strsubr.h>
40 #include <sys/strsun.h>
41 #include <sys/atomic.h>
42 #include <sys/tihdr.h>
43 
44 #include <fs/sockfs/sockcommon.h>
45 #include <fs/sockfs/sockfilter_impl.h>
46 #include <fs/sockfs/socktpi.h>
47 #include <fs/sockfs/sodirect.h>
48 #include <sys/ddi.h>
49 #include <inet/ip.h>
50 #include <sys/time.h>
51 #include <sys/cmn_err.h>
52 
53 #ifdef SOCK_TEST
54 extern int do_useracc;
55 extern clock_t sock_test_timelimit;
56 #endif /* SOCK_TEST */
57 
58 #define	MBLK_PULL_LEN 64
59 uint32_t so_mblk_pull_len = MBLK_PULL_LEN;
60 
61 #ifdef DEBUG
62 boolean_t so_debug_length = B_FALSE;
63 static boolean_t so_check_length(sonode_t *so);
64 #endif
65 
66 static int
so_acceptq_dequeue_locked(struct sonode * so,boolean_t dontblock,struct sonode ** nsop)67 so_acceptq_dequeue_locked(struct sonode *so, boolean_t dontblock,
68     struct sonode **nsop)
69 {
70 	struct sonode *nso = NULL;
71 
72 	*nsop = NULL;
73 	ASSERT(MUTEX_HELD(&so->so_acceptq_lock));
74 	while ((nso = list_remove_head(&so->so_acceptq_list)) == NULL) {
75 		/*
76 		 * No need to check so_error here, because it is not
77 		 * possible for a listening socket to be reset or otherwise
78 		 * disconnected.
79 		 *
80 		 * So now we just need check if it's ok to wait.
81 		 */
82 		if (dontblock)
83 			return (EWOULDBLOCK);
84 		if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
85 			return (EINTR);
86 
87 		if (cv_wait_sig_swap(&so->so_acceptq_cv,
88 		    &so->so_acceptq_lock) == 0)
89 			return (EINTR);
90 	}
91 
92 	ASSERT(nso != NULL);
93 	ASSERT(so->so_acceptq_len > 0);
94 	so->so_acceptq_len--;
95 	nso->so_listener = NULL;
96 
97 	*nsop = nso;
98 
99 	return (0);
100 }
101 
102 /*
103  * int so_acceptq_dequeue(struct sonode *, boolean_t, struct sonode **)
104  *
105  * Pulls a connection off of the accept queue.
106  *
107  * Arguments:
108  *   so	       - listening socket
109  *   dontblock - indicate whether it's ok to sleep if there are no
110  *		 connections on the queue
111  *   nsop      - Value-return argument
112  *
113  * Return values:
114  *   0 when a connection is successfully dequeued, in which case nsop
115  *   is set to point to the new connection. Upon failure a non-zero
116  *   value is returned, and the value of nsop is set to NULL.
117  *
118  * Note:
119  *   so_acceptq_dequeue() may return prematurly if the socket is falling
120  *   back to TPI.
121  */
122 int
so_acceptq_dequeue(struct sonode * so,boolean_t dontblock,struct sonode ** nsop)123 so_acceptq_dequeue(struct sonode *so, boolean_t dontblock,
124     struct sonode **nsop)
125 {
126 	int error;
127 
128 	mutex_enter(&so->so_acceptq_lock);
129 	error = so_acceptq_dequeue_locked(so, dontblock, nsop);
130 	mutex_exit(&so->so_acceptq_lock);
131 
132 	return (error);
133 }
134 
135 static void
so_acceptq_flush_impl(struct sonode * so,list_t * list,boolean_t doclose)136 so_acceptq_flush_impl(struct sonode *so, list_t *list, boolean_t doclose)
137 {
138 	struct sonode *nso;
139 
140 	while ((nso = list_remove_head(list)) != NULL) {
141 		nso->so_listener = NULL;
142 		if (doclose) {
143 			(void) socket_close(nso, 0, CRED());
144 		} else {
145 			/*
146 			 * Only used for fallback - not possible when filters
147 			 * are present.
148 			 */
149 			ASSERT(so->so_filter_active == 0);
150 			/*
151 			 * Since the socket is on the accept queue, there can
152 			 * only be one reference. We drop the reference and
153 			 * just blow off the socket.
154 			 */
155 			ASSERT(nso->so_count == 1);
156 			nso->so_count--;
157 			/* drop the proto ref */
158 			VN_RELE(SOTOV(nso));
159 		}
160 		socket_destroy(nso);
161 	}
162 }
163 /*
164  * void so_acceptq_flush(struct sonode *so)
165  *
166  * Removes all pending connections from a listening socket, and
167  * frees the associated resources.
168  *
169  * Arguments
170  *   so	     - listening socket
171  *   doclose - make a close downcall for each socket on the accept queue
172  *
173  * Return values:
174  *   None.
175  *
176  * Note:
177  *   The caller has to ensure that no calls to so_acceptq_enqueue() or
178  *   so_acceptq_dequeue() occur while the accept queue is being flushed.
179  *   So either the socket needs to be in a state where no operations
180  *   would come in, or so_lock needs to be obtained.
181  */
182 void
so_acceptq_flush(struct sonode * so,boolean_t doclose)183 so_acceptq_flush(struct sonode *so, boolean_t doclose)
184 {
185 	so_acceptq_flush_impl(so, &so->so_acceptq_list, doclose);
186 	so_acceptq_flush_impl(so, &so->so_acceptq_defer, doclose);
187 
188 	so->so_acceptq_len = 0;
189 }
190 
191 int
so_wait_connected_locked(struct sonode * so,boolean_t nonblock,sock_connid_t id)192 so_wait_connected_locked(struct sonode *so, boolean_t nonblock,
193     sock_connid_t id)
194 {
195 	ASSERT(MUTEX_HELD(&so->so_lock));
196 
197 	/*
198 	 * The protocol has notified us that a connection attempt is being
199 	 * made, so before we wait for a notification to arrive we must
200 	 * clear out any errors associated with earlier connection attempts.
201 	 */
202 	if (so->so_error != 0 && SOCK_CONNID_LT(so->so_proto_connid, id))
203 		so->so_error = 0;
204 
205 	while (SOCK_CONNID_LT(so->so_proto_connid, id)) {
206 		if (nonblock)
207 			return (EINPROGRESS);
208 
209 		if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
210 			return (EINTR);
211 
212 		if (cv_wait_sig_swap(&so->so_state_cv, &so->so_lock) == 0)
213 			return (EINTR);
214 	}
215 
216 	if (so->so_error != 0)
217 		return (sogeterr(so, B_TRUE));
218 	/*
219 	 * Under normal circumstances, so_error should contain an error
220 	 * in case the connect failed. However, it is possible for another
221 	 * thread to come in a consume the error, so generate a sensible
222 	 * error in that case.
223 	 */
224 	if ((so->so_state & SS_ISCONNECTED) == 0)
225 		return (ECONNREFUSED);
226 
227 	return (0);
228 }
229 
230 /*
231  * int so_wait_connected(struct sonode *so, boolean_t nonblock,
232  *    sock_connid_t id)
233  *
234  * Wait until the socket is connected or an error has occured.
235  *
236  * Arguments:
237  *   so	      - socket
238  *   nonblock - indicate whether it's ok to sleep if the connection has
239  *		not yet been established
240  *   gen      - generation number that was returned by the protocol
241  *		when the operation was started
242  *
243  * Returns:
244  *   0 if the connection attempt was successful, or an error indicating why
245  *   the connection attempt failed.
246  */
247 int
so_wait_connected(struct sonode * so,boolean_t nonblock,sock_connid_t id)248 so_wait_connected(struct sonode *so, boolean_t nonblock, sock_connid_t id)
249 {
250 	int error;
251 
252 	mutex_enter(&so->so_lock);
253 	error = so_wait_connected_locked(so, nonblock, id);
254 	mutex_exit(&so->so_lock);
255 
256 	return (error);
257 }
258 
259 int
so_snd_wait_qnotfull_locked(struct sonode * so,boolean_t dontblock)260 so_snd_wait_qnotfull_locked(struct sonode *so, boolean_t dontblock)
261 {
262 	int error;
263 
264 	ASSERT(MUTEX_HELD(&so->so_lock));
265 	while (SO_SND_FLOWCTRLD(so)) {
266 		if (so->so_state & SS_CANTSENDMORE)
267 			return (EPIPE);
268 		if (dontblock)
269 			return (EWOULDBLOCK);
270 
271 		if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING))
272 			return (EINTR);
273 
274 		if (so->so_sndtimeo == 0) {
275 			/*
276 			 * Zero means disable timeout.
277 			 */
278 			error = cv_wait_sig(&so->so_snd_cv, &so->so_lock);
279 		} else {
280 			error = cv_reltimedwait_sig(&so->so_snd_cv,
281 			    &so->so_lock, so->so_sndtimeo, TR_CLOCK_TICK);
282 		}
283 		if (error == 0)
284 			return (EINTR);
285 		else if (error == -1)
286 			return (EAGAIN);
287 	}
288 	return (0);
289 }
290 
291 /*
292  * int so_wait_sendbuf(struct sonode *so, boolean_t dontblock)
293  *
294  * Wait for the transport to notify us about send buffers becoming
295  * available.
296  */
297 int
so_snd_wait_qnotfull(struct sonode * so,boolean_t dontblock)298 so_snd_wait_qnotfull(struct sonode *so, boolean_t dontblock)
299 {
300 	int error = 0;
301 
302 	mutex_enter(&so->so_lock);
303 	so->so_snd_wakeup = B_TRUE;
304 	error = so_snd_wait_qnotfull_locked(so, dontblock);
305 	so->so_snd_wakeup = B_FALSE;
306 	mutex_exit(&so->so_lock);
307 
308 	return (error);
309 }
310 
311 void
so_snd_qfull(struct sonode * so)312 so_snd_qfull(struct sonode *so)
313 {
314 	mutex_enter(&so->so_lock);
315 	so->so_snd_qfull = B_TRUE;
316 	mutex_exit(&so->so_lock);
317 }
318 
319 void
so_snd_qnotfull(struct sonode * so)320 so_snd_qnotfull(struct sonode *so)
321 {
322 	mutex_enter(&so->so_lock);
323 	so->so_snd_qfull = B_FALSE;
324 	/* wake up everyone waiting for buffers */
325 	cv_broadcast(&so->so_snd_cv);
326 	mutex_exit(&so->so_lock);
327 }
328 
329 /*
330  * Change the process/process group to which SIGIO is sent.
331  */
332 int
socket_chgpgrp(struct sonode * so,pid_t pid)333 socket_chgpgrp(struct sonode *so, pid_t pid)
334 {
335 	int error;
336 
337 	ASSERT(MUTEX_HELD(&so->so_lock));
338 	if (pid != 0) {
339 		/*
340 		 * Permissions check by sending signal 0.
341 		 * Note that when kill fails it does a
342 		 * set_errno causing the system call to fail.
343 		 */
344 		error = kill(pid, 0);
345 		if (error != 0) {
346 			return (error);
347 		}
348 	}
349 	so->so_pgrp = pid;
350 	return (0);
351 }
352 
353 
354 /*
355  * Generate a SIGIO, for 'writable' events include siginfo structure,
356  * for read events just send the signal.
357  */
358 /*ARGSUSED*/
359 static void
socket_sigproc(proc_t * proc,int event)360 socket_sigproc(proc_t *proc, int event)
361 {
362 	k_siginfo_t info;
363 
364 	ASSERT(event & (SOCKETSIG_WRITE | SOCKETSIG_READ | SOCKETSIG_URG));
365 
366 	if (event & SOCKETSIG_WRITE) {
367 		info.si_signo = SIGPOLL;
368 		info.si_code = POLL_OUT;
369 		info.si_errno = 0;
370 		info.si_fd = 0;
371 		info.si_band = 0;
372 		sigaddq(proc, NULL, &info, KM_NOSLEEP);
373 	}
374 	if (event & SOCKETSIG_READ) {
375 		sigtoproc(proc, NULL, SIGPOLL);
376 	}
377 	if (event & SOCKETSIG_URG) {
378 		sigtoproc(proc, NULL, SIGURG);
379 	}
380 }
381 
382 void
socket_sendsig(struct sonode * so,int event)383 socket_sendsig(struct sonode *so, int event)
384 {
385 	proc_t *proc;
386 
387 	ASSERT(MUTEX_HELD(&so->so_lock));
388 
389 	if (so->so_pgrp == 0 || (!(so->so_state & SS_ASYNC) &&
390 	    event != SOCKETSIG_URG)) {
391 		return;
392 	}
393 
394 	dprint(3, ("sending sig %d to %d\n", event, so->so_pgrp));
395 
396 	if (so->so_pgrp > 0) {
397 		/*
398 		 * XXX This unfortunately still generates
399 		 * a signal when a fd is closed but
400 		 * the proc is active.
401 		 */
402 		mutex_enter(&pidlock);
403 		/*
404 		 * Even if the thread started in another zone, we're receiving
405 		 * on behalf of this socket's zone, so find the proc using the
406 		 * socket's zone ID.
407 		 */
408 		proc = prfind_zone(so->so_pgrp, so->so_zoneid);
409 		if (proc == NULL) {
410 			mutex_exit(&pidlock);
411 			return;
412 		}
413 		mutex_enter(&proc->p_lock);
414 		mutex_exit(&pidlock);
415 		socket_sigproc(proc, event);
416 		mutex_exit(&proc->p_lock);
417 	} else {
418 		/*
419 		 * Send to process group. Hold pidlock across
420 		 * calls to socket_sigproc().
421 		 */
422 		pid_t pgrp = -so->so_pgrp;
423 
424 		mutex_enter(&pidlock);
425 		/*
426 		 * Even if the thread started in another zone, we're receiving
427 		 * on behalf of this socket's zone, so find the pgrp using the
428 		 * socket's zone ID.
429 		 */
430 		proc = pgfind_zone(pgrp, so->so_zoneid);
431 		while (proc != NULL) {
432 			mutex_enter(&proc->p_lock);
433 			socket_sigproc(proc, event);
434 			mutex_exit(&proc->p_lock);
435 			proc = proc->p_pglink;
436 		}
437 		mutex_exit(&pidlock);
438 	}
439 }
440 
441 #define	MIN(a, b) ((a) < (b) ? (a) : (b))
442 /* Copy userdata into a new mblk_t */
443 mblk_t *
socopyinuio(uio_t * uiop,ssize_t iosize,size_t wroff,ssize_t maxblk,size_t tail_len,int * errorp)444 socopyinuio(uio_t *uiop, ssize_t iosize, size_t wroff, ssize_t maxblk,
445     size_t tail_len, int *errorp)
446 {
447 	mblk_t	*head = NULL, **tail = &head;
448 
449 	ASSERT(iosize == INFPSZ || iosize > 0);
450 
451 	if (iosize == INFPSZ || iosize > uiop->uio_resid)
452 		iosize = uiop->uio_resid;
453 
454 	if (maxblk == INFPSZ)
455 		maxblk = iosize;
456 
457 	/* Nothing to do in these cases, so we're done */
458 	if (iosize < 0 || maxblk < 0 || (maxblk == 0 && iosize > 0))
459 		goto done;
460 
461 	/*
462 	 * We will enter the loop below if iosize is 0; it will allocate an
463 	 * empty message block and call uiomove(9F) which will just return.
464 	 * We could avoid that with an extra check but would only slow
465 	 * down the much more likely case where iosize is larger than 0.
466 	 */
467 	do {
468 		ssize_t blocksize;
469 		mblk_t	*mp;
470 
471 		blocksize = MIN(iosize, maxblk);
472 		ASSERT(blocksize >= 0);
473 		mp = allocb(wroff + blocksize + tail_len, BPRI_MED);
474 		if (mp == NULL) {
475 			*errorp = ENOMEM;
476 			return (head);
477 		}
478 		mp->b_rptr += wroff;
479 		mp->b_wptr = mp->b_rptr + blocksize;
480 
481 		*tail = mp;
482 		tail = &mp->b_cont;
483 
484 		/* uiomove(9F) either returns 0 or EFAULT */
485 		if ((*errorp = uiomove(mp->b_rptr, (size_t)blocksize,
486 		    UIO_WRITE, uiop)) != 0) {
487 			ASSERT(*errorp != ENOMEM);
488 			freemsg(head);
489 			return (NULL);
490 		}
491 
492 		iosize -= blocksize;
493 	} while (iosize > 0);
494 
495 done:
496 	*errorp = 0;
497 	return (head);
498 }
499 
500 mblk_t *
socopyoutuio(mblk_t * mp,struct uio * uiop,ssize_t max_read,int * errorp)501 socopyoutuio(mblk_t *mp, struct uio *uiop, ssize_t max_read, int *errorp)
502 {
503 	int error;
504 	ptrdiff_t n;
505 	mblk_t *nmp;
506 
507 	ASSERT(mp->b_wptr >= mp->b_rptr);
508 
509 	/*
510 	 * max_read is the offset of the oobmark and read can not go pass
511 	 * the oobmark.
512 	 */
513 	if (max_read == INFPSZ || max_read > uiop->uio_resid)
514 		max_read = uiop->uio_resid;
515 
516 	do {
517 		if ((n = MIN(max_read, MBLKL(mp))) != 0) {
518 			ASSERT(n > 0);
519 
520 			error = uiomove(mp->b_rptr, n, UIO_READ, uiop);
521 			if (error != 0) {
522 				freemsg(mp);
523 				*errorp = error;
524 				return (NULL);
525 			}
526 		}
527 
528 		mp->b_rptr += n;
529 		max_read -= n;
530 		while (mp != NULL && (mp->b_rptr >= mp->b_wptr)) {
531 			/*
532 			 * get rid of zero length mblks
533 			 */
534 			nmp = mp;
535 			mp = mp->b_cont;
536 			freeb(nmp);
537 		}
538 	} while (mp != NULL && max_read > 0);
539 
540 	*errorp = 0;
541 	return (mp);
542 }
543 
544 static void
so_prepend_msg(struct sonode * so,mblk_t * mp,mblk_t * last_tail)545 so_prepend_msg(struct sonode *so, mblk_t *mp, mblk_t *last_tail)
546 {
547 	ASSERT(last_tail != NULL);
548 	mp->b_next = so->so_rcv_q_head;
549 	mp->b_prev = last_tail;
550 	ASSERT(!(DB_FLAGS(mp) & DBLK_UIOA));
551 
552 	if (so->so_rcv_q_head == NULL) {
553 		ASSERT(so->so_rcv_q_last_head == NULL);
554 		so->so_rcv_q_last_head = mp;
555 #ifdef DEBUG
556 	} else {
557 		ASSERT(!(DB_FLAGS(so->so_rcv_q_head) & DBLK_UIOA));
558 #endif
559 	}
560 	so->so_rcv_q_head = mp;
561 
562 #ifdef DEBUG
563 	if (so_debug_length) {
564 		mutex_enter(&so->so_lock);
565 		ASSERT(so_check_length(so));
566 		mutex_exit(&so->so_lock);
567 	}
568 #endif
569 }
570 
571 /*
572  * Move a mblk chain (mp_head, mp_last_head) to the sonode's rcv queue so it
573  * can be processed by so_dequeue_msg().
574  */
575 void
so_process_new_message(struct sonode * so,mblk_t * mp_head,mblk_t * mp_last_head)576 so_process_new_message(struct sonode *so, mblk_t *mp_head, mblk_t *mp_last_head)
577 {
578 	if (so->so_filter_active > 0 &&
579 	    (mp_head = sof_filter_data_in_proc(so, mp_head,
580 	    &mp_last_head)) == NULL)
581 		return;
582 
583 	ASSERT(mp_head->b_prev != NULL);
584 	if (so->so_rcv_q_head == NULL) {
585 		so->so_rcv_q_head = mp_head;
586 		so->so_rcv_q_last_head = mp_last_head;
587 		ASSERT(so->so_rcv_q_last_head->b_prev != NULL);
588 	} else {
589 		boolean_t flag_equal = ((DB_FLAGS(mp_head) & DBLK_UIOA) ==
590 		    (DB_FLAGS(so->so_rcv_q_last_head) & DBLK_UIOA));
591 
592 		if (mp_head->b_next == NULL &&
593 		    DB_TYPE(mp_head) == M_DATA &&
594 		    DB_TYPE(so->so_rcv_q_last_head) == M_DATA && flag_equal) {
595 			so->so_rcv_q_last_head->b_prev->b_cont = mp_head;
596 			so->so_rcv_q_last_head->b_prev = mp_head->b_prev;
597 			mp_head->b_prev = NULL;
598 		} else if (flag_equal && (DB_FLAGS(mp_head) & DBLK_UIOA)) {
599 			/*
600 			 * Append to last_head if more than one mblks, and both
601 			 * mp_head and last_head are I/OAT mblks.
602 			 */
603 			ASSERT(mp_head->b_next != NULL);
604 			so->so_rcv_q_last_head->b_prev->b_cont = mp_head;
605 			so->so_rcv_q_last_head->b_prev = mp_head->b_prev;
606 			mp_head->b_prev = NULL;
607 
608 			so->so_rcv_q_last_head->b_next = mp_head->b_next;
609 			mp_head->b_next = NULL;
610 			so->so_rcv_q_last_head = mp_last_head;
611 		} else {
612 #ifdef DEBUG
613 			{
614 				mblk_t *tmp_mblk;
615 				tmp_mblk = mp_head;
616 				while (tmp_mblk != NULL) {
617 					ASSERT(tmp_mblk->b_prev != NULL);
618 					tmp_mblk = tmp_mblk->b_next;
619 				}
620 			}
621 #endif
622 			so->so_rcv_q_last_head->b_next = mp_head;
623 			so->so_rcv_q_last_head = mp_last_head;
624 		}
625 	}
626 }
627 
628 /*
629  * Check flow control on a given sonode.  Must have so_lock held, and
630  * this function will release the hold.  Return true if flow control
631  * is cleared.
632  */
633 boolean_t
so_check_flow_control(struct sonode * so)634 so_check_flow_control(struct sonode *so)
635 {
636 	ASSERT(MUTEX_HELD(&so->so_lock));
637 
638 	if (so->so_flowctrld && (so->so_rcv_queued < so->so_rcvlowat &&
639 	    !(so->so_state & SS_FIL_RCV_FLOWCTRL))) {
640 		so->so_flowctrld = B_FALSE;
641 		mutex_exit(&so->so_lock);
642 		/*
643 		 * Open up flow control. SCTP does not have any downcalls, and
644 		 * it will clr flow ctrl in sosctp_recvmsg().
645 		 */
646 		if (so->so_downcalls != NULL &&
647 		    so->so_downcalls->sd_clr_flowctrl != NULL) {
648 			(*so->so_downcalls->sd_clr_flowctrl)
649 			    (so->so_proto_handle);
650 		}
651 		/* filters can start injecting data */
652 		sof_sonode_notify_filters(so, SOF_EV_INJECT_DATA_IN_OK, 0);
653 		return (B_TRUE);
654 	} else {
655 		mutex_exit(&so->so_lock);
656 		return (B_FALSE);
657 	}
658 }
659 
660 int
so_dequeue_msg(struct sonode * so,mblk_t ** mctlp,struct uio * uiop,rval_t * rvalp,int flags)661 so_dequeue_msg(struct sonode *so, mblk_t **mctlp, struct uio *uiop,
662     rval_t *rvalp, int flags)
663 {
664 	mblk_t	*mp, *nmp;
665 	mblk_t	*savemp, *savemptail;
666 	mblk_t	*new_msg_head;
667 	mblk_t	*new_msg_last_head;
668 	mblk_t	*last_tail;
669 	boolean_t partial_read;
670 	boolean_t reset_atmark = B_FALSE;
671 	int more = 0;
672 	int error;
673 	ssize_t oobmark;
674 	sodirect_t *sodp = so->so_direct;
675 
676 	partial_read = B_FALSE;
677 	*mctlp = NULL;
678 again:
679 	mutex_enter(&so->so_lock);
680 again1:
681 #ifdef DEBUG
682 	if (so_debug_length) {
683 		ASSERT(so_check_length(so));
684 	}
685 #endif
686 	if (so->so_state & SS_RCVATMARK) {
687 		/* Check whether the caller is OK to read past the mark */
688 		if (flags & MSG_NOMARK) {
689 			mutex_exit(&so->so_lock);
690 			return (EWOULDBLOCK);
691 		}
692 		reset_atmark = B_TRUE;
693 	}
694 	/*
695 	 * First move messages from the dump area to processing area
696 	 */
697 	if (sodp != NULL) {
698 		if (sodp->sod_enabled) {
699 			if (sodp->sod_uioa.uioa_state & UIOA_ALLOC) {
700 				/* nothing to uioamove */
701 				sodp = NULL;
702 			} else if (sodp->sod_uioa.uioa_state & UIOA_INIT) {
703 				sodp->sod_uioa.uioa_state &= UIOA_CLR;
704 				sodp->sod_uioa.uioa_state |= UIOA_ENABLED;
705 				/*
706 				 * try to uioamove() the data that
707 				 * has already queued.
708 				 */
709 				sod_uioa_so_init(so, sodp, uiop);
710 			}
711 		} else {
712 			sodp = NULL;
713 		}
714 	}
715 	new_msg_head = so->so_rcv_head;
716 	new_msg_last_head = so->so_rcv_last_head;
717 	so->so_rcv_head = NULL;
718 	so->so_rcv_last_head = NULL;
719 	oobmark = so->so_oobmark;
720 	/*
721 	 * We can release the lock as there can only be one reader
722 	 */
723 	mutex_exit(&so->so_lock);
724 
725 	if (new_msg_head != NULL) {
726 		so_process_new_message(so, new_msg_head, new_msg_last_head);
727 	}
728 	savemp = savemptail = NULL;
729 	rvalp->r_vals = 0;
730 	error = 0;
731 	mp = so->so_rcv_q_head;
732 
733 	if (mp != NULL &&
734 	    (so->so_rcv_timer_tid == 0 ||
735 	    so->so_rcv_queued >= so->so_rcv_thresh)) {
736 		partial_read = B_FALSE;
737 
738 		if (flags & MSG_PEEK) {
739 			if ((nmp = dupmsg(mp)) == NULL &&
740 			    (nmp = copymsg(mp)) == NULL) {
741 				size_t size = msgsize(mp);
742 
743 				error = strwaitbuf(size, BPRI_HI);
744 				if (error) {
745 					return (error);
746 				}
747 				goto again;
748 			}
749 			mp = nmp;
750 		} else {
751 			ASSERT(mp->b_prev != NULL);
752 			last_tail = mp->b_prev;
753 			mp->b_prev = NULL;
754 			so->so_rcv_q_head = mp->b_next;
755 			if (so->so_rcv_q_head == NULL) {
756 				so->so_rcv_q_last_head = NULL;
757 			}
758 			mp->b_next = NULL;
759 		}
760 
761 		ASSERT(mctlp != NULL);
762 		/*
763 		 * First process PROTO or PCPROTO blocks, if any.
764 		 */
765 		if (DB_TYPE(mp) != M_DATA) {
766 			*mctlp = mp;
767 			savemp = mp;
768 			savemptail = mp;
769 			ASSERT(DB_TYPE(mp) == M_PROTO ||
770 			    DB_TYPE(mp) == M_PCPROTO);
771 			while (mp->b_cont != NULL &&
772 			    DB_TYPE(mp->b_cont) != M_DATA) {
773 				ASSERT(DB_TYPE(mp->b_cont) == M_PROTO ||
774 				    DB_TYPE(mp->b_cont) == M_PCPROTO);
775 				mp = mp->b_cont;
776 				savemptail = mp;
777 			}
778 			mp = savemptail->b_cont;
779 			savemptail->b_cont = NULL;
780 		}
781 
782 		ASSERT(DB_TYPE(mp) == M_DATA);
783 		/*
784 		 * Now process DATA blocks, if any. Note that for sodirect
785 		 * enabled socket, uio_resid can be 0.
786 		 */
787 		if (uiop->uio_resid >= 0) {
788 			ssize_t copied = 0;
789 
790 			if (sodp != NULL && (DB_FLAGS(mp) & DBLK_UIOA)) {
791 				mutex_enter(&so->so_lock);
792 				ASSERT(uiop == (uio_t *)&sodp->sod_uioa);
793 				copied = sod_uioa_mblk(so, mp);
794 				if (copied > 0)
795 					partial_read = B_TRUE;
796 				mutex_exit(&so->so_lock);
797 				/* mark this mblk as processed */
798 				mp = NULL;
799 			} else {
800 				ssize_t oldresid = uiop->uio_resid;
801 
802 				if (MBLKL(mp) < so_mblk_pull_len) {
803 					if (pullupmsg(mp, -1) == 1) {
804 						last_tail = mp;
805 					}
806 				}
807 				/*
808 				 * Can not read beyond the oobmark
809 				 */
810 				mp = socopyoutuio(mp, uiop,
811 				    oobmark == 0 ? INFPSZ : oobmark, &error);
812 				if (error != 0) {
813 					freemsg(*mctlp);
814 					*mctlp = NULL;
815 					more = 0;
816 					goto done;
817 				}
818 				ASSERT(oldresid >= uiop->uio_resid);
819 				copied = oldresid - uiop->uio_resid;
820 				if (oldresid > uiop->uio_resid)
821 					partial_read = B_TRUE;
822 			}
823 			ASSERT(copied >= 0);
824 			if (copied > 0 && !(flags & MSG_PEEK)) {
825 				mutex_enter(&so->so_lock);
826 				so->so_rcv_queued -= copied;
827 				ASSERT(so->so_oobmark >= 0);
828 				if (so->so_oobmark > 0) {
829 					so->so_oobmark -= copied;
830 					ASSERT(so->so_oobmark >= 0);
831 					if (so->so_oobmark == 0) {
832 						ASSERT(so->so_state &
833 						    SS_OOBPEND);
834 						so->so_oobmark = 0;
835 						so->so_state |= SS_RCVATMARK;
836 					}
837 				}
838 				/*
839 				 * so_check_flow_control() will drop
840 				 * so->so_lock.
841 				 */
842 				rvalp->r_val2 = so_check_flow_control(so);
843 			}
844 		}
845 		if (mp != NULL) { /* more data blocks in msg */
846 			more |= MOREDATA;
847 			if ((flags & (MSG_PEEK|MSG_TRUNC))) {
848 				if (flags & MSG_PEEK) {
849 					freemsg(mp);
850 				} else {
851 					unsigned int msize = msgdsize(mp);
852 
853 					freemsg(mp);
854 					mutex_enter(&so->so_lock);
855 					so->so_rcv_queued -= msize;
856 					/*
857 					 * so_check_flow_control() will drop
858 					 * so->so_lock.
859 					 */
860 					rvalp->r_val2 =
861 					    so_check_flow_control(so);
862 				}
863 			} else if (partial_read && !somsghasdata(mp)) {
864 				/*
865 				 * Avoid queuing a zero-length tail part of
866 				 * a message. partial_read == 1 indicates that
867 				 * we read some of the message.
868 				 */
869 				freemsg(mp);
870 				more &= ~MOREDATA;
871 			} else {
872 				if (savemp != NULL &&
873 				    (flags & MSG_DUPCTRL)) {
874 					mblk_t *nmp;
875 					/*
876 					 * There should only be non data mblks
877 					 */
878 					ASSERT(DB_TYPE(savemp) != M_DATA &&
879 					    DB_TYPE(savemptail) != M_DATA);
880 try_again:
881 					if ((nmp = dupmsg(savemp)) == NULL &&
882 					    (nmp = copymsg(savemp)) == NULL) {
883 
884 						size_t size = msgsize(savemp);
885 
886 						error = strwaitbuf(size,
887 						    BPRI_HI);
888 						if (error != 0) {
889 							/*
890 							 * In case we
891 							 * cannot copy
892 							 * control data
893 							 * free the remaining
894 							 * data.
895 							 */
896 							freemsg(mp);
897 							goto done;
898 						}
899 						goto try_again;
900 					}
901 
902 					ASSERT(nmp != NULL);
903 					ASSERT(DB_TYPE(nmp) != M_DATA);
904 					savemptail->b_cont = mp;
905 					*mctlp = nmp;
906 					mp = savemp;
907 				}
908 				/*
909 				 * putback mp
910 				 */
911 				so_prepend_msg(so, mp, last_tail);
912 			}
913 		}
914 
915 		/* fast check so_rcv_head if there is more data */
916 		if (partial_read && !(so->so_state & SS_RCVATMARK) &&
917 		    *mctlp == NULL && uiop->uio_resid > 0 &&
918 		    !(flags & MSG_PEEK) && so->so_rcv_head != NULL) {
919 			goto again;
920 		}
921 	} else if (!partial_read) {
922 		mutex_enter(&so->so_lock);
923 		if (so->so_error != 0) {
924 			error = sogeterr(so, !(flags & MSG_PEEK));
925 			mutex_exit(&so->so_lock);
926 			return (error);
927 		}
928 
929 		/* See if new data has arrived in the meantime */
930 		if (so->so_rcv_head != NULL)
931 			goto again1;
932 
933 		/*
934 		 * No pending data. Return right away for nonblocking
935 		 * socket, otherwise sleep waiting for data.
936 		 */
937 		if (!(so->so_state & SS_CANTRCVMORE) && uiop->uio_resid > 0) {
938 			if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
939 			    (flags & MSG_DONTWAIT)) {
940 				error = EWOULDBLOCK;
941 			} else {
942 				if (so->so_state & (SS_CLOSING |
943 				    SS_FALLBACK_PENDING)) {
944 					mutex_exit(&so->so_lock);
945 					error = EINTR;
946 					goto done;
947 				}
948 
949 				so->so_rcv_wakeup = B_TRUE;
950 				so->so_rcv_wanted = uiop->uio_resid;
951 				if (so->so_rcvtimeo == 0) {
952 					/*
953 					 * Zero means disable timeout.
954 					 */
955 					error = cv_wait_sig(&so->so_rcv_cv,
956 					    &so->so_lock);
957 				} else {
958 					error = cv_reltimedwait_sig(
959 					    &so->so_rcv_cv, &so->so_lock,
960 					    so->so_rcvtimeo, TR_CLOCK_TICK);
961 				}
962 				so->so_rcv_wakeup = B_FALSE;
963 				so->so_rcv_wanted = 0;
964 
965 				if (error == 0) {
966 					error = EINTR;
967 				} else if (error == -1) {
968 					error = EAGAIN;
969 				} else {
970 					goto again1;
971 				}
972 			}
973 		}
974 		mutex_exit(&so->so_lock);
975 	}
976 	if (reset_atmark && partial_read && !(flags & MSG_PEEK)) {
977 		/*
978 		 * We are passed the mark, update state
979 		 * 4.3BSD and 4.4BSD clears the mark when peeking across it.
980 		 * The draft Posix socket spec states that the mark should
981 		 * not be cleared when peeking. We follow the latter.
982 		 */
983 		mutex_enter(&so->so_lock);
984 		ASSERT(so_verify_oobstate(so));
985 		so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_RCVATMARK);
986 		freemsg(so->so_oobmsg);
987 		so->so_oobmsg = NULL;
988 		ASSERT(so_verify_oobstate(so));
989 		mutex_exit(&so->so_lock);
990 	}
991 	ASSERT(so->so_rcv_wakeup == B_FALSE);
992 done:
993 	if (sodp != NULL) {
994 		mutex_enter(&so->so_lock);
995 		if (sodp->sod_enabled &&
996 		    (sodp->sod_uioa.uioa_state & UIOA_ENABLED)) {
997 			SOD_UIOAFINI(sodp);
998 			if (sodp->sod_uioa.uioa_mbytes > 0) {
999 				ASSERT(so->so_rcv_q_head != NULL ||
1000 				    so->so_rcv_head != NULL);
1001 				so->so_rcv_queued -= sod_uioa_mblk(so, NULL);
1002 				if (error == EWOULDBLOCK)
1003 					error = 0;
1004 			}
1005 		}
1006 		mutex_exit(&so->so_lock);
1007 	}
1008 #ifdef DEBUG
1009 	if (so_debug_length) {
1010 		mutex_enter(&so->so_lock);
1011 		ASSERT(so_check_length(so));
1012 		mutex_exit(&so->so_lock);
1013 	}
1014 #endif
1015 	rvalp->r_val1 = more;
1016 	ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1017 	return (error);
1018 }
1019 
1020 /*
1021  * Enqueue data from the protocol on the socket's rcv queue.
1022  *
1023  * We try to hook new M_DATA mblks onto an existing chain, however,
1024  * that cannot be done if the existing chain has already been
1025  * processed by I/OAT. Non-M_DATA mblks are just linked together via
1026  * b_next. In all cases the b_prev of the enqueued mblk is set to
1027  * point to the last mblk in its b_cont chain.
1028  */
1029 void
so_enqueue_msg(struct sonode * so,mblk_t * mp,size_t msg_size)1030 so_enqueue_msg(struct sonode *so, mblk_t *mp, size_t msg_size)
1031 {
1032 	ASSERT(MUTEX_HELD(&so->so_lock));
1033 
1034 #ifdef DEBUG
1035 	if (so_debug_length) {
1036 		ASSERT(so_check_length(so));
1037 	}
1038 #endif
1039 	so->so_rcv_queued += msg_size;
1040 
1041 	if (so->so_rcv_head == NULL) {
1042 		ASSERT(so->so_rcv_last_head == NULL);
1043 		so->so_rcv_head = mp;
1044 		so->so_rcv_last_head = mp;
1045 	} else if ((DB_TYPE(mp) == M_DATA &&
1046 	    DB_TYPE(so->so_rcv_last_head) == M_DATA) &&
1047 	    ((DB_FLAGS(mp) & DBLK_UIOA) ==
1048 	    (DB_FLAGS(so->so_rcv_last_head) & DBLK_UIOA))) {
1049 		/* Added to the end */
1050 		ASSERT(so->so_rcv_last_head != NULL);
1051 		ASSERT(so->so_rcv_last_head->b_prev != NULL);
1052 		so->so_rcv_last_head->b_prev->b_cont = mp;
1053 	} else {
1054 		/* Start a new end */
1055 		so->so_rcv_last_head->b_next = mp;
1056 		so->so_rcv_last_head = mp;
1057 	}
1058 	while (mp->b_cont != NULL)
1059 		mp = mp->b_cont;
1060 
1061 	so->so_rcv_last_head->b_prev = mp;
1062 #ifdef DEBUG
1063 	if (so_debug_length) {
1064 		ASSERT(so_check_length(so));
1065 	}
1066 #endif
1067 }
1068 
1069 /*
1070  * Return B_TRUE if there is data in the message, B_FALSE otherwise.
1071  */
1072 boolean_t
somsghasdata(mblk_t * mp)1073 somsghasdata(mblk_t *mp)
1074 {
1075 	for (; mp; mp = mp->b_cont)
1076 		if (mp->b_datap->db_type == M_DATA) {
1077 			ASSERT(mp->b_wptr >= mp->b_rptr);
1078 			if (mp->b_wptr > mp->b_rptr)
1079 				return (B_TRUE);
1080 		}
1081 	return (B_FALSE);
1082 }
1083 
1084 /*
1085  * Flush the read side of sockfs.
1086  *
1087  * The caller must be sure that a reader is not already active when the
1088  * buffer is being flushed.
1089  */
1090 void
so_rcv_flush(struct sonode * so)1091 so_rcv_flush(struct sonode *so)
1092 {
1093 	mblk_t  *mp;
1094 
1095 	ASSERT(MUTEX_HELD(&so->so_lock));
1096 
1097 	if (so->so_oobmsg != NULL) {
1098 		freemsg(so->so_oobmsg);
1099 		so->so_oobmsg = NULL;
1100 		so->so_oobmark = 0;
1101 		so->so_state &=
1102 		    ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_HADOOBDATA|SS_RCVATMARK);
1103 	}
1104 
1105 	/*
1106 	 * Free messages sitting in the recv queues
1107 	 */
1108 	while (so->so_rcv_q_head != NULL) {
1109 		mp = so->so_rcv_q_head;
1110 		so->so_rcv_q_head = mp->b_next;
1111 		mp->b_next = mp->b_prev = NULL;
1112 		freemsg(mp);
1113 	}
1114 	while (so->so_rcv_head != NULL) {
1115 		mp = so->so_rcv_head;
1116 		so->so_rcv_head = mp->b_next;
1117 		mp->b_next = mp->b_prev = NULL;
1118 		freemsg(mp);
1119 	}
1120 	so->so_rcv_queued = 0;
1121 	so->so_rcv_q_head = NULL;
1122 	so->so_rcv_q_last_head = NULL;
1123 	so->so_rcv_head = NULL;
1124 	so->so_rcv_last_head = NULL;
1125 }
1126 
1127 /*
1128  * Handle recv* calls that set MSG_OOB or MSG_OOB together with MSG_PEEK.
1129  */
1130 int
sorecvoob(struct sonode * so,struct nmsghdr * msg,struct uio * uiop,int flags,boolean_t oob_inline)1131 sorecvoob(struct sonode *so, struct nmsghdr *msg, struct uio *uiop, int flags,
1132     boolean_t oob_inline)
1133 {
1134 	mblk_t		*mp, *nmp;
1135 	int		error;
1136 
1137 	dprintso(so, 1, ("sorecvoob(%p, %p, 0x%x)\n", (void *)so, (void *)msg,
1138 	    flags));
1139 
1140 	if (msg != NULL) {
1141 		/*
1142 		 * There is never any oob data with addresses or control since
1143 		 * the T_EXDATA_IND does not carry any options.
1144 		 */
1145 		msg->msg_controllen = 0;
1146 		msg->msg_namelen = 0;
1147 		msg->msg_flags = 0;
1148 	}
1149 
1150 	mutex_enter(&so->so_lock);
1151 	ASSERT(so_verify_oobstate(so));
1152 	if (oob_inline ||
1153 	    (so->so_state & (SS_OOBPEND|SS_HADOOBDATA)) != SS_OOBPEND) {
1154 		dprintso(so, 1, ("sorecvoob: inline or data consumed\n"));
1155 		mutex_exit(&so->so_lock);
1156 		return (EINVAL);
1157 	}
1158 	if (!(so->so_state & SS_HAVEOOBDATA)) {
1159 		dprintso(so, 1, ("sorecvoob: no data yet\n"));
1160 		mutex_exit(&so->so_lock);
1161 		return (EWOULDBLOCK);
1162 	}
1163 	ASSERT(so->so_oobmsg != NULL);
1164 	mp = so->so_oobmsg;
1165 	if (flags & MSG_PEEK) {
1166 		/*
1167 		 * Since recv* can not return ENOBUFS we can not use dupmsg.
1168 		 * Instead we revert to the consolidation private
1169 		 * allocb_wait plus bcopy.
1170 		 */
1171 		mblk_t *mp1;
1172 
1173 		mp1 = allocb_wait(msgdsize(mp), BPRI_MED, STR_NOSIG, NULL);
1174 		ASSERT(mp1);
1175 
1176 		while (mp != NULL) {
1177 			ssize_t size;
1178 
1179 			size = MBLKL(mp);
1180 			bcopy(mp->b_rptr, mp1->b_wptr, size);
1181 			mp1->b_wptr += size;
1182 			ASSERT(mp1->b_wptr <= mp1->b_datap->db_lim);
1183 			mp = mp->b_cont;
1184 		}
1185 		mp = mp1;
1186 	} else {
1187 		/*
1188 		 * Update the state indicating that the data has been consumed.
1189 		 * Keep SS_OOBPEND set until data is consumed past the mark.
1190 		 */
1191 		so->so_oobmsg = NULL;
1192 		so->so_state ^= SS_HAVEOOBDATA|SS_HADOOBDATA;
1193 	}
1194 	ASSERT(so_verify_oobstate(so));
1195 	mutex_exit(&so->so_lock);
1196 
1197 	error = 0;
1198 	nmp = mp;
1199 	while (nmp != NULL && uiop->uio_resid > 0) {
1200 		ssize_t n = MBLKL(nmp);
1201 
1202 		n = MIN(n, uiop->uio_resid);
1203 		if (n > 0)
1204 			error = uiomove(nmp->b_rptr, n,
1205 			    UIO_READ, uiop);
1206 		if (error)
1207 			break;
1208 		nmp = nmp->b_cont;
1209 	}
1210 	ASSERT(mp->b_next == NULL && mp->b_prev == NULL);
1211 	freemsg(mp);
1212 	return (error);
1213 }
1214 
1215 /*
1216  * Allocate and initializ sonode
1217  */
1218 /* ARGSUSED */
1219 struct sonode *
socket_sonode_create(struct sockparams * sp,int family,int type,int protocol,int version,int sflags,int * errorp,struct cred * cr)1220 socket_sonode_create(struct sockparams *sp, int family, int type,
1221     int protocol, int version, int sflags, int *errorp, struct cred *cr)
1222 {
1223 	sonode_t *so;
1224 	int	kmflags;
1225 
1226 	/*
1227 	 * Choose the right set of sonodeops based on the upcall and
1228 	 * down call version that the protocol has provided
1229 	 */
1230 	if (SOCK_UC_VERSION != sp->sp_smod_info->smod_uc_version ||
1231 	    SOCK_DC_VERSION != sp->sp_smod_info->smod_dc_version) {
1232 		/*
1233 		 * mismatch
1234 		 */
1235 #ifdef DEBUG
1236 		cmn_err(CE_CONT, "protocol and socket module version mismatch");
1237 #endif
1238 		*errorp = EINVAL;
1239 		return (NULL);
1240 	}
1241 
1242 	kmflags = (sflags & SOCKET_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
1243 
1244 	so = kmem_cache_alloc(socket_cache, kmflags);
1245 	if (so == NULL) {
1246 		*errorp = ENOMEM;
1247 		return (NULL);
1248 	}
1249 
1250 	sonode_init(so, sp, family, type, protocol, &so_sonodeops);
1251 
1252 	if (version == SOV_DEFAULT)
1253 		version = so_default_version;
1254 
1255 	so->so_version = (short)version;
1256 
1257 	/*
1258 	 * set the default values to be INFPSZ
1259 	 * if a protocol desires it can change the value later
1260 	 */
1261 	so->so_proto_props.sopp_rxhiwat = SOCKET_RECVHIWATER;
1262 	so->so_proto_props.sopp_rxlowat = SOCKET_RECVLOWATER;
1263 	so->so_proto_props.sopp_maxpsz = INFPSZ;
1264 	so->so_proto_props.sopp_maxblk = INFPSZ;
1265 
1266 	return (so);
1267 }
1268 
1269 int
socket_init_common(struct sonode * so,struct sonode * pso,int flags,cred_t * cr)1270 socket_init_common(struct sonode *so, struct sonode *pso, int flags, cred_t *cr)
1271 {
1272 	int error = 0;
1273 
1274 	if (pso != NULL) {
1275 		/*
1276 		 * We have a passive open, so inherit basic state from
1277 		 * the parent (listener).
1278 		 *
1279 		 * No need to grab the new sonode's lock, since there is no
1280 		 * one that can have a reference to it.
1281 		 */
1282 		mutex_enter(&pso->so_lock);
1283 
1284 		so->so_state |= SS_ISCONNECTED | (pso->so_state & SS_ASYNC);
1285 		so->so_pgrp = pso->so_pgrp;
1286 		so->so_rcvtimeo = pso->so_rcvtimeo;
1287 		so->so_sndtimeo = pso->so_sndtimeo;
1288 		so->so_xpg_rcvbuf = pso->so_xpg_rcvbuf;
1289 		/*
1290 		 * Make note of the socket level options. TCP and IP level
1291 		 * options are already inherited. We could do all this after
1292 		 * accept is successful but doing it here simplifies code and
1293 		 * no harm done for error case.
1294 		 */
1295 		so->so_options = pso->so_options & (SO_DEBUG|SO_REUSEADDR|
1296 		    SO_KEEPALIVE|SO_DONTROUTE|SO_BROADCAST|SO_USELOOPBACK|
1297 		    SO_OOBINLINE|SO_DGRAM_ERRIND|SO_LINGER);
1298 		so->so_proto_props = pso->so_proto_props;
1299 		so->so_mode = pso->so_mode;
1300 		so->so_pollev = pso->so_pollev & SO_POLLEV_ALWAYS;
1301 
1302 		mutex_exit(&pso->so_lock);
1303 
1304 		/*
1305 		 * If the parent has any filters, try to inherit them.
1306 		 */
1307 		if (pso->so_filter_active > 0 &&
1308 		    (error = sof_sonode_inherit_filters(so, pso)) != 0)
1309 			return (error);
1310 
1311 	} else {
1312 		struct sockparams *sp = so->so_sockparams;
1313 		sock_upcalls_t *upcalls_to_use;
1314 
1315 		/*
1316 		 * Attach automatic filters, if there are any.
1317 		 */
1318 		if (!list_is_empty(&sp->sp_auto_filters) &&
1319 		    (error = sof_sonode_autoattach_filters(so, cr)) != 0)
1320 			return (error);
1321 
1322 		/* OK to attach filters */
1323 		so->so_state |= SS_FILOP_OK;
1324 
1325 		/*
1326 		 * Based on the version number select the right upcalls to
1327 		 * pass down. Currently we only have one version so choose
1328 		 * default
1329 		 */
1330 		upcalls_to_use = &so_upcalls;
1331 
1332 		/* active open, so create a lower handle */
1333 		so->so_proto_handle =
1334 		    sp->sp_smod_info->smod_proto_create_func(so->so_family,
1335 		    so->so_type, so->so_protocol, &so->so_downcalls,
1336 		    &so->so_mode, &error, flags, cr);
1337 
1338 		if (so->so_proto_handle == NULL) {
1339 			ASSERT(error != 0);
1340 			/*
1341 			 * To be safe; if a lower handle cannot be created, and
1342 			 * the proto does not give a reason why, assume there
1343 			 * was a lack of memory.
1344 			 */
1345 			return ((error == 0) ? ENOMEM : error);
1346 		}
1347 		ASSERT(so->so_downcalls != NULL);
1348 		ASSERT(so->so_downcalls->sd_send != NULL ||
1349 		    so->so_downcalls->sd_send_uio != NULL);
1350 		if (so->so_downcalls->sd_recv_uio != NULL) {
1351 			ASSERT(so->so_downcalls->sd_poll != NULL);
1352 			so->so_pollev |= SO_POLLEV_ALWAYS;
1353 		}
1354 
1355 		(*so->so_downcalls->sd_activate)(so->so_proto_handle,
1356 		    (sock_upper_handle_t)so, upcalls_to_use, 0, cr);
1357 
1358 		/* Wildcard */
1359 
1360 		/*
1361 		 * FIXME No need for this, the protocol can deal with it in
1362 		 * sd_create(). Should update ICMP.
1363 		 */
1364 		if (so->so_protocol != so->so_sockparams->sp_protocol) {
1365 			int protocol = so->so_protocol;
1366 			int error;
1367 			/*
1368 			 * Issue SO_PROTOTYPE setsockopt.
1369 			 */
1370 			error = socket_setsockopt(so, SOL_SOCKET, SO_PROTOTYPE,
1371 			    &protocol, (t_uscalar_t)sizeof (protocol), cr);
1372 			if (error) {
1373 				(void) (*so->so_downcalls->sd_close)
1374 				    (so->so_proto_handle, 0, cr);
1375 
1376 				mutex_enter(&so->so_lock);
1377 				so_rcv_flush(so);
1378 				mutex_exit(&so->so_lock);
1379 				/*
1380 				 * Setsockopt often fails with ENOPROTOOPT but
1381 				 * socket() should fail with
1382 				 * EPROTONOSUPPORT/EPROTOTYPE.
1383 				 */
1384 				return (EPROTONOSUPPORT);
1385 			}
1386 		}
1387 	}
1388 
1389 	if (uioasync.enabled)
1390 		sod_sock_init(so);
1391 
1392 	/* put an extra reference on the socket for the protocol */
1393 	VN_HOLD(SOTOV(so));
1394 
1395 	return (0);
1396 }
1397 
1398 /*
1399  * int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1400  *         struct cred *cr, int32_t *rvalp)
1401  *
1402  * Handle ioctls that manipulate basic socket state; non-blocking,
1403  * async, etc.
1404  *
1405  * Returns:
1406  *   < 0  - ioctl was not handle
1407  *  >= 0  - ioctl was handled, if > 0, then it is an errno
1408  *
1409  * Notes:
1410  *   Assumes the standard receive buffer is used to obtain info for
1411  *   NREAD.
1412  */
1413 /* ARGSUSED */
1414 int
socket_ioctl_common(struct sonode * so,int cmd,intptr_t arg,int mode,struct cred * cr,int32_t * rvalp)1415 socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1416     struct cred *cr, int32_t *rvalp)
1417 {
1418 	switch (cmd) {
1419 	case SIOCSQPTR:
1420 		/*
1421 		 * SIOCSQPTR is valid only when helper stream is created
1422 		 * by the protocol.
1423 		 */
1424 
1425 		return (EOPNOTSUPP);
1426 	case FIONBIO: {
1427 		int32_t value;
1428 
1429 		if (so_copyin((void *)arg, &value, sizeof (int32_t),
1430 		    (mode & (int)FKIOCTL)))
1431 			return (EFAULT);
1432 
1433 		mutex_enter(&so->so_lock);
1434 		if (value) {
1435 			so->so_state |= SS_NDELAY;
1436 		} else {
1437 			so->so_state &= ~SS_NDELAY;
1438 		}
1439 		mutex_exit(&so->so_lock);
1440 		return (0);
1441 	}
1442 	case FIOASYNC: {
1443 		int32_t value;
1444 
1445 		if (so_copyin((void *)arg, &value, sizeof (int32_t),
1446 		    (mode & (int)FKIOCTL)))
1447 			return (EFAULT);
1448 
1449 		mutex_enter(&so->so_lock);
1450 
1451 		if (value) {
1452 			/* Turn on SIGIO */
1453 			so->so_state |= SS_ASYNC;
1454 		} else {
1455 			/* Turn off SIGIO */
1456 			so->so_state &= ~SS_ASYNC;
1457 		}
1458 		mutex_exit(&so->so_lock);
1459 
1460 		return (0);
1461 	}
1462 
1463 	case SIOCSPGRP:
1464 	case FIOSETOWN: {
1465 		int error;
1466 		pid_t pid;
1467 
1468 		if (so_copyin((void *)arg, &pid, sizeof (pid_t),
1469 		    (mode & (int)FKIOCTL)))
1470 			return (EFAULT);
1471 
1472 		mutex_enter(&so->so_lock);
1473 		error = (pid != so->so_pgrp) ? socket_chgpgrp(so, pid) : 0;
1474 		mutex_exit(&so->so_lock);
1475 		return (error);
1476 	}
1477 	case SIOCGPGRP:
1478 	case FIOGETOWN:
1479 		if (so_copyout(&so->so_pgrp, (void *)arg,
1480 		    sizeof (pid_t), (mode & (int)FKIOCTL)))
1481 			return (EFAULT);
1482 
1483 		return (0);
1484 	case SIOCATMARK: {
1485 		int retval;
1486 
1487 		/*
1488 		 * Only protocols that support urgent data can handle ATMARK.
1489 		 */
1490 		if ((so->so_mode & SM_EXDATA) == 0)
1491 			return (EINVAL);
1492 
1493 		/*
1494 		 * If the protocol is maintaining its own buffer, then the
1495 		 * request must be passed down.
1496 		 */
1497 		if (so->so_downcalls->sd_recv_uio != NULL)
1498 			return (-1);
1499 
1500 		retval = (so->so_state & SS_RCVATMARK) != 0;
1501 
1502 		if (so_copyout(&retval, (void *)arg, sizeof (int),
1503 		    (mode & (int)FKIOCTL))) {
1504 			return (EFAULT);
1505 		}
1506 		return (0);
1507 	}
1508 
1509 	case FIONREAD: {
1510 		int retval;
1511 
1512 		/*
1513 		 * If the protocol is maintaining its own buffer, then the
1514 		 * request must be passed down.
1515 		 */
1516 		if (so->so_downcalls->sd_recv_uio != NULL)
1517 			return (-1);
1518 
1519 		retval = MIN(so->so_rcv_queued, INT_MAX);
1520 
1521 		if (so_copyout(&retval, (void *)arg,
1522 		    sizeof (retval), (mode & (int)FKIOCTL))) {
1523 			return (EFAULT);
1524 		}
1525 		return (0);
1526 	}
1527 
1528 	case _I_GETPEERCRED: {
1529 		int error = 0;
1530 
1531 		if ((mode & FKIOCTL) == 0)
1532 			return (EINVAL);
1533 
1534 		mutex_enter(&so->so_lock);
1535 		if ((so->so_mode & SM_CONNREQUIRED) == 0) {
1536 			error = ENOTSUP;
1537 		} else if ((so->so_state & SS_ISCONNECTED) == 0) {
1538 			error = ENOTCONN;
1539 		} else if (so->so_peercred != NULL) {
1540 			k_peercred_t *kp = (k_peercred_t *)arg;
1541 			kp->pc_cr = so->so_peercred;
1542 			kp->pc_cpid = so->so_cpid;
1543 			crhold(so->so_peercred);
1544 		} else {
1545 			error = EINVAL;
1546 		}
1547 		mutex_exit(&so->so_lock);
1548 		return (error);
1549 	}
1550 	default:
1551 		return (-1);
1552 	}
1553 }
1554 
1555 /*
1556  * Handle the I_NREAD STREAM ioctl.
1557  */
1558 static int
so_strioc_nread(struct sonode * so,intptr_t arg,int mode,int32_t * rvalp)1559 so_strioc_nread(struct sonode *so, intptr_t arg, int mode, int32_t *rvalp)
1560 {
1561 	size_t size = 0;
1562 	int retval;
1563 	int count = 0;
1564 	mblk_t *mp;
1565 	clock_t wakeup = drv_usectohz(10);
1566 
1567 	if (so->so_downcalls == NULL ||
1568 	    so->so_downcalls->sd_recv_uio != NULL)
1569 		return (EINVAL);
1570 
1571 	mutex_enter(&so->so_lock);
1572 	/* Wait for reader to get out of the way. */
1573 	while (so->so_flag & SOREADLOCKED) {
1574 		/*
1575 		 * If reader is waiting for data, then there should be nothing
1576 		 * on the rcv queue.
1577 		 */
1578 		if (so->so_rcv_wakeup)
1579 			goto out;
1580 
1581 		/* Do a timed sleep, in case the reader goes to sleep. */
1582 		(void) cv_reltimedwait(&so->so_read_cv, &so->so_lock, wakeup,
1583 		    TR_CLOCK_TICK);
1584 	}
1585 
1586 	/*
1587 	 * Since we are holding so_lock no new reader will come in, and the
1588 	 * protocol will not be able to enqueue data. So it's safe to walk
1589 	 * both rcv queues.
1590 	 */
1591 	mp = so->so_rcv_q_head;
1592 	if (mp != NULL) {
1593 		size = msgdsize(so->so_rcv_q_head);
1594 		for (; mp != NULL; mp = mp->b_next)
1595 			count++;
1596 	} else {
1597 		/*
1598 		 * In case the processing list was empty, get the size of the
1599 		 * next msg in line.
1600 		 */
1601 		size = msgdsize(so->so_rcv_head);
1602 	}
1603 
1604 	for (mp = so->so_rcv_head; mp != NULL; mp = mp->b_next)
1605 		count++;
1606 out:
1607 	mutex_exit(&so->so_lock);
1608 
1609 	/*
1610 	 * Drop down from size_t to the "int" required by the
1611 	 * interface.  Cap at INT_MAX.
1612 	 */
1613 	retval = MIN(size, INT_MAX);
1614 	if (so_copyout(&retval, (void *)arg, sizeof (retval),
1615 	    (mode & (int)FKIOCTL))) {
1616 		return (EFAULT);
1617 	} else {
1618 		*rvalp = count;
1619 		return (0);
1620 	}
1621 }
1622 
1623 /*
1624  * Process STREAM ioctls.
1625  *
1626  * Returns:
1627  *   < 0  - ioctl was not handle
1628  *  >= 0  - ioctl was handled, if > 0, then it is an errno
1629  */
1630 int
socket_strioc_common(struct sonode * so,int cmd,intptr_t arg,int mode,struct cred * cr,int32_t * rvalp)1631 socket_strioc_common(struct sonode *so, int cmd, intptr_t arg, int mode,
1632     struct cred *cr, int32_t *rvalp)
1633 {
1634 	int retval;
1635 
1636 	/* Only STREAM iotcls are handled here */
1637 	if ((cmd & 0xffffff00U) != STR)
1638 		return (-1);
1639 
1640 	switch (cmd) {
1641 	case I_CANPUT:
1642 		/*
1643 		 * We return an error for I_CANPUT so that isastream(3C) will
1644 		 * not report the socket as being a STREAM.
1645 		 */
1646 		return (EOPNOTSUPP);
1647 	case I_NREAD:
1648 		/* Avoid doing a fallback for I_NREAD. */
1649 		return (so_strioc_nread(so, arg, mode, rvalp));
1650 	case I_LOOK:
1651 		/* Avoid doing a fallback for I_LOOK. */
1652 		if (so_copyout("sockmod", (void *)arg, strlen("sockmod") + 1,
1653 		    (mode & (int)FKIOCTL))) {
1654 			return (EFAULT);
1655 		}
1656 		return (0);
1657 	default:
1658 		break;
1659 	}
1660 
1661 	/*
1662 	 * Try to fall back to TPI, and if successful, reissue the ioctl.
1663 	 */
1664 	if ((retval = so_tpi_fallback(so, cr)) == 0) {
1665 		/* Reissue the ioctl */
1666 		ASSERT(so->so_rcv_q_head == NULL);
1667 		return (SOP_IOCTL(so, cmd, arg, mode, cr, rvalp));
1668 	} else {
1669 		return (retval);
1670 	}
1671 }
1672 
1673 /*
1674  * This is called for all socket types to verify that the buffer size is large
1675  * enough for the option, and if we can, handle the request as well. Most
1676  * options will be forwarded to the protocol.
1677  */
1678 int
socket_getopt_common(struct sonode * so,int level,int option_name,void * optval,socklen_t * optlenp,int flags)1679 socket_getopt_common(struct sonode *so, int level, int option_name,
1680     void *optval, socklen_t *optlenp, int flags)
1681 {
1682 	if (level != SOL_SOCKET)
1683 		return (-1);
1684 
1685 	switch (option_name) {
1686 	case SO_ERROR:
1687 	case SO_DOMAIN:
1688 	case SO_TYPE:
1689 	case SO_ACCEPTCONN: {
1690 		int32_t value;
1691 		socklen_t optlen = *optlenp;
1692 
1693 		if (optlen < (t_uscalar_t)sizeof (int32_t)) {
1694 			return (EINVAL);
1695 		}
1696 
1697 		switch (option_name) {
1698 		case SO_ERROR:
1699 			mutex_enter(&so->so_lock);
1700 			value = sogeterr(so, B_TRUE);
1701 			mutex_exit(&so->so_lock);
1702 			break;
1703 		case SO_DOMAIN:
1704 			value = so->so_family;
1705 			break;
1706 		case SO_TYPE:
1707 			value = so->so_type;
1708 			break;
1709 		case SO_ACCEPTCONN:
1710 			if (so->so_state & SS_ACCEPTCONN)
1711 				value = SO_ACCEPTCONN;
1712 			else
1713 				value = 0;
1714 			break;
1715 		}
1716 
1717 		bcopy(&value, optval, sizeof (value));
1718 		*optlenp = sizeof (value);
1719 
1720 		return (0);
1721 	}
1722 	case SO_SNDTIMEO:
1723 	case SO_RCVTIMEO: {
1724 		clock_t value;
1725 		socklen_t optlen = *optlenp;
1726 
1727 		if (get_udatamodel() == DATAMODEL_NONE ||
1728 		    get_udatamodel() == DATAMODEL_NATIVE) {
1729 			if (optlen < sizeof (struct timeval))
1730 				return (EINVAL);
1731 		} else {
1732 			if (optlen < sizeof (struct timeval32))
1733 				return (EINVAL);
1734 		}
1735 		if (option_name == SO_RCVTIMEO)
1736 			value = drv_hztousec(so->so_rcvtimeo);
1737 		else
1738 			value = drv_hztousec(so->so_sndtimeo);
1739 
1740 		if (get_udatamodel() == DATAMODEL_NONE ||
1741 		    get_udatamodel() == DATAMODEL_NATIVE) {
1742 			((struct timeval *)(optval))->tv_sec =
1743 			    value / (1000 * 1000);
1744 			((struct timeval *)(optval))->tv_usec =
1745 			    value % (1000 * 1000);
1746 			*optlenp = sizeof (struct timeval);
1747 		} else {
1748 			((struct timeval32 *)(optval))->tv_sec =
1749 			    value / (1000 * 1000);
1750 			((struct timeval32 *)(optval))->tv_usec =
1751 			    value % (1000 * 1000);
1752 			*optlenp = sizeof (struct timeval32);
1753 		}
1754 		return (0);
1755 	}
1756 	case SO_DEBUG:
1757 	case SO_REUSEADDR:
1758 	case SO_KEEPALIVE:
1759 	case SO_DONTROUTE:
1760 	case SO_BROADCAST:
1761 	case SO_USELOOPBACK:
1762 	case SO_OOBINLINE:
1763 	case SO_SNDBUF:
1764 #ifdef notyet
1765 	case SO_SNDLOWAT:
1766 	case SO_RCVLOWAT:
1767 #endif /* notyet */
1768 	case SO_DGRAM_ERRIND: {
1769 		socklen_t optlen = *optlenp;
1770 
1771 		if (optlen < (t_uscalar_t)sizeof (int32_t))
1772 			return (EINVAL);
1773 		break;
1774 	}
1775 	case SO_RCVBUF: {
1776 		socklen_t optlen = *optlenp;
1777 
1778 		if (optlen < (t_uscalar_t)sizeof (int32_t))
1779 			return (EINVAL);
1780 
1781 		if ((flags & _SOGETSOCKOPT_XPG4_2) && so->so_xpg_rcvbuf != 0) {
1782 			/*
1783 			 * XXX If SO_RCVBUF has been set and this is an
1784 			 * XPG 4.2 application then do not ask the transport
1785 			 * since the transport might adjust the value and not
1786 			 * return exactly what was set by the application.
1787 			 * For non-XPG 4.2 application we return the value
1788 			 * that the transport is actually using.
1789 			 */
1790 			*(int32_t *)optval = so->so_xpg_rcvbuf;
1791 			*optlenp = sizeof (so->so_xpg_rcvbuf);
1792 			return (0);
1793 		}
1794 		/*
1795 		 * If the option has not been set then get a default
1796 		 * value from the transport.
1797 		 */
1798 		break;
1799 	}
1800 	case SO_LINGER: {
1801 		socklen_t optlen = *optlenp;
1802 
1803 		if (optlen < (t_uscalar_t)sizeof (struct linger))
1804 			return (EINVAL);
1805 		break;
1806 	}
1807 	case SO_SND_BUFINFO: {
1808 		socklen_t optlen = *optlenp;
1809 
1810 		if (optlen < (t_uscalar_t)sizeof (struct so_snd_bufinfo))
1811 			return (EINVAL);
1812 		((struct so_snd_bufinfo *)(optval))->sbi_wroff =
1813 		    (so->so_proto_props).sopp_wroff;
1814 		((struct so_snd_bufinfo *)(optval))->sbi_maxblk =
1815 		    (so->so_proto_props).sopp_maxblk;
1816 		((struct so_snd_bufinfo *)(optval))->sbi_maxpsz =
1817 		    (so->so_proto_props).sopp_maxpsz;
1818 		((struct so_snd_bufinfo *)(optval))->sbi_tail =
1819 		    (so->so_proto_props).sopp_tail;
1820 		*optlenp = sizeof (struct so_snd_bufinfo);
1821 		return (0);
1822 	}
1823 	case SO_SND_COPYAVOID: {
1824 		sof_instance_t *inst;
1825 
1826 		/*
1827 		 * Avoid zero-copy if there is a filter with a data_out
1828 		 * callback. We could let the operation succeed, but then
1829 		 * the filter would have to copy the data anyway.
1830 		 */
1831 		for (inst = so->so_filter_top; inst != NULL;
1832 		    inst = inst->sofi_next) {
1833 			if (SOF_INTERESTED(inst, data_out))
1834 				return (EOPNOTSUPP);
1835 		}
1836 		break;
1837 	}
1838 
1839 	default:
1840 		break;
1841 	}
1842 
1843 	/* Unknown Option */
1844 	return (-1);
1845 }
1846 
1847 void
socket_sonode_destroy(struct sonode * so)1848 socket_sonode_destroy(struct sonode *so)
1849 {
1850 	sonode_fini(so);
1851 	kmem_cache_free(socket_cache, so);
1852 }
1853 
1854 int
so_zcopy_wait(struct sonode * so)1855 so_zcopy_wait(struct sonode *so)
1856 {
1857 	int error = 0;
1858 
1859 	mutex_enter(&so->so_lock);
1860 	while (!(so->so_copyflag & STZCNOTIFY)) {
1861 		if (so->so_state & SS_CLOSING) {
1862 			mutex_exit(&so->so_lock);
1863 			return (EINTR);
1864 		}
1865 		if (cv_wait_sig(&so->so_copy_cv, &so->so_lock) == 0) {
1866 			error = EINTR;
1867 			break;
1868 		}
1869 	}
1870 	so->so_copyflag &= ~STZCNOTIFY;
1871 	mutex_exit(&so->so_lock);
1872 	return (error);
1873 }
1874 
1875 void
so_timer_callback(void * arg)1876 so_timer_callback(void *arg)
1877 {
1878 	struct sonode *so = (struct sonode *)arg;
1879 
1880 	mutex_enter(&so->so_lock);
1881 
1882 	so->so_rcv_timer_tid = 0;
1883 	if (so->so_rcv_queued > 0) {
1884 		so_notify_data(so, so->so_rcv_queued);
1885 	} else {
1886 		mutex_exit(&so->so_lock);
1887 	}
1888 }
1889 
1890 #ifdef DEBUG
1891 /*
1892  * Verify that the length stored in so_rcv_queued and the length of data blocks
1893  * queued is same.
1894  */
1895 static boolean_t
so_check_length(sonode_t * so)1896 so_check_length(sonode_t *so)
1897 {
1898 	mblk_t *mp = so->so_rcv_q_head;
1899 	int len = 0;
1900 
1901 	ASSERT(MUTEX_HELD(&so->so_lock));
1902 
1903 	if (mp != NULL) {
1904 		len = msgdsize(mp);
1905 		while ((mp = mp->b_next) != NULL)
1906 			len += msgdsize(mp);
1907 	}
1908 	mp = so->so_rcv_head;
1909 	if (mp != NULL) {
1910 		len += msgdsize(mp);
1911 		while ((mp = mp->b_next) != NULL)
1912 			len += msgdsize(mp);
1913 	}
1914 	return ((len == so->so_rcv_queued) ? B_TRUE : B_FALSE);
1915 }
1916 #endif
1917 
1918 int
so_get_mod_version(struct sockparams * sp)1919 so_get_mod_version(struct sockparams *sp)
1920 {
1921 	ASSERT(sp != NULL && sp->sp_smod_info != NULL);
1922 	return (sp->sp_smod_info->smod_version);
1923 }
1924 
1925 /*
1926  * so_start_fallback()
1927  *
1928  * Block new socket operations from coming in, and wait for active operations
1929  * to complete. Threads that are sleeping will be woken up so they can get
1930  * out of the way.
1931  *
1932  * The caller must be a reader on so_fallback_rwlock.
1933  */
1934 static boolean_t
so_start_fallback(struct sonode * so)1935 so_start_fallback(struct sonode *so)
1936 {
1937 	ASSERT(RW_READ_HELD(&so->so_fallback_rwlock));
1938 
1939 	mutex_enter(&so->so_lock);
1940 	if (so->so_state & SS_FALLBACK_PENDING) {
1941 		mutex_exit(&so->so_lock);
1942 		return (B_FALSE);
1943 	}
1944 	so->so_state |= SS_FALLBACK_PENDING;
1945 	/*
1946 	 * Poke all threads that might be sleeping. Any operation that comes
1947 	 * in after the cv_broadcast will observe the fallback pending flag
1948 	 * which cause the call to return where it would normally sleep.
1949 	 */
1950 	cv_broadcast(&so->so_state_cv);		/* threads in connect() */
1951 	cv_broadcast(&so->so_rcv_cv);		/* threads in recvmsg() */
1952 	cv_broadcast(&so->so_snd_cv);		/* threads in sendmsg() */
1953 	mutex_enter(&so->so_acceptq_lock);
1954 	cv_broadcast(&so->so_acceptq_cv);	/* threads in accept() */
1955 	mutex_exit(&so->so_acceptq_lock);
1956 	mutex_exit(&so->so_lock);
1957 
1958 	/*
1959 	 * The main reason for the rw_tryupgrade call is to provide
1960 	 * observability during the fallback process. We want to
1961 	 * be able to see if there are pending operations.
1962 	 */
1963 	if (rw_tryupgrade(&so->so_fallback_rwlock) == 0) {
1964 		/*
1965 		 * It is safe to drop and reaquire the fallback lock, because
1966 		 * we are guaranteed that another fallback cannot take place.
1967 		 */
1968 		rw_exit(&so->so_fallback_rwlock);
1969 		DTRACE_PROBE1(pending__ops__wait, (struct sonode *), so);
1970 		rw_enter(&so->so_fallback_rwlock, RW_WRITER);
1971 		DTRACE_PROBE1(pending__ops__complete, (struct sonode *), so);
1972 	}
1973 
1974 	return (B_TRUE);
1975 }
1976 
1977 /*
1978  * so_end_fallback()
1979  *
1980  * Allow socket opertions back in.
1981  *
1982  * The caller must be a writer on so_fallback_rwlock.
1983  */
1984 static void
so_end_fallback(struct sonode * so)1985 so_end_fallback(struct sonode *so)
1986 {
1987 	ASSERT(RW_ISWRITER(&so->so_fallback_rwlock));
1988 
1989 	mutex_enter(&so->so_lock);
1990 	so->so_state &= ~(SS_FALLBACK_PENDING|SS_FALLBACK_DRAIN);
1991 	mutex_exit(&so->so_lock);
1992 
1993 	rw_downgrade(&so->so_fallback_rwlock);
1994 }
1995 
1996 /*
1997  * so_quiesced_cb()
1998  *
1999  * Callback passed to the protocol during fallback. It is called once
2000  * the endpoint is quiescent.
2001  *
2002  * No requests from the user, no notifications from the protocol, so it
2003  * is safe to synchronize the state. Data can also be moved without
2004  * risk for reordering.
2005  *
2006  * We do not need to hold so_lock, since there can be only one thread
2007  * operating on the sonode.
2008  */
2009 static mblk_t *
so_quiesced_cb(sock_upper_handle_t sock_handle,sock_quiesce_arg_t * arg,struct T_capability_ack * tcap,struct sockaddr * laddr,socklen_t laddrlen,struct sockaddr * faddr,socklen_t faddrlen,short opts)2010 so_quiesced_cb(sock_upper_handle_t sock_handle, sock_quiesce_arg_t *arg,
2011     struct T_capability_ack *tcap,
2012     struct sockaddr *laddr, socklen_t laddrlen,
2013     struct sockaddr *faddr, socklen_t faddrlen, short opts)
2014 {
2015 	struct sonode *so = (struct sonode *)sock_handle;
2016 	boolean_t atmark;
2017 	mblk_t *retmp = NULL, **tailmpp = &retmp;
2018 
2019 	if (tcap != NULL)
2020 		sotpi_update_state(so, tcap, laddr, laddrlen, faddr, faddrlen,
2021 		    opts);
2022 
2023 	/*
2024 	 * Some protocols do not quiece the data path during fallback. Once
2025 	 * we set the SS_FALLBACK_DRAIN flag any attempt to queue data will
2026 	 * fail and the protocol is responsible for saving the data for later
2027 	 * delivery (i.e., once the fallback has completed).
2028 	 */
2029 	mutex_enter(&so->so_lock);
2030 	so->so_state |= SS_FALLBACK_DRAIN;
2031 	SOCKET_TIMER_CANCEL(so);
2032 	mutex_exit(&so->so_lock);
2033 
2034 	if (so->so_rcv_head != NULL) {
2035 		if (so->so_rcv_q_last_head == NULL)
2036 			so->so_rcv_q_head = so->so_rcv_head;
2037 		else
2038 			so->so_rcv_q_last_head->b_next = so->so_rcv_head;
2039 		so->so_rcv_q_last_head = so->so_rcv_last_head;
2040 	}
2041 
2042 	atmark = (so->so_state & SS_RCVATMARK) != 0;
2043 	/*
2044 	 * Clear any OOB state having to do with pending data. The TPI
2045 	 * code path will set the appropriate oob state when we move the
2046 	 * oob data to the STREAM head. We leave SS_HADOOBDATA since the oob
2047 	 * data has already been consumed.
2048 	 */
2049 	so->so_state &= ~(SS_RCVATMARK|SS_OOBPEND|SS_HAVEOOBDATA);
2050 
2051 	ASSERT(so->so_oobmsg != NULL || so->so_oobmark <= so->so_rcv_queued);
2052 
2053 	/*
2054 	 * Move data to the STREAM head.
2055 	 */
2056 	while (so->so_rcv_q_head != NULL) {
2057 		mblk_t *mp = so->so_rcv_q_head;
2058 		size_t mlen = msgdsize(mp);
2059 
2060 		so->so_rcv_q_head = mp->b_next;
2061 		mp->b_next = NULL;
2062 		mp->b_prev = NULL;
2063 
2064 		/*
2065 		 * Send T_EXDATA_IND if we are at the oob mark.
2066 		 */
2067 		if (atmark) {
2068 			struct T_exdata_ind *tei;
2069 			mblk_t *mp1 = arg->soqa_exdata_mp;
2070 
2071 			arg->soqa_exdata_mp = NULL;
2072 			ASSERT(mp1 != NULL);
2073 			mp1->b_datap->db_type = M_PROTO;
2074 			tei = (struct T_exdata_ind *)mp1->b_rptr;
2075 			tei->PRIM_type = T_EXDATA_IND;
2076 			tei->MORE_flag = 0;
2077 			mp1->b_wptr = (uchar_t *)&tei[1];
2078 
2079 			if (IS_SO_OOB_INLINE(so)) {
2080 				mp1->b_cont = mp;
2081 			} else {
2082 				ASSERT(so->so_oobmsg != NULL);
2083 				mp1->b_cont = so->so_oobmsg;
2084 				so->so_oobmsg = NULL;
2085 
2086 				/* process current mp next time around */
2087 				mp->b_next = so->so_rcv_q_head;
2088 				so->so_rcv_q_head = mp;
2089 				mlen = 0;
2090 			}
2091 			mp = mp1;
2092 
2093 			/* we have consumed the oob mark */
2094 			atmark = B_FALSE;
2095 		} else if (so->so_oobmark > 0) {
2096 			/*
2097 			 * Check if the OOB mark is within the current
2098 			 * mblk chain. In that case we have to split it up.
2099 			 */
2100 			if (so->so_oobmark < mlen) {
2101 				mblk_t *urg_mp = mp;
2102 
2103 				atmark = B_TRUE;
2104 				mp = NULL;
2105 				mlen = so->so_oobmark;
2106 
2107 				/*
2108 				 * It is assumed that the OOB mark does
2109 				 * not land within a mblk.
2110 				 */
2111 				do {
2112 					so->so_oobmark -= MBLKL(urg_mp);
2113 					mp = urg_mp;
2114 					urg_mp = urg_mp->b_cont;
2115 				} while (so->so_oobmark > 0);
2116 				mp->b_cont = NULL;
2117 				if (urg_mp != NULL) {
2118 					urg_mp->b_next = so->so_rcv_q_head;
2119 					so->so_rcv_q_head = urg_mp;
2120 				}
2121 			} else {
2122 				so->so_oobmark -= mlen;
2123 				if (so->so_oobmark == 0)
2124 					atmark = B_TRUE;
2125 			}
2126 		}
2127 
2128 		/*
2129 		 * Queue data on the STREAM head.
2130 		 */
2131 		so->so_rcv_queued -= mlen;
2132 		*tailmpp = mp;
2133 		tailmpp = &mp->b_next;
2134 	}
2135 	so->so_rcv_head = NULL;
2136 	so->so_rcv_last_head = NULL;
2137 	so->so_rcv_q_head = NULL;
2138 	so->so_rcv_q_last_head = NULL;
2139 
2140 	/*
2141 	 * Check if the oob byte is at the end of the data stream, or if the
2142 	 * oob byte has not yet arrived. In the latter case we have to send a
2143 	 * SIGURG and a mark indicator to the STREAM head. The mark indicator
2144 	 * is needed to guarantee correct behavior for SIOCATMARK. See block
2145 	 * comment in socktpi.h for more details.
2146 	 */
2147 	if (atmark || so->so_oobmark > 0) {
2148 		mblk_t *mp;
2149 
2150 		if (atmark && so->so_oobmsg != NULL) {
2151 			struct T_exdata_ind *tei;
2152 
2153 			mp = arg->soqa_exdata_mp;
2154 			arg->soqa_exdata_mp = NULL;
2155 			ASSERT(mp != NULL);
2156 			mp->b_datap->db_type = M_PROTO;
2157 			tei = (struct T_exdata_ind *)mp->b_rptr;
2158 			tei->PRIM_type = T_EXDATA_IND;
2159 			tei->MORE_flag = 0;
2160 			mp->b_wptr = (uchar_t *)&tei[1];
2161 
2162 			mp->b_cont = so->so_oobmsg;
2163 			so->so_oobmsg = NULL;
2164 
2165 			*tailmpp = mp;
2166 			tailmpp = &mp->b_next;
2167 		} else {
2168 			/* Send up the signal */
2169 			mp = arg->soqa_exdata_mp;
2170 			arg->soqa_exdata_mp = NULL;
2171 			ASSERT(mp != NULL);
2172 			DB_TYPE(mp) = M_PCSIG;
2173 			*mp->b_wptr++ = (uchar_t)SIGURG;
2174 			*tailmpp = mp;
2175 			tailmpp = &mp->b_next;
2176 
2177 			/* Send up the mark indicator */
2178 			mp = arg->soqa_urgmark_mp;
2179 			arg->soqa_urgmark_mp = NULL;
2180 			mp->b_flag = atmark ? MSGMARKNEXT : MSGNOTMARKNEXT;
2181 			*tailmpp = mp;
2182 			tailmpp = &mp->b_next;
2183 
2184 			so->so_oobmark = 0;
2185 		}
2186 	}
2187 	ASSERT(so->so_oobmark == 0);
2188 	ASSERT(so->so_rcv_queued == 0);
2189 
2190 	return (retmp);
2191 }
2192 
2193 #ifdef DEBUG
2194 /*
2195  * Do an integrity check of the sonode. This should be done if a
2196  * fallback fails after sonode has initially been converted to use
2197  * TPI and subsequently have to be reverted.
2198  *
2199  * Failure to pass the integrity check will panic the system.
2200  */
2201 void
so_integrity_check(struct sonode * cur,struct sonode * orig)2202 so_integrity_check(struct sonode *cur, struct sonode *orig)
2203 {
2204 	VERIFY(cur->so_vnode == orig->so_vnode);
2205 	VERIFY(cur->so_ops == orig->so_ops);
2206 	/*
2207 	 * For so_state we can only VERIFY the state flags in CHECK_STATE.
2208 	 * The other state flags might be affected by a notification from the
2209 	 * protocol.
2210 	 */
2211 #define	CHECK_STATE	(SS_CANTRCVMORE|SS_CANTSENDMORE|SS_NDELAY|SS_NONBLOCK| \
2212 	SS_ASYNC|SS_ACCEPTCONN|SS_SAVEDEOR|SS_RCVATMARK|SS_OOBPEND| \
2213 	SS_HAVEOOBDATA|SS_HADOOBDATA|SS_SENTLASTREADSIG|SS_SENTLASTWRITESIG)
2214 	VERIFY((cur->so_state & (orig->so_state & CHECK_STATE)) ==
2215 	    (orig->so_state & CHECK_STATE));
2216 	VERIFY(cur->so_mode == orig->so_mode);
2217 	VERIFY(cur->so_flag == orig->so_flag);
2218 	VERIFY(cur->so_count == orig->so_count);
2219 	/* Cannot VERIFY so_proto_connid; proto can update it */
2220 	VERIFY(cur->so_sockparams == orig->so_sockparams);
2221 	/* an error might have been recorded, but it can not be lost */
2222 	VERIFY(cur->so_error != 0 || orig->so_error == 0);
2223 	VERIFY(cur->so_family == orig->so_family);
2224 	VERIFY(cur->so_type == orig->so_type);
2225 	VERIFY(cur->so_protocol == orig->so_protocol);
2226 	VERIFY(cur->so_version == orig->so_version);
2227 	/* New conns might have arrived, but none should have been lost */
2228 	VERIFY(cur->so_acceptq_len >= orig->so_acceptq_len);
2229 	VERIFY(list_head(&cur->so_acceptq_list) ==
2230 	    list_head(&orig->so_acceptq_list));
2231 	VERIFY(cur->so_backlog == orig->so_backlog);
2232 	/* New OOB migth have arrived, but mark should not have been lost */
2233 	VERIFY(cur->so_oobmark >= orig->so_oobmark);
2234 	/* Cannot VERIFY so_oobmsg; the proto might have sent up a new one */
2235 	VERIFY(cur->so_pgrp == orig->so_pgrp);
2236 	VERIFY(cur->so_peercred == orig->so_peercred);
2237 	VERIFY(cur->so_cpid == orig->so_cpid);
2238 	VERIFY(cur->so_zoneid == orig->so_zoneid);
2239 	/* New data migth have arrived, but none should have been lost */
2240 	VERIFY(cur->so_rcv_queued >= orig->so_rcv_queued);
2241 	VERIFY(cur->so_rcv_q_head == orig->so_rcv_q_head);
2242 	VERIFY(cur->so_rcv_head == orig->so_rcv_head);
2243 	VERIFY(cur->so_proto_handle == orig->so_proto_handle);
2244 	VERIFY(cur->so_downcalls == orig->so_downcalls);
2245 	/* Cannot VERIFY so_proto_props; they can be updated by proto */
2246 }
2247 #endif
2248 
2249 /*
2250  * so_tpi_fallback()
2251  *
2252  * This is the fallback initation routine; things start here.
2253  *
2254  * Basic strategy:
2255  *   o Block new socket operations from coming in
2256  *   o Allocate/initate info needed by TPI
2257  *   o Quiesce the connection, at which point we sync
2258  *     state and move data
2259  *   o Change operations (sonodeops) associated with the socket
2260  *   o Unblock threads waiting for the fallback to finish
2261  */
2262 int
so_tpi_fallback(struct sonode * so,struct cred * cr)2263 so_tpi_fallback(struct sonode *so, struct cred *cr)
2264 {
2265 	int error;
2266 	queue_t *q;
2267 	struct sockparams *sp;
2268 	struct sockparams *newsp = NULL;
2269 	so_proto_fallback_func_t fbfunc;
2270 	const char *devpath;
2271 	boolean_t direct;
2272 	struct sonode *nso;
2273 	sock_quiesce_arg_t arg = { NULL, NULL };
2274 #ifdef DEBUG
2275 	struct sonode origso;
2276 #endif
2277 	error = 0;
2278 	sp = so->so_sockparams;
2279 	fbfunc = sp->sp_smod_info->smod_proto_fallback_func;
2280 
2281 	/*
2282 	 * Cannot fallback if the socket has active filters or a krecv callback.
2283 	 */
2284 	if (so->so_filter_active > 0 || so->so_krecv_cb != NULL)
2285 		return (EINVAL);
2286 
2287 	switch (so->so_family) {
2288 	case AF_INET:
2289 		devpath = sp->sp_smod_info->smod_fallback_devpath_v4;
2290 		break;
2291 	case AF_INET6:
2292 		devpath = sp->sp_smod_info->smod_fallback_devpath_v6;
2293 		break;
2294 	default:
2295 		return (EINVAL);
2296 	}
2297 
2298 	/*
2299 	 * Fallback can only happen if the socket module has a TPI device
2300 	 * and fallback function.
2301 	 */
2302 	if (devpath == NULL || fbfunc == NULL)
2303 		return (EINVAL);
2304 
2305 	/*
2306 	 * Initiate fallback; upon success we know that no new requests
2307 	 * will come in from the user.
2308 	 */
2309 	if (!so_start_fallback(so))
2310 		return (EAGAIN);
2311 #ifdef DEBUG
2312 	/*
2313 	 * Make a copy of the sonode in case we need to make an integrity
2314 	 * check later on.
2315 	 */
2316 	bcopy(so, &origso, sizeof (*so));
2317 #endif
2318 
2319 	sp->sp_stats.sps_nfallback.value.ui64++;
2320 
2321 	newsp = sockparams_hold_ephemeral_bydev(so->so_family, so->so_type,
2322 	    so->so_protocol, devpath, KM_SLEEP, &error);
2323 	if (error != 0)
2324 		goto out;
2325 
2326 	if (so->so_direct != NULL) {
2327 		sodirect_t *sodp = so->so_direct;
2328 		mutex_enter(&so->so_lock);
2329 
2330 		so->so_direct->sod_enabled = B_FALSE;
2331 		so->so_state &= ~SS_SODIRECT;
2332 		ASSERT(sodp->sod_uioafh == NULL);
2333 		mutex_exit(&so->so_lock);
2334 	}
2335 
2336 	/* Turn sonode into a TPI socket */
2337 	error = sotpi_convert_sonode(so, newsp, &direct, &q, cr);
2338 	if (error != 0)
2339 		goto out;
2340 	/*
2341 	 * When it comes to urgent data we have two cases to deal with;
2342 	 * (1) The oob byte has already arrived, or (2) the protocol has
2343 	 * notified that oob data is pending, but it has not yet arrived.
2344 	 *
2345 	 * For (1) all we need to do is send a T_EXDATA_IND to indicate were
2346 	 * in the byte stream the oob byte is. For (2) we have to send a
2347 	 * SIGURG (M_PCSIG), followed by a zero-length mblk indicating whether
2348 	 * the oob byte will be the next byte from the protocol.
2349 	 *
2350 	 * So in the worst case we need two mblks, one for the signal, another
2351 	 * for mark indication. In that case we use the exdata_mp for the sig.
2352 	 */
2353 	arg.soqa_exdata_mp = allocb_wait(sizeof (struct T_exdata_ind),
2354 	    BPRI_MED, STR_NOSIG, NULL);
2355 	arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, STR_NOSIG, NULL);
2356 
2357 	/*
2358 	 * Now tell the protocol to start using TPI. so_quiesced_cb be
2359 	 * called once it's safe to synchronize state.
2360 	 */
2361 	DTRACE_PROBE1(proto__fallback__begin, struct sonode *, so);
2362 	error = (*fbfunc)(so->so_proto_handle, q, direct, so_quiesced_cb,
2363 	    &arg);
2364 	DTRACE_PROBE1(proto__fallback__end, struct sonode *, so);
2365 
2366 	if (error != 0) {
2367 		/* protocol was unable to do a fallback, revert the sonode */
2368 		sotpi_revert_sonode(so, cr);
2369 		goto out;
2370 	}
2371 
2372 	/*
2373 	 * Walk the accept queue and notify the proto that they should
2374 	 * fall back to TPI. The protocol will send up the T_CONN_IND.
2375 	 */
2376 	nso = list_head(&so->so_acceptq_list);
2377 	while (nso != NULL) {
2378 		int rval;
2379 		struct sonode *next;
2380 
2381 		if (arg.soqa_exdata_mp == NULL) {
2382 			arg.soqa_exdata_mp =
2383 			    allocb_wait(sizeof (struct T_exdata_ind),
2384 			    BPRI_MED, STR_NOSIG, NULL);
2385 		}
2386 		if (arg.soqa_urgmark_mp == NULL) {
2387 			arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED,
2388 			    STR_NOSIG, NULL);
2389 		}
2390 
2391 		DTRACE_PROBE1(proto__fallback__begin, struct sonode *, nso);
2392 		rval = (*fbfunc)(nso->so_proto_handle, NULL, direct,
2393 		    so_quiesced_cb, &arg);
2394 		DTRACE_PROBE1(proto__fallback__end, struct sonode *, nso);
2395 		if (rval != 0) {
2396 			/* Abort the connection */
2397 			zcmn_err(getzoneid(), CE_WARN,
2398 			    "Failed to convert socket in accept queue to TPI. "
2399 			    "Pid = %d\n", curproc->p_pid);
2400 			next = list_next(&so->so_acceptq_list, nso);
2401 			list_remove(&so->so_acceptq_list, nso);
2402 			so->so_acceptq_len--;
2403 
2404 			(void) socket_close(nso, 0, CRED());
2405 			socket_destroy(nso);
2406 			nso = next;
2407 		} else {
2408 			nso = list_next(&so->so_acceptq_list, nso);
2409 		}
2410 	}
2411 
2412 	/*
2413 	 * Now flush the acceptq, this will destroy all sockets. They will
2414 	 * be recreated in sotpi_accept().
2415 	 */
2416 	so_acceptq_flush(so, B_FALSE);
2417 
2418 	mutex_enter(&so->so_lock);
2419 	so->so_state |= SS_FALLBACK_COMP;
2420 	mutex_exit(&so->so_lock);
2421 
2422 	/*
2423 	 * Swap the sonode ops. Socket opertations that come in once this
2424 	 * is done will proceed without blocking.
2425 	 */
2426 	so->so_ops = &sotpi_sonodeops;
2427 
2428 	/*
2429 	 * Wake up any threads stuck in poll. This is needed since the poll
2430 	 * head changes when the fallback happens (moves from the sonode to
2431 	 * the STREAMS head).
2432 	 */
2433 	pollwakeup(&so->so_poll_list, POLLERR);
2434 
2435 	/*
2436 	 * When this non-STREAM socket was created we placed an extra ref on
2437 	 * the associated vnode to support asynchronous close. Drop that ref
2438 	 * here.
2439 	 */
2440 	ASSERT(SOTOV(so)->v_count >= 2);
2441 	VN_RELE(SOTOV(so));
2442 out:
2443 	so_end_fallback(so);
2444 
2445 	if (error != 0) {
2446 #ifdef DEBUG
2447 		so_integrity_check(so, &origso);
2448 #endif
2449 		zcmn_err(getzoneid(), CE_WARN,
2450 		    "Failed to convert socket to TPI (err=%d). Pid = %d\n",
2451 		    error, curproc->p_pid);
2452 		if (newsp != NULL)
2453 			SOCKPARAMS_DEC_REF(newsp);
2454 	}
2455 	if (arg.soqa_exdata_mp != NULL)
2456 		freemsg(arg.soqa_exdata_mp);
2457 	if (arg.soqa_urgmark_mp != NULL)
2458 		freemsg(arg.soqa_urgmark_mp);
2459 
2460 	return (error);
2461 }
2462 
2463 int
so_krecv_set(sonode_t * so,so_krecv_f cb,void * arg)2464 so_krecv_set(sonode_t *so, so_krecv_f cb, void *arg)
2465 {
2466 	int ret;
2467 
2468 	if (cb == NULL && arg != NULL)
2469 		return (EINVAL);
2470 
2471 	SO_BLOCK_FALLBACK(so, so_krecv_set(so, cb, arg));
2472 
2473 	mutex_enter(&so->so_lock);
2474 	if (so->so_state & SS_FALLBACK_COMP) {
2475 		mutex_exit(&so->so_lock);
2476 		SO_UNBLOCK_FALLBACK(so);
2477 		return (ENOTSUP);
2478 	}
2479 
2480 	ret = so_lock_read(so, 0);
2481 	VERIFY(ret == 0);
2482 	/*
2483 	 * Other consumers may actually care about getting extant data delivered
2484 	 * to them, when they come along, they should figure out the best API
2485 	 * for that.
2486 	 */
2487 	so_rcv_flush(so);
2488 
2489 	so->so_krecv_cb = cb;
2490 	so->so_krecv_arg = arg;
2491 
2492 	so_unlock_read(so);
2493 	mutex_exit(&so->so_lock);
2494 	SO_UNBLOCK_FALLBACK(so);
2495 
2496 	return (0);
2497 }
2498 
2499 void
so_krecv_unblock(sonode_t * so)2500 so_krecv_unblock(sonode_t *so)
2501 {
2502 	mutex_enter(&so->so_lock);
2503 	VERIFY(so->so_krecv_cb != NULL);
2504 
2505 	so->so_rcv_queued = 0;
2506 	(void) so_check_flow_control(so);
2507 	/*
2508 	 * so_check_flow_control() always drops so->so_lock, so we won't
2509 	 * need to drop it ourselves.
2510 	 */
2511 }
2512