1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2015, Joyent, Inc.  All rights reserved.
28  * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
29  * Copyright 2022 Garrett D'Amore
30  */
31 
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/sysmacros.h>
36 #include <sys/debug.h>
37 #include <sys/cmn_err.h>
38 
39 #include <sys/stropts.h>
40 #include <sys/socket.h>
41 #include <sys/socketvar.h>
42 
43 #define	_SUN_TPI_VERSION	2
44 #include <sys/tihdr.h>
45 #include <sys/sockio.h>
46 #include <sys/kmem_impl.h>
47 
48 #include <sys/strsubr.h>
49 #include <sys/strsun.h>
50 #include <sys/ddi.h>
51 #include <netinet/in.h>
52 #include <inet/ip.h>
53 
54 #include <fs/sockfs/sockcommon.h>
55 #include <fs/sockfs/sockfilter_impl.h>
56 
57 #include <sys/socket_proto.h>
58 
59 #include <fs/sockfs/socktpi_impl.h>
60 #include <fs/sockfs/sodirect.h>
61 #include <sys/tihdr.h>
62 
63 extern int xnet_skip_checks;
64 extern int xnet_check_print;
65 
66 static void so_queue_oob(struct sonode *, mblk_t *, size_t);
67 
68 
69 /*ARGSUSED*/
70 int
so_accept_notsupp(struct sonode * lso,int fflag,struct cred * cr,struct sonode ** nsop)71 so_accept_notsupp(struct sonode *lso, int fflag,
72     struct cred *cr, struct sonode **nsop)
73 {
74 	return (EOPNOTSUPP);
75 }
76 
77 /*ARGSUSED*/
78 int
so_listen_notsupp(struct sonode * so,int backlog,struct cred * cr)79 so_listen_notsupp(struct sonode *so, int backlog, struct cred *cr)
80 {
81 	return (EOPNOTSUPP);
82 }
83 
84 /*ARGSUSED*/
85 int
so_getsockname_notsupp(struct sonode * so,struct sockaddr * sa,socklen_t * len,struct cred * cr)86 so_getsockname_notsupp(struct sonode *so, struct sockaddr *sa,
87     socklen_t *len, struct cred *cr)
88 {
89 	return (EOPNOTSUPP);
90 }
91 
92 /*ARGSUSED*/
93 int
so_getpeername_notsupp(struct sonode * so,struct sockaddr * addr,socklen_t * addrlen,boolean_t accept,struct cred * cr)94 so_getpeername_notsupp(struct sonode *so, struct sockaddr *addr,
95     socklen_t *addrlen, boolean_t accept, struct cred *cr)
96 {
97 	return (EOPNOTSUPP);
98 }
99 
100 /*ARGSUSED*/
101 int
so_shutdown_notsupp(struct sonode * so,int how,struct cred * cr)102 so_shutdown_notsupp(struct sonode *so, int how, struct cred *cr)
103 {
104 	return (EOPNOTSUPP);
105 }
106 
107 /*ARGSUSED*/
108 int
so_sendmblk_notsupp(struct sonode * so,struct msghdr * msg,int fflag,struct cred * cr,mblk_t ** mpp)109 so_sendmblk_notsupp(struct sonode *so, struct msghdr *msg, int fflag,
110     struct cred *cr, mblk_t **mpp)
111 {
112 	return (EOPNOTSUPP);
113 }
114 
115 /*
116  * Generic Socket Ops
117  */
118 
119 /* ARGSUSED */
120 int
so_init(struct sonode * so,struct sonode * pso,struct cred * cr,int flags)121 so_init(struct sonode *so, struct sonode *pso, struct cred *cr, int flags)
122 {
123 	return (socket_init_common(so, pso, flags, cr));
124 }
125 
126 int
so_bind(struct sonode * so,struct sockaddr * name,socklen_t namelen,int flags,struct cred * cr)127 so_bind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
128     int flags, struct cred *cr)
129 {
130 	int error;
131 
132 	SO_BLOCK_FALLBACK(so, SOP_BIND(so, name, namelen, flags, cr));
133 
134 	ASSERT(flags == _SOBIND_XPG4_2 || flags == _SOBIND_SOCKBSD);
135 
136 	/* X/Open requires this check */
137 	if ((so->so_state & SS_CANTSENDMORE) && !xnet_skip_checks) {
138 		if (xnet_check_print) {
139 			printf("sockfs: X/Open bind state check "
140 			    "caused EINVAL\n");
141 		}
142 		error = EINVAL;
143 		goto done;
144 	}
145 
146 	/*
147 	 * a bind to a NULL address is interpreted as unbind. So just
148 	 * do the downcall.
149 	 */
150 	if (name == NULL)
151 		goto dobind;
152 
153 	switch (so->so_family) {
154 	case AF_INET:
155 		if ((size_t)namelen != sizeof (sin_t)) {
156 			error = name->sa_family != so->so_family ?
157 			    EAFNOSUPPORT : EINVAL;
158 			eprintsoline(so, error);
159 			goto done;
160 		}
161 
162 		if ((flags & _SOBIND_XPG4_2) &&
163 		    (name->sa_family != so->so_family)) {
164 			/*
165 			 * This check has to be made for X/Open
166 			 * sockets however application failures have
167 			 * been observed when it is applied to
168 			 * all sockets.
169 			 */
170 			error = EAFNOSUPPORT;
171 			eprintsoline(so, error);
172 			goto done;
173 		}
174 		/*
175 		 * Force a zero sa_family to match so_family.
176 		 *
177 		 * Some programs like inetd(8) don't set the
178 		 * family field. Other programs leave
179 		 * sin_family set to garbage - SunOS 4.X does
180 		 * not check the family field on a bind.
181 		 * We use the family field that
182 		 * was passed in to the socket() call.
183 		 */
184 		name->sa_family = so->so_family;
185 		break;
186 
187 	case AF_INET6: {
188 #ifdef DEBUG
189 		sin6_t *sin6 = (sin6_t *)name;
190 #endif
191 		if ((size_t)namelen != sizeof (sin6_t)) {
192 			error = name->sa_family != so->so_family ?
193 			    EAFNOSUPPORT : EINVAL;
194 			eprintsoline(so, error);
195 			goto done;
196 		}
197 
198 		if (name->sa_family != so->so_family) {
199 			/*
200 			 * With IPv6 we require the family to match
201 			 * unlike in IPv4.
202 			 */
203 			error = EAFNOSUPPORT;
204 			eprintsoline(so, error);
205 			goto done;
206 		}
207 #ifdef DEBUG
208 		/*
209 		 * Verify that apps don't forget to clear
210 		 * sin6_scope_id etc
211 		 */
212 		if (sin6->sin6_scope_id != 0 &&
213 		    !IN6_IS_ADDR_LINKSCOPE(&sin6->sin6_addr)) {
214 			zcmn_err(getzoneid(), CE_WARN,
215 			    "bind with uninitialized sin6_scope_id "
216 			    "(%d) on socket. Pid = %d\n",
217 			    (int)sin6->sin6_scope_id,
218 			    (int)curproc->p_pid);
219 		}
220 		if (sin6->__sin6_src_id != 0) {
221 			zcmn_err(getzoneid(), CE_WARN,
222 			    "bind with uninitialized __sin6_src_id "
223 			    "(%d) on socket. Pid = %d\n",
224 			    (int)sin6->__sin6_src_id,
225 			    (int)curproc->p_pid);
226 		}
227 #endif /* DEBUG */
228 
229 		break;
230 	}
231 	default:
232 		/* Just pass the request to the protocol */
233 		goto dobind;
234 	}
235 
236 dobind:
237 	if (so->so_filter_active == 0 ||
238 	    (error = sof_filter_bind(so, name, &namelen, cr)) < 0) {
239 		error = (*so->so_downcalls->sd_bind)
240 		    (so->so_proto_handle, name, namelen, cr);
241 	}
242 done:
243 	SO_UNBLOCK_FALLBACK(so);
244 
245 	return (error);
246 }
247 
248 int
so_listen(struct sonode * so,int backlog,struct cred * cr)249 so_listen(struct sonode *so, int backlog, struct cred *cr)
250 {
251 	int	error = 0;
252 
253 	ASSERT(MUTEX_NOT_HELD(&so->so_lock));
254 	SO_BLOCK_FALLBACK(so, SOP_LISTEN(so, backlog, cr));
255 
256 	if ((so)->so_filter_active == 0 ||
257 	    (error = sof_filter_listen(so, &backlog, cr)) < 0)
258 		error = (*so->so_downcalls->sd_listen)(so->so_proto_handle,
259 		    backlog, cr);
260 
261 	SO_UNBLOCK_FALLBACK(so);
262 
263 	return (error);
264 }
265 
266 
267 int
so_connect(struct sonode * so,struct sockaddr * name,socklen_t namelen,int fflag,int flags,struct cred * cr)268 so_connect(struct sonode *so, struct sockaddr *name,
269     socklen_t namelen, int fflag, int flags, struct cred *cr)
270 {
271 	int error = 0;
272 	sock_connid_t id;
273 
274 	ASSERT(MUTEX_NOT_HELD(&so->so_lock));
275 	SO_BLOCK_FALLBACK(so, SOP_CONNECT(so, name, namelen, fflag, flags, cr));
276 
277 	/*
278 	 * If there is a pending error, return error
279 	 * This can happen if a non blocking operation caused an error.
280 	 */
281 
282 	if (so->so_error != 0) {
283 		mutex_enter(&so->so_lock);
284 		error = sogeterr(so, B_TRUE);
285 		mutex_exit(&so->so_lock);
286 		if (error != 0)
287 			goto done;
288 	}
289 
290 	if (so->so_filter_active == 0 ||
291 	    (error = sof_filter_connect(so, (struct sockaddr *)name,
292 	    &namelen, cr)) < 0) {
293 		error = (*so->so_downcalls->sd_connect)(so->so_proto_handle,
294 		    name, namelen, &id, cr);
295 
296 		if (error == EINPROGRESS)
297 			error = so_wait_connected(so,
298 			    fflag & (FNONBLOCK|FNDELAY), id);
299 	}
300 done:
301 	SO_UNBLOCK_FALLBACK(so);
302 	return (error);
303 }
304 
305 /*ARGSUSED*/
306 int
so_accept(struct sonode * so,int fflag,struct cred * cr,struct sonode ** nsop)307 so_accept(struct sonode *so, int fflag, struct cred *cr, struct sonode **nsop)
308 {
309 	int error = 0;
310 	struct sonode *nso;
311 
312 	*nsop = NULL;
313 
314 	SO_BLOCK_FALLBACK(so, SOP_ACCEPT(so, fflag, cr, nsop));
315 	if ((so->so_state & SS_ACCEPTCONN) == 0) {
316 		SO_UNBLOCK_FALLBACK(so);
317 		return ((so->so_type == SOCK_DGRAM || so->so_type == SOCK_RAW) ?
318 		    EOPNOTSUPP : EINVAL);
319 	}
320 
321 	if ((error = so_acceptq_dequeue(so, (fflag & (FNONBLOCK|FNDELAY)),
322 	    &nso)) == 0) {
323 		ASSERT(nso != NULL);
324 
325 		/* finish the accept */
326 		if ((so->so_filter_active > 0 &&
327 		    (error = sof_filter_accept(nso, cr)) > 0) ||
328 		    (error = (*so->so_downcalls->sd_accept)(so->so_proto_handle,
329 		    nso->so_proto_handle, (sock_upper_handle_t)nso, cr)) != 0) {
330 			(void) socket_close(nso, 0, cr);
331 			socket_destroy(nso);
332 		} else {
333 			*nsop = nso;
334 		}
335 	}
336 
337 	SO_UNBLOCK_FALLBACK(so);
338 	return (error);
339 }
340 
341 int
so_sendmsg(struct sonode * so,struct nmsghdr * msg,struct uio * uiop,struct cred * cr)342 so_sendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop,
343     struct cred *cr)
344 {
345 	int error, flags;
346 	boolean_t dontblock;
347 	ssize_t orig_resid;
348 	mblk_t  *mp;
349 
350 	SO_BLOCK_FALLBACK(so, SOP_SENDMSG(so, msg, uiop, cr));
351 
352 	flags = msg->msg_flags;
353 	error = 0;
354 	dontblock = (flags & MSG_DONTWAIT) ||
355 	    (uiop->uio_fmode & (FNONBLOCK|FNDELAY));
356 
357 	if (!(flags & MSG_XPG4_2) && msg->msg_controllen != 0) {
358 		/*
359 		 * Old way of passing fd's is not supported
360 		 */
361 		SO_UNBLOCK_FALLBACK(so);
362 		return (EOPNOTSUPP);
363 	}
364 
365 	if ((so->so_mode & SM_ATOMIC) &&
366 	    uiop->uio_resid > so->so_proto_props.sopp_maxpsz &&
367 	    so->so_proto_props.sopp_maxpsz != -1) {
368 		SO_UNBLOCK_FALLBACK(so);
369 		return (EMSGSIZE);
370 	}
371 
372 	/*
373 	 * For atomic sends we will only do one iteration.
374 	 */
375 	do {
376 		if (so->so_state & SS_CANTSENDMORE) {
377 			error = EPIPE;
378 			break;
379 		}
380 
381 		if (so->so_error != 0) {
382 			mutex_enter(&so->so_lock);
383 			error = sogeterr(so, B_TRUE);
384 			mutex_exit(&so->so_lock);
385 			if (error != 0)
386 				break;
387 		}
388 
389 		/*
390 		 * Send down OOB messages even if the send path is being
391 		 * flow controlled (assuming the protocol supports OOB data).
392 		 */
393 		if (flags & MSG_OOB) {
394 			if ((so->so_mode & SM_EXDATA) == 0) {
395 				error = EOPNOTSUPP;
396 				break;
397 			}
398 		} else if (SO_SND_FLOWCTRLD(so)) {
399 			/*
400 			 * Need to wait until the protocol is ready to receive
401 			 * more data for transmission.
402 			 */
403 			if ((error = so_snd_wait_qnotfull(so, dontblock)) != 0)
404 				break;
405 		}
406 
407 		/*
408 		 * Time to send data to the protocol. We either copy the
409 		 * data into mblks or pass the uio directly to the protocol.
410 		 * We decide what to do based on the available down calls.
411 		 */
412 		if (so->so_downcalls->sd_send_uio != NULL) {
413 			error = (*so->so_downcalls->sd_send_uio)
414 			    (so->so_proto_handle, uiop, msg, cr);
415 			if (error != 0)
416 				break;
417 		} else {
418 			/* save the resid in case of failure */
419 			orig_resid = uiop->uio_resid;
420 
421 			if ((mp = socopyinuio(uiop,
422 			    so->so_proto_props.sopp_maxpsz,
423 			    so->so_proto_props.sopp_wroff,
424 			    so->so_proto_props.sopp_maxblk,
425 			    so->so_proto_props.sopp_tail, &error)) == NULL) {
426 				break;
427 			}
428 			ASSERT(uiop->uio_resid >= 0);
429 
430 			if (so->so_filter_active > 0 &&
431 			    ((mp = SOF_FILTER_DATA_OUT(so, mp, msg, cr,
432 			    &error)) == NULL)) {
433 				if (error != 0)
434 					break;
435 				continue;
436 			}
437 			error = (*so->so_downcalls->sd_send)
438 			    (so->so_proto_handle, mp, msg, cr);
439 			if (error != 0) {
440 				/*
441 				 * The send failed. We do not have to free the
442 				 * mblks, because that is the protocol's
443 				 * responsibility. However, uio_resid must
444 				 * remain accurate, so adjust that here.
445 				 */
446 				uiop->uio_resid = orig_resid;
447 					break;
448 			}
449 		}
450 	} while (uiop->uio_resid > 0);
451 
452 	SO_UNBLOCK_FALLBACK(so);
453 
454 	return (error);
455 }
456 
457 int
so_sendmblk_impl(struct sonode * so,struct nmsghdr * msg,int fflag,struct cred * cr,mblk_t ** mpp,sof_instance_t * fil,boolean_t fil_inject)458 so_sendmblk_impl(struct sonode *so, struct nmsghdr *msg, int fflag,
459     struct cred *cr, mblk_t **mpp, sof_instance_t *fil,
460     boolean_t fil_inject)
461 {
462 	int error;
463 	boolean_t dontblock;
464 	size_t size;
465 	mblk_t *mp = *mpp;
466 
467 	if (so->so_downcalls->sd_send == NULL)
468 		return (EOPNOTSUPP);
469 
470 	error = 0;
471 	dontblock = (msg->msg_flags & MSG_DONTWAIT) ||
472 	    (fflag & (FNONBLOCK|FNDELAY));
473 	size = msgdsize(mp);
474 
475 	if ((so->so_mode & SM_ATOMIC) &&
476 	    size > so->so_proto_props.sopp_maxpsz &&
477 	    so->so_proto_props.sopp_maxpsz != -1) {
478 		SO_UNBLOCK_FALLBACK(so);
479 		return (EMSGSIZE);
480 	}
481 
482 	while (mp != NULL) {
483 		mblk_t *nmp, *last_mblk;
484 		size_t mlen;
485 
486 		if (so->so_state & SS_CANTSENDMORE) {
487 			error = EPIPE;
488 			break;
489 		}
490 		if (so->so_error != 0) {
491 			mutex_enter(&so->so_lock);
492 			error = sogeterr(so, B_TRUE);
493 			mutex_exit(&so->so_lock);
494 			if (error != 0)
495 				break;
496 		}
497 		/* Socket filters are not flow controlled */
498 		if (SO_SND_FLOWCTRLD(so) && !fil_inject) {
499 			/*
500 			 * Need to wait until the protocol is ready to receive
501 			 * more data for transmission.
502 			 */
503 			if ((error = so_snd_wait_qnotfull(so, dontblock)) != 0)
504 				break;
505 		}
506 
507 		/*
508 		 * We only allow so_maxpsz of data to be sent down to
509 		 * the protocol at time.
510 		 */
511 		mlen = MBLKL(mp);
512 		nmp = mp->b_cont;
513 		last_mblk = mp;
514 		while (nmp != NULL) {
515 			mlen += MBLKL(nmp);
516 			if (mlen > so->so_proto_props.sopp_maxpsz) {
517 				last_mblk->b_cont = NULL;
518 				break;
519 			}
520 			last_mblk = nmp;
521 			nmp = nmp->b_cont;
522 		}
523 
524 		if (so->so_filter_active > 0 &&
525 		    (mp = SOF_FILTER_DATA_OUT_FROM(so, fil, mp, msg,
526 		    cr, &error)) == NULL) {
527 			*mpp = mp = nmp;
528 			if (error != 0)
529 				break;
530 			continue;
531 		}
532 		error = (*so->so_downcalls->sd_send)
533 		    (so->so_proto_handle, mp, msg, cr);
534 		if (error != 0) {
535 			/*
536 			 * The send failed. The protocol will free the mblks
537 			 * that were sent down. Let the caller deal with the
538 			 * rest.
539 			 */
540 			*mpp = nmp;
541 			break;
542 		}
543 
544 		*mpp = mp = nmp;
545 	}
546 	/* Let the filter know whether the protocol is flow controlled */
547 	if (fil_inject && error == 0 && SO_SND_FLOWCTRLD(so))
548 		error = ENOSPC;
549 
550 	return (error);
551 }
552 
553 #pragma inline(so_sendmblk_impl)
554 
555 int
so_sendmblk(struct sonode * so,struct nmsghdr * msg,int fflag,struct cred * cr,mblk_t ** mpp)556 so_sendmblk(struct sonode *so, struct nmsghdr *msg, int fflag,
557     struct cred *cr, mblk_t **mpp)
558 {
559 	int error;
560 
561 	SO_BLOCK_FALLBACK(so, SOP_SENDMBLK(so, msg, fflag, cr, mpp));
562 
563 	error = so_sendmblk_impl(so, msg, fflag, cr, mpp, so->so_filter_top,
564 	    B_FALSE);
565 
566 	SO_UNBLOCK_FALLBACK(so);
567 
568 	return (error);
569 }
570 
571 int
so_shutdown(struct sonode * so,int how,struct cred * cr)572 so_shutdown(struct sonode *so, int how, struct cred *cr)
573 {
574 	int error;
575 
576 	SO_BLOCK_FALLBACK(so, SOP_SHUTDOWN(so, how, cr));
577 
578 	/*
579 	 * SunOS 4.X has no check for datagram sockets.
580 	 * 5.X checks that it is connected (ENOTCONN)
581 	 * X/Open requires that we check the connected state.
582 	 */
583 	if (!(so->so_state & SS_ISCONNECTED)) {
584 		if (!xnet_skip_checks) {
585 			error = ENOTCONN;
586 			if (xnet_check_print) {
587 				printf("sockfs: X/Open shutdown check "
588 				    "caused ENOTCONN\n");
589 			}
590 		}
591 		goto done;
592 	}
593 
594 	if (so->so_filter_active == 0 ||
595 	    (error = sof_filter_shutdown(so, &how, cr)) < 0)
596 		error = ((*so->so_downcalls->sd_shutdown)(so->so_proto_handle,
597 		    how, cr));
598 
599 	/*
600 	 * Protocol agreed to shutdown. We need to flush the
601 	 * receive buffer if the receive side is being shutdown.
602 	 */
603 	if (error == 0 && how != SHUT_WR) {
604 		mutex_enter(&so->so_lock);
605 		/* wait for active reader to finish */
606 		(void) so_lock_read(so, 0);
607 
608 		so_rcv_flush(so);
609 
610 		so_unlock_read(so);
611 		mutex_exit(&so->so_lock);
612 	}
613 
614 done:
615 	SO_UNBLOCK_FALLBACK(so);
616 	return (error);
617 }
618 
619 int
so_getsockname(struct sonode * so,struct sockaddr * addr,socklen_t * addrlen,struct cred * cr)620 so_getsockname(struct sonode *so, struct sockaddr *addr,
621     socklen_t *addrlen, struct cred *cr)
622 {
623 	int error;
624 
625 	SO_BLOCK_FALLBACK(so, SOP_GETSOCKNAME(so, addr, addrlen, cr));
626 
627 	if (so->so_filter_active == 0 ||
628 	    (error = sof_filter_getsockname(so, addr, addrlen, cr)) < 0)
629 		error = (*so->so_downcalls->sd_getsockname)
630 		    (so->so_proto_handle, addr, addrlen, cr);
631 
632 	SO_UNBLOCK_FALLBACK(so);
633 	return (error);
634 }
635 
636 int
so_getpeername(struct sonode * so,struct sockaddr * addr,socklen_t * addrlen,boolean_t accept,struct cred * cr)637 so_getpeername(struct sonode *so, struct sockaddr *addr,
638     socklen_t *addrlen, boolean_t accept, struct cred *cr)
639 {
640 	int error;
641 
642 	SO_BLOCK_FALLBACK(so, SOP_GETPEERNAME(so, addr, addrlen, accept, cr));
643 
644 	if (accept) {
645 		error = (*so->so_downcalls->sd_getpeername)
646 		    (so->so_proto_handle, addr, addrlen, cr);
647 	} else if (!(so->so_state & SS_ISCONNECTED)) {
648 		error = ENOTCONN;
649 	} else if ((so->so_state & SS_CANTSENDMORE) && !xnet_skip_checks) {
650 		/* Added this check for X/Open */
651 		error = EINVAL;
652 		if (xnet_check_print) {
653 			printf("sockfs: X/Open getpeername check => EINVAL\n");
654 		}
655 	} else if (so->so_filter_active == 0 ||
656 	    (error = sof_filter_getpeername(so, addr, addrlen, cr)) < 0) {
657 		error = (*so->so_downcalls->sd_getpeername)
658 		    (so->so_proto_handle, addr, addrlen, cr);
659 	}
660 
661 	SO_UNBLOCK_FALLBACK(so);
662 	return (error);
663 }
664 
665 int
so_getsockopt(struct sonode * so,int level,int option_name,void * optval,socklen_t * optlenp,int flags,struct cred * cr)666 so_getsockopt(struct sonode *so, int level, int option_name,
667     void *optval, socklen_t *optlenp, int flags, struct cred *cr)
668 {
669 	int error = 0;
670 
671 	if (level == SOL_FILTER)
672 		return (sof_getsockopt(so, option_name, optval, optlenp, cr));
673 
674 	SO_BLOCK_FALLBACK(so,
675 	    SOP_GETSOCKOPT(so, level, option_name, optval, optlenp, flags, cr));
676 
677 	if ((so->so_filter_active == 0 ||
678 	    (error = sof_filter_getsockopt(so, level, option_name, optval,
679 	    optlenp, cr)) < 0) &&
680 	    (error = socket_getopt_common(so, level, option_name, optval,
681 	    optlenp, flags)) < 0) {
682 		error = (*so->so_downcalls->sd_getsockopt)
683 		    (so->so_proto_handle, level, option_name, optval, optlenp,
684 		    cr);
685 		if (error ==  ENOPROTOOPT) {
686 			if (level == SOL_SOCKET) {
687 				/*
688 				 * If a protocol does not support a particular
689 				 * socket option, set can fail (not allowed)
690 				 * but get can not fail. This is the previous
691 				 * sockfs bahvior.
692 				 */
693 				switch (option_name) {
694 				case SO_LINGER:
695 					if (*optlenp < (t_uscalar_t)
696 					    sizeof (struct linger)) {
697 						error = EINVAL;
698 						break;
699 					}
700 					error = 0;
701 					bzero(optval, sizeof (struct linger));
702 					*optlenp = sizeof (struct linger);
703 					break;
704 				case SO_RCVTIMEO:
705 				case SO_SNDTIMEO:
706 					if (*optlenp < (t_uscalar_t)
707 					    sizeof (struct timeval)) {
708 						error = EINVAL;
709 						break;
710 					}
711 					error = 0;
712 					bzero(optval, sizeof (struct timeval));
713 					*optlenp = sizeof (struct timeval);
714 					break;
715 				case SO_SND_BUFINFO:
716 					if (*optlenp < (t_uscalar_t)
717 					    sizeof (struct so_snd_bufinfo)) {
718 						error = EINVAL;
719 						break;
720 					}
721 					error = 0;
722 					bzero(optval,
723 					    sizeof (struct so_snd_bufinfo));
724 					*optlenp =
725 					    sizeof (struct so_snd_bufinfo);
726 					break;
727 				case SO_DEBUG:
728 				case SO_REUSEADDR:
729 				case SO_KEEPALIVE:
730 				case SO_DONTROUTE:
731 				case SO_BROADCAST:
732 				case SO_USELOOPBACK:
733 				case SO_OOBINLINE:
734 				case SO_DGRAM_ERRIND:
735 				case SO_SNDBUF:
736 				case SO_RCVBUF:
737 					error = 0;
738 					*((int32_t *)optval) = 0;
739 					*optlenp = sizeof (int32_t);
740 					break;
741 				default:
742 					break;
743 				}
744 			}
745 		}
746 	}
747 
748 	SO_UNBLOCK_FALLBACK(so);
749 	return (error);
750 }
751 
752 int
so_setsockopt(struct sonode * so,int level,int option_name,const void * optval,socklen_t optlen,struct cred * cr)753 so_setsockopt(struct sonode *so, int level, int option_name,
754     const void *optval, socklen_t optlen, struct cred *cr)
755 {
756 	int error = 0;
757 	struct timeval tl;
758 	const void *opt = optval;
759 
760 	if (level == SOL_FILTER)
761 		return (sof_setsockopt(so, option_name, optval, optlen, cr));
762 
763 	SO_BLOCK_FALLBACK(so,
764 	    SOP_SETSOCKOPT(so, level, option_name, optval, optlen, cr));
765 
766 	/* X/Open requires this check */
767 	if (so->so_state & SS_CANTSENDMORE && !xnet_skip_checks) {
768 		SO_UNBLOCK_FALLBACK(so);
769 		if (xnet_check_print)
770 			printf("sockfs: X/Open setsockopt check => EINVAL\n");
771 		return (EINVAL);
772 	}
773 
774 	if (so->so_filter_active > 0 &&
775 	    (error = sof_filter_setsockopt(so, level, option_name,
776 	    (void *)optval, &optlen, cr)) >= 0)
777 		goto done;
778 
779 	if (level == SOL_SOCKET) {
780 		switch (option_name) {
781 		case SO_RCVTIMEO:
782 		case SO_SNDTIMEO: {
783 			/*
784 			 * We pass down these two options to protocol in order
785 			 * to support some third part protocols which need to
786 			 * know them. For those protocols which don't care
787 			 * these two options, simply return 0.
788 			 */
789 			clock_t t_usec;
790 
791 			if (get_udatamodel() == DATAMODEL_NONE ||
792 			    get_udatamodel() == DATAMODEL_NATIVE) {
793 				if (optlen != sizeof (struct timeval)) {
794 					error = EINVAL;
795 					goto done;
796 				}
797 				bcopy((struct timeval *)optval, &tl,
798 				    sizeof (struct timeval));
799 			} else {
800 				if (optlen != sizeof (struct timeval32)) {
801 					error = EINVAL;
802 					goto done;
803 				}
804 				TIMEVAL32_TO_TIMEVAL(&tl,
805 				    (struct timeval32 *)optval);
806 			}
807 			opt = &tl;
808 			optlen = sizeof (tl);
809 			t_usec = tl.tv_sec * 1000 * 1000 + tl.tv_usec;
810 			mutex_enter(&so->so_lock);
811 			if (option_name == SO_RCVTIMEO)
812 				so->so_rcvtimeo = drv_usectohz(t_usec);
813 			else
814 				so->so_sndtimeo = drv_usectohz(t_usec);
815 			mutex_exit(&so->so_lock);
816 			break;
817 		}
818 		case SO_RCVBUF:
819 			/*
820 			 * XXX XPG 4.2 applications retrieve SO_RCVBUF from
821 			 * sockfs since the transport might adjust the value
822 			 * and not return exactly what was set by the
823 			 * application.
824 			 */
825 			so->so_xpg_rcvbuf = *(int32_t *)optval;
826 			break;
827 		}
828 	}
829 	error = (*so->so_downcalls->sd_setsockopt)
830 	    (so->so_proto_handle, level, option_name, opt, optlen, cr);
831 done:
832 	SO_UNBLOCK_FALLBACK(so);
833 	return (error);
834 }
835 
836 int
so_ioctl(struct sonode * so,int cmd,intptr_t arg,int mode,struct cred * cr,int32_t * rvalp)837 so_ioctl(struct sonode *so, int cmd, intptr_t arg, int mode,
838     struct cred *cr, int32_t *rvalp)
839 {
840 	int error = 0;
841 
842 	SO_BLOCK_FALLBACK(so, SOP_IOCTL(so, cmd, arg, mode, cr, rvalp));
843 
844 	/*
845 	 * If there is a pending error, return error
846 	 * This can happen if a non blocking operation caused an error.
847 	 */
848 	if (so->so_error != 0) {
849 		mutex_enter(&so->so_lock);
850 		error = sogeterr(so, B_TRUE);
851 		mutex_exit(&so->so_lock);
852 		if (error != 0)
853 			goto done;
854 	}
855 
856 	/*
857 	 * calling strioc can result in the socket falling back to TPI,
858 	 * if that is supported.
859 	 */
860 	if ((so->so_filter_active == 0 ||
861 	    (error = sof_filter_ioctl(so, cmd, arg, mode,
862 	    rvalp, cr)) < 0) &&
863 	    (error = socket_ioctl_common(so, cmd, arg, mode, cr, rvalp)) < 0 &&
864 	    (error = socket_strioc_common(so, cmd, arg, mode, cr, rvalp)) < 0) {
865 		error = (*so->so_downcalls->sd_ioctl)(so->so_proto_handle,
866 		    cmd, arg, mode, rvalp, cr);
867 	}
868 
869 done:
870 	SO_UNBLOCK_FALLBACK(so);
871 
872 	return (error);
873 }
874 
875 int
so_poll(struct sonode * so,short events,int anyyet,short * reventsp,struct pollhead ** phpp)876 so_poll(struct sonode *so, short events, int anyyet, short *reventsp,
877     struct pollhead **phpp)
878 {
879 	int state = so->so_state, mask;
880 	*reventsp = 0;
881 
882 	/*
883 	 * In sockets the errors are represented as input/output events
884 	 */
885 	if (so->so_error != 0 &&
886 	    ((POLLIN|POLLRDNORM|POLLOUT) & events) != 0) {
887 		*reventsp = (POLLIN|POLLRDNORM|POLLOUT) & events;
888 		return (0);
889 	}
890 
891 	/*
892 	 * If the socket is in a state where it can send data
893 	 * turn on POLLWRBAND and POLLOUT events.
894 	 */
895 	if ((so->so_mode & SM_CONNREQUIRED) == 0 || (state & SS_ISCONNECTED)) {
896 		/*
897 		 * out of band data is allowed even if the connection
898 		 * is flow controlled
899 		 */
900 		*reventsp |= POLLWRBAND & events;
901 		if (!SO_SND_FLOWCTRLD(so)) {
902 			/*
903 			 * As long as there is buffer to send data
904 			 * turn on POLLOUT events
905 			 */
906 			*reventsp |= POLLOUT & events;
907 		}
908 	}
909 
910 	/*
911 	 * Turn on POLLIN whenever there is data on the receive queue,
912 	 * or the socket is in a state where no more data will be received.
913 	 * Also, if the socket is accepting connections, flip the bit if
914 	 * there is something on the queue.
915 	 *
916 	 * We do an initial check for events without holding locks. However,
917 	 * if there are no event available, then we redo the check for POLLIN
918 	 * events under the lock.
919 	 */
920 
921 	/* Pending connections */
922 	if (!list_is_empty(&so->so_acceptq_list))
923 		*reventsp |= (POLLIN|POLLRDNORM) & events;
924 
925 	/*
926 	 * If we're looking for POLLRDHUP, indicate it if we have sent the
927 	 * last rx signal for the socket.
928 	 */
929 	if ((events & POLLRDHUP) && (state & SS_SENTLASTREADSIG))
930 		*reventsp |= POLLRDHUP;
931 
932 	/* Data */
933 	/* so_downcalls is null for sctp */
934 	if (so->so_downcalls != NULL && so->so_downcalls->sd_poll != NULL) {
935 		*reventsp |= (*so->so_downcalls->sd_poll)
936 		    (so->so_proto_handle, events & SO_PROTO_POLLEV, anyyet,
937 		    CRED()) & events;
938 		ASSERT((*reventsp & ~events) == 0);
939 		/* do not recheck events */
940 		events &= ~SO_PROTO_POLLEV;
941 	} else {
942 		if (SO_HAVE_DATA(so))
943 			*reventsp |= (POLLIN|POLLRDNORM) & events;
944 
945 		/* Urgent data */
946 		if ((state & SS_OOBPEND) != 0) {
947 			*reventsp |= (POLLRDBAND | POLLPRI) & events;
948 		}
949 
950 		/*
951 		 * If the socket has become disconnected, we set POLLHUP.
952 		 * Note that if we are in this state, we will have set POLLIN
953 		 * (SO_HAVE_DATA() is true on a disconnected socket), but not
954 		 * POLLOUT (SS_ISCONNECTED is false).  This is in keeping with
955 		 * the semantics of POLLHUP, which is defined to be mutually
956 		 * exclusive with respect to POLLOUT but not POLLIN.  We are
957 		 * therefore setting POLLHUP primarily for the benefit of
958 		 * those not polling on POLLIN, as they have no other way of
959 		 * knowing that the socket has been disconnected.
960 		 */
961 		mask = SS_SENTLASTREADSIG | SS_SENTLASTWRITESIG;
962 
963 		if ((state & (mask | SS_ISCONNECTED)) == mask)
964 			*reventsp |= POLLHUP;
965 	}
966 
967 	if ((!*reventsp && !anyyet) || (events & POLLET)) {
968 		/* Check for read events again, but this time under lock */
969 		if (events & (POLLIN|POLLRDNORM)) {
970 			mutex_enter(&so->so_lock);
971 			if (SO_HAVE_DATA(so) ||
972 			    !list_is_empty(&so->so_acceptq_list)) {
973 				if (events & POLLET) {
974 					so->so_pollev |= SO_POLLEV_IN;
975 					*phpp = &so->so_poll_list;
976 				}
977 
978 				mutex_exit(&so->so_lock);
979 				*reventsp |= (POLLIN|POLLRDNORM) & events;
980 
981 				return (0);
982 			} else {
983 				so->so_pollev |= SO_POLLEV_IN;
984 				mutex_exit(&so->so_lock);
985 			}
986 		}
987 		*phpp = &so->so_poll_list;
988 	}
989 	return (0);
990 }
991 
992 /*
993  * Generic Upcalls
994  */
995 void
so_connected(sock_upper_handle_t sock_handle,sock_connid_t id,cred_t * peer_cred,pid_t peer_cpid)996 so_connected(sock_upper_handle_t sock_handle, sock_connid_t id,
997     cred_t *peer_cred, pid_t peer_cpid)
998 {
999 	struct sonode *so = (struct sonode *)sock_handle;
1000 
1001 	mutex_enter(&so->so_lock);
1002 	ASSERT(so->so_proto_handle != NULL);
1003 
1004 	if (peer_cred != NULL) {
1005 		if (so->so_peercred != NULL)
1006 			crfree(so->so_peercred);
1007 		crhold(peer_cred);
1008 		so->so_peercred = peer_cred;
1009 		so->so_cpid = peer_cpid;
1010 	}
1011 
1012 	so->so_proto_connid = id;
1013 	soisconnected(so);
1014 	/*
1015 	 * Wake ones who're waiting for conn to become established.
1016 	 */
1017 	so_notify_connected(so);
1018 }
1019 
1020 int
so_disconnected(sock_upper_handle_t sock_handle,sock_connid_t id,int error)1021 so_disconnected(sock_upper_handle_t sock_handle, sock_connid_t id, int error)
1022 {
1023 	struct sonode *so = (struct sonode *)sock_handle;
1024 	boolean_t connect_failed;
1025 
1026 	mutex_enter(&so->so_lock);
1027 
1028 	/*
1029 	 * If we aren't currently connected, then this isn't a disconnect but
1030 	 * rather a failure to connect.
1031 	 */
1032 	connect_failed = !(so->so_state & SS_ISCONNECTED);
1033 
1034 	so->so_proto_connid = id;
1035 	soisdisconnected(so, error);
1036 	so_notify_disconnected(so, connect_failed, error);
1037 
1038 	return (0);
1039 }
1040 
1041 void
so_opctl(sock_upper_handle_t sock_handle,sock_opctl_action_t action,uintptr_t arg)1042 so_opctl(sock_upper_handle_t sock_handle, sock_opctl_action_t action,
1043     uintptr_t arg)
1044 {
1045 	struct sonode *so = (struct sonode *)sock_handle;
1046 
1047 	switch (action) {
1048 	case SOCK_OPCTL_SHUT_SEND:
1049 		mutex_enter(&so->so_lock);
1050 		socantsendmore(so);
1051 		so_notify_disconnecting(so);
1052 		break;
1053 	case SOCK_OPCTL_SHUT_RECV: {
1054 		mutex_enter(&so->so_lock);
1055 		socantrcvmore(so);
1056 		so_notify_eof(so);
1057 		break;
1058 	}
1059 	case SOCK_OPCTL_ENAB_ACCEPT:
1060 		mutex_enter(&so->so_lock);
1061 		so->so_state |= SS_ACCEPTCONN;
1062 		so->so_backlog = (unsigned int)arg;
1063 		/*
1064 		 * The protocol can stop generating newconn upcalls when
1065 		 * the backlog is full, so to make sure the listener does
1066 		 * not end up with a queue full of deferred connections
1067 		 * we reduce the backlog by one. Thus the listener will
1068 		 * start closing deferred connections before the backlog
1069 		 * is full.
1070 		 */
1071 		if (so->so_filter_active > 0)
1072 			so->so_backlog = MAX(1, so->so_backlog - 1);
1073 		mutex_exit(&so->so_lock);
1074 		break;
1075 	default:
1076 		ASSERT(0);
1077 		break;
1078 	}
1079 }
1080 
1081 void
so_txq_full(sock_upper_handle_t sock_handle,boolean_t qfull)1082 so_txq_full(sock_upper_handle_t sock_handle, boolean_t qfull)
1083 {
1084 	struct sonode *so = (struct sonode *)sock_handle;
1085 
1086 	if (qfull) {
1087 		so_snd_qfull(so);
1088 	} else {
1089 		so_snd_qnotfull(so);
1090 		mutex_enter(&so->so_lock);
1091 		/* so_notify_writable drops so_lock */
1092 		so_notify_writable(so);
1093 	}
1094 }
1095 
1096 sock_upper_handle_t
so_newconn(sock_upper_handle_t parenthandle,sock_lower_handle_t proto_handle,sock_downcalls_t * sock_downcalls,struct cred * peer_cred,pid_t peer_cpid,sock_upcalls_t ** sock_upcallsp)1097 so_newconn(sock_upper_handle_t parenthandle,
1098     sock_lower_handle_t proto_handle, sock_downcalls_t *sock_downcalls,
1099     struct cred *peer_cred, pid_t peer_cpid, sock_upcalls_t **sock_upcallsp)
1100 {
1101 	struct sonode	*so = (struct sonode *)parenthandle;
1102 	struct sonode	*nso;
1103 	int error;
1104 
1105 	ASSERT(proto_handle != NULL);
1106 
1107 	if ((so->so_state & SS_ACCEPTCONN) == 0 ||
1108 	    (so->so_acceptq_len >= so->so_backlog &&
1109 	    (so->so_filter_active == 0 || !sof_sonode_drop_deferred(so)))) {
1110 			return (NULL);
1111 	}
1112 
1113 	nso = socket_newconn(so, proto_handle, sock_downcalls, SOCKET_NOSLEEP,
1114 	    &error);
1115 	if (nso == NULL)
1116 		return (NULL);
1117 
1118 	if (peer_cred != NULL) {
1119 		crhold(peer_cred);
1120 		nso->so_peercred = peer_cred;
1121 		nso->so_cpid = peer_cpid;
1122 	}
1123 	nso->so_listener = so;
1124 
1125 	/*
1126 	 * The new socket (nso), proto_handle and sock_upcallsp are all
1127 	 * valid at this point. But as soon as nso is placed in the accept
1128 	 * queue that can no longer be assumed (since an accept() thread may
1129 	 * pull it off the queue and close the socket).
1130 	 */
1131 	*sock_upcallsp = &so_upcalls;
1132 
1133 	mutex_enter(&so->so_acceptq_lock);
1134 	if (so->so_state & (SS_CLOSING|SS_FALLBACK_PENDING|SS_FALLBACK_COMP)) {
1135 		mutex_exit(&so->so_acceptq_lock);
1136 		ASSERT(nso->so_count == 1);
1137 		nso->so_count--;
1138 		nso->so_listener = NULL;
1139 		/* drop proto ref */
1140 		VN_RELE(SOTOV(nso));
1141 		socket_destroy(nso);
1142 		return (NULL);
1143 	} else {
1144 		so->so_acceptq_len++;
1145 		if (nso->so_state & SS_FIL_DEFER) {
1146 			list_insert_tail(&so->so_acceptq_defer, nso);
1147 			mutex_exit(&so->so_acceptq_lock);
1148 		} else {
1149 			list_insert_tail(&so->so_acceptq_list, nso);
1150 			cv_signal(&so->so_acceptq_cv);
1151 			mutex_exit(&so->so_acceptq_lock);
1152 			mutex_enter(&so->so_lock);
1153 			so_notify_newconn(so);
1154 		}
1155 
1156 		return ((sock_upper_handle_t)nso);
1157 	}
1158 }
1159 
1160 void
so_set_prop(sock_upper_handle_t sock_handle,struct sock_proto_props * soppp)1161 so_set_prop(sock_upper_handle_t sock_handle, struct sock_proto_props *soppp)
1162 {
1163 	struct sonode *so;
1164 
1165 	so = (struct sonode *)sock_handle;
1166 
1167 	mutex_enter(&so->so_lock);
1168 
1169 	if (soppp->sopp_flags & SOCKOPT_MAXBLK)
1170 		so->so_proto_props.sopp_maxblk = soppp->sopp_maxblk;
1171 	if (soppp->sopp_flags & SOCKOPT_WROFF)
1172 		so->so_proto_props.sopp_wroff = soppp->sopp_wroff;
1173 	if (soppp->sopp_flags & SOCKOPT_TAIL)
1174 		so->so_proto_props.sopp_tail = soppp->sopp_tail;
1175 	if (soppp->sopp_flags & SOCKOPT_RCVHIWAT)
1176 		so->so_proto_props.sopp_rxhiwat = soppp->sopp_rxhiwat;
1177 	if (soppp->sopp_flags & SOCKOPT_RCVLOWAT)
1178 		so->so_proto_props.sopp_rxlowat = soppp->sopp_rxlowat;
1179 	if (soppp->sopp_flags & SOCKOPT_MAXPSZ)
1180 		so->so_proto_props.sopp_maxpsz = soppp->sopp_maxpsz;
1181 	if (soppp->sopp_flags & SOCKOPT_MINPSZ)
1182 		so->so_proto_props.sopp_minpsz = soppp->sopp_minpsz;
1183 	if (soppp->sopp_flags & SOCKOPT_ZCOPY) {
1184 		if (soppp->sopp_zcopyflag & ZCVMSAFE) {
1185 			so->so_proto_props.sopp_zcopyflag |= STZCVMSAFE;
1186 			so->so_proto_props.sopp_zcopyflag &= ~STZCVMUNSAFE;
1187 		} else if (soppp->sopp_zcopyflag & ZCVMUNSAFE) {
1188 			so->so_proto_props.sopp_zcopyflag |= STZCVMUNSAFE;
1189 			so->so_proto_props.sopp_zcopyflag &= ~STZCVMSAFE;
1190 		}
1191 
1192 		if (soppp->sopp_zcopyflag & COPYCACHED) {
1193 			so->so_proto_props.sopp_zcopyflag |= STRCOPYCACHED;
1194 		}
1195 	}
1196 	if (soppp->sopp_flags & SOCKOPT_OOBINLINE)
1197 		so->so_proto_props.sopp_oobinline = soppp->sopp_oobinline;
1198 	if (soppp->sopp_flags & SOCKOPT_RCVTIMER)
1199 		so->so_proto_props.sopp_rcvtimer = soppp->sopp_rcvtimer;
1200 	if (soppp->sopp_flags & SOCKOPT_RCVTHRESH)
1201 		so->so_proto_props.sopp_rcvthresh = soppp->sopp_rcvthresh;
1202 	if (soppp->sopp_flags & SOCKOPT_MAXADDRLEN)
1203 		so->so_proto_props.sopp_maxaddrlen = soppp->sopp_maxaddrlen;
1204 	if (soppp->sopp_flags & SOCKOPT_LOOPBACK)
1205 		so->so_proto_props.sopp_loopback = soppp->sopp_loopback;
1206 
1207 	mutex_exit(&so->so_lock);
1208 
1209 	if (so->so_filter_active > 0) {
1210 		sof_instance_t *inst;
1211 		ssize_t maxblk;
1212 		ushort_t wroff, tail;
1213 		maxblk = so->so_proto_props.sopp_maxblk;
1214 		wroff = so->so_proto_props.sopp_wroff;
1215 		tail = so->so_proto_props.sopp_tail;
1216 		for (inst = so->so_filter_bottom; inst != NULL;
1217 		    inst = inst->sofi_prev) {
1218 			if (SOF_INTERESTED(inst, mblk_prop)) {
1219 				(*inst->sofi_ops->sofop_mblk_prop)(
1220 				    (sof_handle_t)inst, inst->sofi_cookie,
1221 				    &maxblk, &wroff, &tail);
1222 			}
1223 		}
1224 		mutex_enter(&so->so_lock);
1225 		so->so_proto_props.sopp_maxblk = maxblk;
1226 		so->so_proto_props.sopp_wroff = wroff;
1227 		so->so_proto_props.sopp_tail = tail;
1228 		mutex_exit(&so->so_lock);
1229 	}
1230 #ifdef DEBUG
1231 	soppp->sopp_flags &= ~(SOCKOPT_MAXBLK | SOCKOPT_WROFF | SOCKOPT_TAIL |
1232 	    SOCKOPT_RCVHIWAT | SOCKOPT_RCVLOWAT | SOCKOPT_MAXPSZ |
1233 	    SOCKOPT_ZCOPY | SOCKOPT_OOBINLINE | SOCKOPT_RCVTIMER |
1234 	    SOCKOPT_RCVTHRESH | SOCKOPT_MAXADDRLEN | SOCKOPT_MINPSZ |
1235 	    SOCKOPT_LOOPBACK);
1236 	ASSERT(soppp->sopp_flags == 0);
1237 #endif
1238 }
1239 
1240 /* ARGSUSED */
1241 ssize_t
so_queue_msg_impl(struct sonode * so,mblk_t * mp,size_t msg_size,int flags,int * errorp,boolean_t * force_pushp,sof_instance_t * filter)1242 so_queue_msg_impl(struct sonode *so, mblk_t *mp,
1243     size_t msg_size, int flags, int *errorp,  boolean_t *force_pushp,
1244     sof_instance_t *filter)
1245 {
1246 	boolean_t force_push = B_TRUE;
1247 	int space_left;
1248 	sodirect_t *sodp = so->so_direct;
1249 
1250 	ASSERT(errorp != NULL);
1251 	*errorp = 0;
1252 	if (mp == NULL) {
1253 		if (so->so_downcalls->sd_recv_uio != NULL) {
1254 			mutex_enter(&so->so_lock);
1255 			/* the notify functions will drop the lock */
1256 			if (flags & MSG_OOB)
1257 				so_notify_oobdata(so, IS_SO_OOB_INLINE(so));
1258 			else
1259 				so_notify_data(so, msg_size);
1260 			return (0);
1261 		}
1262 		ASSERT(msg_size == 0);
1263 		mutex_enter(&so->so_lock);
1264 		goto space_check;
1265 	}
1266 
1267 	ASSERT(mp->b_next == NULL);
1268 	ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_PROTO);
1269 	ASSERT(msg_size == msgdsize(mp));
1270 
1271 	if (DB_TYPE(mp) == M_PROTO && !__TPI_PRIM_ISALIGNED(mp->b_rptr)) {
1272 		/* The read pointer is not aligned correctly for TPI */
1273 		zcmn_err(getzoneid(), CE_WARN,
1274 		    "sockfs: Unaligned TPI message received. rptr = %p\n",
1275 		    (void *)mp->b_rptr);
1276 		freemsg(mp);
1277 		mutex_enter(&so->so_lock);
1278 		if (sodp != NULL)
1279 			SOD_UIOAFINI(sodp);
1280 		goto space_check;
1281 	}
1282 
1283 	if (so->so_filter_active > 0) {
1284 		for (; filter != NULL; filter = filter->sofi_prev) {
1285 			if (!SOF_INTERESTED(filter, data_in))
1286 				continue;
1287 			mp = (*filter->sofi_ops->sofop_data_in)(
1288 			    (sof_handle_t)filter, filter->sofi_cookie, mp,
1289 			    flags, &msg_size);
1290 			ASSERT(msgdsize(mp) == msg_size);
1291 			DTRACE_PROBE2(filter__data, (sof_instance_t), filter,
1292 			    (mblk_t *), mp);
1293 			/* Data was consumed/dropped, just do space check */
1294 			if (msg_size == 0) {
1295 				mutex_enter(&so->so_lock);
1296 				goto space_check;
1297 			}
1298 		}
1299 	}
1300 
1301 	mutex_enter(&so->so_lock);
1302 	if (so->so_krecv_cb != NULL) {
1303 		boolean_t cont;
1304 		so_krecv_f func = so->so_krecv_cb;
1305 		void *arg = so->so_krecv_arg;
1306 
1307 		mutex_exit(&so->so_lock);
1308 		cont = func(so, mp, msg_size, flags & MSG_OOB, arg);
1309 		mutex_enter(&so->so_lock);
1310 		if (cont == B_TRUE) {
1311 			space_left = so->so_rcvbuf;
1312 		} else {
1313 			so->so_rcv_queued = so->so_rcvlowat;
1314 			*errorp = ENOSPC;
1315 			space_left = -1;
1316 		}
1317 		goto done_unlock;
1318 	}
1319 	mutex_exit(&so->so_lock);
1320 
1321 	if (flags & MSG_OOB) {
1322 		so_queue_oob(so, mp, msg_size);
1323 		mutex_enter(&so->so_lock);
1324 		goto space_check;
1325 	}
1326 
1327 	if (force_pushp != NULL)
1328 		force_push = *force_pushp;
1329 
1330 	mutex_enter(&so->so_lock);
1331 	if (so->so_state & (SS_FALLBACK_DRAIN | SS_FALLBACK_COMP)) {
1332 		if (sodp != NULL)
1333 			SOD_DISABLE(sodp);
1334 		mutex_exit(&so->so_lock);
1335 		*errorp = EOPNOTSUPP;
1336 		return (-1);
1337 	}
1338 	if (so->so_state & (SS_CANTRCVMORE | SS_CLOSING)) {
1339 		freemsg(mp);
1340 		if (sodp != NULL)
1341 			SOD_DISABLE(sodp);
1342 		mutex_exit(&so->so_lock);
1343 		return (0);
1344 	}
1345 
1346 	/* process the mblk via I/OAT if capable */
1347 	if (sodp != NULL && sodp->sod_enabled) {
1348 		if (DB_TYPE(mp) == M_DATA) {
1349 			sod_uioa_mblk_init(sodp, mp, msg_size);
1350 		} else {
1351 			SOD_UIOAFINI(sodp);
1352 		}
1353 	}
1354 
1355 	if (mp->b_next == NULL) {
1356 		so_enqueue_msg(so, mp, msg_size);
1357 	} else {
1358 		do {
1359 			mblk_t *nmp;
1360 
1361 			if ((nmp = mp->b_next) != NULL) {
1362 				mp->b_next = NULL;
1363 			}
1364 			so_enqueue_msg(so, mp, msgdsize(mp));
1365 			mp = nmp;
1366 		} while (mp != NULL);
1367 	}
1368 
1369 	space_left = so->so_rcvbuf - so->so_rcv_queued;
1370 	if (space_left <= 0) {
1371 		so->so_flowctrld = B_TRUE;
1372 		*errorp = ENOSPC;
1373 		space_left = -1;
1374 	}
1375 
1376 	if (force_push || so->so_rcv_queued >= so->so_rcv_thresh ||
1377 	    so->so_rcv_queued >= so->so_rcv_wanted) {
1378 		SOCKET_TIMER_CANCEL(so);
1379 		/*
1380 		 * so_notify_data will release the lock
1381 		 */
1382 		so_notify_data(so, so->so_rcv_queued);
1383 
1384 		if (force_pushp != NULL)
1385 			*force_pushp = B_TRUE;
1386 		goto done;
1387 	} else if (so->so_rcv_timer_tid == 0) {
1388 		/* Make sure the recv push timer is running */
1389 		SOCKET_TIMER_START(so);
1390 	}
1391 
1392 done_unlock:
1393 	mutex_exit(&so->so_lock);
1394 done:
1395 	return (space_left);
1396 
1397 space_check:
1398 	space_left = so->so_rcvbuf - so->so_rcv_queued;
1399 	if (space_left <= 0) {
1400 		so->so_flowctrld = B_TRUE;
1401 		*errorp = ENOSPC;
1402 		space_left = -1;
1403 	}
1404 	goto done_unlock;
1405 }
1406 
1407 #pragma	inline(so_queue_msg_impl)
1408 
1409 ssize_t
so_queue_msg(sock_upper_handle_t sock_handle,mblk_t * mp,size_t msg_size,int flags,int * errorp,boolean_t * force_pushp)1410 so_queue_msg(sock_upper_handle_t sock_handle, mblk_t *mp,
1411     size_t msg_size, int flags, int *errorp,  boolean_t *force_pushp)
1412 {
1413 	struct sonode *so = (struct sonode *)sock_handle;
1414 
1415 	return (so_queue_msg_impl(so, mp, msg_size, flags, errorp, force_pushp,
1416 	    so->so_filter_bottom));
1417 }
1418 
1419 /*
1420  * Set the offset of where the oob data is relative to the bytes in
1421  * queued. Also generate SIGURG
1422  */
1423 void
so_signal_oob(sock_upper_handle_t sock_handle,ssize_t offset)1424 so_signal_oob(sock_upper_handle_t sock_handle, ssize_t offset)
1425 {
1426 	struct sonode *so;
1427 
1428 	ASSERT(offset >= 0);
1429 	so = (struct sonode *)sock_handle;
1430 	mutex_enter(&so->so_lock);
1431 	if (so->so_direct != NULL)
1432 		SOD_UIOAFINI(so->so_direct);
1433 
1434 	/*
1435 	 * New urgent data on the way so forget about any old
1436 	 * urgent data.
1437 	 */
1438 	so->so_state &= ~(SS_HAVEOOBDATA|SS_HADOOBDATA);
1439 
1440 	/*
1441 	 * Record that urgent data is pending.
1442 	 */
1443 	so->so_state |= SS_OOBPEND;
1444 
1445 	if (so->so_oobmsg != NULL) {
1446 		dprintso(so, 1, ("sock: discarding old oob\n"));
1447 		freemsg(so->so_oobmsg);
1448 		so->so_oobmsg = NULL;
1449 	}
1450 
1451 	/*
1452 	 * set the offset where the urgent byte is
1453 	 */
1454 	so->so_oobmark = so->so_rcv_queued + offset;
1455 	if (so->so_oobmark == 0)
1456 		so->so_state |= SS_RCVATMARK;
1457 	else
1458 		so->so_state &= ~SS_RCVATMARK;
1459 
1460 	so_notify_oobsig(so);
1461 }
1462 
1463 /*
1464  * Queue the OOB byte
1465  */
1466 static void
so_queue_oob(struct sonode * so,mblk_t * mp,size_t len)1467 so_queue_oob(struct sonode *so, mblk_t *mp, size_t len)
1468 {
1469 	mutex_enter(&so->so_lock);
1470 	if (so->so_direct != NULL)
1471 		SOD_UIOAFINI(so->so_direct);
1472 
1473 	ASSERT(mp != NULL);
1474 	if (!IS_SO_OOB_INLINE(so)) {
1475 		so->so_oobmsg = mp;
1476 		so->so_state |= SS_HAVEOOBDATA;
1477 	} else {
1478 		so_enqueue_msg(so, mp, len);
1479 	}
1480 
1481 	so_notify_oobdata(so, IS_SO_OOB_INLINE(so));
1482 }
1483 
1484 int
so_close(struct sonode * so,int flag,struct cred * cr)1485 so_close(struct sonode *so, int flag, struct cred *cr)
1486 {
1487 	int error;
1488 
1489 	/*
1490 	 * No new data will be enqueued once the CLOSING flag is set.
1491 	 */
1492 	mutex_enter(&so->so_lock);
1493 	so->so_state |= SS_CLOSING;
1494 	ASSERT(so_verify_oobstate(so));
1495 	so_rcv_flush(so);
1496 	mutex_exit(&so->so_lock);
1497 
1498 	if (so->so_filter_active > 0)
1499 		sof_sonode_closing(so);
1500 
1501 	if (so->so_state & SS_ACCEPTCONN) {
1502 		/*
1503 		 * We grab and release the accept lock to ensure that any
1504 		 * thread about to insert a socket in so_newconn completes
1505 		 * before we flush the queue. Any thread calling so_newconn
1506 		 * after we drop the lock will observe the SS_CLOSING flag,
1507 		 * which will stop it from inserting the socket in the queue.
1508 		 */
1509 		mutex_enter(&so->so_acceptq_lock);
1510 		mutex_exit(&so->so_acceptq_lock);
1511 
1512 		so_acceptq_flush(so, B_TRUE);
1513 	}
1514 
1515 	error = (*so->so_downcalls->sd_close)(so->so_proto_handle, flag, cr);
1516 	switch (error) {
1517 	default:
1518 		/* Protocol made a synchronous close; remove proto ref */
1519 		VN_RELE(SOTOV(so));
1520 		break;
1521 	case EINPROGRESS:
1522 		/*
1523 		 * Protocol is in the process of closing, it will make a
1524 		 * 'closed' upcall to remove the reference.
1525 		 */
1526 		error = 0;
1527 		break;
1528 	}
1529 
1530 	return (error);
1531 }
1532 
1533 /*
1534  * Upcall made by the protocol when it's doing an asynchronous close. It
1535  * will drop the protocol's reference on the socket.
1536  */
1537 void
so_closed(sock_upper_handle_t sock_handle)1538 so_closed(sock_upper_handle_t sock_handle)
1539 {
1540 	struct sonode *so = (struct sonode *)sock_handle;
1541 
1542 	VN_RELE(SOTOV(so));
1543 }
1544 
1545 vnode_t *
so_get_vnode(sock_upper_handle_t sock_handle)1546 so_get_vnode(sock_upper_handle_t sock_handle)
1547 {
1548 	sonode_t *so = (sonode_t *)sock_handle;
1549 	vnode_t *vn;
1550 
1551 	vn = SOTOV(so);
1552 	VN_HOLD(vn);
1553 
1554 	return (vn);
1555 }
1556 
1557 void
so_zcopy_notify(sock_upper_handle_t sock_handle)1558 so_zcopy_notify(sock_upper_handle_t sock_handle)
1559 {
1560 	struct sonode *so = (struct sonode *)sock_handle;
1561 
1562 	mutex_enter(&so->so_lock);
1563 	so->so_copyflag |= STZCNOTIFY;
1564 	cv_broadcast(&so->so_copy_cv);
1565 	mutex_exit(&so->so_lock);
1566 }
1567 
1568 void
so_set_error(sock_upper_handle_t sock_handle,int error)1569 so_set_error(sock_upper_handle_t sock_handle, int error)
1570 {
1571 	struct sonode *so = (struct sonode *)sock_handle;
1572 
1573 	mutex_enter(&so->so_lock);
1574 
1575 	soseterror(so, error);
1576 
1577 	so_notify_error(so);
1578 }
1579 
1580 /*
1581  * so_recvmsg - read data from the socket
1582  *
1583  * There are two ways of obtaining data; either we ask the protocol to
1584  * copy directly into the supplied buffer, or we copy data from the
1585  * sonode's receive queue. The decision which one to use depends on
1586  * whether the protocol has a sd_recv_uio down call.
1587  */
1588 int
so_recvmsg(struct sonode * so,struct nmsghdr * msg,struct uio * uiop,struct cred * cr)1589 so_recvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop,
1590     struct cred *cr)
1591 {
1592 	rval_t		rval;
1593 	int		flags = 0;
1594 	t_uscalar_t	controllen, namelen;
1595 	int		error = 0;
1596 	int ret;
1597 	mblk_t		*mctlp = NULL;
1598 	union T_primitives *tpr;
1599 	void		*control;
1600 	ssize_t		saved_resid;
1601 	struct uio	*suiop;
1602 
1603 	SO_BLOCK_FALLBACK(so, SOP_RECVMSG(so, msg, uiop, cr));
1604 
1605 	if ((so->so_state & (SS_ISCONNECTED|SS_CANTRCVMORE)) == 0 &&
1606 	    (so->so_mode & SM_CONNREQUIRED)) {
1607 		SO_UNBLOCK_FALLBACK(so);
1608 		return (ENOTCONN);
1609 	}
1610 
1611 	mutex_enter(&so->so_lock);
1612 	if (so->so_krecv_cb != NULL) {
1613 		mutex_exit(&so->so_lock);
1614 		return (EOPNOTSUPP);
1615 	}
1616 	mutex_exit(&so->so_lock);
1617 
1618 	if (msg->msg_flags & MSG_PEEK)
1619 		msg->msg_flags &= ~MSG_WAITALL;
1620 
1621 	if (so->so_mode & SM_ATOMIC)
1622 		msg->msg_flags |= MSG_TRUNC;
1623 
1624 	if (msg->msg_flags & MSG_OOB) {
1625 		if ((so->so_mode & SM_EXDATA) == 0) {
1626 			error = EOPNOTSUPP;
1627 		} else if (so->so_downcalls->sd_recv_uio != NULL) {
1628 			error = (*so->so_downcalls->sd_recv_uio)
1629 			    (so->so_proto_handle, uiop, msg, cr);
1630 		} else {
1631 			error = sorecvoob(so, msg, uiop, msg->msg_flags,
1632 			    IS_SO_OOB_INLINE(so));
1633 		}
1634 		SO_UNBLOCK_FALLBACK(so);
1635 		return (error);
1636 	}
1637 
1638 	/*
1639 	 * If the protocol has the recv down call, then pass the request
1640 	 * down.
1641 	 */
1642 	if (so->so_downcalls->sd_recv_uio != NULL) {
1643 		error = (*so->so_downcalls->sd_recv_uio)
1644 		    (so->so_proto_handle, uiop, msg, cr);
1645 		SO_UNBLOCK_FALLBACK(so);
1646 		return (error);
1647 	}
1648 
1649 	/*
1650 	 * Reading data from the socket buffer
1651 	 */
1652 	flags = msg->msg_flags;
1653 	msg->msg_flags = 0;
1654 
1655 	/*
1656 	 * Set msg_controllen and msg_namelen to zero here to make it
1657 	 * simpler in the cases that no control or name is returned.
1658 	 */
1659 	controllen = msg->msg_controllen;
1660 	namelen = msg->msg_namelen;
1661 	msg->msg_controllen = 0;
1662 	msg->msg_namelen = 0;
1663 
1664 	mutex_enter(&so->so_lock);
1665 	/* Set SOREADLOCKED */
1666 	error = so_lock_read_intr(so,
1667 	    uiop->uio_fmode | ((flags & MSG_DONTWAIT) ? FNONBLOCK : 0));
1668 	mutex_exit(&so->so_lock);
1669 	if (error) {
1670 		SO_UNBLOCK_FALLBACK(so);
1671 		return (error);
1672 	}
1673 
1674 	suiop = sod_rcv_init(so, flags, &uiop);
1675 retry:
1676 	saved_resid = uiop->uio_resid;
1677 	error = so_dequeue_msg(so, &mctlp, uiop, &rval, flags);
1678 	if (error != 0) {
1679 		goto out;
1680 	}
1681 	/*
1682 	 * For datagrams the MOREDATA flag is used to set MSG_TRUNC.
1683 	 * For non-datagrams MOREDATA is used to set MSG_EOR.
1684 	 */
1685 	ASSERT(!(rval.r_val1 & MORECTL));
1686 	if ((rval.r_val1 & MOREDATA) && (so->so_mode & SM_ATOMIC))
1687 		msg->msg_flags |= MSG_TRUNC;
1688 	if (mctlp == NULL) {
1689 		dprintso(so, 1, ("so_recvmsg: got M_DATA\n"));
1690 
1691 		mutex_enter(&so->so_lock);
1692 		/* Set MSG_EOR based on MOREDATA */
1693 		if (!(rval.r_val1 & MOREDATA)) {
1694 			if (so->so_state & SS_SAVEDEOR) {
1695 				msg->msg_flags |= MSG_EOR;
1696 				so->so_state &= ~SS_SAVEDEOR;
1697 			}
1698 		}
1699 		/*
1700 		 * If some data was received (i.e. not EOF) and the
1701 		 * read/recv* has not been satisfied wait for some more.
1702 		 */
1703 		if ((flags & MSG_WAITALL) && !(msg->msg_flags & MSG_EOR) &&
1704 		    uiop->uio_resid != saved_resid && uiop->uio_resid > 0) {
1705 			mutex_exit(&so->so_lock);
1706 			flags |= MSG_NOMARK;
1707 			goto retry;
1708 		}
1709 
1710 		goto out_locked;
1711 	}
1712 	/* so_queue_msg has already verified length and alignment */
1713 	tpr = (union T_primitives *)mctlp->b_rptr;
1714 	dprintso(so, 1, ("so_recvmsg: type %d\n", tpr->type));
1715 	switch (tpr->type) {
1716 	case T_DATA_IND: {
1717 		/*
1718 		 * Set msg_flags to MSG_EOR based on
1719 		 * MORE_flag and MOREDATA.
1720 		 */
1721 		mutex_enter(&so->so_lock);
1722 		so->so_state &= ~SS_SAVEDEOR;
1723 		if (!(tpr->data_ind.MORE_flag & 1)) {
1724 			if (!(rval.r_val1 & MOREDATA))
1725 				msg->msg_flags |= MSG_EOR;
1726 			else
1727 				so->so_state |= SS_SAVEDEOR;
1728 		}
1729 		freemsg(mctlp);
1730 		/*
1731 		 * If some data was received (i.e. not EOF) and the
1732 		 * read/recv* has not been satisfied wait for some more.
1733 		 */
1734 		if ((flags & MSG_WAITALL) && !(msg->msg_flags & MSG_EOR) &&
1735 		    uiop->uio_resid != saved_resid && uiop->uio_resid > 0) {
1736 			mutex_exit(&so->so_lock);
1737 			flags |= MSG_NOMARK;
1738 			goto retry;
1739 		}
1740 		goto out_locked;
1741 	}
1742 	case T_UNITDATA_IND: {
1743 		void *addr;
1744 		t_uscalar_t addrlen;
1745 		void *abuf;
1746 		t_uscalar_t optlen;
1747 		void *opt;
1748 
1749 		if (namelen != 0) {
1750 			/* Caller wants source address */
1751 			addrlen = tpr->unitdata_ind.SRC_length;
1752 			addr = sogetoff(mctlp, tpr->unitdata_ind.SRC_offset,
1753 			    addrlen, 1);
1754 			if (addr == NULL) {
1755 				freemsg(mctlp);
1756 				error = EPROTO;
1757 				eprintsoline(so, error);
1758 				goto out;
1759 			}
1760 			ASSERT(so->so_family != AF_UNIX);
1761 		}
1762 		optlen = tpr->unitdata_ind.OPT_length;
1763 		if (optlen != 0) {
1764 			t_uscalar_t ncontrollen;
1765 
1766 			/*
1767 			 * Extract any source address option.
1768 			 * Determine how large cmsg buffer is needed.
1769 			 */
1770 			opt = sogetoff(mctlp, tpr->unitdata_ind.OPT_offset,
1771 			    optlen, __TPI_ALIGN_SIZE);
1772 
1773 			if (opt == NULL) {
1774 				freemsg(mctlp);
1775 				error = EPROTO;
1776 				eprintsoline(so, error);
1777 				goto out;
1778 			}
1779 			if (so->so_family == AF_UNIX)
1780 				so_getopt_srcaddr(opt, optlen, &addr, &addrlen);
1781 			ncontrollen = so_cmsglen(mctlp, opt, optlen,
1782 			    !(flags & MSG_XPG4_2));
1783 			if (controllen != 0)
1784 				controllen = ncontrollen;
1785 			else if (ncontrollen != 0)
1786 				msg->msg_flags |= MSG_CTRUNC;
1787 		} else {
1788 			controllen = 0;
1789 		}
1790 
1791 		if (namelen != 0) {
1792 			/*
1793 			 * Return address to caller.
1794 			 * Caller handles truncation if length
1795 			 * exceeds msg_namelen.
1796 			 * NOTE: AF_UNIX NUL termination is ensured by
1797 			 * the sender's copyin_name().
1798 			 */
1799 			abuf = kmem_alloc(addrlen, KM_SLEEP);
1800 
1801 			bcopy(addr, abuf, addrlen);
1802 			msg->msg_name = abuf;
1803 			msg->msg_namelen = addrlen;
1804 		}
1805 
1806 		if (controllen != 0) {
1807 			/*
1808 			 * Return control msg to caller.
1809 			 * Caller handles truncation if length
1810 			 * exceeds msg_controllen.
1811 			 */
1812 			control = kmem_zalloc(controllen, KM_SLEEP);
1813 
1814 			error = so_opt2cmsg(mctlp, opt, optlen,
1815 			    !(flags & MSG_XPG4_2), control, controllen);
1816 			if (error) {
1817 				freemsg(mctlp);
1818 				if (msg->msg_namelen != 0)
1819 					kmem_free(msg->msg_name,
1820 					    msg->msg_namelen);
1821 				kmem_free(control, controllen);
1822 				eprintsoline(so, error);
1823 				goto out;
1824 			}
1825 			msg->msg_control = control;
1826 			msg->msg_controllen = controllen;
1827 		}
1828 
1829 		freemsg(mctlp);
1830 		goto out;
1831 	}
1832 	case T_OPTDATA_IND: {
1833 		struct T_optdata_req *tdr;
1834 		void *opt;
1835 		t_uscalar_t optlen;
1836 
1837 		tdr = (struct T_optdata_req *)mctlp->b_rptr;
1838 		optlen = tdr->OPT_length;
1839 		if (optlen != 0) {
1840 			t_uscalar_t ncontrollen;
1841 			/*
1842 			 * Determine how large cmsg buffer is needed.
1843 			 */
1844 			opt = sogetoff(mctlp,
1845 			    tpr->optdata_ind.OPT_offset, optlen,
1846 			    __TPI_ALIGN_SIZE);
1847 
1848 			if (opt == NULL) {
1849 				freemsg(mctlp);
1850 				error = EPROTO;
1851 				eprintsoline(so, error);
1852 				goto out;
1853 			}
1854 
1855 			ncontrollen = so_cmsglen(mctlp, opt, optlen,
1856 			    !(flags & MSG_XPG4_2));
1857 			if (controllen != 0)
1858 				controllen = ncontrollen;
1859 			else if (ncontrollen != 0)
1860 				msg->msg_flags |= MSG_CTRUNC;
1861 		} else {
1862 			controllen = 0;
1863 		}
1864 
1865 		if (controllen != 0) {
1866 			/*
1867 			 * Return control msg to caller.
1868 			 * Caller handles truncation if length
1869 			 * exceeds msg_controllen.
1870 			 */
1871 			control = kmem_zalloc(controllen, KM_SLEEP);
1872 
1873 			error = so_opt2cmsg(mctlp, opt, optlen,
1874 			    !(flags & MSG_XPG4_2), control, controllen);
1875 			if (error) {
1876 				freemsg(mctlp);
1877 				kmem_free(control, controllen);
1878 				eprintsoline(so, error);
1879 				goto out;
1880 			}
1881 			msg->msg_control = control;
1882 			msg->msg_controllen = controllen;
1883 		}
1884 
1885 		/*
1886 		 * Set msg_flags to MSG_EOR based on
1887 		 * DATA_flag and MOREDATA.
1888 		 */
1889 		mutex_enter(&so->so_lock);
1890 		so->so_state &= ~SS_SAVEDEOR;
1891 		if (!(tpr->data_ind.MORE_flag & 1)) {
1892 			if (!(rval.r_val1 & MOREDATA))
1893 				msg->msg_flags |= MSG_EOR;
1894 			else
1895 				so->so_state |= SS_SAVEDEOR;
1896 		}
1897 		freemsg(mctlp);
1898 		/*
1899 		 * If some data was received (i.e. not EOF) and the
1900 		 * read/recv* has not been satisfied wait for some more.
1901 		 * Not possible to wait if control info was received.
1902 		 */
1903 		if ((flags & MSG_WAITALL) && !(msg->msg_flags & MSG_EOR) &&
1904 		    controllen == 0 &&
1905 		    uiop->uio_resid != saved_resid && uiop->uio_resid > 0) {
1906 			mutex_exit(&so->so_lock);
1907 			flags |= MSG_NOMARK;
1908 			goto retry;
1909 		}
1910 		goto out_locked;
1911 	}
1912 	default:
1913 		cmn_err(CE_CONT, "so_recvmsg bad type %x \n",
1914 		    tpr->type);
1915 		freemsg(mctlp);
1916 		error = EPROTO;
1917 		ASSERT(0);
1918 	}
1919 out:
1920 	mutex_enter(&so->so_lock);
1921 out_locked:
1922 	ret = sod_rcv_done(so, suiop, uiop);
1923 	if (ret != 0 && error == 0)
1924 		error = ret;
1925 
1926 	so_unlock_read(so);	/* Clear SOREADLOCKED */
1927 	mutex_exit(&so->so_lock);
1928 
1929 	SO_UNBLOCK_FALLBACK(so);
1930 
1931 	return (error);
1932 }
1933 
1934 sonodeops_t so_sonodeops = {
1935 	so_init,		/* sop_init	*/
1936 	so_accept,		/* sop_accept   */
1937 	so_bind,		/* sop_bind	*/
1938 	so_listen,		/* sop_listen   */
1939 	so_connect,		/* sop_connect  */
1940 	so_recvmsg,		/* sop_recvmsg  */
1941 	so_sendmsg,		/* sop_sendmsg  */
1942 	so_sendmblk,		/* sop_sendmblk */
1943 	so_getpeername,		/* sop_getpeername */
1944 	so_getsockname,		/* sop_getsockname */
1945 	so_shutdown,		/* sop_shutdown */
1946 	so_getsockopt,		/* sop_getsockopt */
1947 	so_setsockopt,		/* sop_setsockopt */
1948 	so_ioctl,		/* sop_ioctl    */
1949 	so_poll,		/* sop_poll	*/
1950 	so_close,		/* sop_close */
1951 };
1952 
1953 sock_upcalls_t so_upcalls = {
1954 	so_newconn,
1955 	so_connected,
1956 	so_disconnected,
1957 	so_opctl,
1958 	so_queue_msg,
1959 	so_set_prop,
1960 	so_txq_full,
1961 	so_signal_oob,
1962 	so_zcopy_notify,
1963 	so_set_error,
1964 	so_closed,
1965 	so_get_vnode
1966 };
1967