1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2011-2021 Tintri by DDN, Inc. All rights reserved.
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 1982, 1986, 1988, 1991, 1993
29  *	The Regents of the University of California.  All rights reserved.
30  *
31  * Redistribution and use in source and binary forms, with or without
32  * modification, are permitted provided that the following conditions
33  * are met:
34  * 1. Redistributions of source code must retain the above copyright
35  *    notice, this list of conditions and the following disclaimer.
36  * 2. Redistributions in binary form must reproduce the above copyright
37  *    notice, this list of conditions and the following disclaimer in the
38  *    documentation and/or other materials provided with the distribution.
39  * 3. All advertising materials mentioning features or use of this software
40  *    must display the following acknowledgement:
41  *	This product includes software developed by the University of
42  *	California, Berkeley and its contributors.
43  * 4. Neither the name of the University nor the names of its contributors
44  *    may be used to endorse or promote products derived from this software
45  *    without specific prior written permission.
46  *
47  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
48  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
49  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
50  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
51  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
52  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
53  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
55  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
56  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57  * SUCH DAMAGE.
58  *
59  */
60 
61 #include <smbsrv/smb_kproto.h>
62 #include <smbsrv/smb_kstat.h>
63 
64 static kmem_cache_t	*smb_mbc_cache = NULL;
65 static kmem_cache_t	*smb_mbuf_cache = NULL;
66 static kmem_cache_t	*smb_mbufcl_cache = NULL;
67 
68 /* Perhaps move this to libfakekernel? */
69 #ifdef	_FAKE_KERNEL
70 size_t kmem_max_cached = 0x20000;	// UMEM_MAXBUF
71 #endif
72 
73 void
smb_mbc_init(void)74 smb_mbc_init(void)
75 {
76 	if (smb_mbc_cache != NULL)
77 		return;
78 	smb_mbc_cache = kmem_cache_create(SMBSRV_KSTAT_MBC_CACHE,
79 	    sizeof (mbuf_chain_t), 8, NULL, NULL, NULL, NULL, NULL, 0);
80 
81 	smb_mbuf_cache = kmem_cache_create("smb_mbuf_cache",
82 	    sizeof (mbuf_t), 8, NULL, NULL, NULL, NULL, NULL, 0);
83 
84 	smb_mbufcl_cache = kmem_cache_create("smb_mbufcl_cache",
85 	    MCLBYTES, 8, NULL, NULL, NULL, NULL, NULL, 0);
86 }
87 
88 void
smb_mbc_fini(void)89 smb_mbc_fini(void)
90 {
91 	if (smb_mbc_cache != NULL) {
92 		kmem_cache_destroy(smb_mbc_cache);
93 		smb_mbc_cache = NULL;
94 	}
95 	if (smb_mbuf_cache != NULL) {
96 		kmem_cache_destroy(smb_mbuf_cache);
97 		smb_mbuf_cache = NULL;
98 	}
99 	if (smb_mbufcl_cache != NULL) {
100 		kmem_cache_destroy(smb_mbufcl_cache);
101 		smb_mbufcl_cache = NULL;
102 	}
103 }
104 
105 mbuf_chain_t *
smb_mbc_alloc(uint32_t max_bytes)106 smb_mbc_alloc(uint32_t max_bytes)
107 {
108 	mbuf_chain_t	*mbc;
109 	mbuf_t		*m;
110 
111 	mbc = kmem_cache_alloc(smb_mbc_cache, KM_SLEEP);
112 	bzero(mbc, sizeof (*mbc));
113 	mbc->mbc_magic = SMB_MBC_MAGIC;
114 
115 	if (max_bytes != 0) {
116 		MGET(m, M_WAIT, MT_DATA);
117 		m->m_len = 0;
118 		mbc->chain = m;
119 		if (max_bytes > MINCLSIZE)
120 			MCLGET(m, M_WAIT);
121 	}
122 	mbc->max_bytes = max_bytes;
123 	return (mbc);
124 }
125 
126 void
smb_mbc_free(mbuf_chain_t * mbc)127 smb_mbc_free(mbuf_chain_t *mbc)
128 {
129 	SMB_MBC_VALID(mbc);
130 
131 	m_freem(mbc->chain);
132 	mbc->chain = NULL;
133 	mbc->mbc_magic = 0;
134 	kmem_cache_free(smb_mbc_cache, mbc);
135 }
136 
137 /*
138  * smb_mbuf_get
139  *
140  * Allocate mbufs to hold the amount of data specified.
141  * A pointer to the head of the mbuf list is returned.
142  */
143 struct mbuf *
smb_mbuf_get(uchar_t * buf,int nbytes)144 smb_mbuf_get(uchar_t *buf, int nbytes)
145 {
146 	struct mbuf *mhead = 0;
147 	struct mbuf *m = 0;
148 	int count;
149 	int offset = 0;
150 
151 	while (nbytes) {
152 		count = (nbytes > MCLBYTES) ? MCLBYTES : nbytes;
153 		nbytes -= count;
154 
155 		if (mhead == 0) {
156 			MGET(mhead, M_WAIT, MT_DATA);
157 			m = mhead;
158 		} else {
159 			MGET(m->m_next, M_WAIT, MT_DATA);
160 			m = m->m_next;
161 		}
162 
163 		if (count > MLEN) {
164 			MCLGET(m, M_WAIT);
165 		}
166 
167 		m->m_len = count;
168 		bcopy(buf + offset, m->m_data, count);
169 		offset += count;
170 	}
171 	return (mhead);
172 }
173 
174 /*
175  * Build an mbuf with external storage
176  * Like esballoca()
177  */
178 mbuf_t *
smb_mbuf_alloc_ext(caddr_t buf,int len,m_ext_free_t ff,void * arg)179 smb_mbuf_alloc_ext(caddr_t buf, int len, m_ext_free_t ff, void *arg)
180 {
181 	mbuf_t	*m = 0;
182 
183 	MGET(m, M_WAIT, MT_DATA);
184 
185 	/* Like MCLGET(), but external buf. */
186 	m->m_ext.ext_buf = buf;
187 	m->m_data = m->m_ext.ext_buf;
188 	m->m_flags |= M_EXT;
189 	m->m_ext.ext_size = len;
190 	m->m_ext.ext_free = ff;
191 	m->m_ext.ext_arg1 = arg;
192 
193 	m->m_len = len;
194 
195 	return (m);
196 }
197 
198 static void
smb_mbuf_kmem_free(mbuf_t * m)199 smb_mbuf_kmem_free(mbuf_t *m)
200 {
201 	ASSERT((m->m_flags & M_EXT) != 0);
202 
203 	kmem_free(m->m_ext.ext_buf, m->m_ext.ext_size);
204 	/* Caller sets m->m_ext.ext_buf = NULL */
205 }
206 
207 /*
208  * Allocate one mbuf with contiguous (external) storage
209  * obtained via kmem_alloc.  Most places should be using
210  * smb_mbuf_alloc_chain below.
211  */
212 mbuf_t *
smb_mbuf_alloc_kmem(int len)213 smb_mbuf_alloc_kmem(int len)
214 {
215 	mbuf_t *m;
216 	caddr_t buf;
217 
218 	VERIFY(len > 0);
219 #ifdef	DEBUG
220 	if (len > kmem_max_cached) {
221 		debug_enter("fix caller");
222 	}
223 #endif
224 
225 	buf = kmem_alloc(len, KM_SLEEP);
226 	m = smb_mbuf_alloc_ext(buf, len, smb_mbuf_kmem_free, NULL);
227 
228 	return (m);
229 }
230 
231 /*
232  * smb_mbuf_alloc_chain
233  *
234  * Allocate mbufs to hold the amount of data specified.
235  * A pointer to the head of the mbuf chain is returned.
236  *
237  * For large messages, put the large mbufs at the tail, starting with
238  * kmem_max_cached messages (128K) and then a shorter (eg MCLBYTES)
239  * message at the front where we're more likely to need to copy.
240  *
241  * Must limit this to kmem_max_cached, to avoid contention with
242  * allocating from kmem_oversize_arena.
243  */
244 struct mbuf *
smb_mbuf_alloc_chain(int nbytes)245 smb_mbuf_alloc_chain(int nbytes)
246 {
247 	struct mbuf *mhead = NULL;
248 	struct mbuf *m;
249 	int len;	/* m_len for current mbuf */
250 
251 	ASSERT(nbytes > 0);
252 
253 	while (nbytes >= kmem_max_cached) {
254 		len = kmem_max_cached;
255 		m = smb_mbuf_alloc_kmem(len);
256 
257 		/* prepend to chain at mhead */
258 		m->m_next = mhead;
259 		mhead = m;
260 
261 		nbytes -= len;
262 	}
263 
264 	if (nbytes > MCLBYTES) {
265 		len = nbytes;
266 		m = smb_mbuf_alloc_kmem(len);
267 
268 		/* prepend to chain at mhead */
269 		m->m_next = mhead;
270 		mhead = m;
271 
272 		nbytes -= len;
273 	} else if (nbytes > 0) {
274 		ASSERT(nbytes <= MCLBYTES);
275 		len = nbytes;
276 		MGET(m, M_WAIT, MT_DATA);
277 		if (len > MLEN) {
278 			MCLGET(m, M_WAIT);
279 		}
280 		m->m_len = len;
281 
282 		/* prepend to chain at mhead */
283 		m->m_next = mhead;
284 		mhead = m;
285 
286 		nbytes -= len;
287 	}
288 
289 	return (mhead);
290 }
291 
292 /*
293  * Allocate enough mbufs to accommodate the residual count in uio,
294  * and setup the uio_iov to point to them.  Note that uio->uio_iov
295  * is allocated by the call and has MAX_IOVEC elements.  Currently
296  * the uio passed is always part of an smb_vdb_t and starts with
297  * uio->uio_iovcnt = MAX_IOVEC;
298  *
299  * This is used by the various SMB read code paths.  That code is
300  * going to do a disk read into this buffer, so we'd like the
301  * segments to be large.  See smb_mbuf_alloc_chain().
302  */
303 struct mbuf *
smb_mbuf_allocate(struct uio * uio)304 smb_mbuf_allocate(struct uio *uio)
305 {
306 	mbuf_t	*mhead;
307 	int	rc;
308 
309 	ASSERT(uio->uio_resid > 0);
310 	ASSERT(uio->uio_iovcnt == MAX_IOVEC);
311 
312 	mhead = smb_mbuf_alloc_chain(uio->uio_resid);
313 
314 	rc = smb_mbuf_mkuio(mhead, uio);
315 	VERIFY(rc == 0);
316 
317 	return (mhead);
318 }
319 
320 /*
321  * Build an iovec for an mbuf chain.
322  *
323  * The resulting iovec covers uio_resid length in the chain,
324  * which could be shorter than the mbuf chain total length.
325  *
326  * uio->uio_iovcnt is allocated size on entry, used size on return.
327  * Errors if iovec too small or mbuf chain too short.
328  */
329 int
smb_mbuf_mkuio(mbuf_t * m,uio_t * uio)330 smb_mbuf_mkuio(mbuf_t *m, uio_t *uio)
331 {
332 	iovec_t	*iov;
333 	ssize_t off = 0;
334 	int iovcnt = 0;
335 	int tlen;
336 
337 	iov = uio->uio_iov;
338 	while (off < uio->uio_resid) {
339 		if (m == NULL)
340 			return (EFAULT);
341 		if (iovcnt >= uio->uio_iovcnt)
342 			return (E2BIG);
343 		tlen = m->m_len;
344 		if ((off + tlen) > uio->uio_resid)
345 			tlen = (int)(uio->uio_resid - off);
346 		iov->iov_base = m->m_data;
347 		iov->iov_len = tlen;
348 		off += tlen;
349 		m = m->m_next;
350 		iovcnt++;
351 		iov++;
352 	}
353 	uio->uio_iovcnt = iovcnt;
354 
355 	return (0);
356 }
357 
358 /*
359  * Trim an mbuf chain to nbytes.
360  */
361 void
smb_mbuf_trim(struct mbuf * mhead,int nbytes)362 smb_mbuf_trim(struct mbuf *mhead, int nbytes)
363 {
364 	struct mbuf	*m = mhead;
365 
366 	while (m != 0) {
367 		if (nbytes <= m->m_len) {
368 			m->m_len = nbytes;
369 			if (m->m_next != 0) {
370 				m_freem(m->m_next);
371 				m->m_next = 0;
372 			}
373 			break;
374 		}
375 		nbytes -= m->m_len;
376 		m = m->m_next;
377 	}
378 }
379 
380 int
MBC_LENGTH(struct mbuf_chain * MBC)381 MBC_LENGTH(struct mbuf_chain *MBC)
382 {
383 	struct mbuf	*m = (MBC)->chain;
384 	int		used = 0;
385 
386 	while (m != 0) {
387 		used += m->m_len;
388 		m = m->m_next;
389 	}
390 	return (used);
391 }
392 
393 int
MBC_MAXBYTES(struct mbuf_chain * MBC)394 MBC_MAXBYTES(struct mbuf_chain *MBC)
395 {
396 	return (MBC->max_bytes);
397 }
398 
399 void
MBC_SETUP(struct mbuf_chain * MBC,uint32_t max_bytes)400 MBC_SETUP(struct mbuf_chain *MBC, uint32_t max_bytes)
401 {
402 	bzero((MBC), sizeof (struct mbuf_chain));
403 	(MBC)->max_bytes = max_bytes;
404 }
405 
406 /*
407  * Initialize an mbuf chain and allocate the first message (if max_bytes
408  * is specified).  Leave some prepend space at the head.
409  */
410 void
MBC_INIT(struct mbuf_chain * MBC,uint32_t max_bytes)411 MBC_INIT(struct mbuf_chain *MBC, uint32_t max_bytes)
412 {
413 	struct mbuf *m;
414 
415 	bzero((MBC), sizeof (struct mbuf_chain));
416 
417 	if (max_bytes != 0) {
418 		MGET(m, M_WAIT, MT_DATA);
419 		m->m_len = 0;
420 		(MBC)->chain = m;
421 		if (max_bytes > MINCLSIZE)
422 			MCLGET(m, M_WAIT);
423 		m->m_data += MH_PREPEND_SPACE;
424 	}
425 	(MBC)->max_bytes = max_bytes;
426 }
427 
428 void
MBC_FLUSH(struct mbuf_chain * MBC)429 MBC_FLUSH(struct mbuf_chain *MBC)
430 {
431 	extern void	m_freem(struct mbuf *);
432 	struct mbuf	*m;
433 
434 	while ((m = (MBC)->chain) != 0) {
435 		(MBC)->chain = m->m_nextpkt;
436 		m->m_nextpkt = 0;
437 		m_freem(m);
438 	}
439 	MBC_SETUP(MBC, (MBC)->max_bytes);
440 }
441 
442 void
MBC_ATTACH_MBUF(struct mbuf_chain * MBC,struct mbuf * MBUF)443 MBC_ATTACH_MBUF(struct mbuf_chain *MBC, struct mbuf *MBUF)
444 {
445 	if (MBC->chain != 0)
446 		MBC_FLUSH(MBC);
447 
448 	(MBC)->chain_offset = 0;
449 	(MBC)->chain = (MBUF);
450 }
451 
452 void
MBC_APPEND_MBUF(struct mbuf_chain * MBC,struct mbuf * MBUF)453 MBC_APPEND_MBUF(struct mbuf_chain *MBC, struct mbuf *MBUF)
454 {
455 	struct mbuf	*m;
456 
457 	if ((MBC)->chain == 0) {
458 		(MBC)->chain = (MBUF);
459 	} else {
460 		m = (MBC)->chain;
461 		while (m->m_next != 0)
462 			m = m->m_next;
463 		m->m_next = (MBUF);
464 	}
465 }
466 
467 static void /*ARGSUSED*/
mclrefnoop(mbuf_t * m)468 mclrefnoop(mbuf_t *m)
469 {
470 }
471 
472 void
MBC_ATTACH_BUF(struct mbuf_chain * MBC,unsigned char * BUF,int LEN)473 MBC_ATTACH_BUF(struct mbuf_chain *MBC, unsigned char *BUF, int LEN)
474 {
475 	MGET((MBC)->chain, M_WAIT, MT_DATA);
476 	(MBC)->chain_offset = 0;
477 	(MBC)->chain->m_flags |= M_EXT;
478 	(MBC)->chain->m_data = (caddr_t)(BUF);
479 	(MBC)->chain->m_ext.ext_buf = (caddr_t)(BUF);
480 	(MBC)->chain->m_len = (LEN);
481 	(MBC)->chain->m_ext.ext_size = (LEN);
482 	(MBC)->chain->m_ext.ext_free = mclrefnoop;
483 	(MBC)->max_bytes = (LEN);
484 }
485 
486 
487 int
MBC_SHADOW_CHAIN(struct mbuf_chain * submbc,struct mbuf_chain * mbc,int off,int len)488 MBC_SHADOW_CHAIN(struct mbuf_chain *submbc, struct mbuf_chain *mbc,
489     int off, int len)
490 {
491 	int x = off + len;
492 
493 	if (off < 0 || len < 0 || x < 0 ||
494 	    off > mbc->max_bytes || x > mbc->max_bytes)
495 		return (EMSGSIZE);
496 
497 	*submbc = *mbc;
498 	submbc->chain_offset = off;
499 	submbc->max_bytes = x;
500 	submbc->shadow_of = mbc;
501 	return (0);
502 }
503 
504 /*
505  * Trim req_len bytes from the message,
506  * from head if > 0, else from tail.
507  */
508 void
m_adjust(struct mbuf * mp,int req_len)509 m_adjust(struct mbuf *mp, int req_len)
510 {
511 	int len = req_len;
512 	struct mbuf *m;
513 
514 	if ((m = mp) == NULL)
515 		return;
516 
517 	/*
518 	 * We don't use the trim-from-tail case.
519 	 * Using smb_mbuf_trim() for that.
520 	 */
521 	VERIFY(len >= 0);
522 
523 	/*
524 	 * Trim from head.
525 	 */
526 	while (m != NULL && len > 0) {
527 		if (m->m_len <= len) {
528 			len -= m->m_len;
529 			m->m_len = 0;
530 			m = m->m_next;
531 		} else {
532 			m->m_len -= len;
533 			m->m_data += len;
534 			len = 0;
535 		}
536 	}
537 }
538 
539 /*
540  * Arrange to prepend space of size plen to mbuf m.  If a new mbuf must be
541  * allocated, how specifies whether to wait.  If the allocation fails, the
542  * original mbuf chain is freed and m is set to NULL.
543  * BSD had a macro: M_PREPEND(m, plen, how)
544  */
545 
546 struct mbuf *
m_prepend(struct mbuf * m,int plen,int how)547 m_prepend(struct mbuf *m, int plen, int how)
548 {
549 	struct mbuf *mn;
550 
551 	if (M_LEADINGSPACE(m) >= plen) {
552 		m->m_data -= plen;
553 		m->m_len += plen;
554 		return (m);
555 	}
556 	if (m->m_flags & M_PKTHDR) {
557 		MGETHDR(mn, how, m->m_type);
558 	} else {
559 		MGET(mn, how, m->m_type);
560 	}
561 	VERIFY(mn != NULL);
562 	if (m->m_flags & M_PKTHDR) {
563 		// BSD: m_move_pkthdr(mn, m);
564 		// We don't use any pkthdr stuff.
565 		mn->m_pkthdr.len += plen;
566 	}
567 	mn->m_next = m;
568 	m = mn;
569 	if (plen < M_SIZE(m))
570 		M_ALIGN(m, plen);
571 	m->m_len = plen;
572 	DTRACE_PROBE1(prepend_allocated, struct mbuf *, m);
573 	return (m);
574 }
575 
576 /*
577  * Free a single mbuf structure.  Calls m->m_ext.ext_ref() to free any
578  * associated external buffers if present (indicated by m->m_flags & M_EXT)
579  */
580 struct mbuf *
m_free(struct mbuf * m)581 m_free(struct mbuf *m)
582 {
583 	struct mbuf *n;
584 
585 	MFREE(m, n);
586 	return (n);
587 }
588 
589 /*
590  * Free a list of mbufs.  Each mbuf in the list is freed similarly to m_free.
591  */
592 void
m_freem(struct mbuf * m)593 m_freem(struct mbuf *m)
594 {
595 	struct mbuf *n;
596 
597 	if (m == NULL)
598 		return;
599 	/*
600 	 * Lint doesn't like the m = n assignment at the close of the loop
601 	 * but it is correct.  MFREE assigns n = (m)->m_next so the loop
602 	 * is effectively assigning m = (m)->m_next then exiting when
603 	 * m == NULL
604 	 */
605 	do {
606 		MFREE(m, n);
607 	} while ((m = n) != 0);
608 }
609 
610 /*
611  * Mbuffer utility routines.
612  */
613 
614 mbuf_t *
smb_mbuf_alloc(void)615 smb_mbuf_alloc(void)
616 {
617 	mbuf_t *m;
618 
619 	m = kmem_cache_alloc(smb_mbuf_cache, KM_SLEEP);
620 	bzero(m, sizeof (*m));
621 	return (m);
622 }
623 
624 void
smb_mbuf_free(mbuf_t * m)625 smb_mbuf_free(mbuf_t *m)
626 {
627 	kmem_cache_free(smb_mbuf_cache, m);
628 }
629 
630 void *
smb_mbufcl_alloc(void)631 smb_mbufcl_alloc(void)
632 {
633 	void *p;
634 
635 	p = kmem_cache_alloc(smb_mbufcl_cache, KM_SLEEP);
636 	bzero(p, MCLBYTES);
637 	return (p);
638 }
639 
640 void
smb_mbufcl_free(mbuf_t * m)641 smb_mbufcl_free(mbuf_t *m)
642 {
643 	ASSERT((m->m_flags & M_EXT) != 0);
644 	ASSERT(m->m_ext.ext_size == MCLBYTES);
645 
646 	kmem_cache_free(smb_mbufcl_cache, m->m_ext.ext_buf);
647 	/* Caller sets m->m_ext.ext_buf = NULL */
648 }
649 
650 int
smb_mbufcl_ref(void * p,uint_t sz,int incr)651 smb_mbufcl_ref(void *p, uint_t sz, int incr)
652 {
653 	ASSERT3S(sz, ==, MCLBYTES);
654 	if (incr < 0)
655 		kmem_cache_free(smb_mbufcl_cache, p);
656 	return (0);
657 }
658