1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright 2019, Joyent, Inc.
28 * Copyright 2022 Garrett D'Amore
29 */
30
31 /*
32 * Softmac data-path switching:
33 *
34 * - Fast-path model
35 *
36 * When the softmac fast-path is used, a dedicated lower-stream
37 * will be opened over the legacy device for each IP/ARP (upper-)stream
38 * over the softMAC, and all DLPI messages (including control messages
39 * and data messages) will be exchanged between the upper-stream and
40 * the corresponding lower-stream directly. Therefore, the data
41 * demultiplexing, filtering and classification processing will be done
42 * by the lower-stream, and the GLDv3 DLS/MAC layer processing will be
43 * no longer needed.
44 *
45 * - Slow-path model
46 *
47 * Some GLDv3 features requires the GLDv3 DLS/MAC layer processing to
48 * not be bypassed to assure its function correctness. For example,
49 * softmac fast-path must be disabled to support GLDv3 VNIC functionality.
50 * In this case, a shared lower-stream will be opened over the legacy
51 * device, which is responsible for implementing the GLDv3 callbacks
52 * and passing RAW data messages between the legacy devices and the GLDv3
53 * framework.
54 *
55 * By default, the softmac fast-path mode will be used to assure the
56 * performance; MAC clients will be able to request to disable the softmac
57 * fast-path mode to support certain features, and if that succeeds,
58 * the system will fallback to the slow-path softmac data-path model.
59 *
60 *
61 * The details of the softmac data fast-path model is stated as below
62 *
63 * 1. When a stream is opened on a softMAC, the softmac module will takes
64 * over the DLPI processing on this stream;
65 *
66 * 2. For IP/ARP streams over a softMAC, softmac data fast-path will be
67 * used by default, unless fast-path is disabled by any MAC client
68 * explicitly. The softmac module first identifies an IP/ARP stream
69 * by seeing whether there is a SIOCSLIFNAME ioctl sent from upstream,
70 * if there is one, this stream is either an IP or an ARP stream
71 * and will use fast-path potentially;
72 *
73 * 3. When the softmac fast-path is used, an dedicated lower-stream will
74 * be setup for each IP/ARP stream (1-1 mapping). From that point on,
75 * all control and data messages will be exchanged between the IP/ARP
76 * upper-stream and the legacy device through this dedicated
77 * lower-stream. As a result, the DLS/MAC layer processing in GLDv3
78 * will be skipped, and this greatly improves the performance;
79 *
80 * 4. When the softmac data fast-path is disabled by a MAC client (e.g.,
81 * by a VNIC), all the IP/ARP upper streams will try to switch from
82 * the fast-path to the slow-path. The dedicated lower-stream will be
83 * destroyed, and all the control and data-messages will go through the
84 * existing GLDv3 code path and (in the end) the shared lower-stream;
85 *
86 * 5. On the other hand, when the last MAC client cancels its fast-path
87 * disable request, all the IP/ARP streams will try to switch back to
88 * the fast-path mode;
89 *
90 * Step 5 and 6 both rely on the data-path mode switching process
91 * described below:
92 *
93 * 1) To switch the softmac data-path mode (between fast-path and slow-path),
94 * softmac will first send a DL_NOTE_REPLUMB DL_NOTIFY_IND message
95 * upstream over each IP/ARP streams that needs data-path mode switching;
96 *
97 * 2) When IP receives this DL_NOTE_REPLUMB message, it will bring down
98 * all the IP interfaces on the corresponding ill (IP Lower level
99 * structure), and bring up those interfaces over again; this will in
100 * turn cause the ARP to "replumb" the interface.
101 *
102 * During the replumb process, both IP and ARP will send downstream the
103 * necessary DL_DISABMULTI_REQ and DL_UNBIND_REQ messages and cleanup
104 * the old state of the underlying softMAC, following with the necessary
105 * DL_BIND_REQ and DL_ENABMULTI_REQ messages to setup the new state.
106 * Between the cleanup and re-setup process, IP/ARP will also send down
107 * a DL_NOTE_REPLUMB_DONE DL_NOTIFY_CONF messages to the softMAC to
108 * indicate the *switching point*;
109 *
110 * 3) When softmac receives the DL_NOTE_REPLUMB_DONE message, it either
111 * creates or destroys the dedicated lower-stream (depending on which
112 * data-path mode the softMAC switches to), and change the softmac
113 * data-path mode. From then on, softmac will process all the succeeding
114 * control messages (including the DL_BIND_REQ and DL_ENABMULTI_REQ
115 * messages) and data messages based on new data-path mode.
116 */
117
118 #include <sys/types.h>
119 #include <sys/disp.h>
120 #include <sys/callb.h>
121 #include <sys/sysmacros.h>
122 #include <sys/file.h>
123 #include <sys/vlan.h>
124 #include <sys/dld.h>
125 #include <sys/sockio.h>
126 #include <sys/softmac_impl.h>
127 #include <net/if.h>
128
129 static kmutex_t softmac_taskq_lock;
130 static kcondvar_t softmac_taskq_cv;
131 static list_t softmac_taskq_list; /* List of softmac_upper_t */
132 boolean_t softmac_taskq_quit;
133 boolean_t softmac_taskq_done;
134
135 static void softmac_taskq_dispatch();
136 static int softmac_fastpath_setup(softmac_upper_t *);
137 static mac_tx_cookie_t softmac_fastpath_wput_data(softmac_upper_t *, mblk_t *,
138 uintptr_t, uint16_t);
139 static void softmac_datapath_switch_done(softmac_upper_t *);
140
141 void
softmac_fp_init()142 softmac_fp_init()
143 {
144 mutex_init(&softmac_taskq_lock, NULL, MUTEX_DRIVER, NULL);
145 cv_init(&softmac_taskq_cv, NULL, CV_DRIVER, NULL);
146
147 softmac_taskq_quit = B_FALSE;
148 softmac_taskq_done = B_FALSE;
149 list_create(&softmac_taskq_list, sizeof (softmac_upper_t),
150 offsetof(softmac_upper_t, su_taskq_list_node));
151 (void) thread_create(NULL, 0, softmac_taskq_dispatch, NULL, 0,
152 &p0, TS_RUN, minclsyspri);
153 }
154
155 void
softmac_fp_fini()156 softmac_fp_fini()
157 {
158 /*
159 * Request the softmac_taskq thread to quit and wait for it to be done.
160 */
161 mutex_enter(&softmac_taskq_lock);
162 softmac_taskq_quit = B_TRUE;
163 cv_signal(&softmac_taskq_cv);
164 while (!softmac_taskq_done)
165 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
166 mutex_exit(&softmac_taskq_lock);
167 list_destroy(&softmac_taskq_list);
168
169 mutex_destroy(&softmac_taskq_lock);
170 cv_destroy(&softmac_taskq_cv);
171 }
172
173 static boolean_t
check_ip_above(queue_t * q)174 check_ip_above(queue_t *q)
175 {
176 queue_t *next_q;
177 boolean_t ret = B_TRUE;
178
179 claimstr(q);
180 next_q = q->q_next;
181 if (strcmp(next_q->q_qinfo->qi_minfo->mi_idname, "ip") != 0)
182 ret = B_FALSE;
183 releasestr(q);
184 return (ret);
185 }
186
187 /* ARGSUSED */
188 static int
softmac_capab_perim(softmac_upper_t * sup,void * data,uint_t flags)189 softmac_capab_perim(softmac_upper_t *sup, void *data, uint_t flags)
190 {
191 switch (flags) {
192 case DLD_ENABLE:
193 mutex_enter(&sup->su_mutex);
194 break;
195 case DLD_DISABLE:
196 mutex_exit(&sup->su_mutex);
197 break;
198 case DLD_QUERY:
199 return (MUTEX_HELD(&sup->su_mutex));
200 }
201 return (0);
202 }
203
204 static mac_tx_notify_handle_t
softmac_client_tx_notify(softmac_upper_t * sup,mac_tx_notify_t func,void * arg)205 softmac_client_tx_notify(softmac_upper_t *sup, mac_tx_notify_t func, void *arg)
206 {
207 ASSERT(MUTEX_HELD(&sup->su_mutex));
208
209 if (func != NULL) {
210 sup->su_tx_notify_func = func;
211 sup->su_tx_notify_arg = arg;
212 } else {
213 /*
214 * Wait for all tx_notify_func call to be done.
215 */
216 while (sup->su_tx_inprocess != 0)
217 cv_wait(&sup->su_cv, &sup->su_mutex);
218
219 sup->su_tx_notify_func = NULL;
220 sup->su_tx_notify_arg = NULL;
221 }
222 return ((mac_tx_notify_handle_t)sup);
223 }
224
225 static boolean_t
softmac_tx_is_flow_blocked(softmac_upper_t * sup,mac_tx_cookie_t cookie)226 softmac_tx_is_flow_blocked(softmac_upper_t *sup, mac_tx_cookie_t cookie)
227 {
228 ASSERT(cookie == (mac_tx_cookie_t)sup);
229 return (sup->su_tx_busy);
230 }
231
232 static int
softmac_capab_direct(softmac_upper_t * sup,void * data,uint_t flags)233 softmac_capab_direct(softmac_upper_t *sup, void *data, uint_t flags)
234 {
235 dld_capab_direct_t *direct = data;
236 softmac_lower_t *slp = sup->su_slp;
237
238 ASSERT(MUTEX_HELD(&sup->su_mutex));
239
240 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
241
242 switch (flags) {
243 case DLD_ENABLE:
244 if (sup->su_direct)
245 return (0);
246
247 sup->su_direct_rxinfo.slr_rx = (softmac_rx_t)direct->di_rx_cf;
248 sup->su_direct_rxinfo.slr_arg = direct->di_rx_ch;
249 slp->sl_rxinfo = &sup->su_direct_rxinfo;
250 direct->di_tx_df = (uintptr_t)softmac_fastpath_wput_data;
251 direct->di_tx_dh = sup;
252 direct->di_tx_fctl_df = (uintptr_t)softmac_tx_is_flow_blocked;
253 direct->di_tx_fctl_dh = sup;
254 direct->di_tx_cb_df = (uintptr_t)softmac_client_tx_notify;
255 direct->di_tx_cb_dh = sup;
256 sup->su_direct = B_TRUE;
257 return (0);
258
259 case DLD_DISABLE:
260 if (!sup->su_direct)
261 return (0);
262
263 slp->sl_rxinfo = &sup->su_rxinfo;
264 sup->su_direct = B_FALSE;
265 return (0);
266 }
267 return (ENOTSUP);
268 }
269
270 static int
softmac_dld_capab(softmac_upper_t * sup,uint_t type,void * data,uint_t flags)271 softmac_dld_capab(softmac_upper_t *sup, uint_t type, void *data, uint_t flags)
272 {
273 int err;
274
275 /*
276 * Don't enable direct callback capabilities unless the caller is
277 * the IP client. When a module is inserted in a stream (_I_INSERT)
278 * the stack initiates capability disable, but due to races, the
279 * module insertion may complete before the capability disable
280 * completes. So we limit the check to DLD_ENABLE case.
281 */
282 if ((flags == DLD_ENABLE && type != DLD_CAPAB_PERIM) &&
283 !check_ip_above(sup->su_rq)) {
284 return (ENOTSUP);
285 }
286
287 switch (type) {
288 case DLD_CAPAB_DIRECT:
289 err = softmac_capab_direct(sup, data, flags);
290 break;
291
292 case DLD_CAPAB_PERIM:
293 err = softmac_capab_perim(sup, data, flags);
294 break;
295
296 default:
297 err = ENOTSUP;
298 break;
299 }
300 return (err);
301 }
302
303 static void
softmac_capability_advertise(softmac_upper_t * sup,mblk_t * mp)304 softmac_capability_advertise(softmac_upper_t *sup, mblk_t *mp)
305 {
306 dl_capability_ack_t *dlap;
307 dl_capability_sub_t *dlsp;
308 t_uscalar_t subsize;
309 uint8_t *ptr;
310 queue_t *q = sup->su_wq;
311 mblk_t *mp1;
312 softmac_t *softmac = sup->su_softmac;
313 boolean_t dld_capable = B_FALSE;
314 boolean_t hcksum_capable = B_FALSE;
315 boolean_t zcopy_capable = B_FALSE;
316
317 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
318
319 /*
320 * Initially assume no capabilities.
321 */
322 subsize = 0;
323
324 /*
325 * Direct capability negotiation interface between IP and softmac
326 */
327 if (check_ip_above(sup->su_rq)) {
328 dld_capable = B_TRUE;
329 subsize += sizeof (dl_capability_sub_t) +
330 sizeof (dl_capab_dld_t);
331 }
332
333 /*
334 * Check if checksum offload is supported on this MAC.
335 */
336 if (softmac->smac_capab_flags & MAC_CAPAB_HCKSUM) {
337 hcksum_capable = B_TRUE;
338 subsize += sizeof (dl_capability_sub_t) +
339 sizeof (dl_capab_hcksum_t);
340 }
341
342 /*
343 * Check if zerocopy is supported on this interface.
344 */
345 if (!(softmac->smac_capab_flags & MAC_CAPAB_NO_ZCOPY)) {
346 zcopy_capable = B_TRUE;
347 subsize += sizeof (dl_capability_sub_t) +
348 sizeof (dl_capab_zerocopy_t);
349 }
350
351 /*
352 * If there are no capabilities to advertise or if we
353 * can't allocate a response, send a DL_ERROR_ACK.
354 */
355 if ((subsize == 0) || (mp1 = reallocb(mp,
356 sizeof (dl_capability_ack_t) + subsize, 0)) == NULL) {
357 dlerrorack(q, mp, DL_CAPABILITY_REQ, DL_NOTSUPPORTED, 0);
358 return;
359 }
360
361 mp = mp1;
362 DB_TYPE(mp) = M_PROTO;
363 mp->b_wptr = mp->b_rptr + sizeof (dl_capability_ack_t) + subsize;
364 bzero(mp->b_rptr, MBLKL(mp));
365 dlap = (dl_capability_ack_t *)mp->b_rptr;
366 dlap->dl_primitive = DL_CAPABILITY_ACK;
367 dlap->dl_sub_offset = sizeof (dl_capability_ack_t);
368 dlap->dl_sub_length = subsize;
369 ptr = (uint8_t *)&dlap[1];
370
371 /*
372 * IP polling interface.
373 */
374 if (dld_capable) {
375 dl_capab_dld_t dld;
376
377 dlsp = (dl_capability_sub_t *)ptr;
378 dlsp->dl_cap = DL_CAPAB_DLD;
379 dlsp->dl_length = sizeof (dl_capab_dld_t);
380 ptr += sizeof (dl_capability_sub_t);
381
382 bzero(&dld, sizeof (dl_capab_dld_t));
383 dld.dld_version = DLD_CURRENT_VERSION;
384 dld.dld_capab = (uintptr_t)softmac_dld_capab;
385 dld.dld_capab_handle = (uintptr_t)sup;
386
387 dlcapabsetqid(&(dld.dld_mid), sup->su_rq);
388 bcopy(&dld, ptr, sizeof (dl_capab_dld_t));
389 ptr += sizeof (dl_capab_dld_t);
390 }
391
392 /*
393 * TCP/IP checksum offload.
394 */
395 if (hcksum_capable) {
396 dl_capab_hcksum_t hcksum;
397
398 dlsp = (dl_capability_sub_t *)ptr;
399
400 dlsp->dl_cap = DL_CAPAB_HCKSUM;
401 dlsp->dl_length = sizeof (dl_capab_hcksum_t);
402 ptr += sizeof (dl_capability_sub_t);
403
404 bzero(&hcksum, sizeof (dl_capab_hcksum_t));
405 hcksum.hcksum_version = HCKSUM_VERSION_1;
406 hcksum.hcksum_txflags = softmac->smac_hcksum_txflags;
407 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
408 bcopy(&hcksum, ptr, sizeof (dl_capab_hcksum_t));
409 ptr += sizeof (dl_capab_hcksum_t);
410 }
411
412 /*
413 * Zero copy
414 */
415 if (zcopy_capable) {
416 dl_capab_zerocopy_t zcopy;
417
418 dlsp = (dl_capability_sub_t *)ptr;
419
420 dlsp->dl_cap = DL_CAPAB_ZEROCOPY;
421 dlsp->dl_length = sizeof (dl_capab_zerocopy_t);
422 ptr += sizeof (dl_capability_sub_t);
423
424 bzero(&zcopy, sizeof (dl_capab_zerocopy_t));
425 zcopy.zerocopy_version = ZEROCOPY_VERSION_1;
426 zcopy.zerocopy_flags = DL_CAPAB_VMSAFE_MEM;
427 dlcapabsetqid(&(zcopy.zerocopy_mid), sup->su_rq);
428 bcopy(&zcopy, ptr, sizeof (dl_capab_zerocopy_t));
429 ptr += sizeof (dl_capab_zerocopy_t);
430 }
431
432 ASSERT(ptr == mp->b_rptr + sizeof (dl_capability_ack_t) + subsize);
433 qreply(q, mp);
434 }
435
436 static void
softmac_capability_req(softmac_upper_t * sup,mblk_t * mp)437 softmac_capability_req(softmac_upper_t *sup, mblk_t *mp)
438 {
439 dl_capability_req_t *dlp = (dl_capability_req_t *)mp->b_rptr;
440 dl_capability_sub_t *sp;
441 size_t size, len;
442 offset_t off, end;
443 t_uscalar_t dl_err;
444 queue_t *q = sup->su_wq;
445
446 ASSERT(sup->su_mode == SOFTMAC_FASTPATH);
447 if (MBLKL(mp) < sizeof (dl_capability_req_t)) {
448 dl_err = DL_BADPRIM;
449 goto failed;
450 }
451
452 if (!sup->su_bound) {
453 dl_err = DL_OUTSTATE;
454 goto failed;
455 }
456
457 /*
458 * This request is overloaded. If there are no requested capabilities
459 * then we just want to acknowledge with all the capabilities we
460 * support. Otherwise we enable the set of capabilities requested.
461 */
462 if (dlp->dl_sub_length == 0) {
463 softmac_capability_advertise(sup, mp);
464 return;
465 }
466
467 if (!MBLKIN(mp, dlp->dl_sub_offset, dlp->dl_sub_length)) {
468 dl_err = DL_BADPRIM;
469 goto failed;
470 }
471
472 dlp->dl_primitive = DL_CAPABILITY_ACK;
473
474 off = dlp->dl_sub_offset;
475 len = dlp->dl_sub_length;
476
477 /*
478 * Walk the list of capabilities to be enabled.
479 */
480 for (end = off + len; off < end; ) {
481 sp = (dl_capability_sub_t *)(mp->b_rptr + off);
482 size = sizeof (dl_capability_sub_t) + sp->dl_length;
483
484 if (off + size > end ||
485 !IS_P2ALIGNED(off, sizeof (uint32_t))) {
486 dl_err = DL_BADPRIM;
487 goto failed;
488 }
489
490 switch (sp->dl_cap) {
491 /*
492 * TCP/IP checksum offload to hardware.
493 */
494 case DL_CAPAB_HCKSUM: {
495 dl_capab_hcksum_t *hcksump;
496 dl_capab_hcksum_t hcksum;
497
498 hcksump = (dl_capab_hcksum_t *)&sp[1];
499 /*
500 * Copy for alignment.
501 */
502 bcopy(hcksump, &hcksum, sizeof (dl_capab_hcksum_t));
503 dlcapabsetqid(&(hcksum.hcksum_mid), sup->su_rq);
504 bcopy(&hcksum, hcksump, sizeof (dl_capab_hcksum_t));
505 break;
506 }
507
508 default:
509 break;
510 }
511
512 off += size;
513 }
514 qreply(q, mp);
515 return;
516 failed:
517 dlerrorack(q, mp, DL_CAPABILITY_REQ, dl_err, 0);
518 }
519
520 static void
softmac_bind_req(softmac_upper_t * sup,mblk_t * mp)521 softmac_bind_req(softmac_upper_t *sup, mblk_t *mp)
522 {
523 softmac_lower_t *slp = sup->su_slp;
524 softmac_t *softmac = sup->su_softmac;
525 mblk_t *ackmp, *mp1;
526 int err;
527
528 if (MBLKL(mp) < DL_BIND_REQ_SIZE) {
529 freemsg(mp);
530 return;
531 }
532
533 /*
534 * Allocate ackmp incase the underlying driver does not ack timely.
535 */
536 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
537 dlerrorack(sup->su_wq, mp, DL_BIND_REQ, DL_SYSERR, ENOMEM);
538 return;
539 }
540
541 err = softmac_output(slp, mp, DL_BIND_REQ, DL_BIND_ACK, &ackmp);
542 if (ackmp != NULL) {
543 freemsg(mp1);
544 } else {
545 /*
546 * The driver does not ack timely.
547 */
548 ASSERT(err == ENOMSG);
549 ackmp = mp1;
550 }
551 if (err != 0)
552 goto failed;
553
554 /*
555 * Enable capabilities the underlying driver claims to support.
556 */
557 if ((err = softmac_capab_enable(slp)) != 0)
558 goto failed;
559
560 /*
561 * Check whether this softmac is already marked as exclusively used,
562 * e.g., an aggregation is created over it. Fail the BIND_REQ if so.
563 */
564 mutex_enter(&softmac->smac_active_mutex);
565 if (softmac->smac_active) {
566 mutex_exit(&softmac->smac_active_mutex);
567 err = EBUSY;
568 goto failed;
569 }
570 softmac->smac_nactive++;
571 sup->su_active = B_TRUE;
572 mutex_exit(&softmac->smac_active_mutex);
573 sup->su_bound = B_TRUE;
574
575 qreply(sup->su_wq, ackmp);
576 return;
577 failed:
578 if (err != 0) {
579 dlerrorack(sup->su_wq, ackmp, DL_BIND_REQ, DL_SYSERR, err);
580 return;
581 }
582 }
583
584 static void
softmac_unbind_req(softmac_upper_t * sup,mblk_t * mp)585 softmac_unbind_req(softmac_upper_t *sup, mblk_t *mp)
586 {
587 softmac_lower_t *slp = sup->su_slp;
588 softmac_t *softmac = sup->su_softmac;
589 mblk_t *ackmp, *mp1;
590 int err;
591
592 if (MBLKL(mp) < DL_UNBIND_REQ_SIZE) {
593 freemsg(mp);
594 return;
595 }
596
597 if (!sup->su_bound) {
598 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_OUTSTATE, 0);
599 return;
600 }
601
602 /*
603 * Allocate ackmp incase the underlying driver does not ack timely.
604 */
605 if ((mp1 = allocb(sizeof (dl_error_ack_t), BPRI_HI)) == NULL) {
606 dlerrorack(sup->su_wq, mp, DL_UNBIND_REQ, DL_SYSERR, ENOMEM);
607 return;
608 }
609
610 err = softmac_output(slp, mp, DL_UNBIND_REQ, DL_OK_ACK, &ackmp);
611 if (ackmp != NULL) {
612 freemsg(mp1);
613 } else {
614 /*
615 * The driver does not ack timely.
616 */
617 ASSERT(err == ENOMSG);
618 ackmp = mp1;
619 }
620 if (err != 0) {
621 dlerrorack(sup->su_wq, ackmp, DL_UNBIND_REQ, DL_SYSERR, err);
622 return;
623 }
624
625 sup->su_bound = B_FALSE;
626
627 mutex_enter(&softmac->smac_active_mutex);
628 if (sup->su_active) {
629 ASSERT(!softmac->smac_active);
630 softmac->smac_nactive--;
631 sup->su_active = B_FALSE;
632 }
633 mutex_exit(&softmac->smac_active_mutex);
634
635 done:
636 qreply(sup->su_wq, ackmp);
637 }
638
639 /*
640 * Process the non-data mblk.
641 */
642 static void
softmac_wput_single_nondata(softmac_upper_t * sup,mblk_t * mp)643 softmac_wput_single_nondata(softmac_upper_t *sup, mblk_t *mp)
644 {
645 softmac_t *softmac = sup->su_softmac;
646 softmac_lower_t *slp = sup->su_slp;
647 unsigned char dbtype;
648 t_uscalar_t prim;
649
650 dbtype = DB_TYPE(mp);
651 sup->su_is_arp = 0;
652 switch (dbtype) {
653 case M_CTL:
654 sup->su_is_arp = 1;
655 /* FALLTHROUGH */
656 case M_IOCTL: {
657 uint32_t expected_mode;
658
659 if (((struct iocblk *)(mp->b_rptr))->ioc_cmd != SIOCSLIFNAME)
660 break;
661
662 /*
663 * Nak the M_IOCTL based on the STREAMS specification.
664 */
665 if (dbtype == M_IOCTL)
666 miocnak(sup->su_wq, mp, 0, EINVAL);
667 else
668 freemsg(mp);
669
670 /*
671 * This stream is either IP or ARP. See whether
672 * we need to setup a dedicated-lower-stream for it.
673 */
674 mutex_enter(&softmac->smac_fp_mutex);
675
676 expected_mode = DATAPATH_MODE(softmac);
677 if (expected_mode == SOFTMAC_SLOWPATH)
678 sup->su_mode = SOFTMAC_SLOWPATH;
679 list_insert_head(&softmac->smac_sup_list, sup);
680 mutex_exit(&softmac->smac_fp_mutex);
681
682 /*
683 * Setup the fast-path dedicated lower stream if fast-path
684 * is expected. Note that no lock is held here, and if
685 * smac_expected_mode is changed from SOFTMAC_FASTPATH to
686 * SOFTMAC_SLOWPATH, the DL_NOTE_REPLUMB message used for
687 * data-path switching would already be queued and will
688 * be processed by softmac_wput_single_nondata() later.
689 */
690 if (expected_mode == SOFTMAC_FASTPATH)
691 (void) softmac_fastpath_setup(sup);
692 return;
693 }
694 case M_PROTO:
695 case M_PCPROTO:
696 if (MBLKL(mp) < sizeof (t_uscalar_t)) {
697 freemsg(mp);
698 return;
699 }
700 prim = ((union DL_primitives *)mp->b_rptr)->dl_primitive;
701 switch (prim) {
702 case DL_NOTIFY_IND:
703 if (MBLKL(mp) < sizeof (dl_notify_ind_t) ||
704 ((dl_notify_ind_t *)mp->b_rptr)->dl_notification !=
705 DL_NOTE_REPLUMB) {
706 freemsg(mp);
707 return;
708 }
709 /*
710 * This DL_NOTE_REPLUMB message is initiated
711 * and queued by the softmac itself, when the
712 * sup is trying to switching its datapath mode
713 * between SOFTMAC_SLOWPATH and SOFTMAC_FASTPATH.
714 * Send this message upstream.
715 */
716 qreply(sup->su_wq, mp);
717 return;
718 case DL_NOTIFY_CONF:
719 if (MBLKL(mp) < sizeof (dl_notify_conf_t) ||
720 ((dl_notify_conf_t *)mp->b_rptr)->dl_notification !=
721 DL_NOTE_REPLUMB_DONE) {
722 freemsg(mp);
723 return;
724 }
725 /*
726 * This is an indication from IP/ARP that the
727 * fastpath->slowpath switch is done.
728 */
729 freemsg(mp);
730 softmac_datapath_switch_done(sup);
731 return;
732 }
733 break;
734 }
735
736 /*
737 * No need to hold lock to check su_mode, since su_mode updating only
738 * operation is is serialized by softmac_wput_nondata_task().
739 */
740 if (sup->su_mode != SOFTMAC_FASTPATH) {
741 (void) dld_wput(sup->su_wq, mp);
742 return;
743 }
744
745 /*
746 * Fastpath non-data message processing. Most of non-data messages
747 * can be directly passed down to the dedicated-lower-stream, aside
748 * from the following M_PROTO/M_PCPROTO messages.
749 */
750 switch (dbtype) {
751 case M_PROTO:
752 case M_PCPROTO:
753 switch (prim) {
754 case DL_BIND_REQ:
755 softmac_bind_req(sup, mp);
756 break;
757 case DL_UNBIND_REQ:
758 softmac_unbind_req(sup, mp);
759 break;
760 case DL_CAPABILITY_REQ:
761 softmac_capability_req(sup, mp);
762 break;
763 default:
764 putnext(slp->sl_wq, mp);
765 break;
766 }
767 break;
768 default:
769 putnext(slp->sl_wq, mp);
770 break;
771 }
772 }
773
774 /*
775 * The worker thread which processes non-data messages. Note we only process
776 * one message at one time in order to be able to "flush" the queued message
777 * and serialize the processing.
778 */
779 static void
softmac_wput_nondata_task(void * arg)780 softmac_wput_nondata_task(void *arg)
781 {
782 softmac_upper_t *sup = arg;
783 mblk_t *mp;
784
785 mutex_enter(&sup->su_disp_mutex);
786
787 while (sup->su_pending_head != NULL) {
788 if (sup->su_closing)
789 break;
790
791 SOFTMAC_DQ_PENDING(sup, &mp);
792 mutex_exit(&sup->su_disp_mutex);
793 softmac_wput_single_nondata(sup, mp);
794 mutex_enter(&sup->su_disp_mutex);
795 }
796
797 /*
798 * If the stream is closing, flush all queued messages and inform
799 * the stream to be closed.
800 */
801 freemsgchain(sup->su_pending_head);
802 sup->su_pending_head = sup->su_pending_tail = NULL;
803 sup->su_dlpi_pending = B_FALSE;
804 cv_signal(&sup->su_disp_cv);
805 mutex_exit(&sup->su_disp_mutex);
806 }
807
808 /*
809 * Kernel thread to handle taskq dispatch failures in softmac_wput_nondata().
810 * This thread is started when the softmac module is first loaded.
811 */
812 static void
softmac_taskq_dispatch(void)813 softmac_taskq_dispatch(void)
814 {
815 callb_cpr_t cprinfo;
816 softmac_upper_t *sup;
817
818 CALLB_CPR_INIT(&cprinfo, &softmac_taskq_lock, callb_generic_cpr,
819 "softmac_taskq_dispatch");
820 mutex_enter(&softmac_taskq_lock);
821
822 while (!softmac_taskq_quit) {
823 sup = list_head(&softmac_taskq_list);
824 while (sup != NULL) {
825 list_remove(&softmac_taskq_list, sup);
826 sup->su_taskq_scheduled = B_FALSE;
827 mutex_exit(&softmac_taskq_lock);
828 VERIFY(taskq_dispatch(system_taskq,
829 softmac_wput_nondata_task, sup, TQ_SLEEP) !=
830 TASKQID_INVALID);
831 mutex_enter(&softmac_taskq_lock);
832 sup = list_head(&softmac_taskq_list);
833 }
834
835 CALLB_CPR_SAFE_BEGIN(&cprinfo);
836 cv_wait(&softmac_taskq_cv, &softmac_taskq_lock);
837 CALLB_CPR_SAFE_END(&cprinfo, &softmac_taskq_lock);
838 }
839
840 softmac_taskq_done = B_TRUE;
841 cv_signal(&softmac_taskq_cv);
842 CALLB_CPR_EXIT(&cprinfo);
843 thread_exit();
844 }
845
846 void
softmac_wput_nondata(softmac_upper_t * sup,mblk_t * mp)847 softmac_wput_nondata(softmac_upper_t *sup, mblk_t *mp)
848 {
849 /*
850 * The processing of the message might block. Enqueue the
851 * message for later processing.
852 */
853 mutex_enter(&sup->su_disp_mutex);
854
855 if (sup->su_closing) {
856 mutex_exit(&sup->su_disp_mutex);
857 freemsg(mp);
858 return;
859 }
860
861 SOFTMAC_EQ_PENDING(sup, mp);
862
863 if (sup->su_dlpi_pending) {
864 mutex_exit(&sup->su_disp_mutex);
865 return;
866 }
867 sup->su_dlpi_pending = B_TRUE;
868 mutex_exit(&sup->su_disp_mutex);
869
870 if (taskq_dispatch(system_taskq, softmac_wput_nondata_task,
871 sup, TQ_NOSLEEP) != TASKQID_INVALID) {
872 return;
873 }
874
875 mutex_enter(&softmac_taskq_lock);
876 if (!sup->su_taskq_scheduled) {
877 list_insert_tail(&softmac_taskq_list, sup);
878 cv_signal(&softmac_taskq_cv);
879 }
880 sup->su_taskq_scheduled = B_TRUE;
881 mutex_exit(&softmac_taskq_lock);
882 }
883
884 /*
885 * Setup the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
886 */
887 static int
softmac_fastpath_setup(softmac_upper_t * sup)888 softmac_fastpath_setup(softmac_upper_t *sup)
889 {
890 softmac_t *softmac = sup->su_softmac;
891 softmac_lower_t *slp;
892 int err;
893
894 err = softmac_lower_setup(softmac, sup, &slp);
895
896 mutex_enter(&sup->su_mutex);
897 /*
898 * Wait for all data messages to be processed so that we can change
899 * the su_mode.
900 */
901 while (sup->su_tx_inprocess != 0)
902 cv_wait(&sup->su_cv, &sup->su_mutex);
903
904 ASSERT(sup->su_mode != SOFTMAC_FASTPATH);
905 ASSERT(sup->su_slp == NULL);
906 if (err != 0) {
907 sup->su_mode = SOFTMAC_SLOWPATH;
908 } else {
909 sup->su_slp = slp;
910 sup->su_mode = SOFTMAC_FASTPATH;
911 }
912 mutex_exit(&sup->su_mutex);
913 return (err);
914 }
915
916 /*
917 * Tear down the dedicated-lower-stream (fast-path) for the IP/ARP upperstream.
918 */
919 static void
softmac_fastpath_tear(softmac_upper_t * sup)920 softmac_fastpath_tear(softmac_upper_t *sup)
921 {
922 mutex_enter(&sup->su_mutex);
923 /*
924 * Wait for all data messages in the dedicated-lower-stream
925 * to be processed.
926 */
927 while (sup->su_tx_inprocess != 0)
928 cv_wait(&sup->su_cv, &sup->su_mutex);
929
930 /*
931 * Note that this function is called either when the stream is closed,
932 * or the stream is unbound (fastpath-slowpath-switch). Therefore,
933 * No need to call the tx_notify callback.
934 */
935 sup->su_tx_notify_func = NULL;
936 sup->su_tx_notify_arg = NULL;
937 if (sup->su_tx_busy) {
938 ASSERT(sup->su_tx_flow_mp == NULL);
939 VERIFY((sup->su_tx_flow_mp = getq(sup->su_wq)) != NULL);
940 sup->su_tx_busy = B_FALSE;
941 }
942
943 sup->su_mode = SOFTMAC_SLOWPATH;
944
945 /*
946 * Destroy the dedicated-lower-stream. Note that slp is destroyed
947 * when lh is closed.
948 */
949 (void) ldi_close(sup->su_slp->sl_lh, FREAD|FWRITE, kcred);
950 sup->su_slp = NULL;
951 mutex_exit(&sup->su_mutex);
952 }
953
954 void
softmac_wput_data(softmac_upper_t * sup,mblk_t * mp)955 softmac_wput_data(softmac_upper_t *sup, mblk_t *mp)
956 {
957 /*
958 * No lock is required to access the su_mode field since the data
959 * traffic is quiesce by IP when the data-path mode is in the
960 * process of switching.
961 */
962 if (sup->su_mode != SOFTMAC_FASTPATH)
963 (void) dld_wput(sup->su_wq, mp);
964 else
965 (void) softmac_fastpath_wput_data(sup, mp, (uintptr_t)NULL, 0);
966 }
967
968 /*ARGSUSED*/
969 static mac_tx_cookie_t
softmac_fastpath_wput_data(softmac_upper_t * sup,mblk_t * mp,uintptr_t f_hint,uint16_t flag)970 softmac_fastpath_wput_data(softmac_upper_t *sup, mblk_t *mp, uintptr_t f_hint,
971 uint16_t flag)
972 {
973 queue_t *wq = sup->su_slp->sl_wq;
974
975 /*
976 * This function is called from IP, only the MAC_DROP_ON_NO_DESC
977 * flag can be specified.
978 */
979 ASSERT((flag & ~MAC_DROP_ON_NO_DESC) == 0);
980 ASSERT(mp->b_next == NULL);
981
982 /*
983 * Check wether the dedicated-lower-stream is able to handle more
984 * messages, and enable the flow-control if it is not.
985 *
986 * Note that in order not to introduce any packet reordering, we
987 * always send the message down to the dedicated-lower-stream:
988 *
989 * If the flow-control is already enabled, but we still get
990 * the messages from the upper-stream, it means that the upper
991 * stream does not respect STREAMS flow-control (e.g., TCP). Simply
992 * pass the message down to the lower-stream in that case.
993 */
994 if (SOFTMAC_CANPUTNEXT(wq)) {
995 putnext(wq, mp);
996 return ((mac_tx_cookie_t)NULL);
997 }
998
999 if (sup->su_tx_busy) {
1000 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1001 freemsg(mp);
1002 else
1003 putnext(wq, mp);
1004 return ((mac_tx_cookie_t)sup);
1005 }
1006
1007 mutex_enter(&sup->su_mutex);
1008 if (!sup->su_tx_busy) {
1009 /*
1010 * If DLD_CAPAB_DIRECT is enabled, the notify callback will be
1011 * called when the flow control can be disabled. Otherwise,
1012 * put the tx_flow_mp into the wq to make use of the old
1013 * streams flow control.
1014 */
1015 ASSERT(sup->su_tx_flow_mp != NULL);
1016 (void) putq(sup->su_wq, sup->su_tx_flow_mp);
1017 sup->su_tx_flow_mp = NULL;
1018 sup->su_tx_busy = B_TRUE;
1019 qenable(wq);
1020 }
1021 mutex_exit(&sup->su_mutex);
1022
1023 if ((flag & MAC_DROP_ON_NO_DESC) != 0)
1024 freemsg(mp);
1025 else
1026 putnext(wq, mp);
1027 return ((mac_tx_cookie_t)sup);
1028 }
1029
1030 boolean_t
softmac_active_set(void * arg)1031 softmac_active_set(void *arg)
1032 {
1033 softmac_t *softmac = arg;
1034
1035 mutex_enter(&softmac->smac_active_mutex);
1036 if (softmac->smac_nactive != 0) {
1037 mutex_exit(&softmac->smac_active_mutex);
1038 return (B_FALSE);
1039 }
1040 softmac->smac_active = B_TRUE;
1041 mutex_exit(&softmac->smac_active_mutex);
1042 return (B_TRUE);
1043 }
1044
1045 void
softmac_active_clear(void * arg)1046 softmac_active_clear(void *arg)
1047 {
1048 softmac_t *softmac = arg;
1049
1050 mutex_enter(&softmac->smac_active_mutex);
1051 ASSERT(softmac->smac_active && (softmac->smac_nactive == 0));
1052 softmac->smac_active = B_FALSE;
1053 mutex_exit(&softmac->smac_active_mutex);
1054 }
1055
1056 /*
1057 * Disable/reenable fastpath on given softmac. This request could come from a
1058 * MAC client or directly from administrators.
1059 */
1060 int
softmac_datapath_switch(softmac_t * softmac,boolean_t disable,boolean_t admin)1061 softmac_datapath_switch(softmac_t *softmac, boolean_t disable, boolean_t admin)
1062 {
1063 softmac_upper_t *sup;
1064 mblk_t *head = NULL, *tail = NULL, *mp;
1065 list_t reqlist;
1066 softmac_switch_req_t *req;
1067 uint32_t current_mode, expected_mode;
1068 int err = 0;
1069
1070 mutex_enter(&softmac->smac_fp_mutex);
1071
1072 current_mode = DATAPATH_MODE(softmac);
1073 if (admin) {
1074 if (softmac->smac_fastpath_admin_disabled == disable) {
1075 mutex_exit(&softmac->smac_fp_mutex);
1076 return (0);
1077 }
1078 softmac->smac_fastpath_admin_disabled = disable;
1079 } else if (disable) {
1080 softmac->smac_fp_disable_clients++;
1081 } else {
1082 ASSERT(softmac->smac_fp_disable_clients != 0);
1083 softmac->smac_fp_disable_clients--;
1084 }
1085
1086 expected_mode = DATAPATH_MODE(softmac);
1087 if (current_mode == expected_mode) {
1088 mutex_exit(&softmac->smac_fp_mutex);
1089 return (0);
1090 }
1091
1092 /*
1093 * The expected mode is different from whatever datapath mode
1094 * this softmac is expected from last request, enqueue the data-path
1095 * switch request.
1096 */
1097 list_create(&reqlist, sizeof (softmac_switch_req_t),
1098 offsetof(softmac_switch_req_t, ssq_req_list_node));
1099
1100 /*
1101 * Allocate all DL_NOTIFY_IND messages and request structures that
1102 * are required to switch each IP/ARP stream to the expected mode.
1103 */
1104 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1105 sup = list_next(&softmac->smac_sup_list, sup)) {
1106 dl_notify_ind_t *dlip;
1107
1108 req = kmem_alloc(sizeof (softmac_switch_req_t), KM_NOSLEEP);
1109 if (req == NULL)
1110 break;
1111
1112 req->ssq_expected_mode = expected_mode;
1113 if (sup->su_is_arp) {
1114 list_insert_tail(&reqlist, req);
1115 continue;
1116 }
1117 /*
1118 * Allocate the DL_NOTE_REPLUMB message.
1119 */
1120 if ((mp = allocb(sizeof (dl_notify_ind_t), BPRI_LO)) == NULL) {
1121 kmem_free(req, sizeof (softmac_switch_req_t));
1122 break;
1123 }
1124
1125 list_insert_tail(&reqlist, req);
1126
1127 mp->b_wptr = mp->b_rptr + sizeof (dl_notify_ind_t);
1128 mp->b_datap->db_type = M_PROTO;
1129 bzero(mp->b_rptr, sizeof (dl_notify_ind_t));
1130 dlip = (dl_notify_ind_t *)mp->b_rptr;
1131 dlip->dl_primitive = DL_NOTIFY_IND;
1132 dlip->dl_notification = DL_NOTE_REPLUMB;
1133 if (head == NULL) {
1134 head = tail = mp;
1135 } else {
1136 tail->b_next = mp;
1137 tail = mp;
1138 }
1139 }
1140
1141 /*
1142 * Note that it is fine if the expected data-path mode is fast-path
1143 * and some of streams fails to switch. Only return failure if we
1144 * are expected to switch to the slow-path.
1145 */
1146 if (sup != NULL && expected_mode == SOFTMAC_SLOWPATH) {
1147 err = ENOMEM;
1148 goto fail;
1149 }
1150
1151 /*
1152 * Start switching for each IP/ARP stream. The switching operation
1153 * will eventually succeed and there is no need to wait for it
1154 * to finish.
1155 */
1156 for (sup = list_head(&softmac->smac_sup_list); sup != NULL;
1157 sup = list_next(&softmac->smac_sup_list, sup)) {
1158 if (!sup->su_is_arp) {
1159 mp = head->b_next;
1160 head->b_next = NULL;
1161 softmac_wput_nondata(sup, head);
1162 head = mp;
1163 }
1164 /*
1165 * Add the switch request to the requests list of the stream.
1166 */
1167 req = list_head(&reqlist);
1168 ASSERT(req != NULL);
1169 list_remove(&reqlist, req);
1170 list_insert_tail(&sup->su_req_list, req);
1171 }
1172
1173 mutex_exit(&softmac->smac_fp_mutex);
1174 ASSERT(list_is_empty(&reqlist));
1175 list_destroy(&reqlist);
1176 return (0);
1177 fail:
1178 if (admin) {
1179 softmac->smac_fastpath_admin_disabled = !disable;
1180 } else if (disable) {
1181 softmac->smac_fp_disable_clients--;
1182 } else {
1183 softmac->smac_fp_disable_clients++;
1184 }
1185
1186 mutex_exit(&softmac->smac_fp_mutex);
1187 while ((req = list_head(&reqlist)) != NULL) {
1188 list_remove(&reqlist, req);
1189 kmem_free(req, sizeof (softmac_switch_req_t));
1190 }
1191 freemsgchain(head);
1192 list_destroy(&reqlist);
1193 return (err);
1194 }
1195
1196 int
softmac_fastpath_disable(void * arg)1197 softmac_fastpath_disable(void *arg)
1198 {
1199 return (softmac_datapath_switch((softmac_t *)arg, B_TRUE, B_FALSE));
1200 }
1201
1202 void
softmac_fastpath_enable(void * arg)1203 softmac_fastpath_enable(void *arg)
1204 {
1205 VERIFY(softmac_datapath_switch((softmac_t *)arg, B_FALSE,
1206 B_FALSE) == 0);
1207 }
1208
1209 void
softmac_upperstream_close(softmac_upper_t * sup)1210 softmac_upperstream_close(softmac_upper_t *sup)
1211 {
1212 softmac_t *softmac = sup->su_softmac;
1213 softmac_switch_req_t *req;
1214
1215 mutex_enter(&softmac->smac_fp_mutex);
1216
1217 if (sup->su_mode == SOFTMAC_FASTPATH)
1218 softmac_fastpath_tear(sup);
1219
1220 if (sup->su_mode != SOFTMAC_UNKNOWN) {
1221 list_remove(&softmac->smac_sup_list, sup);
1222 sup->su_mode = SOFTMAC_UNKNOWN;
1223 }
1224
1225 /*
1226 * Cleanup all the switch requests queueed on this stream.
1227 */
1228 while ((req = list_head(&sup->su_req_list)) != NULL) {
1229 list_remove(&sup->su_req_list, req);
1230 kmem_free(req, sizeof (softmac_switch_req_t));
1231 }
1232 mutex_exit(&softmac->smac_fp_mutex);
1233 }
1234
1235 /*
1236 * Handle the DL_NOTE_REPLUMB_DONE indication from IP/ARP. Change the upper
1237 * stream from the fastpath mode to the slowpath mode.
1238 */
1239 static void
softmac_datapath_switch_done(softmac_upper_t * sup)1240 softmac_datapath_switch_done(softmac_upper_t *sup)
1241 {
1242 softmac_t *softmac = sup->su_softmac;
1243 softmac_switch_req_t *req;
1244 uint32_t expected_mode;
1245
1246 mutex_enter(&softmac->smac_fp_mutex);
1247 req = list_head(&sup->su_req_list);
1248 list_remove(&sup->su_req_list, req);
1249 expected_mode = req->ssq_expected_mode;
1250 kmem_free(req, sizeof (softmac_switch_req_t));
1251
1252 if (expected_mode == sup->su_mode) {
1253 mutex_exit(&softmac->smac_fp_mutex);
1254 return;
1255 }
1256
1257 ASSERT(!sup->su_bound);
1258 mutex_exit(&softmac->smac_fp_mutex);
1259
1260 /*
1261 * It is fine if the expected mode is fast-path and we fail
1262 * to enable fastpath on this stream.
1263 */
1264 if (expected_mode == SOFTMAC_SLOWPATH)
1265 softmac_fastpath_tear(sup);
1266 else
1267 (void) softmac_fastpath_setup(sup);
1268 }
1269