1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2018 Joyent, Inc.
24  * Copyright 2020 RackTop Systems.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/callb.h>
29 #include <sys/cpupart.h>
30 #include <sys/pool.h>
31 #include <sys/pool_pset.h>
32 #include <sys/sdt.h>
33 #include <sys/strsubr.h>
34 #include <sys/strsun.h>
35 #include <sys/vlan.h>
36 #include <inet/ipsec_impl.h>
37 #include <inet/ip_impl.h>
38 #include <inet/sadb.h>
39 #include <inet/ipsecesp.h>
40 #include <inet/ipsecah.h>
41 
42 #include <sys/mac_impl.h>
43 #include <sys/mac_client_impl.h>
44 #include <sys/mac_client_priv.h>
45 #include <sys/mac_soft_ring.h>
46 #include <sys/mac_flow_impl.h>
47 #include <sys/mac_stat.h>
48 
49 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t);
50 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *);
51 static void mac_srs_poll_unbind(mac_soft_ring_set_t *);
52 static void mac_srs_worker_unbind(mac_soft_ring_set_t *);
53 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t);
54 
55 static int mac_srs_cpu_setup(cpu_setup_t, int, void *);
56 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t);
57 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t);
58 static void mac_srs_threads_unbind(mac_soft_ring_set_t *);
59 static void mac_srs_add_glist(mac_soft_ring_set_t *);
60 static void mac_srs_remove_glist(mac_soft_ring_set_t *);
61 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *);
62 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *);
63 
64 static int mac_compute_soft_ring_count(flow_entry_t *, int, int);
65 static void mac_walk_srs_and_bind(int);
66 static void mac_walk_srs_and_unbind(int);
67 
68 extern boolean_t mac_latency_optimize;
69 
70 static kmem_cache_t *mac_srs_cache;
71 kmem_cache_t *mac_soft_ring_cache;
72 
73 /*
74  * The duration in msec we wait before signalling the soft ring
75  * worker thread in case packets get queued.
76  */
77 uint32_t mac_soft_ring_worker_wait = 0;
78 
79 /*
80  * A global tunable for turning polling on/off. By default, dynamic
81  * polling is always on and is always very beneficial. It should be
82  * turned off with absolute care and for the rare workload (very
83  * low latency sensitive traffic).
84  */
85 int mac_poll_enable = B_TRUE;
86 
87 /*
88  * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency.
89  * Large values could end up in consuming lot of system memory and cause
90  * system hang.
91  */
92 int mac_soft_ring_max_q_cnt = 1024;
93 int mac_soft_ring_min_q_cnt = 256;
94 int mac_soft_ring_poll_thres = 16;
95 
96 boolean_t mac_tx_serialize = B_FALSE;
97 
98 /*
99  * mac_tx_srs_hiwat is the queue depth threshold at which callers of
100  * mac_tx() will be notified of flow control condition.
101  *
102  * TCP does not honour flow control condition sent up by mac_tx().
103  * Thus provision is made for TCP to allow more packets to be queued
104  * in SRS upto a maximum of mac_tx_srs_max_q_cnt.
105  *
106  * Note that mac_tx_srs_hiwat is always be lesser than
107  * mac_tx_srs_max_q_cnt.
108  */
109 uint32_t mac_tx_srs_max_q_cnt = 100000;
110 uint32_t mac_tx_srs_hiwat = 1000;
111 
112 /*
113  * mac_rx_soft_ring_count, mac_soft_ring_10gig_count:
114  *
115  * Global tunables that determines the number of soft rings to be used for
116  * fanning out incoming traffic on a link. These count will be used only
117  * when no explicit set of CPUs was assigned to the data-links.
118  *
119  * mac_rx_soft_ring_count tunable will come into effect only if
120  * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by
121  * default only for sun4v platforms.
122  *
123  * mac_rx_soft_ring_10gig_count will come into effect if you are running on a
124  * 10Gbps link and is not dependent upon mac_soft_ring_enable.
125  *
126  * The number of soft rings for fanout for a link or a flow is determined
127  * by mac_compute_soft_ring_count() routine. This routine will take into
128  * account mac_soft_ring_enable, mac_rx_soft_ring_count and
129  * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link.
130  *
131  * If a bandwidth is specified, the determination of the number of soft
132  * rings is based on specified bandwidth, CPU speed and number of CPUs in
133  * the system.
134  */
135 uint_t mac_rx_soft_ring_count = 8;
136 uint_t mac_rx_soft_ring_10gig_count = 8;
137 
138 /*
139  * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added
140  * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The
141  * list is used to walk the list of all MAC threads when a CPU is
142  * coming online or going offline.
143  */
144 static mac_soft_ring_set_t *mac_srs_g_list = NULL;
145 static krwlock_t mac_srs_g_lock;
146 
147 /*
148  * Whether the SRS threads should be bound, or not.
149  */
150 boolean_t mac_srs_thread_bind = B_TRUE;
151 
152 /*
153  * Whether Rx/Tx interrupts should be re-targeted. Disabled by default.
154  * dladm command would override this.
155  */
156 boolean_t mac_tx_intr_retarget = B_FALSE;
157 boolean_t mac_rx_intr_retarget = B_FALSE;
158 
159 /*
160  * If cpu bindings are specified by user, then Tx SRS and its soft
161  * rings should also be bound to the CPUs specified by user. The
162  * CPUs for Tx bindings are at the end of the cpu list provided by
163  * the user. If enough CPUs are not available (for Tx and Rx
164  * SRSes), then the CPUs are shared by both Tx and Rx SRSes.
165  */
166 #define	BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) {			\
167 	processorid_t cpuid;						\
168 	int i;								\
169 	mac_soft_ring_t *softring;					\
170 	mac_cpus_t *srs_cpu;						\
171 									\
172 	srs_cpu = &mac_tx_srs->srs_cpu;					\
173 	cpuid = srs_cpu->mc_tx_fanout_cpus[0];				\
174 	mac_srs_worker_bind(mac_tx_srs, cpuid);				\
175 	if (MAC_TX_SOFT_RINGS(mac_tx_srs)) {				\
176 		for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) {	\
177 			cpuid = srs_cpu->mc_tx_fanout_cpus[i];		\
178 			softring = mac_tx_srs->srs_tx_soft_rings[i];	\
179 			if (cpuid != -1) {				\
180 				(void) mac_soft_ring_bind(softring,	\
181 				    cpuid);				\
182 			}						\
183 		}							\
184 	}								\
185 }
186 
187 /*
188  * Re-targeting is allowed only for exclusive group or for primary.
189  */
190 #define	RETARGETABLE_CLIENT(group, mcip)				\
191 	((((group) != NULL) &&						\
192 	    ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) ||	\
193 	    mac_is_primary_client(mcip))
194 
195 #define	MAC_RING_RETARGETABLE(ring)					\
196 	(((ring) != NULL) &&						\
197 	    ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) &&		\
198 	    !((ring)->mr_info.mri_intr.mi_ddi_shared))
199 
200 
201 /* INIT and FINI ROUTINES */
202 
203 void
mac_soft_ring_init(void)204 mac_soft_ring_init(void)
205 {
206 	mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache",
207 	    sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
208 
209 	mac_srs_cache = kmem_cache_create("mac_srs_cache",
210 	    sizeof (mac_soft_ring_set_t),
211 	    64, NULL, NULL, NULL, NULL, NULL, 0);
212 
213 	rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL);
214 	mutex_enter(&cpu_lock);
215 	register_cpu_setup_func(mac_srs_cpu_setup, NULL);
216 	mutex_exit(&cpu_lock);
217 }
218 
219 void
mac_soft_ring_finish(void)220 mac_soft_ring_finish(void)
221 {
222 	mutex_enter(&cpu_lock);
223 	unregister_cpu_setup_func(mac_srs_cpu_setup, NULL);
224 	mutex_exit(&cpu_lock);
225 	rw_destroy(&mac_srs_g_lock);
226 	kmem_cache_destroy(mac_soft_ring_cache);
227 	kmem_cache_destroy(mac_srs_cache);
228 }
229 
230 static void
mac_srs_soft_rings_free(mac_soft_ring_set_t * mac_srs)231 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs)
232 {
233 	mac_soft_ring_t	*softring, *next, *head;
234 
235 	/*
236 	 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from
237 	 * DR. The callbacks from DR are called with cpu_lock held, and hence
238 	 * can't wait to grab the mac perimeter. The soft ring list is hence
239 	 * protected for read access by srs_lock. Changing the soft ring list
240 	 * needs the mac perimeter and the srs_lock.
241 	 */
242 	mutex_enter(&mac_srs->srs_lock);
243 
244 	head = mac_srs->srs_soft_ring_head;
245 	mac_srs->srs_soft_ring_head = NULL;
246 	mac_srs->srs_soft_ring_tail = NULL;
247 	mac_srs->srs_soft_ring_count = 0;
248 
249 	mutex_exit(&mac_srs->srs_lock);
250 
251 	for (softring = head; softring != NULL; softring = next) {
252 		next = softring->s_ring_next;
253 		mac_soft_ring_free(softring);
254 	}
255 }
256 
257 static void
mac_srs_add_glist(mac_soft_ring_set_t * mac_srs)258 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs)
259 {
260 	ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL);
261 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
262 
263 	rw_enter(&mac_srs_g_lock, RW_WRITER);
264 	mutex_enter(&mac_srs->srs_lock);
265 
266 	ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0);
267 
268 	if (mac_srs_g_list == NULL) {
269 		mac_srs_g_list = mac_srs;
270 	} else {
271 		mac_srs->srs_next = mac_srs_g_list;
272 		mac_srs_g_list->srs_prev = mac_srs;
273 		mac_srs->srs_prev = NULL;
274 		mac_srs_g_list = mac_srs;
275 	}
276 	mac_srs->srs_state |= SRS_IN_GLIST;
277 
278 	mutex_exit(&mac_srs->srs_lock);
279 	rw_exit(&mac_srs_g_lock);
280 }
281 
282 static void
mac_srs_remove_glist(mac_soft_ring_set_t * mac_srs)283 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs)
284 {
285 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
286 
287 	rw_enter(&mac_srs_g_lock, RW_WRITER);
288 	mutex_enter(&mac_srs->srs_lock);
289 
290 	ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0);
291 
292 	if (mac_srs == mac_srs_g_list) {
293 		mac_srs_g_list = mac_srs->srs_next;
294 		if (mac_srs_g_list != NULL)
295 			mac_srs_g_list->srs_prev = NULL;
296 	} else {
297 		mac_srs->srs_prev->srs_next = mac_srs->srs_next;
298 		if (mac_srs->srs_next != NULL)
299 			mac_srs->srs_next->srs_prev = mac_srs->srs_prev;
300 	}
301 	mac_srs->srs_state &= ~SRS_IN_GLIST;
302 
303 	mutex_exit(&mac_srs->srs_lock);
304 	rw_exit(&mac_srs_g_lock);
305 }
306 
307 /* POLLING SETUP AND TEAR DOWN ROUTINES */
308 
309 /*
310  * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart
311  *
312  * These routines are used to call back into the upper layer
313  * (primarily TCP squeue) to stop polling the soft rings or
314  * restart polling.
315  */
316 void
mac_srs_client_poll_quiesce(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)317 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip,
318     mac_soft_ring_set_t *mac_srs)
319 {
320 	mac_soft_ring_t	*softring;
321 
322 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
323 
324 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
325 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
326 		return;
327 	}
328 
329 	for (softring = mac_srs->srs_soft_ring_head;
330 	    softring != NULL; softring = softring->s_ring_next) {
331 		if ((softring->s_ring_type & ST_RING_TCP) &&
332 		    (softring->s_ring_rx_arg2 != NULL)) {
333 			mcip->mci_resource_quiesce(mcip->mci_resource_arg,
334 			    softring->s_ring_rx_arg2);
335 		}
336 	}
337 }
338 
339 void
mac_srs_client_poll_restart(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)340 mac_srs_client_poll_restart(mac_client_impl_t *mcip,
341     mac_soft_ring_set_t *mac_srs)
342 {
343 	mac_soft_ring_t	*softring;
344 
345 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
346 
347 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
348 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
349 		return;
350 	}
351 
352 	for (softring = mac_srs->srs_soft_ring_head;
353 	    softring != NULL; softring = softring->s_ring_next) {
354 		if ((softring->s_ring_type & ST_RING_TCP) &&
355 		    (softring->s_ring_rx_arg2 != NULL)) {
356 			mcip->mci_resource_restart(mcip->mci_resource_arg,
357 			    softring->s_ring_rx_arg2);
358 		}
359 	}
360 }
361 
362 /*
363  * Register the given SRS and associated soft rings with the consumer and
364  * enable the polling interface used by the consumer.(i.e IP) over this
365  * SRS and associated soft rings.
366  */
367 void
mac_srs_client_poll_enable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)368 mac_srs_client_poll_enable(mac_client_impl_t *mcip,
369     mac_soft_ring_set_t *mac_srs)
370 {
371 	mac_rx_fifo_t		mrf;
372 	mac_soft_ring_t		*softring;
373 
374 	ASSERT(mac_srs->srs_mcip == mcip);
375 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
376 
377 	if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE))
378 		return;
379 
380 	bzero(&mrf, sizeof (mac_rx_fifo_t));
381 	mrf.mrf_type = MAC_RX_FIFO;
382 
383 	/*
384 	 * A SRS is capable of acting as a soft ring for cases
385 	 * where no fanout is needed. This is the case for userland
386 	 * flows.
387 	 */
388 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
389 		return;
390 
391 	mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
392 	mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
393 	mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
394 	mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED;
395 
396 	softring = mac_srs->srs_soft_ring_head;
397 	while (softring != NULL) {
398 		if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) {
399 			/*
400 			 * TCP and UDP support DLS bypass. Squeue polling
401 			 * support implies DLS bypass since the squeue poll
402 			 * path does not have DLS processing.
403 			 */
404 			mac_soft_ring_dls_bypass(softring,
405 			    mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg);
406 		}
407 		/*
408 		 * Non-TCP protocols don't support squeues. Hence we don't
409 		 * make any ring addition callbacks for non-TCP rings
410 		 */
411 		if (!(softring->s_ring_type & ST_RING_TCP)) {
412 			softring->s_ring_rx_arg2 = NULL;
413 			softring = softring->s_ring_next;
414 			continue;
415 		}
416 		mrf.mrf_rx_arg = softring;
417 		mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
418 		mrf.mrf_cpu_id = softring->s_ring_cpuid;
419 		mrf.mrf_flow_priority = mac_srs->srs_pri;
420 
421 		softring->s_ring_rx_arg2 = mcip->mci_resource_add(
422 		    mcip->mci_resource_arg, (mac_resource_t *)&mrf);
423 
424 		softring = softring->s_ring_next;
425 	}
426 }
427 
428 /*
429  * Unregister the given SRS and associated soft rings with the consumer and
430  * disable the polling interface used by the consumer.(i.e IP) over this
431  * SRS and associated soft rings.
432  */
433 void
mac_srs_client_poll_disable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)434 mac_srs_client_poll_disable(mac_client_impl_t *mcip,
435     mac_soft_ring_set_t *mac_srs)
436 {
437 	mac_soft_ring_t		*softring;
438 
439 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
440 
441 	/*
442 	 * A SRS is capable of acting as a soft ring for cases
443 	 * where no protocol fanout is needed. This is the case
444 	 * for userland flows. Nothing to do here.
445 	 */
446 	if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
447 		return;
448 
449 	mutex_enter(&mac_srs->srs_lock);
450 	if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
451 		ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
452 		mutex_exit(&mac_srs->srs_lock);
453 		return;
454 	}
455 	mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS);
456 	mutex_exit(&mac_srs->srs_lock);
457 
458 	/*
459 	 * DLS bypass is now disabled in the case of both TCP and UDP.
460 	 * Reset the soft ring callbacks to the standard 'mac_rx_deliver'
461 	 * callback. In addition, in the case of TCP, invoke IP's callback
462 	 * for ring removal.
463 	 */
464 	for (softring = mac_srs->srs_soft_ring_head;
465 	    softring != NULL; softring = softring->s_ring_next) {
466 		if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP)))
467 			continue;
468 
469 		if ((softring->s_ring_type & ST_RING_TCP) &&
470 		    softring->s_ring_rx_arg2 != NULL) {
471 			mcip->mci_resource_remove(mcip->mci_resource_arg,
472 			    softring->s_ring_rx_arg2);
473 		}
474 
475 		mutex_enter(&softring->s_ring_lock);
476 		while (softring->s_ring_state & S_RING_PROC) {
477 			softring->s_ring_state |= S_RING_CLIENT_WAIT;
478 			cv_wait(&softring->s_ring_client_cv,
479 			    &softring->s_ring_lock);
480 		}
481 		softring->s_ring_state &= ~S_RING_CLIENT_WAIT;
482 		softring->s_ring_rx_arg2 = NULL;
483 		softring->s_ring_rx_func = mac_rx_deliver;
484 		softring->s_ring_rx_arg1 = mcip;
485 		mutex_exit(&softring->s_ring_lock);
486 	}
487 }
488 
489 /*
490  * Enable or disable poll capability of the SRS on the underlying Rx ring.
491  *
492  * There is a need to enable or disable the poll capability of an SRS over an
493  * Rx ring depending on the number of mac clients sharing the ring and also
494  * whether user flows are configured on it. However the poll state is actively
495  * manipulated by the SRS worker and poll threads and uncoordinated changes by
496  * yet another thread to the underlying capability can surprise them leading
497  * to assert failures. Instead we quiesce the SRS, make the changes and then
498  * restart the SRS.
499  */
500 static void
mac_srs_poll_state_change(mac_soft_ring_set_t * mac_srs,boolean_t turn_off_poll_capab,mac_rx_func_t rx_func)501 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs,
502     boolean_t turn_off_poll_capab, mac_rx_func_t rx_func)
503 {
504 	boolean_t	need_restart = B_FALSE;
505 	mac_srs_rx_t	*srs_rx = &mac_srs->srs_rx;
506 	mac_ring_t	*ring;
507 
508 	if (!SRS_QUIESCED(mac_srs)) {
509 		mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE);
510 		need_restart = B_TRUE;
511 	}
512 
513 	ring = mac_srs->srs_ring;
514 	if ((ring != NULL) &&
515 	    (ring->mr_classify_type == MAC_HW_CLASSIFIER)) {
516 		if (turn_off_poll_capab)
517 			mac_srs->srs_state &= ~SRS_POLLING_CAPAB;
518 		else if (mac_poll_enable)
519 			mac_srs->srs_state |= SRS_POLLING_CAPAB;
520 	}
521 	srs_rx->sr_lower_proc = rx_func;
522 
523 	if (need_restart)
524 		mac_rx_srs_restart(mac_srs);
525 }
526 
527 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */
528 
529 /*
530  * Return the next CPU to be used to bind a MAC kernel thread.
531  * If a cpupart is specified, the cpu chosen must be from that
532  * cpu partition.
533  */
534 static processorid_t
mac_next_bind_cpu(cpupart_t * cpupart)535 mac_next_bind_cpu(cpupart_t *cpupart)
536 {
537 	static cpu_t		*cp = NULL;
538 	cpu_t			*cp_start;
539 
540 	ASSERT(MUTEX_HELD(&cpu_lock));
541 
542 	if (cp == NULL)
543 		cp = cpu_list;
544 
545 	cp = cp->cpu_next_onln;
546 	cp_start = cp;
547 
548 	do {
549 		if ((cpupart == NULL) || (cp->cpu_part == cpupart))
550 			return (cp->cpu_id);
551 
552 	} while ((cp = cp->cpu_next_onln) != cp_start);
553 
554 	return (-1);	/* No matching CPU found online */
555 }
556 
557 /* ARGSUSED */
558 static int
mac_srs_cpu_setup(cpu_setup_t what,int id,void * arg)559 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg)
560 {
561 	ASSERT(MUTEX_HELD(&cpu_lock));
562 	switch (what) {
563 	case CPU_CONFIG:
564 	case CPU_ON:
565 	case CPU_CPUPART_IN:
566 		mac_walk_srs_and_bind(id);
567 		break;
568 
569 	case CPU_UNCONFIG:
570 	case CPU_OFF:
571 	case CPU_CPUPART_OUT:
572 		mac_walk_srs_and_unbind(id);
573 		break;
574 
575 	default:
576 		break;
577 	}
578 	return (0);
579 }
580 
581 /*
582  * mac_compute_soft_ring_count():
583  *
584  * This routine computes the number of soft rings needed to handle incoming
585  * load given a flow_entry.
586  *
587  * The routine does the following:
588  * 1) soft rings will be created if mac_soft_ring_enable is set.
589  * 2) If the underlying link is a 10Gbps link, then soft rings will be
590  * created even if mac_soft_ring_enable is not set. The number of soft
591  * rings, so created,  will equal mac_rx_soft_ring_10gig_count.
592  * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the
593  * mac_rx_soft_ring_10gig_count number of soft rings will be created for a
594  * 10Gbps link.
595  *
596  * If a bandwidth limit is specified, the number that gets computed is
597  * dependent upon CPU speed, the number of Rx rings configured, and
598  * the bandwidth limit.
599  * If more Rx rings are available, less number of soft rings is needed.
600  *
601  * mac_use_bw_heuristic is another "hidden" variable that can be used to
602  * override the default use of soft ring count computation. Depending upon
603  * the usefulness of it, mac_use_bw_heuristic can later be made into a
604  * data-link property or removed altogether.
605  *
606  * TODO: Cleanup and tighten some of the assumptions.
607  */
608 boolean_t mac_check_overlay = B_TRUE;
609 boolean_t mac_use_bw_heuristic = B_TRUE;
610 static int
mac_compute_soft_ring_count(flow_entry_t * flent,int rx_srs_cnt,int maxcpus)611 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus)
612 {
613 	uint64_t cpu_speed, bw = 0;
614 	int srings = 0;
615 	boolean_t bw_enabled = B_FALSE;
616 	mac_client_impl_t *mcip = flent->fe_mcip;
617 
618 	ASSERT(!(flent->fe_type & FLOW_USER));
619 	if (flent->fe_resource_props.mrp_mask & MRP_MAXBW &&
620 	    mac_use_bw_heuristic) {
621 		/* bandwidth enabled */
622 		bw_enabled = B_TRUE;
623 		bw = flent->fe_resource_props.mrp_maxbw;
624 	}
625 	if (!bw_enabled) {
626 		/* No bandwidth enabled */
627 		if (mac_soft_ring_enable)
628 			srings = mac_rx_soft_ring_count;
629 
630 		/* Is this a 10Gig link? */
631 		flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip,
632 		    MAC_STAT_IFSPEED);
633 		/* convert to Mbps */
634 		if (((flent->fe_nic_speed)/1000000) > 1000 &&
635 		    mac_rx_soft_ring_10gig_count > 0) {
636 			/* This is a 10Gig link */
637 			srings = mac_rx_soft_ring_10gig_count;
638 			/*
639 			 * Use 2 times mac_rx_soft_ring_10gig_count for
640 			 * sun4v systems.
641 			 */
642 			if (mac_soft_ring_enable)
643 				srings = srings * 2;
644 		} else if (mac_check_overlay == B_TRUE &&
645 		    (mcip->mci_state_flags & MCIS_IS_VNIC) != 0) {
646 			/* Is this a VNIC on an overlay? */
647 			mac_handle_t mh = (mac_handle_t)mcip->mci_mip;
648 			if (mac_is_overlay(mh) == B_TRUE) {
649 				srings = mac_rx_soft_ring_10gig_count;
650 			}
651 		}
652 
653 
654 	} else {
655 		/*
656 		 * Soft ring computation using CPU speed and specified
657 		 * bandwidth limit.
658 		 */
659 		/* Assumption: all CPUs have the same frequency */
660 		cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock;
661 
662 		/* cpu_speed is in MHz; make bw in units of Mbps.  */
663 		bw = bw/1000000;
664 
665 		if (bw >= 1000) {
666 			/*
667 			 * bw is greater than or equal to 1Gbps.
668 			 * The number of soft rings required is a function
669 			 * of bandwidth and CPU speed. To keep this simple,
670 			 * let's use this rule: 1GHz CPU can handle 1Gbps.
671 			 * If bw is less than 1 Gbps, then there is no need
672 			 * for soft rings. Assumption is that CPU speeds
673 			 * (on modern systems) are at least 1GHz.
674 			 */
675 			srings = bw/cpu_speed;
676 			if (srings <= 1 && mac_soft_ring_enable) {
677 				/*
678 				 * Give at least 2 soft rings
679 				 * for sun4v systems
680 				 */
681 				srings = 2;
682 			}
683 		}
684 	}
685 	/*
686 	 * If the flent has multiple Rx SRSs, then each SRS need not
687 	 * have that many soft rings on top of it. The number of
688 	 * soft rings for each Rx SRS is found by dividing srings by
689 	 * rx_srs_cnt.
690 	 */
691 	if (rx_srs_cnt > 1) {
692 		int remainder;
693 
694 		remainder = srings%rx_srs_cnt;
695 		srings = srings/rx_srs_cnt;
696 		if (remainder != 0)
697 			srings++;
698 		/*
699 		 * Fanning out to 1 soft ring is not very useful.
700 		 * Set it as well to 0 and mac_srs_fanout_init()
701 		 * will take care of creating a single soft ring
702 		 * for proto fanout.
703 		 */
704 		if (srings == 1)
705 			srings = 0;
706 	}
707 	/* Do some more massaging */
708 	srings = min(srings, maxcpus);
709 	srings = min(srings, MAX_SR_FANOUT);
710 	return (srings);
711 }
712 
713 /*
714  * mac_tx_cpu_init:
715  * set up CPUs for Tx interrupt re-targeting and Tx worker
716  * thread binding
717  */
718 static void
mac_tx_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp,cpupart_t * cpupart)719 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp,
720     cpupart_t *cpupart)
721 {
722 	mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
723 	mac_srs_tx_t *srs_tx = &tx_srs->srs_tx;
724 	mac_cpus_t *srs_cpu = &tx_srs->srs_cpu;
725 	mac_soft_ring_t *sringp;
726 	mac_ring_t *ring;
727 	processorid_t worker_cpuid;
728 	boolean_t retargetable_client = B_FALSE;
729 	int i, j;
730 
731 	if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group,
732 	    flent->fe_mcip)) {
733 		retargetable_client = B_TRUE;
734 	}
735 
736 	if (MAC_TX_SOFT_RINGS(tx_srs)) {
737 		if (mrp != NULL)
738 			j = mrp->mrp_ncpus - 1;
739 		for (i = 0; i < tx_srs->srs_tx_ring_count; i++) {
740 			if (mrp != NULL) {
741 				if (j < 0)
742 					j = mrp->mrp_ncpus - 1;
743 				worker_cpuid = mrp->mrp_cpu[j];
744 			} else {
745 				/*
746 				 * Bind interrupt to the next CPU available
747 				 * and leave the worker unbound.
748 				 */
749 				worker_cpuid = -1;
750 			}
751 			sringp = tx_srs->srs_tx_soft_rings[i];
752 			ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
753 			srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid;
754 			if (MAC_RING_RETARGETABLE(ring) &&
755 			    retargetable_client) {
756 				mutex_enter(&cpu_lock);
757 				srs_cpu->mc_tx_intr_cpu[i] =
758 				    (mrp != NULL) ? mrp->mrp_cpu[j] :
759 				    (mac_tx_intr_retarget ?
760 				    mac_next_bind_cpu(cpupart) : -1);
761 				mutex_exit(&cpu_lock);
762 			} else {
763 				srs_cpu->mc_tx_intr_cpu[i] = -1;
764 			}
765 			if (mrp != NULL)
766 				j--;
767 		}
768 	} else {
769 		/* Tx mac_ring_handle_t is stored in st_arg2 */
770 		srs_cpu->mc_tx_fanout_cpus[0] =
771 		    (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1;
772 		ring = (mac_ring_t *)srs_tx->st_arg2;
773 		if (MAC_RING_RETARGETABLE(ring) && retargetable_client) {
774 			mutex_enter(&cpu_lock);
775 			srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ?
776 			    mrp->mrp_cpu[mrp->mrp_ncpus - 1] :
777 			    (mac_tx_intr_retarget ?
778 			    mac_next_bind_cpu(cpupart) : -1);
779 			mutex_exit(&cpu_lock);
780 		} else {
781 			srs_cpu->mc_tx_intr_cpu[0] = -1;
782 		}
783 	}
784 }
785 
786 /*
787  * Assignment of user specified CPUs to a link.
788  *
789  * Minimum CPUs required to get an optimal assignmet:
790  * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize
791  * flag is set -- one for polling, one for fanout soft ring.
792  * If mac_latency_optimize is not set, then 3 CPUs are needed -- one
793  * for polling, one for SRS worker thread and one for fanout soft ring.
794  *
795  * The CPUs needed for Tx side is equal to the number of Tx rings
796  * the link is using.
797  *
798  * mac_flow_user_cpu_init() categorizes the CPU assignment depending
799  * upon the number of CPUs in 3 different buckets.
800  *
801  * In the first bucket, the most optimal case is handled. The user has
802  * passed enough number of CPUs and every thread gets its own CPU.
803  *
804  * The second and third are the sub-optimal cases. Enough CPUs are not
805  * available.
806  *
807  * The second bucket handles the case where atleast one distinct CPU is
808  * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx
809  * SRS or soft rings).
810  *
811  * In the third case (worst case scenario), specified CPU count is less
812  * than the Rx rings configured for the link. In this case, we round
813  * robin the CPUs among the Rx SRSes and Tx SRS/soft rings.
814  */
815 static void
mac_flow_user_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp)816 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp)
817 {
818 	mac_soft_ring_set_t *rx_srs, *tx_srs;
819 	int i, srs_cnt;
820 	mac_cpus_t *srs_cpu;
821 	int no_of_cpus, cpu_cnt;
822 	int rx_srs_cnt, reqd_rx_cpu_cnt;
823 	int fanout_cpu_cnt, reqd_tx_cpu_cnt;
824 	int reqd_poll_worker_cnt, fanout_cnt_per_srs;
825 	mac_resource_props_t *emrp = &flent->fe_effective_props;
826 
827 	ASSERT(mrp->mrp_fanout_mode == MCM_CPUS);
828 	/*
829 	 * The check for nbc_ncpus to be within limits for
830 	 * the user specified case was done earlier and if
831 	 * not within limits, an error would have been
832 	 * returned to the user.
833 	 */
834 	ASSERT(mrp->mrp_ncpus > 0);
835 
836 	no_of_cpus = mrp->mrp_ncpus;
837 
838 	if (mrp->mrp_rx_intr_cpu != -1) {
839 		/*
840 		 * interrupt has been re-targetted. Poll
841 		 * thread needs to be bound to interrupt
842 		 * CPU.
843 		 *
844 		 * Find where in the list is the intr
845 		 * CPU and swap it with the first one.
846 		 * We will be using the first CPU in the
847 		 * list for poll.
848 		 */
849 		for (i = 0; i < no_of_cpus; i++) {
850 			if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu)
851 				break;
852 		}
853 		mrp->mrp_cpu[i] = mrp->mrp_cpu[0];
854 		mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu;
855 	}
856 
857 	/*
858 	 * Requirements:
859 	 * The number of CPUs that each Rx ring needs is dependent
860 	 * upon mac_latency_optimize flag.
861 	 * 1) If set, atleast 2 CPUs are needed -- one for
862 	 * polling, one for fanout soft ring.
863 	 * 2) If not set, then atleast 3 CPUs are needed -- one
864 	 * for polling, one for srs worker thread, and one for
865 	 * fanout soft ring.
866 	 */
867 	rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ?
868 	    (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt;
869 	reqd_rx_cpu_cnt = mac_latency_optimize ?
870 	    (rx_srs_cnt * 2) : (rx_srs_cnt * 3);
871 
872 	/* How many CPUs are needed for Tx side? */
873 	tx_srs = flent->fe_tx_srs;
874 	reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ?
875 	    tx_srs->srs_tx_ring_count : 1;
876 
877 	/* CPUs needed for Rx SRSes poll and worker threads */
878 	reqd_poll_worker_cnt = mac_latency_optimize ?
879 	    rx_srs_cnt : rx_srs_cnt * 2;
880 
881 	/* Has the user provided enough CPUs? */
882 	if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) {
883 		/*
884 		 * Best case scenario. There is enough CPUs. All
885 		 * Rx rings will get their own set of CPUs plus
886 		 * Tx soft rings will get their own.
887 		 */
888 		/*
889 		 * fanout_cpu_cnt is the number of CPUs available
890 		 * for Rx side fanout soft rings.
891 		 */
892 		fanout_cpu_cnt = no_of_cpus -
893 		    reqd_poll_worker_cnt - reqd_tx_cpu_cnt;
894 
895 		/*
896 		 * Divide fanout_cpu_cnt by rx_srs_cnt to find
897 		 * out how many fanout soft rings each Rx SRS
898 		 * can have.
899 		 */
900 		fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt;
901 
902 		/* fanout_cnt_per_srs should not be >  MAX_SR_FANOUT */
903 		fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT);
904 
905 		/* Do the assignment for the default Rx ring */
906 		cpu_cnt = 0;
907 		rx_srs = flent->fe_rx_srs[0];
908 		ASSERT(rx_srs->srs_ring == NULL);
909 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
910 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
911 		srs_cpu = &rx_srs->srs_cpu;
912 		srs_cpu->mc_ncpus = no_of_cpus;
913 		bcopy(mrp->mrp_cpu,
914 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
915 		srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
916 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
917 		/* Retarget the interrupt to the same CPU as the poll */
918 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
919 		srs_cpu->mc_rx_workerid = (mac_latency_optimize ?
920 		    srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]);
921 		for (i = 0; i < fanout_cnt_per_srs; i++)
922 			srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++];
923 
924 		/* Do the assignment for h/w Rx SRSes */
925 		if (flent->fe_rx_srs_cnt > 1) {
926 			cpu_cnt = 0;
927 			for (srs_cnt = 1;
928 			    srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
929 				rx_srs = flent->fe_rx_srs[srs_cnt];
930 				ASSERT(rx_srs->srs_ring != NULL);
931 				if (rx_srs->srs_fanout_state ==
932 				    SRS_FANOUT_INIT) {
933 					rx_srs->srs_fanout_state =
934 					    SRS_FANOUT_REINIT;
935 				}
936 				srs_cpu = &rx_srs->srs_cpu;
937 				srs_cpu->mc_ncpus = no_of_cpus;
938 				bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
939 				    sizeof (srs_cpu->mc_cpus));
940 				srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
941 				/* The first CPU in the list is the intr CPU */
942 				srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
943 				srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
944 				srs_cpu->mc_rx_workerid =
945 				    (mac_latency_optimize ?
946 				    srs_cpu->mc_rx_pollid :
947 				    mrp->mrp_cpu[cpu_cnt++]);
948 				for (i = 0; i < fanout_cnt_per_srs; i++) {
949 					srs_cpu->mc_rx_fanout_cpus[i] =
950 					    mrp->mrp_cpu[cpu_cnt++];
951 				}
952 				ASSERT(cpu_cnt <= no_of_cpus);
953 			}
954 		}
955 		goto tx_cpu_init;
956 	}
957 
958 	/*
959 	 * Sub-optimal case.
960 	 * We have the following information:
961 	 * no_of_cpus - no. of cpus that user passed.
962 	 * rx_srs_cnt - no. of rx rings.
963 	 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3
964 	 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side.
965 	 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2
966 	 */
967 	/*
968 	 * If we bind the Rx fanout soft rings to the same CPUs
969 	 * as poll/worker, would that be enough?
970 	 */
971 	if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) {
972 		boolean_t worker_assign = B_FALSE;
973 
974 		/*
975 		 * If mac_latency_optimize is not set, are there
976 		 * enough CPUs to assign a CPU for worker also?
977 		 */
978 		if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt))
979 			worker_assign = B_TRUE;
980 		/*
981 		 * Zero'th Rx SRS is the default Rx ring. It is not
982 		 * associated with h/w Rx ring.
983 		 */
984 		rx_srs = flent->fe_rx_srs[0];
985 		ASSERT(rx_srs->srs_ring == NULL);
986 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
987 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
988 		cpu_cnt = 0;
989 		srs_cpu = &rx_srs->srs_cpu;
990 		srs_cpu->mc_ncpus = no_of_cpus;
991 		bcopy(mrp->mrp_cpu,
992 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
993 		srs_cpu->mc_rx_fanout_cnt = 1;
994 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
995 		/* Retarget the interrupt to the same CPU as the poll */
996 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
997 		srs_cpu->mc_rx_workerid =
998 		    ((!mac_latency_optimize && worker_assign) ?
999 		    mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid);
1000 
1001 		srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1002 
1003 		/* Do CPU bindings for SRSes having h/w Rx rings */
1004 		if (flent->fe_rx_srs_cnt > 1) {
1005 			cpu_cnt = 0;
1006 			for (srs_cnt = 1;
1007 			    srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1008 				rx_srs = flent->fe_rx_srs[srs_cnt];
1009 				ASSERT(rx_srs->srs_ring != NULL);
1010 				if (rx_srs->srs_fanout_state ==
1011 				    SRS_FANOUT_INIT) {
1012 					rx_srs->srs_fanout_state =
1013 					    SRS_FANOUT_REINIT;
1014 				}
1015 				srs_cpu = &rx_srs->srs_cpu;
1016 				srs_cpu->mc_ncpus = no_of_cpus;
1017 				bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
1018 				    sizeof (srs_cpu->mc_cpus));
1019 				srs_cpu->mc_rx_pollid =
1020 				    mrp->mrp_cpu[cpu_cnt];
1021 				srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1022 				srs_cpu->mc_rx_workerid =
1023 				    ((!mac_latency_optimize && worker_assign) ?
1024 				    mrp->mrp_cpu[++cpu_cnt] :
1025 				    srs_cpu->mc_rx_pollid);
1026 				srs_cpu->mc_rx_fanout_cnt = 1;
1027 				srs_cpu->mc_rx_fanout_cpus[0] =
1028 				    mrp->mrp_cpu[cpu_cnt];
1029 				cpu_cnt++;
1030 				ASSERT(cpu_cnt <= no_of_cpus);
1031 			}
1032 		}
1033 		goto tx_cpu_init;
1034 	}
1035 
1036 	/*
1037 	 * Real sub-optimal case. Not enough CPUs for poll and
1038 	 * Tx soft rings. Do a round robin assignment where
1039 	 * each Rx SRS will get the same CPU for poll, worker
1040 	 * and fanout soft ring.
1041 	 */
1042 	cpu_cnt = 0;
1043 	for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1044 		rx_srs = flent->fe_rx_srs[srs_cnt];
1045 		srs_cpu = &rx_srs->srs_cpu;
1046 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1047 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1048 		srs_cpu->mc_ncpus = no_of_cpus;
1049 		bcopy(mrp->mrp_cpu,
1050 		    srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
1051 		srs_cpu->mc_rx_fanout_cnt = 1;
1052 		srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt];
1053 		/* Retarget the interrupt to the same CPU as the poll */
1054 		srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
1055 		srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt];
1056 		srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1057 		if (++cpu_cnt >= no_of_cpus)
1058 			cpu_cnt = 0;
1059 	}
1060 
1061 tx_cpu_init:
1062 	mac_tx_cpu_init(flent, mrp, NULL);
1063 
1064 	/*
1065 	 * Copy the user specified CPUs to the effective CPUs
1066 	 */
1067 	for (i = 0; i < mrp->mrp_ncpus; i++) {
1068 		emrp->mrp_cpu[i] = mrp->mrp_cpu[i];
1069 	}
1070 	emrp->mrp_ncpus = mrp->mrp_ncpus;
1071 	emrp->mrp_mask = mrp->mrp_mask;
1072 	bzero(emrp->mrp_pool, MAXPATHLEN);
1073 }
1074 
1075 /*
1076  * mac_flow_cpu_init():
1077  *
1078  * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in
1079  * the CPU binding information in srs_cpu for all Rx SRSes associated
1080  * with a flent.
1081  */
1082 static void
mac_flow_cpu_init(flow_entry_t * flent,cpupart_t * cpupart)1083 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart)
1084 {
1085 	mac_soft_ring_set_t *rx_srs;
1086 	processorid_t cpuid;
1087 	int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0;
1088 	mac_cpus_t *srs_cpu;
1089 	mac_resource_props_t *emrp = &flent->fe_effective_props;
1090 
1091 	/*
1092 	 * The maximum number of CPUs available can either be
1093 	 * the number of CPUs in the pool or the number of CPUs
1094 	 * in the system.
1095 	 */
1096 	maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
1097 	/*
1098 	 * We cannot exceed the hard limit imposed by data structures.
1099 	 * Leave space for polling CPU and the SRS worker thread when
1100 	 * "mac_latency_optimize" is not set.
1101 	 */
1102 	maxcpus = MIN(maxcpus, MRP_NCPUS - 2);
1103 
1104 	/*
1105 	 * Compute the number of soft rings needed on top for each Rx
1106 	 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS
1107 	 * associated with h/w Rx rings. Soft ring count needed for
1108 	 * each h/w Rx SRS is computed and the same is applied to
1109 	 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[]
1110 	 * is the software classified Rx SRS.
1111 	 */
1112 	soft_ring_cnt = mac_compute_soft_ring_count(flent,
1113 	    flent->fe_rx_srs_cnt - 1, maxcpus);
1114 	if (soft_ring_cnt == 0) {
1115 		/*
1116 		 * Even when soft_ring_cnt is 0, we still need
1117 		 * to create a soft ring for TCP, UDP and
1118 		 * OTHER. So set it to 1.
1119 		 */
1120 		soft_ring_cnt = 1;
1121 	}
1122 
1123 	emrp->mrp_ncpus = 0;
1124 	for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt &&
1125 	    emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) {
1126 		rx_srs = flent->fe_rx_srs[srs_cnt];
1127 		srs_cpu = &rx_srs->srs_cpu;
1128 		if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1129 			rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1130 		srs_cpu->mc_ncpus = soft_ring_cnt;
1131 		srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt;
1132 		mutex_enter(&cpu_lock);
1133 		for (j = 0; j < soft_ring_cnt; j++) {
1134 			cpuid = mac_next_bind_cpu(cpupart);
1135 			srs_cpu->mc_cpus[j] = cpuid;
1136 			srs_cpu->mc_rx_fanout_cpus[j] = cpuid;
1137 		}
1138 		cpuid = mac_next_bind_cpu(cpupart);
1139 		srs_cpu->mc_rx_pollid = cpuid;
1140 		srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ?
1141 		    srs_cpu->mc_rx_pollid : -1);
1142 		/* increment ncpus to account for polling cpu */
1143 		srs_cpu->mc_ncpus++;
1144 		srs_cpu->mc_cpus[j++] = cpuid;
1145 		if (!mac_latency_optimize) {
1146 			cpuid = mac_next_bind_cpu(cpupart);
1147 			srs_cpu->mc_ncpus++;
1148 			srs_cpu->mc_cpus[j++] = cpuid;
1149 		}
1150 		srs_cpu->mc_rx_workerid = cpuid;
1151 		mutex_exit(&cpu_lock);
1152 
1153 		/*
1154 		 * Copy fanout CPUs to fe_effective_props without duplicates.
1155 		 */
1156 		for (i = 0; i < srs_cpu->mc_ncpus &&
1157 		    emrp->mrp_ncpus < MRP_NCPUS; i++) {
1158 			for (j = 0; j < emrp->mrp_ncpus; j++) {
1159 				if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i])
1160 					break;
1161 			}
1162 			if (j == emrp->mrp_ncpus) {
1163 				emrp->mrp_cpu[emrp->mrp_ncpus++] =
1164 				    srs_cpu->mc_cpus[i];
1165 			}
1166 		}
1167 	}
1168 
1169 	mac_tx_cpu_init(flent, NULL, cpupart);
1170 }
1171 
1172 /*
1173  * DATAPATH SETUP ROUTINES
1174  * (setup SRS and set/update FANOUT, B/W and PRIORITY)
1175  */
1176 
1177 /*
1178  * mac_srs_fanout_list_alloc:
1179  *
1180  * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of
1181  * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of
1182  * array space is needed to store Tx soft rings. Thus we allocate so
1183  * much array space for srs_tx_soft_rings.
1184  *
1185  * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth
1186  * of space to st_soft_rings. This array is used for quick access to
1187  * soft ring associated with a pseudo Tx ring based on the pseudo
1188  * ring's index (mr_index).
1189  */
1190 static void
mac_srs_fanout_list_alloc(mac_soft_ring_set_t * mac_srs)1191 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs)
1192 {
1193 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
1194 
1195 	if (mac_srs->srs_type & SRST_TX) {
1196 		mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **)
1197 		    kmem_zalloc(sizeof (mac_soft_ring_t *) *
1198 		    MAX_RINGS_PER_GROUP, KM_SLEEP);
1199 		if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
1200 			mac_srs_tx_t *tx = &mac_srs->srs_tx;
1201 
1202 			tx->st_soft_rings = (mac_soft_ring_t **)
1203 			    kmem_zalloc(sizeof (mac_soft_ring_t *) *
1204 			    MAX_RINGS_PER_GROUP, KM_SLEEP);
1205 		}
1206 	} else {
1207 		mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **)
1208 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1209 		    KM_SLEEP);
1210 		mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **)
1211 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1212 		    KM_SLEEP);
1213 		mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **)
1214 		    kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
1215 		    KM_SLEEP);
1216 	}
1217 }
1218 
1219 static void
mac_srs_worker_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1220 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1221 {
1222 	cpu_t *cp;
1223 	boolean_t clear = B_FALSE;
1224 
1225 	ASSERT(MUTEX_HELD(&cpu_lock));
1226 
1227 	if (!mac_srs_thread_bind)
1228 		return;
1229 
1230 	cp = cpu_get(cpuid);
1231 	if (cp == NULL || !cpu_is_online(cp))
1232 		return;
1233 
1234 	mutex_enter(&mac_srs->srs_lock);
1235 	mac_srs->srs_state |= SRS_WORKER_BOUND;
1236 	if (mac_srs->srs_worker_cpuid != -1)
1237 		clear = B_TRUE;
1238 	mac_srs->srs_worker_cpuid = cpuid;
1239 	mutex_exit(&mac_srs->srs_lock);
1240 
1241 	if (clear)
1242 		thread_affinity_clear(mac_srs->srs_worker);
1243 
1244 	thread_affinity_set(mac_srs->srs_worker, cpuid);
1245 	DTRACE_PROBE1(worker__CPU, processorid_t, cpuid);
1246 }
1247 
1248 static void
mac_srs_poll_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1249 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1250 {
1251 	cpu_t *cp;
1252 	boolean_t clear = B_FALSE;
1253 
1254 	ASSERT(MUTEX_HELD(&cpu_lock));
1255 
1256 	if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL)
1257 		return;
1258 
1259 	cp = cpu_get(cpuid);
1260 	if (cp == NULL || !cpu_is_online(cp))
1261 		return;
1262 
1263 	mutex_enter(&mac_srs->srs_lock);
1264 	mac_srs->srs_state |= SRS_POLL_BOUND;
1265 	if (mac_srs->srs_poll_cpuid != -1)
1266 		clear = B_TRUE;
1267 	mac_srs->srs_poll_cpuid = cpuid;
1268 	mutex_exit(&mac_srs->srs_lock);
1269 
1270 	if (clear)
1271 		thread_affinity_clear(mac_srs->srs_poll_thr);
1272 
1273 	thread_affinity_set(mac_srs->srs_poll_thr, cpuid);
1274 	DTRACE_PROBE1(poll__CPU, processorid_t, cpuid);
1275 }
1276 
1277 /*
1278  * Re-target interrupt to the passed CPU. If re-target is successful,
1279  * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1.
1280  */
1281 void
mac_rx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1282 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1283 {
1284 	cpu_t *cp;
1285 	mac_ring_t *ring = mac_srs->srs_ring;
1286 	mac_intr_t *mintr = &ring->mr_info.mri_intr;
1287 	flow_entry_t *flent = mac_srs->srs_flent;
1288 	boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip);
1289 
1290 	ASSERT(MUTEX_HELD(&cpu_lock));
1291 
1292 	/*
1293 	 * Don't re-target the interrupt for these cases:
1294 	 * 1) ring is NULL
1295 	 * 2) the interrupt is shared (mi_ddi_shared)
1296 	 * 3) ddi_handle is NULL and !primary
1297 	 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2
1298 	 * Case 3 & 4 are because of mac_client_intr_cpu() routine.
1299 	 * This routine will re-target fixed interrupt for primary
1300 	 * mac client if the client has only one ring. In that
1301 	 * case, mc_rx_intr_cpu will already have the correct value.
1302 	 */
1303 	if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 ||
1304 	    (mintr->mi_ddi_handle == NULL && !primary) || (primary &&
1305 	    mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) {
1306 		mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1307 		return;
1308 	}
1309 
1310 	if (mintr->mi_ddi_handle == NULL)
1311 		return;
1312 
1313 	cp = cpu_get(cpuid);
1314 	if (cp == NULL || !cpu_is_online(cp))
1315 		return;
1316 
1317 	/* Drop the cpu_lock as set_intr_affinity() holds it */
1318 	mutex_exit(&cpu_lock);
1319 	if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS)
1320 		mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid;
1321 	else
1322 		mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
1323 	mutex_enter(&cpu_lock);
1324 }
1325 
1326 /*
1327  * Re-target Tx interrupts
1328  */
1329 void
mac_tx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs)1330 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs)
1331 {
1332 	cpu_t *cp;
1333 	mac_ring_t *ring;
1334 	mac_intr_t *mintr;
1335 	mac_soft_ring_t *sringp;
1336 	mac_srs_tx_t *srs_tx;
1337 	mac_cpus_t *srs_cpu;
1338 	processorid_t cpuid;
1339 	int i;
1340 
1341 	ASSERT(MUTEX_HELD(&cpu_lock));
1342 
1343 	srs_cpu = &mac_srs->srs_cpu;
1344 	if (MAC_TX_SOFT_RINGS(mac_srs)) {
1345 		for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
1346 			sringp = mac_srs->srs_tx_soft_rings[i];
1347 			ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
1348 			cpuid = srs_cpu->mc_tx_intr_cpu[i];
1349 			cp = cpu_get(cpuid);
1350 			if (cp == NULL || !cpu_is_online(cp) ||
1351 			    !MAC_RING_RETARGETABLE(ring)) {
1352 				srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1353 				continue;
1354 			}
1355 			mintr = &ring->mr_info.mri_intr;
1356 			/*
1357 			 * Drop the cpu_lock as set_intr_affinity()
1358 			 * holds it
1359 			 */
1360 			mutex_exit(&cpu_lock);
1361 			if (set_intr_affinity(mintr->mi_ddi_handle,
1362 			    cpuid) == DDI_SUCCESS) {
1363 				srs_cpu->mc_tx_retargeted_cpu[i] = cpuid;
1364 			} else {
1365 				srs_cpu->mc_tx_retargeted_cpu[i] = -1;
1366 			}
1367 			mutex_enter(&cpu_lock);
1368 		}
1369 	} else {
1370 		cpuid = srs_cpu->mc_tx_intr_cpu[0];
1371 		cp = cpu_get(cpuid);
1372 		if (cp == NULL || !cpu_is_online(cp)) {
1373 			srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1374 			return;
1375 		}
1376 		srs_tx = &mac_srs->srs_tx;
1377 		ring = (mac_ring_t *)srs_tx->st_arg2;
1378 		if (MAC_RING_RETARGETABLE(ring)) {
1379 			mintr = &ring->mr_info.mri_intr;
1380 			mutex_exit(&cpu_lock);
1381 			if ((set_intr_affinity(mintr->mi_ddi_handle,
1382 			    cpuid) == DDI_SUCCESS)) {
1383 				srs_cpu->mc_tx_retargeted_cpu[0] = cpuid;
1384 			} else {
1385 				srs_cpu->mc_tx_retargeted_cpu[0] = -1;
1386 			}
1387 			mutex_enter(&cpu_lock);
1388 		}
1389 	}
1390 }
1391 
1392 /*
1393  * When a CPU comes back online, bind the MAC kernel threads which
1394  * were previously bound to that CPU, and had to be unbound because
1395  * the CPU was going away.
1396  *
1397  * These functions are called with cpu_lock held and hence we can't
1398  * cv_wait to grab the mac perimeter. Since these functions walk the soft
1399  * ring list of an SRS without being in the perimeter, the list itself
1400  * is protected by the SRS lock.
1401  */
1402 static void
mac_walk_srs_and_bind(int cpuid)1403 mac_walk_srs_and_bind(int cpuid)
1404 {
1405 	mac_soft_ring_set_t *mac_srs;
1406 	mac_soft_ring_t *soft_ring;
1407 
1408 	rw_enter(&mac_srs_g_lock, RW_READER);
1409 
1410 	if ((mac_srs = mac_srs_g_list) == NULL)
1411 		goto done;
1412 
1413 	for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
1414 		if (mac_srs->srs_worker_cpuid == -1 &&
1415 		    mac_srs->srs_worker_cpuid_save == cpuid) {
1416 			mac_srs->srs_worker_cpuid_save = -1;
1417 			mac_srs_worker_bind(mac_srs, cpuid);
1418 		}
1419 
1420 		if (!(mac_srs->srs_type & SRST_TX)) {
1421 			if (mac_srs->srs_poll_cpuid == -1 &&
1422 			    mac_srs->srs_poll_cpuid_save == cpuid) {
1423 				mac_srs->srs_poll_cpuid_save = -1;
1424 				mac_srs_poll_bind(mac_srs, cpuid);
1425 			}
1426 		}
1427 
1428 		/* Next tackle the soft rings associated with the srs */
1429 		mutex_enter(&mac_srs->srs_lock);
1430 		for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
1431 		    soft_ring = soft_ring->s_ring_next) {
1432 			if (soft_ring->s_ring_cpuid == -1 &&
1433 			    soft_ring->s_ring_cpuid_save == cpuid) {
1434 				soft_ring->s_ring_cpuid_save = -1;
1435 				(void) mac_soft_ring_bind(soft_ring, cpuid);
1436 			}
1437 		}
1438 		mutex_exit(&mac_srs->srs_lock);
1439 	}
1440 done:
1441 	rw_exit(&mac_srs_g_lock);
1442 }
1443 
1444 /*
1445  * Change the priority of the SRS's poll and worker thread. Additionally,
1446  * update the priority of the worker threads for the SRS's soft rings.
1447  * Need to modify any associated squeue threads.
1448  */
1449 void
mac_update_srs_priority(mac_soft_ring_set_t * mac_srs,pri_t prival)1450 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival)
1451 {
1452 	mac_soft_ring_t		*ringp;
1453 
1454 	mac_srs->srs_pri = prival;
1455 	thread_lock(mac_srs->srs_worker);
1456 	(void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0);
1457 	thread_unlock(mac_srs->srs_worker);
1458 	if (mac_srs->srs_poll_thr != NULL) {
1459 		thread_lock(mac_srs->srs_poll_thr);
1460 		(void) thread_change_pri(mac_srs->srs_poll_thr,
1461 		    mac_srs->srs_pri, 0);
1462 		thread_unlock(mac_srs->srs_poll_thr);
1463 	}
1464 	if ((ringp = mac_srs->srs_soft_ring_head) == NULL)
1465 		return;
1466 	while (ringp != mac_srs->srs_soft_ring_tail) {
1467 		thread_lock(ringp->s_ring_worker);
1468 		(void) thread_change_pri(ringp->s_ring_worker,
1469 		    mac_srs->srs_pri, 0);
1470 		thread_unlock(ringp->s_ring_worker);
1471 		ringp = ringp->s_ring_next;
1472 	}
1473 	ASSERT(ringp == mac_srs->srs_soft_ring_tail);
1474 	thread_lock(ringp->s_ring_worker);
1475 	(void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0);
1476 	thread_unlock(ringp->s_ring_worker);
1477 }
1478 
1479 /*
1480  * Change the receive bandwidth limit.
1481  */
1482 static void
mac_rx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1483 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1484 {
1485 	mac_soft_ring_t		*softring;
1486 
1487 	mutex_enter(&srs->srs_lock);
1488 	mutex_enter(&srs->srs_bw->mac_bw_lock);
1489 
1490 	if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1491 		/* Reset bandwidth limit */
1492 		if (srs->srs_type & SRST_BW_CONTROL) {
1493 			softring = srs->srs_soft_ring_head;
1494 			while (softring != NULL) {
1495 				softring->s_ring_type &= ~ST_RING_BW_CTL;
1496 				softring = softring->s_ring_next;
1497 			}
1498 			srs->srs_type &= ~SRST_BW_CONTROL;
1499 			srs->srs_drain_func = mac_rx_srs_drain;
1500 		}
1501 	} else {
1502 		/* Set/Modify bandwidth limit */
1503 		srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1504 		/*
1505 		 * Give twice the queuing capability before
1506 		 * dropping packets. The unit is bytes/tick.
1507 		 */
1508 		srs->srs_bw->mac_bw_drop_threshold =
1509 		    srs->srs_bw->mac_bw_limit << 1;
1510 		if (!(srs->srs_type & SRST_BW_CONTROL)) {
1511 			softring = srs->srs_soft_ring_head;
1512 			while (softring != NULL) {
1513 				softring->s_ring_type |= ST_RING_BW_CTL;
1514 				softring = softring->s_ring_next;
1515 			}
1516 			srs->srs_type |= SRST_BW_CONTROL;
1517 			srs->srs_drain_func = mac_rx_srs_drain_bw;
1518 		}
1519 	}
1520 done:
1521 	mutex_exit(&srs->srs_bw->mac_bw_lock);
1522 	mutex_exit(&srs->srs_lock);
1523 }
1524 
1525 /* Change the transmit bandwidth limit */
1526 static void
mac_tx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1527 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1528 {
1529 	uint32_t		tx_mode, ring_info = 0;
1530 	mac_srs_tx_t		*srs_tx = &srs->srs_tx;
1531 	mac_client_impl_t	*mcip = srs->srs_mcip;
1532 
1533 	/*
1534 	 * We need to quiesce/restart the client here because mac_tx() and
1535 	 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing
1536 	 * st_mode and related fields, which are modified by the code below.
1537 	 */
1538 	mac_tx_client_quiesce((mac_client_handle_t)mcip);
1539 
1540 	mutex_enter(&srs->srs_lock);
1541 	mutex_enter(&srs->srs_bw->mac_bw_lock);
1542 
1543 	tx_mode = srs_tx->st_mode;
1544 	if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1545 		/* Reset bandwidth limit */
1546 		if (tx_mode == SRS_TX_BW) {
1547 			if (srs_tx->st_arg2 != NULL)
1548 				ring_info = mac_hwring_getinfo(srs_tx->st_arg2);
1549 			if (mac_tx_serialize ||
1550 			    (ring_info & MAC_RING_TX_SERIALIZE)) {
1551 				srs_tx->st_mode = SRS_TX_SERIALIZE;
1552 			} else {
1553 				srs_tx->st_mode = SRS_TX_DEFAULT;
1554 			}
1555 		} else if (tx_mode == SRS_TX_BW_FANOUT) {
1556 			srs_tx->st_mode = SRS_TX_FANOUT;
1557 		} else if (tx_mode == SRS_TX_BW_AGGR) {
1558 			srs_tx->st_mode = SRS_TX_AGGR;
1559 		}
1560 		srs->srs_type &= ~SRST_BW_CONTROL;
1561 	} else {
1562 		/* Set/Modify bandwidth limit */
1563 		srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1564 		/*
1565 		 * Give twice the queuing capability before
1566 		 * dropping packets. The unit is bytes/tick.
1567 		 */
1568 		srs->srs_bw->mac_bw_drop_threshold =
1569 		    srs->srs_bw->mac_bw_limit << 1;
1570 		srs->srs_type |= SRST_BW_CONTROL;
1571 		if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT &&
1572 		    tx_mode != SRS_TX_BW_AGGR) {
1573 			if (tx_mode == SRS_TX_SERIALIZE ||
1574 			    tx_mode == SRS_TX_DEFAULT) {
1575 				srs_tx->st_mode = SRS_TX_BW;
1576 			} else if (tx_mode == SRS_TX_FANOUT) {
1577 				srs_tx->st_mode = SRS_TX_BW_FANOUT;
1578 			} else if (tx_mode == SRS_TX_AGGR) {
1579 				srs_tx->st_mode = SRS_TX_BW_AGGR;
1580 			} else {
1581 				ASSERT(0);
1582 			}
1583 		}
1584 	}
1585 done:
1586 	srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode);
1587 	mutex_exit(&srs->srs_bw->mac_bw_lock);
1588 	mutex_exit(&srs->srs_lock);
1589 
1590 	mac_tx_client_restart((mac_client_handle_t)mcip);
1591 }
1592 
1593 /*
1594  * The uber function that deals with any update to bandwidth limits.
1595  */
1596 void
mac_srs_update_bwlimit(flow_entry_t * flent,mac_resource_props_t * mrp)1597 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp)
1598 {
1599 	int			count;
1600 
1601 	for (count = 0; count < flent->fe_rx_srs_cnt; count++)
1602 		mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp);
1603 	mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp);
1604 }
1605 
1606 /*
1607  * When the first sub-flow is added to a link, we disable polling on the
1608  * link and also modify the entry point to mac_rx_srs_subflow_process().
1609  * (polling is disabled because with the subflow added, accounting
1610  * for polling needs additional logic, it is assumed that when a subflow is
1611  * added, we can take some hit as a result of disabling polling rather than
1612  * adding more complexity - if this becomes a perf. issue we need to
1613  * re-rvaluate this logic).  When the last subflow is removed, we turn back
1614  * polling and also reset the entry point to mac_rx_srs_process().
1615  *
1616  * In the future if there are multiple SRS, we can simply
1617  * take one and give it to the flow rather than disabling polling and
1618  * resetting the entry point.
1619  */
1620 void
mac_client_update_classifier(mac_client_impl_t * mcip,boolean_t enable)1621 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable)
1622 {
1623 	flow_entry_t		*flent = mcip->mci_flent;
1624 	int			i;
1625 	mac_impl_t		*mip = mcip->mci_mip;
1626 	mac_rx_func_t		rx_func;
1627 	uint_t			rx_srs_cnt;
1628 	boolean_t		enable_classifier;
1629 
1630 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1631 
1632 	enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable;
1633 
1634 	rx_func = enable_classifier ? mac_rx_srs_subflow_process :
1635 	    mac_rx_srs_process;
1636 
1637 	/* Tell mac_srs_poll_state_change to disable polling if necessary */
1638 	if (mip->mi_state_flags & MIS_POLL_DISABLE)
1639 		enable_classifier = B_TRUE;
1640 
1641 	/*
1642 	 * If receive function has already been configured correctly for
1643 	 * current subflow configuration, do nothing.
1644 	 */
1645 	if (flent->fe_cb_fn == (flow_fn_t)rx_func)
1646 		return;
1647 
1648 	rx_srs_cnt = flent->fe_rx_srs_cnt;
1649 	for (i = 0; i < rx_srs_cnt; i++) {
1650 		ASSERT(flent->fe_rx_srs[i] != NULL);
1651 		mac_srs_poll_state_change(flent->fe_rx_srs[i],
1652 		    enable_classifier, rx_func);
1653 	}
1654 
1655 	/*
1656 	 * Change the S/W classifier so that we can land in the
1657 	 * correct processing function with correct argument.
1658 	 * If all subflows have been removed we can revert to
1659 	 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process().
1660 	 */
1661 	mutex_enter(&flent->fe_lock);
1662 	flent->fe_cb_fn = (flow_fn_t)rx_func;
1663 	flent->fe_cb_arg1 = (void *)mip;
1664 	flent->fe_cb_arg2 = flent->fe_rx_srs[0];
1665 	mutex_exit(&flent->fe_lock);
1666 }
1667 
1668 static void
mac_srs_update_fanout_list(mac_soft_ring_set_t * mac_srs)1669 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs)
1670 {
1671 	int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0;
1672 	mac_soft_ring_t *softring;
1673 
1674 	softring = mac_srs->srs_soft_ring_head;
1675 	if (softring == NULL) {
1676 		ASSERT(mac_srs->srs_soft_ring_count == 0);
1677 		mac_srs->srs_tcp_ring_count = 0;
1678 		mac_srs->srs_udp_ring_count = 0;
1679 		mac_srs->srs_oth_ring_count = 0;
1680 		mac_srs->srs_tx_ring_count = 0;
1681 		return;
1682 	}
1683 
1684 	while (softring != NULL) {
1685 		if (softring->s_ring_type & ST_RING_TCP) {
1686 			mac_srs->srs_tcp_soft_rings[tcp_count++] = softring;
1687 		} else if (softring->s_ring_type & ST_RING_UDP) {
1688 			mac_srs->srs_udp_soft_rings[udp_count++] = softring;
1689 		} else if (softring->s_ring_type & ST_RING_OTH) {
1690 			mac_srs->srs_oth_soft_rings[oth_count++] = softring;
1691 		} else {
1692 			ASSERT(softring->s_ring_type & ST_RING_TX);
1693 			mac_srs->srs_tx_soft_rings[tx_count++] = softring;
1694 		}
1695 		softring = softring->s_ring_next;
1696 	}
1697 
1698 	ASSERT(mac_srs->srs_soft_ring_count ==
1699 	    (tcp_count + udp_count + oth_count + tx_count));
1700 	mac_srs->srs_tcp_ring_count = tcp_count;
1701 	mac_srs->srs_udp_ring_count = udp_count;
1702 	mac_srs->srs_oth_ring_count = oth_count;
1703 	mac_srs->srs_tx_ring_count = tx_count;
1704 }
1705 
1706 void
mac_srs_create_proto_softrings(int id,uint16_t type,pri_t pri,mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs,processorid_t cpuid,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,boolean_t set_bypass)1707 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri,
1708     mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs,
1709     processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1,
1710     mac_resource_handle_t x_arg2, boolean_t set_bypass)
1711 {
1712 	mac_soft_ring_t	*softring;
1713 	mac_rx_fifo_t	mrf;
1714 
1715 	bzero(&mrf, sizeof (mac_rx_fifo_t));
1716 	mrf.mrf_type = MAC_RX_FIFO;
1717 	mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
1718 	mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
1719 	mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
1720 	mrf.mrf_flow_priority = pri;
1721 
1722 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1723 	    (type|ST_RING_TCP), pri, mcip, mac_srs,
1724 	    cpuid, rx_func, x_arg1, x_arg2);
1725 	softring->s_ring_rx_arg2 = NULL;
1726 
1727 	/*
1728 	 * TCP and UDP support DLS bypass. In addition TCP
1729 	 * squeue can also poll their corresponding soft rings.
1730 	 */
1731 	if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1732 		mac_soft_ring_dls_bypass(softring,
1733 		    mcip->mci_direct_rx_fn,
1734 		    mcip->mci_direct_rx_arg);
1735 
1736 		mrf.mrf_rx_arg = softring;
1737 		mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
1738 
1739 		/*
1740 		 * Make a call in IP to get a TCP squeue assigned to
1741 		 * this softring to maintain full CPU locality through
1742 		 * the stack and allow the squeue to be able to poll
1743 		 * the softring so the flow control can be pushed
1744 		 * all the way to H/W.
1745 		 */
1746 		softring->s_ring_rx_arg2 =
1747 		    mcip->mci_resource_add((void *)mcip->mci_resource_arg,
1748 		    (mac_resource_t *)&mrf);
1749 	}
1750 
1751 	/*
1752 	 * Non-TCP protocols don't support squeues. Hence we
1753 	 * don't make any ring addition callbacks for non-TCP
1754 	 * rings. Now create the UDP softring and allow it to
1755 	 * bypass the DLS layer.
1756 	 */
1757 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1758 	    (type|ST_RING_UDP), pri, mcip, mac_srs,
1759 	    cpuid, rx_func, x_arg1, x_arg2);
1760 	softring->s_ring_rx_arg2 = NULL;
1761 
1762 	if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1763 		mac_soft_ring_dls_bypass(softring,
1764 		    mcip->mci_direct_rx_fn,
1765 		    mcip->mci_direct_rx_arg);
1766 	}
1767 
1768 	/* Create the Oth softrings which has to go through the DLS */
1769 	softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
1770 	    (type|ST_RING_OTH), pri, mcip, mac_srs,
1771 	    cpuid, rx_func, x_arg1, x_arg2);
1772 	softring->s_ring_rx_arg2 = NULL;
1773 }
1774 
1775 /*
1776  * This routine associates a CPU or a set of CPU to process incoming
1777  * traffic from a mac client. If multiple CPUs are specified, then
1778  * so many soft rings are created with each soft ring worker thread
1779  * bound to a CPU in the set. Each soft ring in turn will be
1780  * associated with an squeue and the squeue will be moved to the
1781  * same CPU as that of the soft ring's.
1782  */
1783 static void
mac_srs_fanout_modify(mac_client_impl_t * mcip,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs)1784 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func,
1785     void *x_arg1, mac_resource_handle_t x_arg2,
1786     mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs)
1787 {
1788 	mac_soft_ring_t *softring;
1789 	uint32_t soft_ring_flag = 0;
1790 	processorid_t cpuid = -1;
1791 	int i, srings_present, new_fanout_cnt;
1792 	mac_cpus_t *srs_cpu;
1793 
1794 	/* fanout state is REINIT. Set it back to INIT */
1795 	ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT);
1796 	mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1797 
1798 	/* how many are present right now */
1799 	srings_present = mac_rx_srs->srs_tcp_ring_count;
1800 	/* new request */
1801 	srs_cpu = &mac_rx_srs->srs_cpu;
1802 	new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt;
1803 
1804 	mutex_enter(&mac_rx_srs->srs_lock);
1805 	if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1806 		soft_ring_flag |= ST_RING_BW_CTL;
1807 	mutex_exit(&mac_rx_srs->srs_lock);
1808 
1809 	if (new_fanout_cnt > srings_present) {
1810 		/* soft rings increased */
1811 		mutex_enter(&mac_rx_srs->srs_lock);
1812 		mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1813 		mutex_exit(&mac_rx_srs->srs_lock);
1814 
1815 		for (i = mac_rx_srs->srs_tcp_ring_count;
1816 		    i < new_fanout_cnt; i++) {
1817 			/*
1818 			 * Create the protocol softrings and set the
1819 			 * DLS bypass where possible.
1820 			 */
1821 			mac_srs_create_proto_softrings(i, soft_ring_flag,
1822 			    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1823 			    rx_func, x_arg1, x_arg2, B_TRUE);
1824 		}
1825 		mac_srs_update_fanout_list(mac_rx_srs);
1826 	} else if (new_fanout_cnt < srings_present) {
1827 		/* soft rings decreased */
1828 		if (new_fanout_cnt == 1) {
1829 			mutex_enter(&mac_rx_srs->srs_lock);
1830 			mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP;
1831 			ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO);
1832 			mutex_exit(&mac_rx_srs->srs_lock);
1833 		}
1834 		/* Get rid of extra soft rings */
1835 		for (i = new_fanout_cnt;
1836 		    i < mac_rx_srs->srs_tcp_ring_count; i++) {
1837 			softring = mac_rx_srs->srs_tcp_soft_rings[i];
1838 			if (softring->s_ring_rx_arg2 != NULL) {
1839 				mcip->mci_resource_remove(
1840 				    (void *)mcip->mci_resource_arg,
1841 				    softring->s_ring_rx_arg2);
1842 			}
1843 			mac_soft_ring_remove(mac_rx_srs,
1844 			    mac_rx_srs->srs_tcp_soft_rings[i]);
1845 			mac_soft_ring_remove(mac_rx_srs,
1846 			    mac_rx_srs->srs_udp_soft_rings[i]);
1847 			mac_soft_ring_remove(mac_rx_srs,
1848 			    mac_rx_srs->srs_oth_soft_rings[i]);
1849 		}
1850 		mac_srs_update_fanout_list(mac_rx_srs);
1851 	}
1852 
1853 	ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count);
1854 	mutex_enter(&cpu_lock);
1855 	for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) {
1856 		cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1857 		(void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i],
1858 		    cpuid);
1859 		(void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i],
1860 		    cpuid);
1861 		(void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i],
1862 		    cpuid);
1863 		softring = mac_rx_srs->srs_tcp_soft_rings[i];
1864 		if (softring->s_ring_rx_arg2 != NULL) {
1865 			mcip->mci_resource_bind((void *)mcip->mci_resource_arg,
1866 			    softring->s_ring_rx_arg2, cpuid);
1867 		}
1868 	}
1869 
1870 	mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1871 	mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1872 	mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1873 	/*
1874 	 * Bind Tx srs and soft ring threads too. Let's bind tx
1875 	 * srs to the last cpu in mrp list.
1876 	 */
1877 	if (mac_tx_srs != NULL) {
1878 		BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1879 		mac_tx_srs_retarget_intr(mac_tx_srs);
1880 	}
1881 	mutex_exit(&cpu_lock);
1882 }
1883 
1884 /*
1885  * Bind SRS threads and soft rings to CPUs/create fanout list.
1886  */
1887 void
mac_srs_fanout_init(mac_client_impl_t * mcip,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs,cpupart_t * cpupart)1888 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
1889     mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
1890     mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs,
1891     cpupart_t *cpupart)
1892 {
1893 	int		i;
1894 	processorid_t	cpuid;
1895 	uint32_t	soft_ring_flag = 0;
1896 	int soft_ring_cnt;
1897 	mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu;
1898 
1899 	/*
1900 	 * Remove the no soft ring flag and we will adjust it
1901 	 * appropriately further down.
1902 	 */
1903 	mutex_enter(&mac_rx_srs->srs_lock);
1904 	mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS;
1905 	mutex_exit(&mac_rx_srs->srs_lock);
1906 
1907 	ASSERT(mac_rx_srs->srs_soft_ring_head == NULL);
1908 
1909 	if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1910 		soft_ring_flag |= ST_RING_BW_CTL;
1911 
1912 	ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT);
1913 	mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1914 	/*
1915 	 * Ring count can be 0 if no fanout is required and no cpu
1916 	 * were specified. Leave the SRS worker and poll thread
1917 	 * unbound
1918 	 */
1919 	ASSERT(mrp != NULL);
1920 	soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt;
1921 
1922 	/* Step 1: bind cpu contains cpu list where threads need to bind */
1923 	if (soft_ring_cnt > 0) {
1924 		mutex_enter(&cpu_lock);
1925 		for (i = 0; i < soft_ring_cnt; i++) {
1926 			cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1927 			/* Create the protocol softrings */
1928 			mac_srs_create_proto_softrings(i, soft_ring_flag,
1929 			    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1930 			    rx_func, x_arg1, x_arg2, B_FALSE);
1931 		}
1932 		mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
1933 		mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
1934 		mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1935 		/*
1936 		 * Bind Tx srs and soft ring threads too.
1937 		 * Let's bind tx srs to the last cpu in
1938 		 * mrp list.
1939 		 */
1940 		if (mac_tx_srs == NULL) {
1941 			mutex_exit(&cpu_lock);
1942 			goto alldone;
1943 		}
1944 
1945 		BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
1946 		mac_tx_srs_retarget_intr(mac_tx_srs);
1947 		mutex_exit(&cpu_lock);
1948 	} else {
1949 		mutex_enter(&cpu_lock);
1950 		/*
1951 		 * For a subflow, mrp_workerid and mrp_pollid
1952 		 * is not set.
1953 		 */
1954 		mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid);
1955 		mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid);
1956 		mutex_exit(&cpu_lock);
1957 		goto no_softrings;
1958 	}
1959 
1960 alldone:
1961 	if (soft_ring_cnt > 1)
1962 		mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1963 	mac_srs_update_fanout_list(mac_rx_srs);
1964 	mac_srs_client_poll_enable(mcip, mac_rx_srs);
1965 	return;
1966 
1967 no_softrings:
1968 	if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) {
1969 		mutex_enter(&cpu_lock);
1970 		cpuid = mac_next_bind_cpu(cpupart);
1971 		/* Create the protocol softrings */
1972 		mac_srs_create_proto_softrings(0, soft_ring_flag,
1973 		    mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1974 		    rx_func, x_arg1, x_arg2, B_FALSE);
1975 		mutex_exit(&cpu_lock);
1976 	} else {
1977 		/*
1978 		 * This is the case when there is no fanout which is
1979 		 * true for subflows.
1980 		 */
1981 		mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS;
1982 	}
1983 	mac_srs_update_fanout_list(mac_rx_srs);
1984 	mac_srs_client_poll_enable(mcip, mac_rx_srs);
1985 }
1986 
1987 /*
1988  * Calls mac_srs_fanout_init() or modify() depending upon whether
1989  * the SRS is getting initialized or re-initialized.
1990  */
1991 void
mac_fanout_setup(mac_client_impl_t * mcip,flow_entry_t * flent,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,cpupart_t * cpupart)1992 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
1993     mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1,
1994     mac_resource_handle_t x_arg2, cpupart_t *cpupart)
1995 {
1996 	mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs;
1997 	int i, rx_srs_cnt;
1998 
1999 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
2000 
2001 	/*
2002 	 * Aggr ports do not have SRSes. This function should never be
2003 	 * called on an aggr port.
2004 	 */
2005 	ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2006 	mac_rx_srs = flent->fe_rx_srs[0];
2007 
2008 	/*
2009 	 * Set up the fanout on the tx side only once, with the
2010 	 * first rx SRS. The CPU binding, fanout, and bandwidth
2011 	 * criteria are common to both RX and TX, so
2012 	 * initializing them along side avoids redundant code.
2013 	 */
2014 	mac_tx_srs = flent->fe_tx_srs;
2015 	rx_srs_cnt = flent->fe_rx_srs_cnt;
2016 
2017 	/* No fanout for subflows */
2018 	if (flent->fe_type & FLOW_USER) {
2019 		mac_srs_fanout_init(mcip, mrp, rx_func,
2020 		    x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2021 		    cpupart);
2022 		return;
2023 	}
2024 
2025 	if (mrp->mrp_mask & MRP_CPUS_USERSPEC)
2026 		mac_flow_user_cpu_init(flent, mrp);
2027 	else
2028 		mac_flow_cpu_init(flent, cpupart);
2029 
2030 	mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt;
2031 
2032 	/*
2033 	 * Set up fanout for both SW (0th SRS) and HW classified
2034 	 * SRS (the rest of Rx SRSs in flent).
2035 	 */
2036 	for (i = 0; i < rx_srs_cnt; i++) {
2037 		mac_rx_srs = flent->fe_rx_srs[i];
2038 		if (i != 0)
2039 			mac_tx_srs = NULL;
2040 		switch (mac_rx_srs->srs_fanout_state) {
2041 		case SRS_FANOUT_UNINIT:
2042 			mac_srs_fanout_init(mcip, mrp, rx_func,
2043 			    x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
2044 			    cpupart);
2045 			break;
2046 		case SRS_FANOUT_INIT:
2047 			break;
2048 		case SRS_FANOUT_REINIT:
2049 			mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE);
2050 			mac_srs_fanout_modify(mcip, rx_func, x_arg1,
2051 			    x_arg2, mac_rx_srs, mac_tx_srs);
2052 			mac_rx_srs_restart(mac_rx_srs);
2053 			break;
2054 		default:
2055 			VERIFY(mac_rx_srs->srs_fanout_state <=
2056 			    SRS_FANOUT_REINIT);
2057 			break;
2058 		}
2059 	}
2060 }
2061 
2062 /*
2063  * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is
2064  * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side
2065  * processing is created.
2066  *
2067  * Details on Rx SRS:
2068  * Create a SRS and also add the necessary soft rings for TCP and
2069  * non-TCP based on fanout type and count specified.
2070  *
2071  * mac_soft_ring_fanout, mac_srs_fanout_modify (?),
2072  * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need
2073  * to be heavily modified.
2074  *
2075  * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear.
2076  */
2077 mac_soft_ring_set_t *
mac_srs_create(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t srs_type,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_ring_t * ring)2078 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type,
2079     mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
2080     mac_ring_t *ring)
2081 {
2082 	mac_soft_ring_set_t	*mac_srs;
2083 	mac_srs_rx_t		*srs_rx;
2084 	mac_srs_tx_t		*srs_tx;
2085 	mac_bw_ctl_t		*mac_bw;
2086 	mac_resource_props_t	*mrp;
2087 	boolean_t		is_tx_srs = ((srs_type & SRST_TX) != 0);
2088 
2089 	mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP);
2090 	bzero(mac_srs, sizeof (mac_soft_ring_set_t));
2091 	srs_rx = &mac_srs->srs_rx;
2092 	srs_tx = &mac_srs->srs_tx;
2093 
2094 	mutex_enter(&flent->fe_lock);
2095 
2096 	/*
2097 	 * Get the bandwidth control structure from the flent. Get
2098 	 * rid of any residual values in the control structure for
2099 	 * the tx bw struct and also for the rx, if the rx srs is
2100 	 * the 1st one being brought up (the rx bw ctl struct may
2101 	 * be shared by multiple SRSs)
2102 	 */
2103 	if (is_tx_srs) {
2104 		mac_srs->srs_bw = &flent->fe_tx_bw;
2105 		bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2106 		flent->fe_tx_srs = mac_srs;
2107 	} else {
2108 		/*
2109 		 * The bw counter (stored in the flent) is shared
2110 		 * by SRS's within an rx group.
2111 		 */
2112 		mac_srs->srs_bw = &flent->fe_rx_bw;
2113 		/* First rx SRS, clear the bw structure */
2114 		if (flent->fe_rx_srs_cnt == 0)
2115 			bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2116 
2117 		/*
2118 		 * It is better to panic here rather than just assert because
2119 		 * on a non-debug kernel we might end up courrupting memory
2120 		 * and making it difficult to debug.
2121 		 */
2122 		if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) {
2123 			panic("Array Overrun detected due to MAC client %p "
2124 			    " having more rings than %d", (void *)mcip,
2125 			    MAX_RINGS_PER_GROUP);
2126 		}
2127 		flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs;
2128 		flent->fe_rx_srs_cnt++;
2129 	}
2130 	mac_srs->srs_flent = flent;
2131 	mutex_exit(&flent->fe_lock);
2132 
2133 	mac_srs->srs_state = 0;
2134 	mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS);
2135 	mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1;
2136 	mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1;
2137 	mac_srs->srs_mcip = mcip;
2138 	mac_srs_fanout_list_alloc(mac_srs);
2139 
2140 	/*
2141 	 * For a flow we use the underlying MAC client's priority range with
2142 	 * the priority value to find an absolute priority value. For a MAC
2143 	 * client we use the MAC client's maximum priority as the value.
2144 	 */
2145 	mrp = &flent->fe_effective_props;
2146 	if ((mac_srs->srs_type & SRST_FLOW) != 0) {
2147 		mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri,
2148 		    mcip->mci_max_pri, mrp->mrp_priority);
2149 	} else {
2150 		mac_srs->srs_pri = mcip->mci_max_pri;
2151 	}
2152 	/*
2153 	 * We need to insert the SRS in the global list before
2154 	 * binding the SRS and SR threads. Otherwise there is a
2155 	 * is a small window where the cpu reconfig callbacks
2156 	 * may miss the SRS in the list walk and DR could fail
2157 	 * as there are bound threads.
2158 	 */
2159 	mac_srs_add_glist(mac_srs);
2160 
2161 	/* Initialize bw limit */
2162 	if ((mrp->mrp_mask & MRP_MAXBW) != 0) {
2163 		mac_srs->srs_drain_func = mac_rx_srs_drain_bw;
2164 
2165 		mac_bw = mac_srs->srs_bw;
2166 		mutex_enter(&mac_bw->mac_bw_lock);
2167 		mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
2168 
2169 		/*
2170 		 * Give twice the queuing capability before
2171 		 * dropping packets. The unit is bytes/tick.
2172 		 */
2173 		mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1;
2174 		mutex_exit(&mac_bw->mac_bw_lock);
2175 		mac_srs->srs_type |= SRST_BW_CONTROL;
2176 	} else {
2177 		mac_srs->srs_drain_func = mac_rx_srs_drain;
2178 	}
2179 
2180 	/*
2181 	 * We use the following policy to control Receive
2182 	 * Side Dynamic Polling:
2183 	 * 1) We switch to poll mode anytime the processing thread causes
2184 	 *    a backlog to build up in SRS and its associated Soft Rings
2185 	 *    (sr_poll_pkt_cnt > 0).
2186 	 * 2) As long as the backlog stays under the low water mark
2187 	 *    (sr_lowat), we poll the H/W for more packets.
2188 	 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we
2189 	 *    stay in poll mode but don't poll the H/W for more packets.
2190 	 * 4) Anytime in polling mode, if we poll the H/W for packets and
2191 	 *    find nothing plus we have an existing backlog
2192 	 *    (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll
2193 	 *    the H/W for packets anymore (let the polling thread go to sleep).
2194 	 * 5) Once the backlog is relieved (packets are processed) we reenable
2195 	 *    polling (by signalling the poll thread) only when the backlog
2196 	 *    dips below sr_poll_thres.
2197 	 * 6) sr_hiwat is used exclusively when we are not polling capable
2198 	 *    and is used to decide when to drop packets so the SRS queue
2199 	 *    length doesn't grow infinitely.
2200 	 */
2201 	if (!is_tx_srs) {
2202 		srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt;
2203 		/* Low water mark needs to be less than high water mark */
2204 		srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <=
2205 		    mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt :
2206 		    (mac_soft_ring_max_q_cnt >> 2);
2207 		/* Poll threshold need to be half of low water mark or less */
2208 		srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <=
2209 		    (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres :
2210 		    (srs_rx->sr_lowat >> 1);
2211 		if (mac_latency_optimize)
2212 			mac_srs->srs_state |= SRS_LATENCY_OPT;
2213 		else
2214 			mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2215 	}
2216 
2217 	/*
2218 	 * Create the srs_worker with twice the stack of a normal kernel thread
2219 	 * to reduce the likelihood of stack overflows in receive-side
2220 	 * processing.  (The larger stacks are not the only precaution taken
2221 	 * against stack overflows; see the use of mac_rx_srs_stack_needed
2222 	 * in mac_sched.c).
2223 	 */
2224 	mac_srs->srs_worker = thread_create(NULL, default_stksize << 1,
2225 	    mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri);
2226 
2227 	if (is_tx_srs) {
2228 		/* Handle everything about Tx SRS and return */
2229 		mac_srs->srs_drain_func = mac_tx_srs_drain;
2230 		srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt;
2231 		srs_tx->st_hiwat =
2232 		    (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ?
2233 		    mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat;
2234 		srs_tx->st_arg1 = x_arg1;
2235 		srs_tx->st_arg2 = x_arg2;
2236 		goto done;
2237 	}
2238 
2239 	if ((srs_type & SRST_FLOW) != 0 ||
2240 	    FLOW_TAB_EMPTY(mcip->mci_subflow_tab))
2241 		srs_rx->sr_lower_proc = mac_rx_srs_process;
2242 	else
2243 		srs_rx->sr_lower_proc = mac_rx_srs_subflow_process;
2244 
2245 	srs_rx->sr_func = rx_func;
2246 	srs_rx->sr_arg1 = x_arg1;
2247 	srs_rx->sr_arg2 = x_arg2;
2248 
2249 	if (ring != NULL) {
2250 		uint_t ring_info;
2251 
2252 		/* Is the mac_srs created over the RX default group? */
2253 		if (ring->mr_gh == (mac_group_handle_t)
2254 		    MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) {
2255 			mac_srs->srs_type |= SRST_DEFAULT_GRP;
2256 		}
2257 		mac_srs->srs_ring = ring;
2258 		ring->mr_srs = mac_srs;
2259 		ring->mr_classify_type = MAC_HW_CLASSIFIER;
2260 		ring->mr_flag |= MR_INCIPIENT;
2261 
2262 		if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) &&
2263 		    FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable)
2264 			mac_srs->srs_state |= SRS_POLLING_CAPAB;
2265 
2266 		mac_srs->srs_poll_thr = thread_create(NULL, 0,
2267 		    mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN,
2268 		    mac_srs->srs_pri);
2269 		/*
2270 		 * Some drivers require serialization and don't send
2271 		 * packet chains in interrupt context. For such
2272 		 * drivers, we should always queue in the soft ring
2273 		 * so that we get a chance to switch into polling
2274 		 * mode under backlog.
2275 		 */
2276 		ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring);
2277 		if (ring_info & MAC_RING_RX_ENQUEUE)
2278 			mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2279 	}
2280 done:
2281 	mac_srs_stat_create(mac_srs);
2282 	return (mac_srs);
2283 }
2284 
2285 /*
2286  * Figure out the number of soft rings required. Its dependant on
2287  * if protocol fanout is required (for LINKs), global settings
2288  * require us to do fanout for performance (based on mac_soft_ring_enable),
2289  * or user has specifically requested fanout.
2290  */
2291 static uint32_t
mac_find_fanout(flow_entry_t * flent,uint32_t link_type)2292 mac_find_fanout(flow_entry_t *flent, uint32_t link_type)
2293 {
2294 	uint32_t			fanout_type;
2295 	mac_resource_props_t		*mrp = &flent->fe_effective_props;
2296 
2297 	/* no fanout for subflows */
2298 	switch (link_type) {
2299 	case SRST_FLOW:
2300 		fanout_type = SRST_NO_SOFT_RINGS;
2301 		break;
2302 	case SRST_LINK:
2303 		fanout_type = SRST_FANOUT_PROTO;
2304 		break;
2305 	}
2306 
2307 	/* A primary NIC/link is being plumbed */
2308 	if (flent->fe_type & FLOW_PRIMARY_MAC) {
2309 		if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) {
2310 			fanout_type |= SRST_FANOUT_SRC_IP;
2311 		}
2312 	} else if (flent->fe_type & FLOW_VNIC) {
2313 		/* A VNIC is being created */
2314 		if (mrp != NULL && mrp->mrp_ncpus > 0) {
2315 			fanout_type |= SRST_FANOUT_SRC_IP;
2316 		}
2317 	}
2318 
2319 	return (fanout_type);
2320 }
2321 
2322 /*
2323  * Change a group from h/w to s/w classification.
2324  */
2325 void
mac_rx_switch_grp_to_sw(mac_group_t * group)2326 mac_rx_switch_grp_to_sw(mac_group_t *group)
2327 {
2328 	mac_ring_t		*ring;
2329 	mac_soft_ring_set_t	*mac_srs;
2330 
2331 	for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) {
2332 		if (ring->mr_classify_type == MAC_HW_CLASSIFIER) {
2333 			/*
2334 			 * Remove the SRS associated with the HW ring.
2335 			 * As a result, polling will be disabled.
2336 			 */
2337 			mac_srs = ring->mr_srs;
2338 			ASSERT(mac_srs != NULL);
2339 			mac_rx_srs_remove(mac_srs);
2340 			ring->mr_srs = NULL;
2341 		}
2342 
2343 		if (ring->mr_state != MR_INUSE)
2344 			(void) mac_start_ring(ring);
2345 
2346 		/*
2347 		 * We need to perform SW classification
2348 		 * for packets landing in these rings
2349 		 */
2350 		ring->mr_flag = 0;
2351 		ring->mr_classify_type = MAC_SW_CLASSIFIER;
2352 	}
2353 }
2354 
2355 /*
2356  * Create the Rx SRS for S/W classifier and for each ring in the
2357  * group (if exclusive group). Also create the Tx SRS.
2358  */
2359 void
mac_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2360 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2361     uint32_t link_type)
2362 {
2363 	cpupart_t		*cpupart;
2364 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
2365 	mac_resource_props_t	*emrp = MCIP_EFFECTIVE_PROPS(mcip);
2366 	boolean_t		use_default = B_FALSE;
2367 
2368 	mac_rx_srs_group_setup(mcip, flent, link_type);
2369 	mac_tx_srs_group_setup(mcip, flent, link_type);
2370 
2371 	/* Aggr ports don't have SRSes; thus there is no soft ring fanout. */
2372 	if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
2373 		return;
2374 
2375 	pool_lock();
2376 	cpupart = mac_pset_find(mrp, &use_default);
2377 	mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip),
2378 	    mac_rx_deliver, mcip, NULL, cpupart);
2379 	mac_set_pool_effective(use_default, cpupart, mrp, emrp);
2380 	pool_unlock();
2381 }
2382 
2383 /*
2384  * Set up the Rx SRSes. If there is no group associated with the
2385  * client, then only setup SW classification. If the client has
2386  * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an
2387  * SRS for each HW ring. If the client is sharing a group, then make
2388  * sure to teardown the HW SRSes.
2389  */
2390 void
mac_rx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2391 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2392     uint32_t link_type)
2393 {
2394 	mac_impl_t		*mip = mcip->mci_mip;
2395 	mac_soft_ring_set_t	*mac_srs;
2396 	mac_ring_t		*ring;
2397 	uint32_t		fanout_type;
2398 	mac_group_t		*rx_group = flent->fe_rx_ring_group;
2399 	boolean_t		no_unicast;
2400 
2401 	/*
2402 	 * If this is an an aggr port, then don't setup Rx SRS and Rx
2403 	 * soft rings as they won't be used. However, we still need to
2404 	 * start the rings to receive data on them.
2405 	 */
2406 	if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) {
2407 		if (rx_group == NULL)
2408 			return;
2409 
2410 		for (ring = rx_group->mrg_rings; ring != NULL;
2411 		    ring = ring->mr_next) {
2412 			if (ring->mr_state != MR_INUSE)
2413 				(void) mac_start_ring(ring);
2414 		}
2415 
2416 		return;
2417 	}
2418 
2419 	/*
2420 	 * Aggr ports should never have SRSes.
2421 	 */
2422 	ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2423 
2424 	fanout_type = mac_find_fanout(flent, link_type);
2425 	no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0;
2426 
2427 	/* Create the SRS for SW classification if none exists */
2428 	if (flent->fe_rx_srs[0] == NULL) {
2429 		ASSERT(flent->fe_rx_srs_cnt == 0);
2430 		mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type,
2431 		    mac_rx_deliver, mcip, NULL, NULL);
2432 		mutex_enter(&flent->fe_lock);
2433 		flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc;
2434 		flent->fe_cb_arg1 = (void *)mip;
2435 		flent->fe_cb_arg2 = (void *)mac_srs;
2436 		mutex_exit(&flent->fe_lock);
2437 	}
2438 
2439 	if (rx_group == NULL)
2440 		return;
2441 
2442 	/*
2443 	 * If the group is marked RESERVED then setup an SRS and
2444 	 * fanout for each HW ring.
2445 	 */
2446 	switch (rx_group->mrg_state) {
2447 	case MAC_GROUP_STATE_RESERVED:
2448 		for (ring = rx_group->mrg_rings; ring != NULL;
2449 		    ring = ring->mr_next) {
2450 			uint16_t vid = i_mac_flow_vid(mcip->mci_flent);
2451 
2452 			switch (ring->mr_state) {
2453 			case MR_INUSE:
2454 			case MR_FREE:
2455 				if (ring->mr_srs != NULL)
2456 					break;
2457 				if (ring->mr_state != MR_INUSE)
2458 					(void) mac_start_ring(ring);
2459 
2460 				/*
2461 				 * If a client requires SW VLAN
2462 				 * filtering or has no unicast address
2463 				 * then we don't create any HW ring
2464 				 * SRSes.
2465 				 */
2466 				if ((!MAC_GROUP_HW_VLAN(rx_group) &&
2467 				    vid != VLAN_ID_NONE) || no_unicast)
2468 					break;
2469 
2470 				/*
2471 				 * When a client has exclusive use of
2472 				 * a group, and that group's traffic
2473 				 * is fully HW classified, we create
2474 				 * an SRS for each HW ring in order to
2475 				 * make use of dynamic polling of said
2476 				 * HW rings.
2477 				 */
2478 				mac_srs = mac_srs_create(mcip, flent,
2479 				    fanout_type | link_type,
2480 				    mac_rx_deliver, mcip, NULL, ring);
2481 				break;
2482 			default:
2483 				cmn_err(CE_PANIC,
2484 				    "srs_setup: mcip = %p "
2485 				    "trying to add UNKNOWN ring = %p\n",
2486 				    (void *)mcip, (void *)ring);
2487 				break;
2488 			}
2489 		}
2490 		break;
2491 	case MAC_GROUP_STATE_SHARED:
2492 		/*
2493 		 * When a group is shared by multiple clients, we must
2494 		 * use SW classifiction to ensure packets are
2495 		 * delivered to the correct client.
2496 		 */
2497 		mac_rx_switch_grp_to_sw(rx_group);
2498 		break;
2499 	default:
2500 		ASSERT(B_FALSE);
2501 		break;
2502 	}
2503 }
2504 
2505 /*
2506  * Set up the TX SRS.
2507  */
2508 void
mac_tx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2509 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2510     uint32_t link_type)
2511 {
2512 	/*
2513 	 * If this is an exclusive client (e.g. an aggr port), then
2514 	 * don't setup Tx SRS and Tx soft rings as they won't be used.
2515 	 * However, we still need to start the rings to send data
2516 	 * across them.
2517 	 */
2518 	if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
2519 		mac_ring_t		*ring;
2520 		mac_group_t		*grp;
2521 
2522 		grp = (mac_group_t *)flent->fe_tx_ring_group;
2523 
2524 		if (grp == NULL)
2525 			return;
2526 
2527 		for (ring = grp->mrg_rings; ring != NULL;
2528 		    ring = ring->mr_next) {
2529 			if (ring->mr_state != MR_INUSE)
2530 				(void) mac_start_ring(ring);
2531 		}
2532 
2533 		return;
2534 	}
2535 
2536 	/*
2537 	 * Aggr ports should never have SRSes.
2538 	 */
2539 	ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2540 
2541 	if (flent->fe_tx_srs == NULL) {
2542 		(void) mac_srs_create(mcip, flent, SRST_TX | link_type,
2543 		    NULL, mcip, NULL, NULL);
2544 	}
2545 
2546 	mac_tx_srs_setup(mcip, flent);
2547 }
2548 
2549 /*
2550  * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown
2551  * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set
2552  * when we wish to move a MAC client from one group to another. In
2553  * that case, we need to release the current HW SRSes but keep the SW
2554  * SRS for continued traffic classifiction.
2555  */
2556 void
mac_rx_srs_group_teardown(flow_entry_t * flent,boolean_t hwonly)2557 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly)
2558 {
2559 	mac_soft_ring_set_t	*mac_srs;
2560 	int			i;
2561 	int			count = flent->fe_rx_srs_cnt;
2562 
2563 	for (i = 0; i < count; i++) {
2564 		if (i == 0 && hwonly)
2565 			continue;
2566 		mac_srs = flent->fe_rx_srs[i];
2567 		mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED);
2568 		mac_srs_free(mac_srs);
2569 		flent->fe_rx_srs[i] = NULL;
2570 		flent->fe_rx_srs_cnt--;
2571 	}
2572 
2573 	/*
2574 	 * If we are only tearing down the HW SRSes then there must be
2575 	 * one SRS left for SW classification. Otherwise we are tearing
2576 	 * down both HW and SW and there should be no SRSes left.
2577 	 */
2578 	if (hwonly)
2579 		VERIFY3S(flent->fe_rx_srs_cnt, ==, 1);
2580 	else
2581 		VERIFY3S(flent->fe_rx_srs_cnt, ==, 0);
2582 }
2583 
2584 /*
2585  * Remove the TX SRS.
2586  */
2587 void
mac_tx_srs_group_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2588 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
2589     uint32_t link_type)
2590 {
2591 	mac_soft_ring_set_t	*tx_srs;
2592 	mac_srs_tx_t		*tx;
2593 
2594 	if ((tx_srs = flent->fe_tx_srs) == NULL)
2595 		return;
2596 
2597 	tx = &tx_srs->srs_tx;
2598 	switch (link_type) {
2599 	case SRST_FLOW:
2600 		/*
2601 		 * For flows, we need to work with passed
2602 		 * flent to find the Rx/Tx SRS.
2603 		 */
2604 		mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED);
2605 		break;
2606 	case SRST_LINK:
2607 		mac_tx_client_condemn((mac_client_handle_t)mcip);
2608 		if (tx->st_arg2 != NULL) {
2609 			ASSERT(tx_srs->srs_type & SRST_TX);
2610 			/*
2611 			 * The ring itself will be stopped when
2612 			 * we release the group or in the
2613 			 * mac_datapath_teardown (for the default
2614 			 * group)
2615 			 */
2616 			tx->st_arg2 = NULL;
2617 		}
2618 		break;
2619 	default:
2620 		ASSERT(B_FALSE);
2621 		break;
2622 	}
2623 	mac_srs_free(tx_srs);
2624 	flent->fe_tx_srs = NULL;
2625 }
2626 
2627 /*
2628  * This is the group state machine.
2629  *
2630  * The state of an Rx group is given by
2631  * the following table. The default group and its rings are started in
2632  * mac_start itself and the default group stays in SHARED state until
2633  * mac_stop at which time the group and rings are stopped and and it
2634  * reverts to the Registered state.
2635  *
2636  * Typically this function is called on a group after adding or removing a
2637  * client from it, to find out what should be the new state of the group.
2638  * If the new state is RESERVED, then the client that owns this group
2639  * exclusively is also returned. Note that adding or removing a client from
2640  * a group could also impact the default group and the caller needs to
2641  * evaluate the effect on the default group.
2642  *
2643  * Group type		# of clients	mi_nactiveclients	Group State
2644  *			in the group
2645  *
2646  * Non-default		0		N.A.			REGISTERED
2647  * Non-default		1		N.A.			RESERVED
2648  *
2649  * Default		0		N.A.			SHARED
2650  * Default		1		1			RESERVED
2651  * Default		1		> 1			SHARED
2652  * Default		> 1		N.A.			SHARED
2653  *
2654  * For a TX group, the following is the state table.
2655  *
2656  * Group type		# of clients	Group State
2657  *			in the group
2658  *
2659  * Non-default		0		REGISTERED
2660  * Non-default		1		RESERVED
2661  *
2662  * Default		0		REGISTERED
2663  * Default		1		RESERVED
2664  * Default		> 1		SHARED
2665  */
2666 mac_group_state_t
mac_group_next_state(mac_group_t * grp,mac_client_impl_t ** group_only_mcip,mac_group_t * defgrp,boolean_t rx_group)2667 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip,
2668     mac_group_t *defgrp, boolean_t rx_group)
2669 {
2670 	mac_impl_t		*mip = (mac_impl_t *)grp->mrg_mh;
2671 
2672 	*group_only_mcip = NULL;
2673 
2674 	/* Non-default group */
2675 
2676 	if (grp != defgrp) {
2677 		if (MAC_GROUP_NO_CLIENT(grp))
2678 			return (MAC_GROUP_STATE_REGISTERED);
2679 
2680 		*group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2681 		if (*group_only_mcip != NULL)
2682 			return (MAC_GROUP_STATE_RESERVED);
2683 
2684 		return (MAC_GROUP_STATE_SHARED);
2685 	}
2686 
2687 	/* Default group */
2688 
2689 	if (MAC_GROUP_NO_CLIENT(grp)) {
2690 		if (rx_group)
2691 			return (MAC_GROUP_STATE_SHARED);
2692 		else
2693 			return (MAC_GROUP_STATE_REGISTERED);
2694 	}
2695 	*group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2696 	if (*group_only_mcip == NULL)
2697 		return (MAC_GROUP_STATE_SHARED);
2698 
2699 	if (rx_group && mip->mi_nactiveclients != 1)
2700 		return (MAC_GROUP_STATE_SHARED);
2701 
2702 	ASSERT(*group_only_mcip != NULL);
2703 	return (MAC_GROUP_STATE_RESERVED);
2704 }
2705 
2706 /*
2707  * OVERVIEW NOTES FOR DATAPATH
2708  * ===========================
2709  *
2710  * Create an SRS and setup the corresponding flow function and args.
2711  * Add a classification rule for the flow specified by 'flent' and program
2712  * the hardware classifier when applicable.
2713  *
2714  * Rx ring assignment, SRS, polling and B/W enforcement
2715  * ----------------------------------------------------
2716  *
2717  * We try to use H/W classification on NIC and assign traffic to a
2718  * MAC address to a particular Rx ring. There is a 1-1 mapping
2719  * between a SRS and a Rx ring. The SRS (short for soft ring set)
2720  * dynamically switches the underlying Rx ring between interrupt
2721  * and polling mode and enforces any specified B/W control.
2722  *
2723  * There is always a SRS created and tied to each H/W and S/W rule.
2724  * Whenever we create a H/W rule, we always add the the same rule to
2725  * S/W classifier and tie a SRS to it.
2726  *
2727  * In case a B/W control is specified, its broken into bytes
2728  * per ticks and as soon as the quota for a tick is exhausted,
2729  * the underlying Rx ring is forced into poll mode for remianing
2730  * tick. The SRS poll thread only polls for bytes that are
2731  * allowed to come in the SRS. We typically let 4x the configured
2732  * B/W worth of packets to come in the SRS (to prevent unnecessary
2733  * drops due to bursts) but only process the specified amount.
2734  *
2735  * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more
2736  * Rx rings (and corresponding SRSs) assigned to it. The SRS
2737  * in turn can have softrings to do protocol level fanout or
2738  * softrings to do S/W based fanout or both. In case the NIC
2739  * has no Rx rings, we do S/W classification to respective SRS.
2740  * The S/W classification rule is always setup and ready. This
2741  * allows the MAC layer to reassign Rx rings whenever needed
2742  * but packets still continue to flow via the default path and
2743  * getting S/W classified to correct SRS.
2744  *
2745  * In other cases where a NIC or VNIC is plumbed, our goal is use
2746  * H/W classifier and get two Rx ring assigned for the Link. One
2747  * for TCP and one for UDP|SCTP. The respective SRS still do the
2748  * polling on the Rx ring. For Link that is plumbed for IP, there
2749  * is a TCP squeue which also does polling and can control the
2750  * the Rx ring directly (where SRS is just pass through). For
2751  * the following cases, the SRS does the polling underneath.
2752  * 1) non IP based Links (Links which are not plumbed via ifconfig)
2753  *    and paths which have no IP squeues (UDP & SCTP)
2754  * 2) If B/W control is specified on the Link
2755  * 3) If S/W fanout is secified
2756  *
2757  * Note1: As of current implementation, we try to assign only 1 Rx
2758  * ring per Link and more than 1 Rx ring for primary Link for
2759  * H/W based fanout. We always create following softrings per SRS:
2760  * 1) TCP softring which is polled by TCP squeue where possible
2761  *    (and also bypasses DLS)
2762  * 2) UDP/SCTP based which bypasses DLS
2763  * 3) OTH softring which goes via DLS (currently deal with IPv6
2764  *    and non TCP/UDP/SCTP for IPv4 packets).
2765  *
2766  * It is necessary to create 3 softrings since SRS has to poll
2767  * the single Rx ring underneath and enforce any link level B/W
2768  * control (we can't switch the Rx ring in poll mode just based
2769  * on TCP squeue if the same Rx ring is sharing UDP and other
2770  * traffic as well). Once polling is done and any Link level B/W
2771  * control is specified, the packets are assigned to respective
2772  * softring based on protocol. Since TCP has IP based squeue
2773  * which benefits by polling, we separate TCP packets into
2774  * its own softring which can be polled by IP squeue. We need
2775  * to separate out UDP/SCTP to UDP softring since it can bypass
2776  * the DLS layer which has heavy performance advanatges and we
2777  * need a softring (OTH) for rest.
2778  *
2779  * ToDo: The 3 softrings for protocol are needed only till we can
2780  * get rid of DLS from datapath, make IPv4 and IPv6 paths
2781  * symmetric (deal with mac_header_info for v6 and polling for
2782  * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues
2783  * are generic), and bring SAP based classification to MAC layer
2784  *
2785  * H/W and S/W based fanout and multiple Rx rings per Link
2786  * -------------------------------------------------------
2787  *
2788  * In case, fanout is requested (or determined automatically based
2789  * on Link speed and processor speed), we try to assign multiple
2790  * Rx rings per Link with their respective SRS. In this case
2791  * the NIC should be capable of fanning out incoming packets between
2792  * the assigned Rx rings (H/W based fanout). All the SRS
2793  * individually switch their Rx ring between interrupt and polling
2794  * mode but share a common B/W control counter in case of Link
2795  * level B/W is specified.
2796  *
2797  * If S/W based fanout is specified in lieu of H/W based fanout,
2798  * the Link SRS creates the specified number of softrings for
2799  * each protocol (TCP, UDP, OTH). Incoming packets are fanned
2800  * out to the correct softring based on their protocol and
2801  * protocol specific hash function.
2802  *
2803  * Primary and non primary MAC clients
2804  * -----------------------------------
2805  *
2806  * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links
2807  * and are a Layer 2 construct.
2808  *
2809  * Primary NIC:
2810  *	The Link that owns the primary MAC address and typically
2811  *	is used as the data NIC in non virtualized cases. As such
2812  *	H/W resources are preferntially given to primary NIC. As
2813  *	far as code is concerned, there is no difference in the
2814  *	primary NIC vs VNICs. They are all treated as Links.
2815  *	At the very first call to mac_unicast_add() we program the S/W
2816  *	classifier for the primary MAC address, get a soft ring set
2817  *	(and soft rings based on 'ip_soft_ring_cnt')
2818  *	and a Rx ring assigned for polling to get enabled.
2819  *	When IP get plumbed and negotiates polling, we can
2820  *	let squeue do the polling on TCP softring.
2821  *
2822  * VNICs:
2823  *	Same as any other Link. As long as the H/W resource assignments
2824  *	are equal, the data path and setup for all Links is same.
2825  *
2826  * Flows:
2827  *	Can be configured on Links. They have their own SRS and the
2828  *	S/W classifier is programmed appropriately based on the flow.
2829  *	The flows typically deal with layer 3 and above and
2830  *	creates a soft ring set specific to the flow. The receive
2831  *	side function is switched from mac_rx_srs_process to
2832  *	mac_rx_srs_subflow_process which first tries to assign the
2833  *	packet to appropriate flow SRS and failing which assigns it
2834  *	to link SRS. This allows us to avoid the layered approach
2835  *	which gets complex.
2836  *
2837  * By the time mac_datapath_setup() completes, we already have the
2838  * soft rings set, Rx rings, soft rings, etc figured out and both H/W
2839  * and S/W classifiers programmed. IP is not plumbed yet (and might
2840  * never be for Virtual Machines guest OS path). When IP is plumbed
2841  * (for both NIC and VNIC), we do a capability negotiation for polling
2842  * and upcall functions etc.
2843  *
2844  * Rx ring Assignement NOTES
2845  * -------------------------
2846  *
2847  * For NICs which have only 1 Rx ring (we treat  NICs with no Rx rings
2848  * as NIC with a single default ring), we assign the only ring to
2849  * primary Link. The primary Link SRS can do polling on it as long as
2850  * it is the only link in use and we compare the MAC address for unicast
2851  * packets before accepting an incoming packet (there is no need for S/W
2852  * classification in this case). We disable polling on the only ring the
2853  * moment 2nd link gets created (the polling remains enabled even though
2854  * there are broadcast and * multicast flows created).
2855  *
2856  * If the NIC has more than 1 Rx ring, we assign the default ring (the
2857  * 1st ring) to deal with broadcast, multicast and traffic for other
2858  * NICs which needs S/W classification. We assign the primary mac
2859  * addresses to another ring by specifiying a classification rule for
2860  * primary unicast MAC address to the selected ring. The primary Link
2861  * (and its SRS) can continue to poll the assigned Rx ring at all times
2862  * independantly.
2863  *
2864  * Note: In future, if no fanout is specified, we try to assign 2 Rx
2865  * rings for the primary Link with the primary MAC address + TCP going
2866  * to one ring and primary MAC address + UDP|SCTP going to other ring.
2867  * Any remaining traffic for primary MAC address can go to the default
2868  * Rx ring and get S/W classified. This way the respective SRSs don't
2869  * need to do proto fanout and don't need to have softrings at all and
2870  * can poll their respective Rx rings.
2871  *
2872  * As an optimization, when a new NIC or VNIC is created, we can get
2873  * only one Rx ring and make it a TCP specific Rx ring and use the
2874  * H/W default Rx ring for the rest (this Rx ring is never polled).
2875  *
2876  * For clients that don't have MAC address, but want to receive and
2877  * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath.
2878  * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we
2879  * always give the default group and use software classification (i.e.
2880  * even if this is the only client in the default group, we will
2881  * leave group as shared).
2882  */
2883 
2884 int
mac_datapath_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2885 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2886     uint32_t link_type)
2887 {
2888 	mac_impl_t		*mip = mcip->mci_mip;
2889 	mac_group_t		*rgroup = NULL;
2890 	mac_group_t		*tgroup = NULL;
2891 	mac_group_t		*default_rgroup;
2892 	mac_group_t		*default_tgroup;
2893 	int			err;
2894 	uint16_t		vid;
2895 	uint8_t			*mac_addr;
2896 	mac_group_state_t	next_state;
2897 	mac_client_impl_t	*group_only_mcip;
2898 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
2899 	mac_resource_props_t	*emrp = MCIP_EFFECTIVE_PROPS(mcip);
2900 	boolean_t		rxhw;
2901 	boolean_t		txhw;
2902 	boolean_t		use_default = B_FALSE;
2903 	cpupart_t		*cpupart;
2904 	boolean_t		no_unicast;
2905 	boolean_t		isprimary = flent->fe_type & FLOW_PRIMARY_MAC;
2906 	mac_client_impl_t	*reloc_pmcip = NULL;
2907 	boolean_t		use_hw;
2908 
2909 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2910 
2911 	switch (link_type) {
2912 	case SRST_FLOW:
2913 		mac_srs_group_setup(mcip, flent, link_type);
2914 		return (0);
2915 
2916 	case SRST_LINK:
2917 		no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2918 		mac_addr = flent->fe_flow_desc.fd_dst_mac;
2919 
2920 		/* Default RX group */
2921 		default_rgroup = MAC_DEFAULT_RX_GROUP(mip);
2922 
2923 		/* Default TX group */
2924 		default_tgroup = MAC_DEFAULT_TX_GROUP(mip);
2925 
2926 		if (no_unicast) {
2927 			rgroup = default_rgroup;
2928 			tgroup = default_tgroup;
2929 			goto grp_found;
2930 		}
2931 		rxhw = (mrp->mrp_mask & MRP_RX_RINGS) &&
2932 		    (mrp->mrp_nrxrings > 0 ||
2933 		    (mrp->mrp_mask & MRP_RXRINGS_UNSPEC));
2934 		txhw = (mrp->mrp_mask & MRP_TX_RINGS) &&
2935 		    (mrp->mrp_ntxrings > 0 ||
2936 		    (mrp->mrp_mask & MRP_TXRINGS_UNSPEC));
2937 
2938 		/*
2939 		 * All the rings initially belong to the default group
2940 		 * under dynamic grouping. The primary client uses the
2941 		 * default group when it is the only client. The
2942 		 * default group is also used as the destination for
2943 		 * all multicast and broadcast traffic of all clients.
2944 		 * Therefore, the primary client loses its ability to
2945 		 * poll the softrings on addition of a second client.
2946 		 * To avoid a performance penalty, MAC will move the
2947 		 * primary client to a dedicated group when it can.
2948 		 *
2949 		 * When using static grouping, the primary client
2950 		 * begins life on a non-default group. There is
2951 		 * no moving needed upon addition of a second client.
2952 		 */
2953 		if (!isprimary && mip->mi_nactiveclients == 2 &&
2954 		    (group_only_mcip = mac_primary_client_handle(mip)) !=
2955 		    NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
2956 			reloc_pmcip = mac_check_primary_relocation(
2957 			    group_only_mcip, rxhw);
2958 		}
2959 
2960 		/*
2961 		 * Check to see if we can get an exclusive group for
2962 		 * this mac address or if there already exists a
2963 		 * group that has this mac address (case of VLANs).
2964 		 * If no groups are available, use the default group.
2965 		 */
2966 		rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE);
2967 		if (rgroup == NULL && rxhw) {
2968 			err = ENOSPC;
2969 			goto setup_failed;
2970 		} else if (rgroup == NULL) {
2971 			rgroup = default_rgroup;
2972 		}
2973 
2974 		/*
2975 		 * If we are adding a second client to a
2976 		 * non-default group then we need to move the
2977 		 * existing client to the default group and
2978 		 * add the new client to the default group as
2979 		 * well.
2980 		 */
2981 		if (rgroup != default_rgroup &&
2982 		    rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) {
2983 			group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup);
2984 			err = mac_rx_switch_group(group_only_mcip, rgroup,
2985 			    default_rgroup);
2986 
2987 			if (err != 0)
2988 				goto setup_failed;
2989 
2990 			rgroup = default_rgroup;
2991 		}
2992 
2993 		/*
2994 		 * Check to see if we can get an exclusive group for
2995 		 * this mac client. If no groups are available, use
2996 		 * the default group.
2997 		 */
2998 		tgroup = mac_reserve_tx_group(mcip, B_FALSE);
2999 		if (tgroup == NULL && txhw) {
3000 			if (rgroup != NULL && rgroup != default_rgroup)
3001 				mac_release_rx_group(mcip, rgroup);
3002 			err = ENOSPC;
3003 			goto setup_failed;
3004 		} else if (tgroup == NULL) {
3005 			tgroup = default_tgroup;
3006 		}
3007 
3008 		/*
3009 		 * Some NICs don't support any Rx rings, so there may not
3010 		 * even be a default group.
3011 		 */
3012 	grp_found:
3013 		if (rgroup != NULL) {
3014 			if (rgroup != default_rgroup &&
3015 			    MAC_GROUP_NO_CLIENT(rgroup) &&
3016 			    (rxhw || mcip->mci_share != 0)) {
3017 				MAC_RX_GRP_RESERVED(mip);
3018 				if (mip->mi_rx_group_type ==
3019 				    MAC_GROUP_TYPE_DYNAMIC) {
3020 					MAC_RX_RING_RESERVED(mip,
3021 					    rgroup->mrg_cur_count);
3022 				}
3023 			}
3024 
3025 			flent->fe_rx_ring_group = rgroup;
3026 			/*
3027 			 * Add the client to the group and update the
3028 			 * group's state. If rgroup != default_group
3029 			 * then the rgroup should only ever have one
3030 			 * client and be in the RESERVED state. But no
3031 			 * matter what, the default_rgroup will enter
3032 			 * the SHARED state since it has to receive
3033 			 * all broadcast and multicast traffic. This
3034 			 * case is handled later in the function.
3035 			 */
3036 			mac_group_add_client(rgroup, mcip);
3037 			next_state = mac_group_next_state(rgroup,
3038 			    &group_only_mcip, default_rgroup, B_TRUE);
3039 			mac_set_group_state(rgroup, next_state);
3040 		}
3041 
3042 		if (tgroup != NULL) {
3043 			if (tgroup != default_tgroup &&
3044 			    MAC_GROUP_NO_CLIENT(tgroup) &&
3045 			    (txhw || mcip->mci_share != 0)) {
3046 				MAC_TX_GRP_RESERVED(mip);
3047 				if (mip->mi_tx_group_type ==
3048 				    MAC_GROUP_TYPE_DYNAMIC) {
3049 					MAC_TX_RING_RESERVED(mip,
3050 					    tgroup->mrg_cur_count);
3051 				}
3052 			}
3053 			flent->fe_tx_ring_group = tgroup;
3054 			mac_group_add_client(tgroup, mcip);
3055 			next_state = mac_group_next_state(tgroup,
3056 			    &group_only_mcip, default_tgroup, B_FALSE);
3057 			tgroup->mrg_state = next_state;
3058 		}
3059 
3060 		/* We are setting up minimal datapath only */
3061 		if (no_unicast) {
3062 			mac_srs_group_setup(mcip, flent, link_type);
3063 			break;
3064 		}
3065 
3066 		/* Program software classification. */
3067 		if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0)
3068 			goto setup_failed;
3069 
3070 		/* Program hardware classification. */
3071 		vid = i_mac_flow_vid(flent);
3072 		use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0;
3073 		err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw);
3074 
3075 		if (err != 0)
3076 			goto setup_failed;
3077 
3078 		mcip->mci_unicast = mac_find_macaddr(mip, mac_addr);
3079 		VERIFY3P(mcip->mci_unicast, !=, NULL);
3080 
3081 		/*
3082 		 * Setup the Rx and Tx SRSes. If the client has a
3083 		 * reserved group, then mac_srs_group_setup() creates
3084 		 * the required SRSes for the HW rings. If we have a
3085 		 * shared group, mac_srs_group_setup() dismantles the
3086 		 * HW SRSes of the previously exclusive group.
3087 		 */
3088 		mac_srs_group_setup(mcip, flent, link_type);
3089 
3090 		/* (Re)init the v6 token & local addr used by link protection */
3091 		mac_protect_update_mac_token(mcip);
3092 		break;
3093 
3094 	default:
3095 		ASSERT(B_FALSE);
3096 		break;
3097 	}
3098 
3099 	/*
3100 	 * All broadcast and multicast traffic is received only on the default
3101 	 * group. If we have setup the datapath for a non-default group above
3102 	 * then move the default group to shared state to allow distribution of
3103 	 * incoming broadcast traffic to the other groups and dismantle the
3104 	 * SRSes over the default group.
3105 	 */
3106 	if (rgroup != NULL) {
3107 		if (rgroup != default_rgroup) {
3108 			if (default_rgroup->mrg_state ==
3109 			    MAC_GROUP_STATE_RESERVED) {
3110 				group_only_mcip = MAC_GROUP_ONLY_CLIENT(
3111 				    default_rgroup);
3112 				ASSERT(group_only_mcip != NULL &&
3113 				    mip->mi_nactiveclients > 1);
3114 
3115 				mac_set_group_state(default_rgroup,
3116 				    MAC_GROUP_STATE_SHARED);
3117 				mac_rx_srs_group_setup(group_only_mcip,
3118 				    group_only_mcip->mci_flent, SRST_LINK);
3119 				pool_lock();
3120 				cpupart = mac_pset_find(mrp, &use_default);
3121 				mac_fanout_setup(group_only_mcip,
3122 				    group_only_mcip->mci_flent,
3123 				    MCIP_RESOURCE_PROPS(group_only_mcip),
3124 				    mac_rx_deliver, group_only_mcip, NULL,
3125 				    cpupart);
3126 				mac_set_pool_effective(use_default, cpupart,
3127 				    mrp, emrp);
3128 				pool_unlock();
3129 			}
3130 			ASSERT(default_rgroup->mrg_state ==
3131 			    MAC_GROUP_STATE_SHARED);
3132 		}
3133 
3134 		/*
3135 		 * A VLAN MAC client on a reserved group still
3136 		 * requires SW classification if the MAC doesn't
3137 		 * provide VLAN HW filtering.
3138 		 *
3139 		 * Clients with no unicast address also require SW
3140 		 * classification.
3141 		 */
3142 		if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED &&
3143 		    ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) ||
3144 		    no_unicast)) {
3145 			mac_rx_switch_grp_to_sw(rgroup);
3146 		}
3147 
3148 	}
3149 
3150 	mac_set_rings_effective(mcip);
3151 	return (0);
3152 
3153 setup_failed:
3154 	/* Switch the primary back to default group */
3155 	if (reloc_pmcip != NULL) {
3156 		(void) mac_rx_switch_group(reloc_pmcip,
3157 		    reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup);
3158 	}
3159 	mac_datapath_teardown(mcip, flent, link_type);
3160 	return (err);
3161 }
3162 
3163 void
mac_datapath_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)3164 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
3165     uint32_t link_type)
3166 {
3167 	mac_impl_t		*mip = mcip->mci_mip;
3168 	mac_group_t		*group = NULL;
3169 	mac_client_impl_t	*grp_only_mcip;
3170 	flow_entry_t		*group_only_flent;
3171 	mac_group_t		*default_group;
3172 	boolean_t		check_default_group = B_FALSE;
3173 	mac_group_state_t	next_state;
3174 	mac_resource_props_t	*mrp = MCIP_RESOURCE_PROPS(mcip);
3175 	uint16_t		vid;
3176 
3177 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
3178 
3179 	switch (link_type) {
3180 	case SRST_FLOW:
3181 		mac_rx_srs_group_teardown(flent, B_FALSE);
3182 		mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW);
3183 		return;
3184 
3185 	case SRST_LINK:
3186 		/* Stop sending packets */
3187 		mac_tx_client_block(mcip);
3188 		group = flent->fe_rx_ring_group;
3189 		vid = i_mac_flow_vid(flent);
3190 
3191 		/*
3192 		 * Stop the packet flow from the hardware by disabling
3193 		 * any hardware filters assigned to this client.
3194 		 */
3195 		if (mcip->mci_unicast != NULL) {
3196 			int err;
3197 
3198 			err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid);
3199 
3200 			if (err != 0) {
3201 				cmn_err(CE_WARN, "%s: failed to remove a MAC HW"
3202 				    " filters because of error 0x%x",
3203 				    mip->mi_name, err);
3204 			}
3205 
3206 			mcip->mci_unicast = NULL;
3207 		}
3208 
3209 		/* Stop the packets coming from the S/W classifier */
3210 		mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE);
3211 		mac_flow_wait(flent, FLOW_DRIVER_UPCALL);
3212 
3213 		/* Quiesce and destroy all the SRSes. */
3214 		mac_rx_srs_group_teardown(flent, B_FALSE);
3215 		mac_tx_srs_group_teardown(mcip, flent, SRST_LINK);
3216 
3217 		ASSERT3P(mcip->mci_flent, ==, flent);
3218 		ASSERT3P(flent->fe_next, ==, NULL);
3219 
3220 		/*
3221 		 * Release our hold on the group as well. We need
3222 		 * to check if the shared group has only one client
3223 		 * left who can use it exclusively. Also, if we
3224 		 * were the last client, release the group.
3225 		 */
3226 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3227 		if (group != NULL) {
3228 			mac_group_remove_client(group, mcip);
3229 			next_state = mac_group_next_state(group,
3230 			    &grp_only_mcip, default_group, B_TRUE);
3231 
3232 			if (next_state == MAC_GROUP_STATE_RESERVED) {
3233 				/*
3234 				 * Only one client left on this RX group.
3235 				 */
3236 				VERIFY3P(grp_only_mcip, !=, NULL);
3237 				mac_set_group_state(group,
3238 				    MAC_GROUP_STATE_RESERVED);
3239 				group_only_flent = grp_only_mcip->mci_flent;
3240 
3241 				/*
3242 				 * The only remaining client has exclusive
3243 				 * access on the group. Allow it to
3244 				 * dynamically poll the H/W rings etc.
3245 				 */
3246 				mac_rx_srs_group_setup(grp_only_mcip,
3247 				    group_only_flent, SRST_LINK);
3248 				mac_fanout_setup(grp_only_mcip,
3249 				    group_only_flent,
3250 				    MCIP_RESOURCE_PROPS(grp_only_mcip),
3251 				    mac_rx_deliver, grp_only_mcip, NULL, NULL);
3252 				mac_rx_group_unmark(group, MR_INCIPIENT);
3253 				mac_set_rings_effective(grp_only_mcip);
3254 			} else if (next_state == MAC_GROUP_STATE_REGISTERED) {
3255 				/*
3256 				 * This is a non-default group being freed up.
3257 				 * We need to reevaluate the default group
3258 				 * to see if the primary client can get
3259 				 * exclusive access to the default group.
3260 				 */
3261 				VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip));
3262 				if (mrp->mrp_mask & MRP_RX_RINGS) {
3263 					MAC_RX_GRP_RELEASED(mip);
3264 					if (mip->mi_rx_group_type ==
3265 					    MAC_GROUP_TYPE_DYNAMIC) {
3266 						MAC_RX_RING_RELEASED(mip,
3267 						    group->mrg_cur_count);
3268 					}
3269 				}
3270 				mac_release_rx_group(mcip, group);
3271 				mac_set_group_state(group,
3272 				    MAC_GROUP_STATE_REGISTERED);
3273 				check_default_group = B_TRUE;
3274 			} else {
3275 				VERIFY3S(next_state, ==,
3276 				    MAC_GROUP_STATE_SHARED);
3277 				mac_set_group_state(group,
3278 				    MAC_GROUP_STATE_SHARED);
3279 				mac_rx_group_unmark(group, MR_CONDEMNED);
3280 			}
3281 			flent->fe_rx_ring_group = NULL;
3282 		}
3283 		/*
3284 		 * Remove the client from the TX group. Additionally, if
3285 		 * this a non-default group, then we also need to release
3286 		 * the group.
3287 		 */
3288 		group = flent->fe_tx_ring_group;
3289 		default_group = MAC_DEFAULT_TX_GROUP(mip);
3290 		if (group != NULL) {
3291 			mac_group_remove_client(group, mcip);
3292 			next_state = mac_group_next_state(group,
3293 			    &grp_only_mcip, default_group, B_FALSE);
3294 			if (next_state == MAC_GROUP_STATE_REGISTERED) {
3295 				if (group != default_group) {
3296 					if (mrp->mrp_mask & MRP_TX_RINGS) {
3297 						MAC_TX_GRP_RELEASED(mip);
3298 						if (mip->mi_tx_group_type ==
3299 						    MAC_GROUP_TYPE_DYNAMIC) {
3300 							MAC_TX_RING_RELEASED(
3301 							    mip, group->
3302 							    mrg_cur_count);
3303 						}
3304 					}
3305 					mac_release_tx_group(mcip, group);
3306 					/*
3307 					 * If the default group is reserved,
3308 					 * then we need to set the effective
3309 					 * rings as we would have given
3310 					 * back some rings when the group
3311 					 * was released
3312 					 */
3313 					if (mip->mi_tx_group_type ==
3314 					    MAC_GROUP_TYPE_DYNAMIC &&
3315 					    default_group->mrg_state ==
3316 					    MAC_GROUP_STATE_RESERVED) {
3317 						grp_only_mcip =
3318 						    MAC_GROUP_ONLY_CLIENT
3319 						    (default_group);
3320 						mac_set_rings_effective(
3321 						    grp_only_mcip);
3322 					}
3323 				} else {
3324 					mac_ring_t	*ring;
3325 					int		cnt;
3326 					int		ringcnt;
3327 
3328 					/*
3329 					 * Stop all the rings except the
3330 					 * default ring.
3331 					 */
3332 					ringcnt = group->mrg_cur_count;
3333 					ring = group->mrg_rings;
3334 					for (cnt = 0; cnt < ringcnt; cnt++) {
3335 						if (ring->mr_state ==
3336 						    MR_INUSE && ring !=
3337 						    (mac_ring_t *)
3338 						    mip->mi_default_tx_ring) {
3339 							mac_stop_ring(ring);
3340 							ring->mr_flag = 0;
3341 						}
3342 						ring = ring->mr_next;
3343 					}
3344 				}
3345 			} else if (next_state == MAC_GROUP_STATE_RESERVED) {
3346 				mac_set_rings_effective(grp_only_mcip);
3347 			}
3348 			flent->fe_tx_ring_group = NULL;
3349 			group->mrg_state = next_state;
3350 		}
3351 		break;
3352 	default:
3353 		ASSERT(B_FALSE);
3354 		break;
3355 	}
3356 
3357 	/*
3358 	 * The mac client using the default group gets exclusive access to the
3359 	 * default group if and only if it is the sole client on the entire
3360 	 * mip. If so set the group state to reserved, and set up the SRSes
3361 	 * over the default group.
3362 	 */
3363 	if (check_default_group) {
3364 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3365 		VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED);
3366 		next_state = mac_group_next_state(default_group,
3367 		    &grp_only_mcip, default_group, B_TRUE);
3368 		if (next_state == MAC_GROUP_STATE_RESERVED) {
3369 			VERIFY3P(grp_only_mcip, !=, NULL);
3370 			VERIFY3U(mip->mi_nactiveclients, ==, 1);
3371 			mac_set_group_state(default_group,
3372 			    MAC_GROUP_STATE_RESERVED);
3373 			mac_rx_srs_group_setup(grp_only_mcip,
3374 			    grp_only_mcip->mci_flent, SRST_LINK);
3375 			mac_fanout_setup(grp_only_mcip,
3376 			    grp_only_mcip->mci_flent,
3377 			    MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver,
3378 			    grp_only_mcip, NULL, NULL);
3379 			mac_rx_group_unmark(default_group, MR_INCIPIENT);
3380 			mac_set_rings_effective(grp_only_mcip);
3381 		}
3382 	}
3383 
3384 	/*
3385 	 * If the primary is the only one left and the MAC supports
3386 	 * dynamic grouping, we need to see if the primary needs to
3387 	 * be moved to the default group so that it can use all the
3388 	 * H/W rings.
3389 	 */
3390 	if (!(flent->fe_type & FLOW_PRIMARY_MAC) &&
3391 	    mip->mi_nactiveclients == 1 &&
3392 	    mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
3393 		default_group = MAC_DEFAULT_RX_GROUP(mip);
3394 		grp_only_mcip = mac_primary_client_handle(mip);
3395 		if (grp_only_mcip == NULL)
3396 			return;
3397 		group_only_flent = grp_only_mcip->mci_flent;
3398 		mrp = MCIP_RESOURCE_PROPS(grp_only_mcip);
3399 		/*
3400 		 * If the primary has an explicit property set, leave it
3401 		 * alone.
3402 		 */
3403 		if (mrp->mrp_mask & MRP_RX_RINGS)
3404 			return;
3405 		/*
3406 		 * Switch the primary to the default group.
3407 		 */
3408 		(void) mac_rx_switch_group(grp_only_mcip,
3409 		    group_only_flent->fe_rx_ring_group, default_group);
3410 	}
3411 }
3412 
3413 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */
3414 
3415 static void
mac_srs_fanout_list_free(mac_soft_ring_set_t * mac_srs)3416 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs)
3417 {
3418 	if (mac_srs->srs_type & SRST_TX) {
3419 		mac_srs_tx_t *tx;
3420 
3421 		ASSERT(mac_srs->srs_tcp_soft_rings == NULL);
3422 		ASSERT(mac_srs->srs_udp_soft_rings == NULL);
3423 		ASSERT(mac_srs->srs_oth_soft_rings == NULL);
3424 		ASSERT(mac_srs->srs_tx_soft_rings != NULL);
3425 		kmem_free(mac_srs->srs_tx_soft_rings,
3426 		    sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3427 		mac_srs->srs_tx_soft_rings = NULL;
3428 		tx = &mac_srs->srs_tx;
3429 		if (tx->st_soft_rings != NULL) {
3430 			kmem_free(tx->st_soft_rings,
3431 			    sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
3432 		}
3433 	} else {
3434 		ASSERT(mac_srs->srs_tx_soft_rings == NULL);
3435 		ASSERT(mac_srs->srs_tcp_soft_rings != NULL);
3436 		kmem_free(mac_srs->srs_tcp_soft_rings,
3437 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3438 		mac_srs->srs_tcp_soft_rings = NULL;
3439 		ASSERT(mac_srs->srs_udp_soft_rings != NULL);
3440 		kmem_free(mac_srs->srs_udp_soft_rings,
3441 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3442 		mac_srs->srs_udp_soft_rings = NULL;
3443 		ASSERT(mac_srs->srs_oth_soft_rings != NULL);
3444 		kmem_free(mac_srs->srs_oth_soft_rings,
3445 		    sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
3446 		mac_srs->srs_oth_soft_rings = NULL;
3447 	}
3448 }
3449 
3450 /*
3451  * An RX SRS is attached to at most one mac_ring.
3452  * A TX SRS  has no  rings.
3453  */
3454 static void
mac_srs_ring_free(mac_soft_ring_set_t * mac_srs)3455 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs)
3456 {
3457 	mac_client_impl_t	*mcip;
3458 	mac_ring_t		*ring;
3459 	flow_entry_t		*flent;
3460 
3461 	ring = mac_srs->srs_ring;
3462 	if (mac_srs->srs_type & SRST_TX) {
3463 		ASSERT(ring == NULL);
3464 		return;
3465 	}
3466 
3467 	if (ring == NULL)
3468 		return;
3469 
3470 	/*
3471 	 * Broadcast flows don't have a client impl association, but they
3472 	 * use only soft rings.
3473 	 */
3474 	flent = mac_srs->srs_flent;
3475 	mcip = flent->fe_mcip;
3476 	ASSERT(mcip != NULL);
3477 
3478 	ring->mr_classify_type = MAC_NO_CLASSIFIER;
3479 	ring->mr_srs = NULL;
3480 }
3481 
3482 /*
3483  * Physical unlink and free of the data structures happen below. This is
3484  * driven from mac_flow_destroy(), on the last refrele of a flow.
3485  *
3486  * Assumes Rx srs is 1-1 mapped with an ring.
3487  */
3488 void
mac_srs_free(mac_soft_ring_set_t * mac_srs)3489 mac_srs_free(mac_soft_ring_set_t *mac_srs)
3490 {
3491 	ASSERT(mac_srs->srs_mcip == NULL ||
3492 	    MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3493 	ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE |
3494 	    SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE));
3495 
3496 	mac_drop_chain(mac_srs->srs_first, "SRS free");
3497 	mac_srs_ring_free(mac_srs);
3498 	mac_srs_soft_rings_free(mac_srs);
3499 	mac_srs_fanout_list_free(mac_srs);
3500 
3501 	mac_srs->srs_bw = NULL;
3502 	mac_srs_stat_delete(mac_srs);
3503 	kmem_cache_free(mac_srs_cache, mac_srs);
3504 }
3505 
3506 static void
mac_srs_soft_rings_quiesce(mac_soft_ring_set_t * mac_srs,uint_t s_ring_flag)3507 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag)
3508 {
3509 	mac_soft_ring_t	*softring;
3510 
3511 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3512 
3513 	mac_srs_soft_rings_signal(mac_srs, s_ring_flag);
3514 	if (s_ring_flag == S_RING_CONDEMNED) {
3515 		while (mac_srs->srs_soft_ring_condemned_count !=
3516 		    mac_srs->srs_soft_ring_count)
3517 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3518 	} else {
3519 		while (mac_srs->srs_soft_ring_quiesced_count !=
3520 		    mac_srs->srs_soft_ring_count)
3521 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3522 	}
3523 	mutex_exit(&mac_srs->srs_lock);
3524 
3525 	for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3526 	    softring = softring->s_ring_next) {
3527 		(void) untimeout(softring->s_ring_tid);
3528 		softring->s_ring_tid = NULL;
3529 	}
3530 
3531 	(void) untimeout(mac_srs->srs_tid);
3532 	mac_srs->srs_tid = NULL;
3533 
3534 	mutex_enter(&mac_srs->srs_lock);
3535 }
3536 
3537 /*
3538  * The block comment above mac_rx_classify_flow_state_change explains the
3539  * background. At this point upcalls from the driver (both hardware classified
3540  * and software classified) have been cut off. We now need to quiesce the
3541  * SRS worker, poll, and softring threads. The SRS worker thread serves as
3542  * the master controller. The steps involved are described below in the function
3543  */
3544 void
mac_srs_worker_quiesce(mac_soft_ring_set_t * mac_srs)3545 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs)
3546 {
3547 	uint_t			s_ring_flag;
3548 	uint_t			srs_poll_wait_flag;
3549 
3550 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3551 	ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE));
3552 
3553 	if (mac_srs->srs_state & SRS_CONDEMNED) {
3554 		s_ring_flag = S_RING_CONDEMNED;
3555 		srs_poll_wait_flag = SRS_POLL_THR_EXITED;
3556 	} else {
3557 		s_ring_flag = S_RING_QUIESCE;
3558 		srs_poll_wait_flag = SRS_POLL_THR_QUIESCED;
3559 	}
3560 
3561 	/*
3562 	 * In the case of Rx SRS wait till the poll thread is done.
3563 	 */
3564 	if ((mac_srs->srs_type & SRST_TX) == 0 &&
3565 	    mac_srs->srs_poll_thr != NULL) {
3566 		while (!(mac_srs->srs_state & srs_poll_wait_flag))
3567 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3568 
3569 		/*
3570 		 * Turn off polling as part of the quiesce operation.
3571 		 */
3572 		MAC_SRS_POLLING_OFF(mac_srs);
3573 		mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS);
3574 	}
3575 
3576 	/*
3577 	 * Then signal the soft ring worker threads to quiesce or quit
3578 	 * as needed and then wait till that happens.
3579 	 */
3580 	mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag);
3581 
3582 	if (mac_srs->srs_state & SRS_CONDEMNED)
3583 		mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE);
3584 	else
3585 		mac_srs->srs_state |= SRS_QUIESCE_DONE;
3586 	cv_signal(&mac_srs->srs_quiesce_done_cv);
3587 }
3588 
3589 /*
3590  * Signal an SRS to start a temporary quiesce, or permanent removal, or restart
3591  * a quiesced SRS by setting the appropriate flags and signaling the SRS worker
3592  * or poll thread. This function is internal to the quiescing logic and is
3593  * called internally from the SRS quiesce or flow quiesce or client quiesce
3594  * higher level functions.
3595  */
3596 void
mac_srs_signal(mac_soft_ring_set_t * mac_srs,uint_t srs_flag)3597 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag)
3598 {
3599 	mac_ring_t	*ring;
3600 
3601 	ring = mac_srs->srs_ring;
3602 	ASSERT(ring == NULL || ring->mr_refcnt == 0);
3603 
3604 	if (srs_flag == SRS_CONDEMNED) {
3605 		/*
3606 		 * The SRS is going away. We need to unbind the SRS and SR
3607 		 * threads before removing from the global SRS list. Otherwise
3608 		 * there is a small window where the cpu reconfig callbacks
3609 		 * may miss the SRS in the list walk and DR could fail since
3610 		 * there are still bound threads.
3611 		 */
3612 		mac_srs_threads_unbind(mac_srs);
3613 		mac_srs_remove_glist(mac_srs);
3614 	}
3615 	/*
3616 	 * Wakeup the SRS worker and poll threads.
3617 	 */
3618 	mutex_enter(&mac_srs->srs_lock);
3619 	mac_srs->srs_state |= srs_flag;
3620 	cv_signal(&mac_srs->srs_async);
3621 	cv_signal(&mac_srs->srs_cv);
3622 	mutex_exit(&mac_srs->srs_lock);
3623 }
3624 
3625 /*
3626  * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
3627  * from the driver are done, then the Rx SRS is quiesced and only then can
3628  * we signal the soft rings. Thus this function can't be called arbitrarily
3629  * without satisfying the prerequisites. On the Tx side, the threads from
3630  * top need to quiesced, then the Tx SRS and only then can we signal the
3631  * Tx soft rings.
3632  */
3633 static void
mac_srs_soft_rings_signal(mac_soft_ring_set_t * mac_srs,uint_t sr_flag)3634 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag)
3635 {
3636 	mac_soft_ring_t		*softring;
3637 
3638 	for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3639 	    softring = softring->s_ring_next)
3640 		mac_soft_ring_signal(softring, sr_flag);
3641 }
3642 
3643 /*
3644  * The block comment above mac_rx_classify_flow_state_change explains the
3645  * background. At this point the SRS is quiesced and we need to restart the
3646  * SRS worker, poll, and softring threads. The SRS worker thread serves as
3647  * the master controller. The steps involved are described below in the function
3648  */
3649 void
mac_srs_worker_restart(mac_soft_ring_set_t * mac_srs)3650 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs)
3651 {
3652 	boolean_t	iam_rx_srs;
3653 	mac_soft_ring_t	*softring;
3654 
3655 	ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3656 	if ((mac_srs->srs_type & SRST_TX) != 0) {
3657 		iam_rx_srs = B_FALSE;
3658 		ASSERT((mac_srs->srs_state &
3659 		    (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3660 		    (SRS_QUIESCE_DONE | SRS_QUIESCE));
3661 	} else {
3662 		iam_rx_srs = B_TRUE;
3663 		ASSERT((mac_srs->srs_state &
3664 		    (SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3665 		    (SRS_QUIESCE_DONE | SRS_QUIESCE));
3666 		if (mac_srs->srs_poll_thr != NULL) {
3667 			ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) ==
3668 			    SRS_POLL_THR_QUIESCED);
3669 		}
3670 	}
3671 
3672 	/*
3673 	 * Signal any quiesced soft ring workers to restart and wait for the
3674 	 * soft ring down count to come down to zero.
3675 	 */
3676 	if (mac_srs->srs_soft_ring_quiesced_count != 0) {
3677 		for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3678 		    softring = softring->s_ring_next) {
3679 			if (!(softring->s_ring_state & S_RING_QUIESCE))
3680 				continue;
3681 			mac_soft_ring_signal(softring, S_RING_RESTART);
3682 		}
3683 		while (mac_srs->srs_soft_ring_quiesced_count != 0)
3684 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3685 	}
3686 
3687 	mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART);
3688 	if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) {
3689 		/*
3690 		 * Signal the poll thread and ask it to restart. Wait till it
3691 		 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets
3692 		 * cleared.
3693 		 */
3694 		mac_srs->srs_state |= SRS_POLL_THR_RESTART;
3695 		cv_signal(&mac_srs->srs_cv);
3696 		while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED)
3697 			cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3698 		ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART));
3699 	}
3700 	/* Wake up any waiter waiting for the restart to complete */
3701 	mac_srs->srs_state |= SRS_RESTART_DONE;
3702 	cv_signal(&mac_srs->srs_quiesce_done_cv);
3703 }
3704 
3705 static void
mac_srs_worker_unbind(mac_soft_ring_set_t * mac_srs)3706 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs)
3707 {
3708 	mutex_enter(&mac_srs->srs_lock);
3709 	if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) {
3710 		ASSERT(mac_srs->srs_worker_cpuid == -1);
3711 		mutex_exit(&mac_srs->srs_lock);
3712 		return;
3713 	}
3714 
3715 	mac_srs->srs_worker_cpuid = -1;
3716 	mac_srs->srs_state &= ~SRS_WORKER_BOUND;
3717 	thread_affinity_clear(mac_srs->srs_worker);
3718 	mutex_exit(&mac_srs->srs_lock);
3719 }
3720 
3721 static void
mac_srs_poll_unbind(mac_soft_ring_set_t * mac_srs)3722 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs)
3723 {
3724 	mutex_enter(&mac_srs->srs_lock);
3725 	if (mac_srs->srs_poll_thr == NULL ||
3726 	    (mac_srs->srs_state & SRS_POLL_BOUND) == 0) {
3727 		ASSERT(mac_srs->srs_poll_cpuid == -1);
3728 		mutex_exit(&mac_srs->srs_lock);
3729 		return;
3730 	}
3731 
3732 	mac_srs->srs_poll_cpuid = -1;
3733 	mac_srs->srs_state &= ~SRS_POLL_BOUND;
3734 	thread_affinity_clear(mac_srs->srs_poll_thr);
3735 	mutex_exit(&mac_srs->srs_lock);
3736 }
3737 
3738 static void
mac_srs_threads_unbind(mac_soft_ring_set_t * mac_srs)3739 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs)
3740 {
3741 	mac_soft_ring_t	*soft_ring;
3742 
3743 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3744 
3745 	mutex_enter(&cpu_lock);
3746 	mac_srs_worker_unbind(mac_srs);
3747 	if (!(mac_srs->srs_type & SRST_TX))
3748 		mac_srs_poll_unbind(mac_srs);
3749 
3750 	for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3751 	    soft_ring = soft_ring->s_ring_next) {
3752 		mac_soft_ring_unbind(soft_ring);
3753 	}
3754 	mutex_exit(&cpu_lock);
3755 }
3756 
3757 /*
3758  * When a CPU is going away, unbind all MAC threads which are bound
3759  * to that CPU. The affinity of the thread to the CPU is saved to allow
3760  * the thread to be rebound to the CPU if it comes back online.
3761  */
3762 static void
mac_walk_srs_and_unbind(int cpuid)3763 mac_walk_srs_and_unbind(int cpuid)
3764 {
3765 	mac_soft_ring_set_t *mac_srs;
3766 	mac_soft_ring_t *soft_ring;
3767 
3768 	rw_enter(&mac_srs_g_lock, RW_READER);
3769 
3770 	if ((mac_srs = mac_srs_g_list) == NULL)
3771 		goto done;
3772 
3773 	for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
3774 		if (mac_srs->srs_worker_cpuid == cpuid) {
3775 			mac_srs->srs_worker_cpuid_save = cpuid;
3776 			mac_srs_worker_unbind(mac_srs);
3777 		}
3778 
3779 		if (!(mac_srs->srs_type & SRST_TX)) {
3780 			if (mac_srs->srs_poll_cpuid == cpuid) {
3781 				mac_srs->srs_poll_cpuid_save = cpuid;
3782 				mac_srs_poll_unbind(mac_srs);
3783 			}
3784 		}
3785 
3786 		/* Next tackle the soft rings associated with the srs */
3787 		mutex_enter(&mac_srs->srs_lock);
3788 		for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3789 		    soft_ring = soft_ring->s_ring_next) {
3790 			if (soft_ring->s_ring_cpuid == cpuid) {
3791 				soft_ring->s_ring_cpuid_save = cpuid;
3792 				mac_soft_ring_unbind(soft_ring);
3793 			}
3794 		}
3795 		mutex_exit(&mac_srs->srs_lock);
3796 	}
3797 done:
3798 	rw_exit(&mac_srs_g_lock);
3799 }
3800 
3801 /* TX SETUP and TEARDOWN ROUTINES */
3802 
3803 /*
3804  * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring()
3805  * handle the case where the number of rings is one. I.e. there is
3806  * a ring pointed to by mac_srs->srs_tx_arg2.
3807  */
3808 void
mac_tx_srs_add_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3809 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3810 {
3811 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
3812 	mac_soft_ring_t *soft_ring;
3813 	int count = mac_srs->srs_tx_ring_count;
3814 	uint32_t soft_ring_type = ST_RING_TX;
3815 	uint_t ring_info;
3816 
3817 	ASSERT(mac_srs->srs_state & SRS_QUIESCE);
3818 	ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring);
3819 	if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE))
3820 		soft_ring_type |= ST_RING_WORKER_ONLY;
3821 	soft_ring = mac_soft_ring_create(count, 0,
3822 	    soft_ring_type, maxclsyspri, mcip, mac_srs, -1,
3823 	    NULL, mcip, (mac_resource_handle_t)tx_ring);
3824 	mac_srs->srs_tx_ring_count++;
3825 	mac_srs_update_fanout_list(mac_srs);
3826 	/*
3827 	 * put this soft ring in quiesce mode too so when we restart
3828 	 * all soft rings in the srs are in the same state.
3829 	 */
3830 	mac_soft_ring_signal(soft_ring, S_RING_QUIESCE);
3831 }
3832 
3833 static void
mac_soft_ring_remove(mac_soft_ring_set_t * mac_srs,mac_soft_ring_t * softring)3834 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring)
3835 {
3836 	int sringcnt;
3837 
3838 	mutex_enter(&mac_srs->srs_lock);
3839 	sringcnt = mac_srs->srs_soft_ring_count;
3840 	ASSERT(sringcnt > 0);
3841 	mac_soft_ring_signal(softring, S_RING_CONDEMNED);
3842 
3843 	ASSERT(mac_srs->srs_soft_ring_condemned_count == 0);
3844 	while (mac_srs->srs_soft_ring_condemned_count != 1)
3845 		cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3846 
3847 	if (softring == mac_srs->srs_soft_ring_head) {
3848 		mac_srs->srs_soft_ring_head = softring->s_ring_next;
3849 		if (mac_srs->srs_soft_ring_head != NULL) {
3850 			mac_srs->srs_soft_ring_head->s_ring_prev = NULL;
3851 		} else {
3852 			mac_srs->srs_soft_ring_tail = NULL;
3853 		}
3854 	} else {
3855 		softring->s_ring_prev->s_ring_next =
3856 		    softring->s_ring_next;
3857 		if (softring->s_ring_next != NULL) {
3858 			softring->s_ring_next->s_ring_prev =
3859 			    softring->s_ring_prev;
3860 		} else {
3861 			mac_srs->srs_soft_ring_tail =
3862 			    softring->s_ring_prev;
3863 		}
3864 	}
3865 	mac_srs->srs_soft_ring_count--;
3866 
3867 	mac_srs->srs_soft_ring_condemned_count--;
3868 	mutex_exit(&mac_srs->srs_lock);
3869 
3870 	mac_soft_ring_free(softring);
3871 }
3872 
3873 void
mac_tx_srs_del_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3874 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3875 {
3876 	int i;
3877 	mac_soft_ring_t *soft_ring, *remove_sring;
3878 	mac_client_impl_t *mcip = mac_srs->srs_mcip;
3879 
3880 	mutex_enter(&mac_srs->srs_lock);
3881 	for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
3882 		soft_ring =  mac_srs->srs_tx_soft_rings[i];
3883 		if (soft_ring->s_ring_tx_arg2 == tx_ring)
3884 			break;
3885 	}
3886 	mutex_exit(&mac_srs->srs_lock);
3887 	ASSERT(i < mac_srs->srs_tx_ring_count);
3888 	remove_sring = soft_ring;
3889 	/*
3890 	 * In the case of aggr, the soft ring associated with a Tx ring
3891 	 * is also stored in st_soft_rings[] array. That entry should
3892 	 * be removed.
3893 	 */
3894 	if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
3895 		mac_srs_tx_t *tx = &mac_srs->srs_tx;
3896 
3897 		ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring);
3898 		tx->st_soft_rings[tx_ring->mr_index] = NULL;
3899 	}
3900 	mac_soft_ring_remove(mac_srs, remove_sring);
3901 	mac_srs_update_fanout_list(mac_srs);
3902 }
3903 
3904 /*
3905  * mac_tx_srs_setup():
3906  * Used to setup Tx rings. If no free Tx ring is available, then default
3907  * Tx ring is used.
3908  */
3909 void
mac_tx_srs_setup(mac_client_impl_t * mcip,flow_entry_t * flent)3910 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent)
3911 {
3912 	mac_impl_t		*mip = mcip->mci_mip;
3913 	mac_soft_ring_set_t	*tx_srs = flent->fe_tx_srs;
3914 	int			i;
3915 	int			tx_ring_count = 0;
3916 	uint32_t		soft_ring_type;
3917 	mac_group_t		*grp = NULL;
3918 	mac_ring_t		*ring;
3919 	mac_srs_tx_t		*tx = &tx_srs->srs_tx;
3920 	boolean_t		is_aggr;
3921 	uint_t			ring_info = 0;
3922 
3923 	is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0;
3924 	grp = flent->fe_tx_ring_group;
3925 	if (grp == NULL) {
3926 		ring = (mac_ring_t *)mip->mi_default_tx_ring;
3927 		goto no_group;
3928 	}
3929 	tx_ring_count = grp->mrg_cur_count;
3930 	ring = grp->mrg_rings;
3931 	/*
3932 	 * An attempt is made to reserve 'tx_ring_count' number
3933 	 * of Tx rings. If tx_ring_count is 0, default Tx ring
3934 	 * is used. If it is 1, an attempt is made to reserve one
3935 	 * Tx ring. In both the cases, the ring information is
3936 	 * stored in Tx SRS. If multiple Tx rings are specified,
3937 	 * then each Tx ring will have a Tx-side soft ring. All
3938 	 * these soft rings will be hang off Tx SRS.
3939 	 */
3940 	switch (grp->mrg_state) {
3941 		case MAC_GROUP_STATE_SHARED:
3942 		case MAC_GROUP_STATE_RESERVED:
3943 			if (tx_ring_count <= 1 && !is_aggr) {
3944 no_group:
3945 				if (ring != NULL &&
3946 				    ring->mr_state != MR_INUSE) {
3947 					(void) mac_start_ring(ring);
3948 					ring_info = mac_hwring_getinfo(
3949 					    (mac_ring_handle_t)ring);
3950 				}
3951 				tx->st_arg2 = (void *)ring;
3952 				mac_tx_srs_stat_recreate(tx_srs, B_FALSE);
3953 				if (tx_srs->srs_type & SRST_BW_CONTROL) {
3954 					tx->st_mode = SRS_TX_BW;
3955 				} else if (mac_tx_serialize ||
3956 				    (ring_info & MAC_RING_TX_SERIALIZE)) {
3957 					tx->st_mode = SRS_TX_SERIALIZE;
3958 				} else {
3959 					tx->st_mode = SRS_TX_DEFAULT;
3960 				}
3961 				break;
3962 			}
3963 			soft_ring_type = ST_RING_TX;
3964 			if (tx_srs->srs_type & SRST_BW_CONTROL) {
3965 				tx->st_mode = is_aggr ?
3966 				    SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT;
3967 			} else {
3968 				tx->st_mode = is_aggr ? SRS_TX_AGGR :
3969 				    SRS_TX_FANOUT;
3970 			}
3971 			for (i = 0; i < tx_ring_count; i++) {
3972 				ASSERT(ring != NULL);
3973 				switch (ring->mr_state) {
3974 				case MR_INUSE:
3975 				case MR_FREE:
3976 					ASSERT(ring->mr_srs == NULL);
3977 
3978 					if (ring->mr_state != MR_INUSE)
3979 						(void) mac_start_ring(ring);
3980 					ring_info = mac_hwring_getinfo(
3981 					    (mac_ring_handle_t)ring);
3982 					if (mac_tx_serialize || (ring_info &
3983 					    MAC_RING_TX_SERIALIZE)) {
3984 						soft_ring_type |=
3985 						    ST_RING_WORKER_ONLY;
3986 					}
3987 					(void) mac_soft_ring_create(i, 0,
3988 					    soft_ring_type, maxclsyspri,
3989 					    mcip, tx_srs, -1, NULL, mcip,
3990 					    (mac_resource_handle_t)ring);
3991 					break;
3992 				default:
3993 					cmn_err(CE_PANIC,
3994 					    "srs_setup: mcip = %p "
3995 					    "trying to add UNKNOWN ring = %p\n",
3996 					    (void *)mcip, (void *)ring);
3997 					break;
3998 				}
3999 				ring = ring->mr_next;
4000 			}
4001 			mac_srs_update_fanout_list(tx_srs);
4002 			break;
4003 		default:
4004 			ASSERT(B_FALSE);
4005 			break;
4006 	}
4007 	tx->st_func = mac_tx_get_func(tx->st_mode);
4008 	if (is_aggr) {
4009 		VERIFY(i_mac_capab_get((mac_handle_t)mip,
4010 		    MAC_CAPAB_AGGR, &tx->st_capab_aggr));
4011 	}
4012 	DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs,
4013 	    int, tx->st_mode, int, tx_srs->srs_tx_ring_count);
4014 }
4015 
4016 /*
4017  * Update the fanout of a client if its recorded link speed doesn't match
4018  * its current link speed.
4019  */
4020 void
mac_fanout_recompute_client(mac_client_impl_t * mcip,cpupart_t * cpupart)4021 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart)
4022 {
4023 	uint64_t link_speed;
4024 	mac_resource_props_t *mcip_mrp;
4025 	flow_entry_t *flent = mcip->mci_flent;
4026 	mac_soft_ring_set_t *rx_srs;
4027 	mac_cpus_t *srs_cpu;
4028 	int soft_ring_count, maxcpus;
4029 
4030 	ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
4031 
4032 	link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip,
4033 	    MAC_STAT_IFSPEED);
4034 
4035 	if ((link_speed != 0) &&
4036 	    (link_speed != mcip->mci_flent->fe_nic_speed)) {
4037 		mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
4038 		/*
4039 		 * Before calling mac_fanout_setup(), check to see if
4040 		 * the SRSes already have the right number of soft
4041 		 * rings. mac_fanout_setup() is a heavy duty operation
4042 		 * where new cpu bindings are done for SRS and soft
4043 		 * ring threads and interrupts re-targeted.
4044 		 */
4045 		maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
4046 		soft_ring_count = mac_compute_soft_ring_count(flent,
4047 		    flent->fe_rx_srs_cnt - 1, maxcpus);
4048 		/*
4049 		 * If soft_ring_count returned by
4050 		 * mac_compute_soft_ring_count() is 0, bump it
4051 		 * up by 1 because we always have atleast one
4052 		 * TCP, UDP, and OTH soft ring associated with
4053 		 * an SRS.
4054 		 */
4055 		soft_ring_count = (soft_ring_count == 0) ?
4056 		    1 : soft_ring_count;
4057 		rx_srs = flent->fe_rx_srs[0];
4058 		srs_cpu = &rx_srs->srs_cpu;
4059 		if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) {
4060 			mac_fanout_setup(mcip, flent, mcip_mrp,
4061 			    mac_rx_deliver, mcip, NULL, cpupart);
4062 		}
4063 	}
4064 }
4065 
4066 /*
4067  * Walk through the list of MAC clients for the MAC.
4068  * For each active MAC client, recompute the number of soft rings
4069  * associated with every client, only if current speed is different
4070  * from the speed that was previously used for soft ring computation.
4071  * If the cable is disconnected whlie the NIC is started, we would get
4072  * notification with speed set to 0. We do not recompute in that case.
4073  */
4074 void
mac_fanout_recompute(mac_impl_t * mip)4075 mac_fanout_recompute(mac_impl_t *mip)
4076 {
4077 	mac_client_impl_t	*mcip;
4078 	cpupart_t		*cpupart;
4079 	boolean_t		use_default;
4080 	mac_resource_props_t	*mrp, *emrp;
4081 
4082 	i_mac_perim_enter(mip);
4083 	if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 ||
4084 	    mip->mi_linkstate != LINK_STATE_UP) {
4085 		i_mac_perim_exit(mip);
4086 		return;
4087 	}
4088 
4089 	for (mcip = mip->mi_clients_list; mcip != NULL;
4090 	    mcip = mcip->mci_client_next) {
4091 		/* Aggr port clients don't have SRSes. */
4092 		if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
4093 			continue;
4094 
4095 		if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 ||
4096 		    !MCIP_DATAPATH_SETUP(mcip))
4097 			continue;
4098 		mrp = MCIP_RESOURCE_PROPS(mcip);
4099 		emrp = MCIP_EFFECTIVE_PROPS(mcip);
4100 		use_default = B_FALSE;
4101 		pool_lock();
4102 		cpupart = mac_pset_find(mrp, &use_default);
4103 		mac_fanout_recompute_client(mcip, cpupart);
4104 		mac_set_pool_effective(use_default, cpupart, mrp, emrp);
4105 		pool_unlock();
4106 	}
4107 
4108 	i_mac_perim_exit(mip);
4109 }
4110 
4111 /*
4112  * Given a MAC, change the polling state for all its MAC clients.  'enable' is
4113  * B_TRUE to enable polling or B_FALSE to disable.  Polling is enabled by
4114  * default.
4115  */
4116 void
mac_poll_state_change(mac_handle_t mh,boolean_t enable)4117 mac_poll_state_change(mac_handle_t mh, boolean_t enable)
4118 {
4119 	mac_impl_t *mip = (mac_impl_t *)mh;
4120 	mac_client_impl_t *mcip;
4121 
4122 	i_mac_perim_enter(mip);
4123 	if (enable)
4124 		mip->mi_state_flags &= ~MIS_POLL_DISABLE;
4125 	else
4126 		mip->mi_state_flags |= MIS_POLL_DISABLE;
4127 	for (mcip = mip->mi_clients_list; mcip != NULL;
4128 	    mcip = mcip->mci_client_next)
4129 		mac_client_update_classifier(mcip, B_TRUE);
4130 	i_mac_perim_exit(mip);
4131 }
4132