1da14cebeSEric Cheng /*
2da14cebeSEric Cheng * CDDL HEADER START
3da14cebeSEric Cheng *
4da14cebeSEric Cheng * The contents of this file are subject to the terms of the
5da14cebeSEric Cheng * Common Development and Distribution License (the "License").
6da14cebeSEric Cheng * You may not use this file except in compliance with the License.
7da14cebeSEric Cheng *
8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing.
10da14cebeSEric Cheng * See the License for the specific language governing permissions
11da14cebeSEric Cheng * and limitations under the License.
12da14cebeSEric Cheng *
13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each
14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15da14cebeSEric Cheng * If applicable, add the following below this CDDL HEADER, with the
16da14cebeSEric Cheng * fields enclosed by brackets "[]" replaced with your own identifying
17da14cebeSEric Cheng * information: Portions Copyright [yyyy] [name of copyright owner]
18da14cebeSEric Cheng *
19da14cebeSEric Cheng * CDDL HEADER END
20da14cebeSEric Cheng */
21da14cebeSEric Cheng /*
225adf34bdSRajagopal Kunhappan * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
2384de666eSRyan Zezeski * Copyright 2018 Joyent, Inc.
243714f7beSPaul Winder * Copyright 2020 RackTop Systems.
25da14cebeSEric Cheng */
26da14cebeSEric Cheng
27da14cebeSEric Cheng #include <sys/types.h>
28da14cebeSEric Cheng #include <sys/callb.h>
290dc2366fSVenugopal Iyer #include <sys/cpupart.h>
300dc2366fSVenugopal Iyer #include <sys/pool.h>
310dc2366fSVenugopal Iyer #include <sys/pool_pset.h>
32da14cebeSEric Cheng #include <sys/sdt.h>
33da14cebeSEric Cheng #include <sys/strsubr.h>
34da14cebeSEric Cheng #include <sys/strsun.h>
35da14cebeSEric Cheng #include <sys/vlan.h>
36da14cebeSEric Cheng #include <inet/ipsec_impl.h>
37da14cebeSEric Cheng #include <inet/ip_impl.h>
38da14cebeSEric Cheng #include <inet/sadb.h>
39da14cebeSEric Cheng #include <inet/ipsecesp.h>
40da14cebeSEric Cheng #include <inet/ipsecah.h>
41da14cebeSEric Cheng
42da14cebeSEric Cheng #include <sys/mac_impl.h>
43da14cebeSEric Cheng #include <sys/mac_client_impl.h>
44da14cebeSEric Cheng #include <sys/mac_client_priv.h>
45da14cebeSEric Cheng #include <sys/mac_soft_ring.h>
46da14cebeSEric Cheng #include <sys/mac_flow_impl.h>
470dc2366fSVenugopal Iyer #include <sys/mac_stat.h>
48da14cebeSEric Cheng
49da14cebeSEric Cheng static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t);
50da14cebeSEric Cheng static void mac_srs_update_fanout_list(mac_soft_ring_set_t *);
51da14cebeSEric Cheng static void mac_srs_poll_unbind(mac_soft_ring_set_t *);
52da14cebeSEric Cheng static void mac_srs_worker_unbind(mac_soft_ring_set_t *);
53da14cebeSEric Cheng static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t);
54da14cebeSEric Cheng
55da14cebeSEric Cheng static int mac_srs_cpu_setup(cpu_setup_t, int, void *);
56da14cebeSEric Cheng static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t);
57da14cebeSEric Cheng static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t);
58da14cebeSEric Cheng static void mac_srs_threads_unbind(mac_soft_ring_set_t *);
59da14cebeSEric Cheng static void mac_srs_add_glist(mac_soft_ring_set_t *);
60da14cebeSEric Cheng static void mac_srs_remove_glist(mac_soft_ring_set_t *);
61da14cebeSEric Cheng static void mac_srs_fanout_list_free(mac_soft_ring_set_t *);
62da14cebeSEric Cheng static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *);
63da14cebeSEric Cheng
640dc2366fSVenugopal Iyer static int mac_compute_soft_ring_count(flow_entry_t *, int, int);
65da14cebeSEric Cheng static void mac_walk_srs_and_bind(int);
66da14cebeSEric Cheng static void mac_walk_srs_and_unbind(int);
67da14cebeSEric Cheng
68da14cebeSEric Cheng extern boolean_t mac_latency_optimize;
69da14cebeSEric Cheng
70da14cebeSEric Cheng static kmem_cache_t *mac_srs_cache;
71da14cebeSEric Cheng kmem_cache_t *mac_soft_ring_cache;
72da14cebeSEric Cheng
73da14cebeSEric Cheng /*
74da14cebeSEric Cheng * The duration in msec we wait before signalling the soft ring
75da14cebeSEric Cheng * worker thread in case packets get queued.
76da14cebeSEric Cheng */
77ae6aa22aSVenugopal Iyer uint32_t mac_soft_ring_worker_wait = 0;
78ae6aa22aSVenugopal Iyer
79ae6aa22aSVenugopal Iyer /*
80ae6aa22aSVenugopal Iyer * A global tunable for turning polling on/off. By default, dynamic
81ae6aa22aSVenugopal Iyer * polling is always on and is always very beneficial. It should be
82ae6aa22aSVenugopal Iyer * turned off with absolute care and for the rare workload (very
83ae6aa22aSVenugopal Iyer * low latency sensitive traffic).
84ae6aa22aSVenugopal Iyer */
85ae6aa22aSVenugopal Iyer int mac_poll_enable = B_TRUE;
86da14cebeSEric Cheng
87da14cebeSEric Cheng /*
88da14cebeSEric Cheng * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency.
89da14cebeSEric Cheng * Large values could end up in consuming lot of system memory and cause
90da14cebeSEric Cheng * system hang.
91da14cebeSEric Cheng */
92ae6aa22aSVenugopal Iyer int mac_soft_ring_max_q_cnt = 1024;
93ae6aa22aSVenugopal Iyer int mac_soft_ring_min_q_cnt = 256;
94ae6aa22aSVenugopal Iyer int mac_soft_ring_poll_thres = 16;
95da14cebeSEric Cheng
96ae6aa22aSVenugopal Iyer boolean_t mac_tx_serialize = B_FALSE;
97da14cebeSEric Cheng
98da14cebeSEric Cheng /*
99da14cebeSEric Cheng * mac_tx_srs_hiwat is the queue depth threshold at which callers of
100da14cebeSEric Cheng * mac_tx() will be notified of flow control condition.
101da14cebeSEric Cheng *
102da14cebeSEric Cheng * TCP does not honour flow control condition sent up by mac_tx().
103da14cebeSEric Cheng * Thus provision is made for TCP to allow more packets to be queued
104da14cebeSEric Cheng * in SRS upto a maximum of mac_tx_srs_max_q_cnt.
105da14cebeSEric Cheng *
106da14cebeSEric Cheng * Note that mac_tx_srs_hiwat is always be lesser than
107da14cebeSEric Cheng * mac_tx_srs_max_q_cnt.
108da14cebeSEric Cheng */
109ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_max_q_cnt = 100000;
110ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_hiwat = 1000;
111da14cebeSEric Cheng
112da14cebeSEric Cheng /*
113da14cebeSEric Cheng * mac_rx_soft_ring_count, mac_soft_ring_10gig_count:
114da14cebeSEric Cheng *
115da14cebeSEric Cheng * Global tunables that determines the number of soft rings to be used for
116da14cebeSEric Cheng * fanning out incoming traffic on a link. These count will be used only
117da14cebeSEric Cheng * when no explicit set of CPUs was assigned to the data-links.
118da14cebeSEric Cheng *
119da14cebeSEric Cheng * mac_rx_soft_ring_count tunable will come into effect only if
120da14cebeSEric Cheng * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by
121da14cebeSEric Cheng * default only for sun4v platforms.
122da14cebeSEric Cheng *
123da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count will come into effect if you are running on a
124da14cebeSEric Cheng * 10Gbps link and is not dependent upon mac_soft_ring_enable.
125da14cebeSEric Cheng *
126da14cebeSEric Cheng * The number of soft rings for fanout for a link or a flow is determined
127da14cebeSEric Cheng * by mac_compute_soft_ring_count() routine. This routine will take into
128da14cebeSEric Cheng * account mac_soft_ring_enable, mac_rx_soft_ring_count and
129da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link.
130da14cebeSEric Cheng *
131da14cebeSEric Cheng * If a bandwidth is specified, the determination of the number of soft
132da14cebeSEric Cheng * rings is based on specified bandwidth, CPU speed and number of CPUs in
133da14cebeSEric Cheng * the system.
134da14cebeSEric Cheng */
135ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_count = 8;
136ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_10gig_count = 8;
137da14cebeSEric Cheng
138da14cebeSEric Cheng /*
139da14cebeSEric Cheng * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added
140da14cebeSEric Cheng * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The
141da14cebeSEric Cheng * list is used to walk the list of all MAC threads when a CPU is
142da14cebeSEric Cheng * coming online or going offline.
143da14cebeSEric Cheng */
144da14cebeSEric Cheng static mac_soft_ring_set_t *mac_srs_g_list = NULL;
145da14cebeSEric Cheng static krwlock_t mac_srs_g_lock;
146da14cebeSEric Cheng
147da14cebeSEric Cheng /*
148da14cebeSEric Cheng * Whether the SRS threads should be bound, or not.
149da14cebeSEric Cheng */
150ae6aa22aSVenugopal Iyer boolean_t mac_srs_thread_bind = B_TRUE;
151da14cebeSEric Cheng
152da14cebeSEric Cheng /*
1530dc2366fSVenugopal Iyer * Whether Rx/Tx interrupts should be re-targeted. Disabled by default.
1540dc2366fSVenugopal Iyer * dladm command would override this.
155da14cebeSEric Cheng */
1560dc2366fSVenugopal Iyer boolean_t mac_tx_intr_retarget = B_FALSE;
1570dc2366fSVenugopal Iyer boolean_t mac_rx_intr_retarget = B_FALSE;
158da14cebeSEric Cheng
159da14cebeSEric Cheng /*
160da14cebeSEric Cheng * If cpu bindings are specified by user, then Tx SRS and its soft
161da14cebeSEric Cheng * rings should also be bound to the CPUs specified by user. The
162da14cebeSEric Cheng * CPUs for Tx bindings are at the end of the cpu list provided by
163da14cebeSEric Cheng * the user. If enough CPUs are not available (for Tx and Rx
164da14cebeSEric Cheng * SRSes), then the CPUs are shared by both Tx and Rx SRSes.
165da14cebeSEric Cheng */
166da14cebeSEric Cheng #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \
167da14cebeSEric Cheng processorid_t cpuid; \
1680dc2366fSVenugopal Iyer int i; \
169da14cebeSEric Cheng mac_soft_ring_t *softring; \
1700dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; \
171da14cebeSEric Cheng \
1720dc2366fSVenugopal Iyer srs_cpu = &mac_tx_srs->srs_cpu; \
1730dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \
1740dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_tx_srs, cpuid); \
1750dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \
1760dc2366fSVenugopal Iyer for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \
1770dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \
1780dc2366fSVenugopal Iyer softring = mac_tx_srs->srs_tx_soft_rings[i]; \
1790dc2366fSVenugopal Iyer if (cpuid != -1) { \
1800dc2366fSVenugopal Iyer (void) mac_soft_ring_bind(softring, \
1810dc2366fSVenugopal Iyer cpuid); \
1820dc2366fSVenugopal Iyer } \
183da14cebeSEric Cheng } \
184da14cebeSEric Cheng } \
185da14cebeSEric Cheng }
186da14cebeSEric Cheng
1870dc2366fSVenugopal Iyer /*
1880dc2366fSVenugopal Iyer * Re-targeting is allowed only for exclusive group or for primary.
1890dc2366fSVenugopal Iyer */
1900dc2366fSVenugopal Iyer #define RETARGETABLE_CLIENT(group, mcip) \
1910dc2366fSVenugopal Iyer ((((group) != NULL) && \
1920dc2366fSVenugopal Iyer ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \
1930dc2366fSVenugopal Iyer mac_is_primary_client(mcip))
1940dc2366fSVenugopal Iyer
1950dc2366fSVenugopal Iyer #define MAC_RING_RETARGETABLE(ring) \
1960dc2366fSVenugopal Iyer (((ring) != NULL) && \
1970dc2366fSVenugopal Iyer ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \
1980dc2366fSVenugopal Iyer !((ring)->mr_info.mri_intr.mi_ddi_shared))
1990dc2366fSVenugopal Iyer
2000dc2366fSVenugopal Iyer
201da14cebeSEric Cheng /* INIT and FINI ROUTINES */
202da14cebeSEric Cheng
203da14cebeSEric Cheng void
mac_soft_ring_init(void)204da14cebeSEric Cheng mac_soft_ring_init(void)
205da14cebeSEric Cheng {
206da14cebeSEric Cheng mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache",
207da14cebeSEric Cheng sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0);
208da14cebeSEric Cheng
209da14cebeSEric Cheng mac_srs_cache = kmem_cache_create("mac_srs_cache",
210da14cebeSEric Cheng sizeof (mac_soft_ring_set_t),
211da14cebeSEric Cheng 64, NULL, NULL, NULL, NULL, NULL, 0);
212da14cebeSEric Cheng
213da14cebeSEric Cheng rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL);
214da14cebeSEric Cheng mutex_enter(&cpu_lock);
215da14cebeSEric Cheng register_cpu_setup_func(mac_srs_cpu_setup, NULL);
216da14cebeSEric Cheng mutex_exit(&cpu_lock);
217da14cebeSEric Cheng }
218da14cebeSEric Cheng
219da14cebeSEric Cheng void
mac_soft_ring_finish(void)220da14cebeSEric Cheng mac_soft_ring_finish(void)
221da14cebeSEric Cheng {
222da14cebeSEric Cheng mutex_enter(&cpu_lock);
223da14cebeSEric Cheng unregister_cpu_setup_func(mac_srs_cpu_setup, NULL);
224da14cebeSEric Cheng mutex_exit(&cpu_lock);
225da14cebeSEric Cheng rw_destroy(&mac_srs_g_lock);
226da14cebeSEric Cheng kmem_cache_destroy(mac_soft_ring_cache);
227da14cebeSEric Cheng kmem_cache_destroy(mac_srs_cache);
228da14cebeSEric Cheng }
229da14cebeSEric Cheng
230da14cebeSEric Cheng static void
mac_srs_soft_rings_free(mac_soft_ring_set_t * mac_srs)2310dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs)
232da14cebeSEric Cheng {
233da14cebeSEric Cheng mac_soft_ring_t *softring, *next, *head;
234da14cebeSEric Cheng
235da14cebeSEric Cheng /*
236da14cebeSEric Cheng * Synchronize with mac_walk_srs_bind/unbind which are callbacks from
237da14cebeSEric Cheng * DR. The callbacks from DR are called with cpu_lock held, and hence
238da14cebeSEric Cheng * can't wait to grab the mac perimeter. The soft ring list is hence
239da14cebeSEric Cheng * protected for read access by srs_lock. Changing the soft ring list
240da14cebeSEric Cheng * needs the mac perimeter and the srs_lock.
241da14cebeSEric Cheng */
242da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
243da14cebeSEric Cheng
244da14cebeSEric Cheng head = mac_srs->srs_soft_ring_head;
245da14cebeSEric Cheng mac_srs->srs_soft_ring_head = NULL;
246da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL;
247da14cebeSEric Cheng mac_srs->srs_soft_ring_count = 0;
248da14cebeSEric Cheng
249da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
250da14cebeSEric Cheng
251da14cebeSEric Cheng for (softring = head; softring != NULL; softring = next) {
252da14cebeSEric Cheng next = softring->s_ring_next;
2530dc2366fSVenugopal Iyer mac_soft_ring_free(softring);
254da14cebeSEric Cheng }
255da14cebeSEric Cheng }
256da14cebeSEric Cheng
257da14cebeSEric Cheng static void
mac_srs_add_glist(mac_soft_ring_set_t * mac_srs)258da14cebeSEric Cheng mac_srs_add_glist(mac_soft_ring_set_t *mac_srs)
259da14cebeSEric Cheng {
260da14cebeSEric Cheng ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL);
261da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
262da14cebeSEric Cheng
263da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER);
264da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
265da14cebeSEric Cheng
266da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0);
267da14cebeSEric Cheng
268da14cebeSEric Cheng if (mac_srs_g_list == NULL) {
269da14cebeSEric Cheng mac_srs_g_list = mac_srs;
270da14cebeSEric Cheng } else {
271da14cebeSEric Cheng mac_srs->srs_next = mac_srs_g_list;
272da14cebeSEric Cheng mac_srs_g_list->srs_prev = mac_srs;
273da14cebeSEric Cheng mac_srs->srs_prev = NULL;
274da14cebeSEric Cheng mac_srs_g_list = mac_srs;
275da14cebeSEric Cheng }
276da14cebeSEric Cheng mac_srs->srs_state |= SRS_IN_GLIST;
277da14cebeSEric Cheng
278da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
279da14cebeSEric Cheng rw_exit(&mac_srs_g_lock);
280da14cebeSEric Cheng }
281da14cebeSEric Cheng
282da14cebeSEric Cheng static void
mac_srs_remove_glist(mac_soft_ring_set_t * mac_srs)283da14cebeSEric Cheng mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs)
284da14cebeSEric Cheng {
285da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
286da14cebeSEric Cheng
287da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER);
288da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
289da14cebeSEric Cheng
290da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0);
291da14cebeSEric Cheng
292da14cebeSEric Cheng if (mac_srs == mac_srs_g_list) {
293da14cebeSEric Cheng mac_srs_g_list = mac_srs->srs_next;
294da14cebeSEric Cheng if (mac_srs_g_list != NULL)
295da14cebeSEric Cheng mac_srs_g_list->srs_prev = NULL;
296da14cebeSEric Cheng } else {
297da14cebeSEric Cheng mac_srs->srs_prev->srs_next = mac_srs->srs_next;
298da14cebeSEric Cheng if (mac_srs->srs_next != NULL)
299da14cebeSEric Cheng mac_srs->srs_next->srs_prev = mac_srs->srs_prev;
300da14cebeSEric Cheng }
301da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_IN_GLIST;
302da14cebeSEric Cheng
303da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
304da14cebeSEric Cheng rw_exit(&mac_srs_g_lock);
305da14cebeSEric Cheng }
306da14cebeSEric Cheng
307da14cebeSEric Cheng /* POLLING SETUP AND TEAR DOWN ROUTINES */
308da14cebeSEric Cheng
309da14cebeSEric Cheng /*
310da14cebeSEric Cheng * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart
311da14cebeSEric Cheng *
312da14cebeSEric Cheng * These routines are used to call back into the upper layer
313da14cebeSEric Cheng * (primarily TCP squeue) to stop polling the soft rings or
314da14cebeSEric Cheng * restart polling.
315da14cebeSEric Cheng */
316da14cebeSEric Cheng void
mac_srs_client_poll_quiesce(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)317da14cebeSEric Cheng mac_srs_client_poll_quiesce(mac_client_impl_t *mcip,
318da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs)
319da14cebeSEric Cheng {
320da14cebeSEric Cheng mac_soft_ring_t *softring;
321da14cebeSEric Cheng
322da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
323da14cebeSEric Cheng
324da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
325da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
326da14cebeSEric Cheng return;
327da14cebeSEric Cheng }
328da14cebeSEric Cheng
329da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head;
330da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) {
331da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) &&
332da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) {
333da14cebeSEric Cheng mcip->mci_resource_quiesce(mcip->mci_resource_arg,
334da14cebeSEric Cheng softring->s_ring_rx_arg2);
335da14cebeSEric Cheng }
336da14cebeSEric Cheng }
337da14cebeSEric Cheng }
338da14cebeSEric Cheng
339da14cebeSEric Cheng void
mac_srs_client_poll_restart(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)340da14cebeSEric Cheng mac_srs_client_poll_restart(mac_client_impl_t *mcip,
341da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs)
342da14cebeSEric Cheng {
343da14cebeSEric Cheng mac_soft_ring_t *softring;
344da14cebeSEric Cheng
345da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
346da14cebeSEric Cheng
347da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
348da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
349da14cebeSEric Cheng return;
350da14cebeSEric Cheng }
351da14cebeSEric Cheng
352da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head;
353da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) {
354da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) &&
355da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) {
356da14cebeSEric Cheng mcip->mci_resource_restart(mcip->mci_resource_arg,
357da14cebeSEric Cheng softring->s_ring_rx_arg2);
358da14cebeSEric Cheng }
359da14cebeSEric Cheng }
360da14cebeSEric Cheng }
361da14cebeSEric Cheng
362da14cebeSEric Cheng /*
363da14cebeSEric Cheng * Register the given SRS and associated soft rings with the consumer and
364da14cebeSEric Cheng * enable the polling interface used by the consumer.(i.e IP) over this
365da14cebeSEric Cheng * SRS and associated soft rings.
366da14cebeSEric Cheng */
367da14cebeSEric Cheng void
mac_srs_client_poll_enable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)368da14cebeSEric Cheng mac_srs_client_poll_enable(mac_client_impl_t *mcip,
369da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs)
370da14cebeSEric Cheng {
371da14cebeSEric Cheng mac_rx_fifo_t mrf;
372da14cebeSEric Cheng mac_soft_ring_t *softring;
373da14cebeSEric Cheng
374da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == mcip);
375da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
376da14cebeSEric Cheng
377da14cebeSEric Cheng if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE))
378da14cebeSEric Cheng return;
379da14cebeSEric Cheng
380da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t));
381da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO;
382da14cebeSEric Cheng
383da14cebeSEric Cheng /*
384da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases
385da14cebeSEric Cheng * where no fanout is needed. This is the case for userland
386da14cebeSEric Cheng * flows.
387da14cebeSEric Cheng */
388da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
389da14cebeSEric Cheng return;
390da14cebeSEric Cheng
391da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
392da14cebeSEric Cheng mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
393da14cebeSEric Cheng mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
394da14cebeSEric Cheng mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED;
395da14cebeSEric Cheng
396da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head;
397da14cebeSEric Cheng while (softring != NULL) {
398da14cebeSEric Cheng if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) {
399da14cebeSEric Cheng /*
400da14cebeSEric Cheng * TCP and UDP support DLS bypass. Squeue polling
401da14cebeSEric Cheng * support implies DLS bypass since the squeue poll
402da14cebeSEric Cheng * path does not have DLS processing.
403da14cebeSEric Cheng */
404da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring,
405da14cebeSEric Cheng mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg);
406da14cebeSEric Cheng }
407da14cebeSEric Cheng /*
408da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we don't
409da14cebeSEric Cheng * make any ring addition callbacks for non-TCP rings
410da14cebeSEric Cheng */
411da14cebeSEric Cheng if (!(softring->s_ring_type & ST_RING_TCP)) {
412da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL;
413da14cebeSEric Cheng softring = softring->s_ring_next;
414da14cebeSEric Cheng continue;
415da14cebeSEric Cheng }
416da14cebeSEric Cheng mrf.mrf_rx_arg = softring;
417da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
418da14cebeSEric Cheng mrf.mrf_cpu_id = softring->s_ring_cpuid;
419da14cebeSEric Cheng mrf.mrf_flow_priority = mac_srs->srs_pri;
420da14cebeSEric Cheng
421da14cebeSEric Cheng softring->s_ring_rx_arg2 = mcip->mci_resource_add(
422da14cebeSEric Cheng mcip->mci_resource_arg, (mac_resource_t *)&mrf);
423da14cebeSEric Cheng
424da14cebeSEric Cheng softring = softring->s_ring_next;
425da14cebeSEric Cheng }
426da14cebeSEric Cheng }
427da14cebeSEric Cheng
428da14cebeSEric Cheng /*
429da14cebeSEric Cheng * Unregister the given SRS and associated soft rings with the consumer and
430da14cebeSEric Cheng * disable the polling interface used by the consumer.(i.e IP) over this
431da14cebeSEric Cheng * SRS and associated soft rings.
432da14cebeSEric Cheng */
433da14cebeSEric Cheng void
mac_srs_client_poll_disable(mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs)434da14cebeSEric Cheng mac_srs_client_poll_disable(mac_client_impl_t *mcip,
435da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs)
436da14cebeSEric Cheng {
437da14cebeSEric Cheng mac_soft_ring_t *softring;
438da14cebeSEric Cheng
439da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
440da14cebeSEric Cheng
441da14cebeSEric Cheng /*
442da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases
443da14cebeSEric Cheng * where no protocol fanout is needed. This is the case
444da14cebeSEric Cheng * for userland flows. Nothing to do here.
445da14cebeSEric Cheng */
446da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS)
447da14cebeSEric Cheng return;
448da14cebeSEric Cheng
449da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
450da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) {
451da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS));
452da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
453da14cebeSEric Cheng return;
454da14cebeSEric Cheng }
455da14cebeSEric Cheng mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS);
456da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
457da14cebeSEric Cheng
458da14cebeSEric Cheng /*
459da14cebeSEric Cheng * DLS bypass is now disabled in the case of both TCP and UDP.
460da14cebeSEric Cheng * Reset the soft ring callbacks to the standard 'mac_rx_deliver'
461da14cebeSEric Cheng * callback. In addition, in the case of TCP, invoke IP's callback
462da14cebeSEric Cheng * for ring removal.
463da14cebeSEric Cheng */
464da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head;
465da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) {
466da14cebeSEric Cheng if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP)))
467da14cebeSEric Cheng continue;
468da14cebeSEric Cheng
469da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) &&
470da14cebeSEric Cheng softring->s_ring_rx_arg2 != NULL) {
471da14cebeSEric Cheng mcip->mci_resource_remove(mcip->mci_resource_arg,
472da14cebeSEric Cheng softring->s_ring_rx_arg2);
473da14cebeSEric Cheng }
474da14cebeSEric Cheng
475da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock);
476da14cebeSEric Cheng while (softring->s_ring_state & S_RING_PROC) {
477da14cebeSEric Cheng softring->s_ring_state |= S_RING_CLIENT_WAIT;
478da14cebeSEric Cheng cv_wait(&softring->s_ring_client_cv,
479da14cebeSEric Cheng &softring->s_ring_lock);
480da14cebeSEric Cheng }
481da14cebeSEric Cheng softring->s_ring_state &= ~S_RING_CLIENT_WAIT;
482da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL;
483da14cebeSEric Cheng softring->s_ring_rx_func = mac_rx_deliver;
484da14cebeSEric Cheng softring->s_ring_rx_arg1 = mcip;
485da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock);
486da14cebeSEric Cheng }
487da14cebeSEric Cheng }
488da14cebeSEric Cheng
489da14cebeSEric Cheng /*
490da14cebeSEric Cheng * Enable or disable poll capability of the SRS on the underlying Rx ring.
491da14cebeSEric Cheng *
492da14cebeSEric Cheng * There is a need to enable or disable the poll capability of an SRS over an
493da14cebeSEric Cheng * Rx ring depending on the number of mac clients sharing the ring and also
494da14cebeSEric Cheng * whether user flows are configured on it. However the poll state is actively
495da14cebeSEric Cheng * manipulated by the SRS worker and poll threads and uncoordinated changes by
496da14cebeSEric Cheng * yet another thread to the underlying capability can surprise them leading
497da14cebeSEric Cheng * to assert failures. Instead we quiesce the SRS, make the changes and then
498da14cebeSEric Cheng * restart the SRS.
499da14cebeSEric Cheng */
500da14cebeSEric Cheng static void
mac_srs_poll_state_change(mac_soft_ring_set_t * mac_srs,boolean_t turn_off_poll_capab,mac_rx_func_t rx_func)501da14cebeSEric Cheng mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs,
502da14cebeSEric Cheng boolean_t turn_off_poll_capab, mac_rx_func_t rx_func)
503da14cebeSEric Cheng {
504da14cebeSEric Cheng boolean_t need_restart = B_FALSE;
505da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx;
506da14cebeSEric Cheng mac_ring_t *ring;
507da14cebeSEric Cheng
508da14cebeSEric Cheng if (!SRS_QUIESCED(mac_srs)) {
509da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE);
510da14cebeSEric Cheng need_restart = B_TRUE;
511da14cebeSEric Cheng }
512da14cebeSEric Cheng
513da14cebeSEric Cheng ring = mac_srs->srs_ring;
514da14cebeSEric Cheng if ((ring != NULL) &&
515da14cebeSEric Cheng (ring->mr_classify_type == MAC_HW_CLASSIFIER)) {
516da14cebeSEric Cheng if (turn_off_poll_capab)
517da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLLING_CAPAB;
518ae6aa22aSVenugopal Iyer else if (mac_poll_enable)
519da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB;
520da14cebeSEric Cheng }
521da14cebeSEric Cheng srs_rx->sr_lower_proc = rx_func;
522da14cebeSEric Cheng
523da14cebeSEric Cheng if (need_restart)
524da14cebeSEric Cheng mac_rx_srs_restart(mac_srs);
525da14cebeSEric Cheng }
526da14cebeSEric Cheng
527da14cebeSEric Cheng /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */
528da14cebeSEric Cheng
529da14cebeSEric Cheng /*
530da14cebeSEric Cheng * Return the next CPU to be used to bind a MAC kernel thread.
5310dc2366fSVenugopal Iyer * If a cpupart is specified, the cpu chosen must be from that
5320dc2366fSVenugopal Iyer * cpu partition.
533da14cebeSEric Cheng */
534da14cebeSEric Cheng static processorid_t
mac_next_bind_cpu(cpupart_t * cpupart)5350dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart_t *cpupart)
536da14cebeSEric Cheng {
5370dc2366fSVenugopal Iyer static cpu_t *cp = NULL;
5380dc2366fSVenugopal Iyer cpu_t *cp_start;
539da14cebeSEric Cheng
540da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock));
541da14cebeSEric Cheng
5420dc2366fSVenugopal Iyer if (cp == NULL)
5430dc2366fSVenugopal Iyer cp = cpu_list;
5440dc2366fSVenugopal Iyer
5450dc2366fSVenugopal Iyer cp = cp->cpu_next_onln;
5460dc2366fSVenugopal Iyer cp_start = cp;
5470dc2366fSVenugopal Iyer
5480dc2366fSVenugopal Iyer do {
5490dc2366fSVenugopal Iyer if ((cpupart == NULL) || (cp->cpu_part == cpupart))
5500dc2366fSVenugopal Iyer return (cp->cpu_id);
551da14cebeSEric Cheng
5520dc2366fSVenugopal Iyer } while ((cp = cp->cpu_next_onln) != cp_start);
5530dc2366fSVenugopal Iyer
55436f99a58SToomas Soome return (-1); /* No matching CPU found online */
555da14cebeSEric Cheng }
556da14cebeSEric Cheng
557da14cebeSEric Cheng /* ARGSUSED */
558da14cebeSEric Cheng static int
mac_srs_cpu_setup(cpu_setup_t what,int id,void * arg)559da14cebeSEric Cheng mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg)
560da14cebeSEric Cheng {
561da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock));
562da14cebeSEric Cheng switch (what) {
563da14cebeSEric Cheng case CPU_CONFIG:
564da14cebeSEric Cheng case CPU_ON:
565da14cebeSEric Cheng case CPU_CPUPART_IN:
566da14cebeSEric Cheng mac_walk_srs_and_bind(id);
567da14cebeSEric Cheng break;
568da14cebeSEric Cheng
569da14cebeSEric Cheng case CPU_UNCONFIG:
570da14cebeSEric Cheng case CPU_OFF:
571da14cebeSEric Cheng case CPU_CPUPART_OUT:
572da14cebeSEric Cheng mac_walk_srs_and_unbind(id);
573da14cebeSEric Cheng break;
574da14cebeSEric Cheng
575da14cebeSEric Cheng default:
576da14cebeSEric Cheng break;
577da14cebeSEric Cheng }
578da14cebeSEric Cheng return (0);
579da14cebeSEric Cheng }
580da14cebeSEric Cheng
581da14cebeSEric Cheng /*
582da14cebeSEric Cheng * mac_compute_soft_ring_count():
583da14cebeSEric Cheng *
584da14cebeSEric Cheng * This routine computes the number of soft rings needed to handle incoming
585da14cebeSEric Cheng * load given a flow_entry.
586da14cebeSEric Cheng *
587da14cebeSEric Cheng * The routine does the following:
588da14cebeSEric Cheng * 1) soft rings will be created if mac_soft_ring_enable is set.
589da14cebeSEric Cheng * 2) If the underlying link is a 10Gbps link, then soft rings will be
590da14cebeSEric Cheng * created even if mac_soft_ring_enable is not set. The number of soft
591da14cebeSEric Cheng * rings, so created, will equal mac_rx_soft_ring_10gig_count.
592da14cebeSEric Cheng * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the
593da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count number of soft rings will be created for a
594da14cebeSEric Cheng * 10Gbps link.
595da14cebeSEric Cheng *
596da14cebeSEric Cheng * If a bandwidth limit is specified, the number that gets computed is
597da14cebeSEric Cheng * dependent upon CPU speed, the number of Rx rings configured, and
598da14cebeSEric Cheng * the bandwidth limit.
599da14cebeSEric Cheng * If more Rx rings are available, less number of soft rings is needed.
600da14cebeSEric Cheng *
601da14cebeSEric Cheng * mac_use_bw_heuristic is another "hidden" variable that can be used to
602da14cebeSEric Cheng * override the default use of soft ring count computation. Depending upon
603da14cebeSEric Cheng * the usefulness of it, mac_use_bw_heuristic can later be made into a
604da14cebeSEric Cheng * data-link property or removed altogether.
605da14cebeSEric Cheng *
606da14cebeSEric Cheng * TODO: Cleanup and tighten some of the assumptions.
607da14cebeSEric Cheng */
608*36589d6bSRobert Mustacchi boolean_t mac_check_overlay = B_TRUE;
609da14cebeSEric Cheng boolean_t mac_use_bw_heuristic = B_TRUE;
610da14cebeSEric Cheng static int
mac_compute_soft_ring_count(flow_entry_t * flent,int rx_srs_cnt,int maxcpus)6110dc2366fSVenugopal Iyer mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus)
612da14cebeSEric Cheng {
613da14cebeSEric Cheng uint64_t cpu_speed, bw = 0;
614da14cebeSEric Cheng int srings = 0;
615da14cebeSEric Cheng boolean_t bw_enabled = B_FALSE;
616*36589d6bSRobert Mustacchi mac_client_impl_t *mcip = flent->fe_mcip;
617da14cebeSEric Cheng
618da14cebeSEric Cheng ASSERT(!(flent->fe_type & FLOW_USER));
619da14cebeSEric Cheng if (flent->fe_resource_props.mrp_mask & MRP_MAXBW &&
620da14cebeSEric Cheng mac_use_bw_heuristic) {
621da14cebeSEric Cheng /* bandwidth enabled */
622da14cebeSEric Cheng bw_enabled = B_TRUE;
623da14cebeSEric Cheng bw = flent->fe_resource_props.mrp_maxbw;
624da14cebeSEric Cheng }
625da14cebeSEric Cheng if (!bw_enabled) {
626da14cebeSEric Cheng /* No bandwidth enabled */
627da14cebeSEric Cheng if (mac_soft_ring_enable)
628da14cebeSEric Cheng srings = mac_rx_soft_ring_count;
629da14cebeSEric Cheng
630da14cebeSEric Cheng /* Is this a 10Gig link? */
631da14cebeSEric Cheng flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip,
632da14cebeSEric Cheng MAC_STAT_IFSPEED);
633da14cebeSEric Cheng /* convert to Mbps */
634da14cebeSEric Cheng if (((flent->fe_nic_speed)/1000000) > 1000 &&
635da14cebeSEric Cheng mac_rx_soft_ring_10gig_count > 0) {
636da14cebeSEric Cheng /* This is a 10Gig link */
637da14cebeSEric Cheng srings = mac_rx_soft_ring_10gig_count;
638da14cebeSEric Cheng /*
639da14cebeSEric Cheng * Use 2 times mac_rx_soft_ring_10gig_count for
640da14cebeSEric Cheng * sun4v systems.
641da14cebeSEric Cheng */
642da14cebeSEric Cheng if (mac_soft_ring_enable)
643da14cebeSEric Cheng srings = srings * 2;
644*36589d6bSRobert Mustacchi } else if (mac_check_overlay == B_TRUE &&
645*36589d6bSRobert Mustacchi (mcip->mci_state_flags & MCIS_IS_VNIC) != 0) {
646*36589d6bSRobert Mustacchi /* Is this a VNIC on an overlay? */
647*36589d6bSRobert Mustacchi mac_handle_t mh = (mac_handle_t)mcip->mci_mip;
648*36589d6bSRobert Mustacchi if (mac_is_overlay(mh) == B_TRUE) {
649*36589d6bSRobert Mustacchi srings = mac_rx_soft_ring_10gig_count;
650*36589d6bSRobert Mustacchi }
651da14cebeSEric Cheng }
652*36589d6bSRobert Mustacchi
653*36589d6bSRobert Mustacchi
654da14cebeSEric Cheng } else {
655da14cebeSEric Cheng /*
656da14cebeSEric Cheng * Soft ring computation using CPU speed and specified
657da14cebeSEric Cheng * bandwidth limit.
658da14cebeSEric Cheng */
659da14cebeSEric Cheng /* Assumption: all CPUs have the same frequency */
660da14cebeSEric Cheng cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock;
661da14cebeSEric Cheng
662da14cebeSEric Cheng /* cpu_speed is in MHz; make bw in units of Mbps. */
663da14cebeSEric Cheng bw = bw/1000000;
664da14cebeSEric Cheng
665da14cebeSEric Cheng if (bw >= 1000) {
666da14cebeSEric Cheng /*
667da14cebeSEric Cheng * bw is greater than or equal to 1Gbps.
668da14cebeSEric Cheng * The number of soft rings required is a function
669da14cebeSEric Cheng * of bandwidth and CPU speed. To keep this simple,
670da14cebeSEric Cheng * let's use this rule: 1GHz CPU can handle 1Gbps.
671da14cebeSEric Cheng * If bw is less than 1 Gbps, then there is no need
672da14cebeSEric Cheng * for soft rings. Assumption is that CPU speeds
673da14cebeSEric Cheng * (on modern systems) are at least 1GHz.
674da14cebeSEric Cheng */
675da14cebeSEric Cheng srings = bw/cpu_speed;
676da14cebeSEric Cheng if (srings <= 1 && mac_soft_ring_enable) {
677da14cebeSEric Cheng /*
678da14cebeSEric Cheng * Give at least 2 soft rings
679da14cebeSEric Cheng * for sun4v systems
680da14cebeSEric Cheng */
681da14cebeSEric Cheng srings = 2;
682da14cebeSEric Cheng }
683da14cebeSEric Cheng }
684da14cebeSEric Cheng }
685da14cebeSEric Cheng /*
686da14cebeSEric Cheng * If the flent has multiple Rx SRSs, then each SRS need not
687da14cebeSEric Cheng * have that many soft rings on top of it. The number of
688da14cebeSEric Cheng * soft rings for each Rx SRS is found by dividing srings by
689da14cebeSEric Cheng * rx_srs_cnt.
690da14cebeSEric Cheng */
691da14cebeSEric Cheng if (rx_srs_cnt > 1) {
692da14cebeSEric Cheng int remainder;
693da14cebeSEric Cheng
694da14cebeSEric Cheng remainder = srings%rx_srs_cnt;
695da14cebeSEric Cheng srings = srings/rx_srs_cnt;
696da14cebeSEric Cheng if (remainder != 0)
697da14cebeSEric Cheng srings++;
698da14cebeSEric Cheng /*
699da14cebeSEric Cheng * Fanning out to 1 soft ring is not very useful.
700da14cebeSEric Cheng * Set it as well to 0 and mac_srs_fanout_init()
701da14cebeSEric Cheng * will take care of creating a single soft ring
702da14cebeSEric Cheng * for proto fanout.
703da14cebeSEric Cheng */
704da14cebeSEric Cheng if (srings == 1)
705da14cebeSEric Cheng srings = 0;
706da14cebeSEric Cheng }
707da14cebeSEric Cheng /* Do some more massaging */
7080dc2366fSVenugopal Iyer srings = min(srings, maxcpus);
709da14cebeSEric Cheng srings = min(srings, MAX_SR_FANOUT);
710da14cebeSEric Cheng return (srings);
711da14cebeSEric Cheng }
712da14cebeSEric Cheng
7130dc2366fSVenugopal Iyer /*
7140dc2366fSVenugopal Iyer * mac_tx_cpu_init:
7150dc2366fSVenugopal Iyer * set up CPUs for Tx interrupt re-targeting and Tx worker
7160dc2366fSVenugopal Iyer * thread binding
7170dc2366fSVenugopal Iyer */
7180dc2366fSVenugopal Iyer static void
mac_tx_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp,cpupart_t * cpupart)7190dc2366fSVenugopal Iyer mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp,
7200dc2366fSVenugopal Iyer cpupart_t *cpupart)
7210dc2366fSVenugopal Iyer {
7220dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
7230dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx = &tx_srs->srs_tx;
7240dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu = &tx_srs->srs_cpu;
7250dc2366fSVenugopal Iyer mac_soft_ring_t *sringp;
7260dc2366fSVenugopal Iyer mac_ring_t *ring;
7270dc2366fSVenugopal Iyer processorid_t worker_cpuid;
7280dc2366fSVenugopal Iyer boolean_t retargetable_client = B_FALSE;
7290dc2366fSVenugopal Iyer int i, j;
7300dc2366fSVenugopal Iyer
7310dc2366fSVenugopal Iyer if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group,
7320dc2366fSVenugopal Iyer flent->fe_mcip)) {
7330dc2366fSVenugopal Iyer retargetable_client = B_TRUE;
7340dc2366fSVenugopal Iyer }
7350dc2366fSVenugopal Iyer
7360dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(tx_srs)) {
7370dc2366fSVenugopal Iyer if (mrp != NULL)
7380dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1;
7390dc2366fSVenugopal Iyer for (i = 0; i < tx_srs->srs_tx_ring_count; i++) {
7400dc2366fSVenugopal Iyer if (mrp != NULL) {
7410dc2366fSVenugopal Iyer if (j < 0)
7420dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1;
7430dc2366fSVenugopal Iyer worker_cpuid = mrp->mrp_cpu[j];
7440dc2366fSVenugopal Iyer } else {
7450dc2366fSVenugopal Iyer /*
7460dc2366fSVenugopal Iyer * Bind interrupt to the next CPU available
7470dc2366fSVenugopal Iyer * and leave the worker unbound.
7480dc2366fSVenugopal Iyer */
7490dc2366fSVenugopal Iyer worker_cpuid = -1;
7500dc2366fSVenugopal Iyer }
7510dc2366fSVenugopal Iyer sringp = tx_srs->srs_tx_soft_rings[i];
7520dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
7530dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid;
7540dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) &&
7550dc2366fSVenugopal Iyer retargetable_client) {
7560dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
7570dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] =
7580dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[j] :
7590dc2366fSVenugopal Iyer (mac_tx_intr_retarget ?
7600dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1);
7610dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
7620dc2366fSVenugopal Iyer } else {
7630dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = -1;
7640dc2366fSVenugopal Iyer }
7650dc2366fSVenugopal Iyer if (mrp != NULL)
7660dc2366fSVenugopal Iyer j--;
7670dc2366fSVenugopal Iyer }
7680dc2366fSVenugopal Iyer } else {
7690dc2366fSVenugopal Iyer /* Tx mac_ring_handle_t is stored in st_arg2 */
7700dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[0] =
7710dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1;
7720dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2;
7730dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && retargetable_client) {
7740dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
7750dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ?
7760dc2366fSVenugopal Iyer mrp->mrp_cpu[mrp->mrp_ncpus - 1] :
7770dc2366fSVenugopal Iyer (mac_tx_intr_retarget ?
7780dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1);
7790dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
7800dc2366fSVenugopal Iyer } else {
7810dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = -1;
7820dc2366fSVenugopal Iyer }
7830dc2366fSVenugopal Iyer }
7840dc2366fSVenugopal Iyer }
7850dc2366fSVenugopal Iyer
786da14cebeSEric Cheng /*
787da14cebeSEric Cheng * Assignment of user specified CPUs to a link.
788da14cebeSEric Cheng *
789da14cebeSEric Cheng * Minimum CPUs required to get an optimal assignmet:
790da14cebeSEric Cheng * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize
791da14cebeSEric Cheng * flag is set -- one for polling, one for fanout soft ring.
792da14cebeSEric Cheng * If mac_latency_optimize is not set, then 3 CPUs are needed -- one
793da14cebeSEric Cheng * for polling, one for SRS worker thread and one for fanout soft ring.
794da14cebeSEric Cheng *
795da14cebeSEric Cheng * The CPUs needed for Tx side is equal to the number of Tx rings
796da14cebeSEric Cheng * the link is using.
797da14cebeSEric Cheng *
798da14cebeSEric Cheng * mac_flow_user_cpu_init() categorizes the CPU assignment depending
799da14cebeSEric Cheng * upon the number of CPUs in 3 different buckets.
800da14cebeSEric Cheng *
801da14cebeSEric Cheng * In the first bucket, the most optimal case is handled. The user has
802da14cebeSEric Cheng * passed enough number of CPUs and every thread gets its own CPU.
803da14cebeSEric Cheng *
804da14cebeSEric Cheng * The second and third are the sub-optimal cases. Enough CPUs are not
805da14cebeSEric Cheng * available.
806da14cebeSEric Cheng *
807da14cebeSEric Cheng * The second bucket handles the case where atleast one distinct CPU is
808da14cebeSEric Cheng * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx
809da14cebeSEric Cheng * SRS or soft rings).
810da14cebeSEric Cheng *
811da14cebeSEric Cheng * In the third case (worst case scenario), specified CPU count is less
812da14cebeSEric Cheng * than the Rx rings configured for the link. In this case, we round
813da14cebeSEric Cheng * robin the CPUs among the Rx SRSes and Tx SRS/soft rings.
814da14cebeSEric Cheng */
815da14cebeSEric Cheng static void
mac_flow_user_cpu_init(flow_entry_t * flent,mac_resource_props_t * mrp)816da14cebeSEric Cheng mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp)
817da14cebeSEric Cheng {
818da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs, *tx_srs;
819da14cebeSEric Cheng int i, srs_cnt;
820da14cebeSEric Cheng mac_cpus_t *srs_cpu;
821da14cebeSEric Cheng int no_of_cpus, cpu_cnt;
822da14cebeSEric Cheng int rx_srs_cnt, reqd_rx_cpu_cnt;
823da14cebeSEric Cheng int fanout_cpu_cnt, reqd_tx_cpu_cnt;
824da14cebeSEric Cheng int reqd_poll_worker_cnt, fanout_cnt_per_srs;
8250dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props;
826da14cebeSEric Cheng
827da14cebeSEric Cheng ASSERT(mrp->mrp_fanout_mode == MCM_CPUS);
828da14cebeSEric Cheng /*
829da14cebeSEric Cheng * The check for nbc_ncpus to be within limits for
830da14cebeSEric Cheng * the user specified case was done earlier and if
831da14cebeSEric Cheng * not within limits, an error would have been
832da14cebeSEric Cheng * returned to the user.
833da14cebeSEric Cheng */
8345adf34bdSRajagopal Kunhappan ASSERT(mrp->mrp_ncpus > 0);
835da14cebeSEric Cheng
836da14cebeSEric Cheng no_of_cpus = mrp->mrp_ncpus;
837da14cebeSEric Cheng
8380dc2366fSVenugopal Iyer if (mrp->mrp_rx_intr_cpu != -1) {
839da14cebeSEric Cheng /*
840da14cebeSEric Cheng * interrupt has been re-targetted. Poll
841da14cebeSEric Cheng * thread needs to be bound to interrupt
8420dc2366fSVenugopal Iyer * CPU.
843da14cebeSEric Cheng *
844da14cebeSEric Cheng * Find where in the list is the intr
845da14cebeSEric Cheng * CPU and swap it with the first one.
846da14cebeSEric Cheng * We will be using the first CPU in the
847da14cebeSEric Cheng * list for poll.
848da14cebeSEric Cheng */
849da14cebeSEric Cheng for (i = 0; i < no_of_cpus; i++) {
8500dc2366fSVenugopal Iyer if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu)
851da14cebeSEric Cheng break;
852da14cebeSEric Cheng }
853da14cebeSEric Cheng mrp->mrp_cpu[i] = mrp->mrp_cpu[0];
8540dc2366fSVenugopal Iyer mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu;
855da14cebeSEric Cheng }
856da14cebeSEric Cheng
857da14cebeSEric Cheng /*
858da14cebeSEric Cheng * Requirements:
859da14cebeSEric Cheng * The number of CPUs that each Rx ring needs is dependent
860da14cebeSEric Cheng * upon mac_latency_optimize flag.
861da14cebeSEric Cheng * 1) If set, atleast 2 CPUs are needed -- one for
862da14cebeSEric Cheng * polling, one for fanout soft ring.
863da14cebeSEric Cheng * 2) If not set, then atleast 3 CPUs are needed -- one
864da14cebeSEric Cheng * for polling, one for srs worker thread, and one for
865da14cebeSEric Cheng * fanout soft ring.
866da14cebeSEric Cheng */
867da14cebeSEric Cheng rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ?
868da14cebeSEric Cheng (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt;
869da14cebeSEric Cheng reqd_rx_cpu_cnt = mac_latency_optimize ?
870da14cebeSEric Cheng (rx_srs_cnt * 2) : (rx_srs_cnt * 3);
871da14cebeSEric Cheng
872da14cebeSEric Cheng /* How many CPUs are needed for Tx side? */
873da14cebeSEric Cheng tx_srs = flent->fe_tx_srs;
8740dc2366fSVenugopal Iyer reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ?
8750dc2366fSVenugopal Iyer tx_srs->srs_tx_ring_count : 1;
876da14cebeSEric Cheng
877da14cebeSEric Cheng /* CPUs needed for Rx SRSes poll and worker threads */
878da14cebeSEric Cheng reqd_poll_worker_cnt = mac_latency_optimize ?
879da14cebeSEric Cheng rx_srs_cnt : rx_srs_cnt * 2;
880da14cebeSEric Cheng
881da14cebeSEric Cheng /* Has the user provided enough CPUs? */
882da14cebeSEric Cheng if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) {
883da14cebeSEric Cheng /*
884da14cebeSEric Cheng * Best case scenario. There is enough CPUs. All
885da14cebeSEric Cheng * Rx rings will get their own set of CPUs plus
886da14cebeSEric Cheng * Tx soft rings will get their own.
887da14cebeSEric Cheng */
888da14cebeSEric Cheng /*
889da14cebeSEric Cheng * fanout_cpu_cnt is the number of CPUs available
890da14cebeSEric Cheng * for Rx side fanout soft rings.
891da14cebeSEric Cheng */
892da14cebeSEric Cheng fanout_cpu_cnt = no_of_cpus -
893da14cebeSEric Cheng reqd_poll_worker_cnt - reqd_tx_cpu_cnt;
894da14cebeSEric Cheng
895da14cebeSEric Cheng /*
896da14cebeSEric Cheng * Divide fanout_cpu_cnt by rx_srs_cnt to find
897da14cebeSEric Cheng * out how many fanout soft rings each Rx SRS
898da14cebeSEric Cheng * can have.
899da14cebeSEric Cheng */
900da14cebeSEric Cheng fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt;
901da14cebeSEric Cheng
9025adf34bdSRajagopal Kunhappan /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */
9035adf34bdSRajagopal Kunhappan fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT);
9045adf34bdSRajagopal Kunhappan
905da14cebeSEric Cheng /* Do the assignment for the default Rx ring */
906da14cebeSEric Cheng cpu_cnt = 0;
907da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0];
908da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL);
909da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
910da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
911da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu;
912da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus;
913da14cebeSEric Cheng bcopy(mrp->mrp_cpu,
914da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
9150dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
9160dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
9170dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */
9180dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
9190dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = (mac_latency_optimize ?
9200dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]);
921da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++)
9220dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++];
923da14cebeSEric Cheng
924da14cebeSEric Cheng /* Do the assignment for h/w Rx SRSes */
925da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) {
926da14cebeSEric Cheng cpu_cnt = 0;
927da14cebeSEric Cheng for (srs_cnt = 1;
928da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
929da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt];
930da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL);
931da14cebeSEric Cheng if (rx_srs->srs_fanout_state ==
932da14cebeSEric Cheng SRS_FANOUT_INIT) {
933da14cebeSEric Cheng rx_srs->srs_fanout_state =
934da14cebeSEric Cheng SRS_FANOUT_REINIT;
935da14cebeSEric Cheng }
936da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu;
937da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus;
938da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
939da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus));
9400dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs;
941da14cebeSEric Cheng /* The first CPU in the list is the intr CPU */
9420dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
9430dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
9440dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid =
9450dc2366fSVenugopal Iyer (mac_latency_optimize ?
9460dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid :
9470dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++]);
948da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) {
9490dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] =
950da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++];
951da14cebeSEric Cheng }
952da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus);
953da14cebeSEric Cheng }
954da14cebeSEric Cheng }
9550dc2366fSVenugopal Iyer goto tx_cpu_init;
956da14cebeSEric Cheng }
957da14cebeSEric Cheng
958da14cebeSEric Cheng /*
959da14cebeSEric Cheng * Sub-optimal case.
960da14cebeSEric Cheng * We have the following information:
961da14cebeSEric Cheng * no_of_cpus - no. of cpus that user passed.
962da14cebeSEric Cheng * rx_srs_cnt - no. of rx rings.
963da14cebeSEric Cheng * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3
964da14cebeSEric Cheng * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side.
965da14cebeSEric Cheng * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2
966da14cebeSEric Cheng */
967da14cebeSEric Cheng /*
968da14cebeSEric Cheng * If we bind the Rx fanout soft rings to the same CPUs
969da14cebeSEric Cheng * as poll/worker, would that be enough?
970da14cebeSEric Cheng */
971da14cebeSEric Cheng if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) {
972da14cebeSEric Cheng boolean_t worker_assign = B_FALSE;
973da14cebeSEric Cheng
974da14cebeSEric Cheng /*
975da14cebeSEric Cheng * If mac_latency_optimize is not set, are there
976da14cebeSEric Cheng * enough CPUs to assign a CPU for worker also?
977da14cebeSEric Cheng */
978da14cebeSEric Cheng if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt))
979da14cebeSEric Cheng worker_assign = B_TRUE;
980da14cebeSEric Cheng /*
981da14cebeSEric Cheng * Zero'th Rx SRS is the default Rx ring. It is not
982da14cebeSEric Cheng * associated with h/w Rx ring.
983da14cebeSEric Cheng */
984da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0];
985da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL);
986da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
987da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
988da14cebeSEric Cheng cpu_cnt = 0;
989da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu;
990da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus;
991da14cebeSEric Cheng bcopy(mrp->mrp_cpu,
992da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
9930dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1;
9940dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++];
9950dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */
9960dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
9970dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid =
9980dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ?
9990dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid);
10000dc2366fSVenugopal Iyer
10010dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1002da14cebeSEric Cheng
1003da14cebeSEric Cheng /* Do CPU bindings for SRSes having h/w Rx rings */
1004da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) {
1005da14cebeSEric Cheng cpu_cnt = 0;
1006da14cebeSEric Cheng for (srs_cnt = 1;
1007da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1008da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt];
1009da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL);
1010da14cebeSEric Cheng if (rx_srs->srs_fanout_state ==
1011da14cebeSEric Cheng SRS_FANOUT_INIT) {
1012da14cebeSEric Cheng rx_srs->srs_fanout_state =
1013da14cebeSEric Cheng SRS_FANOUT_REINIT;
1014da14cebeSEric Cheng }
1015da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu;
1016da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus;
1017da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus,
1018da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus));
10190dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid =
1020da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt];
10210dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
10220dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid =
10230dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ?
10240dc2366fSVenugopal Iyer mrp->mrp_cpu[++cpu_cnt] :
10250dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid);
10260dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1;
10270dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] =
1028da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt];
1029da14cebeSEric Cheng cpu_cnt++;
1030da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus);
1031da14cebeSEric Cheng }
1032da14cebeSEric Cheng }
10330dc2366fSVenugopal Iyer goto tx_cpu_init;
1034da14cebeSEric Cheng }
1035da14cebeSEric Cheng
1036da14cebeSEric Cheng /*
1037da14cebeSEric Cheng * Real sub-optimal case. Not enough CPUs for poll and
1038da14cebeSEric Cheng * Tx soft rings. Do a round robin assignment where
1039da14cebeSEric Cheng * each Rx SRS will get the same CPU for poll, worker
1040da14cebeSEric Cheng * and fanout soft ring.
1041da14cebeSEric Cheng */
1042da14cebeSEric Cheng cpu_cnt = 0;
1043da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) {
1044da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt];
1045da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu;
1046da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
1047da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
1048da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus;
1049da14cebeSEric Cheng bcopy(mrp->mrp_cpu,
1050da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus));
10510dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1;
10520dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt];
10530dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */
10540dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid;
10550dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt];
10560dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt];
1057da14cebeSEric Cheng if (++cpu_cnt >= no_of_cpus)
1058da14cebeSEric Cheng cpu_cnt = 0;
1059da14cebeSEric Cheng }
10600dc2366fSVenugopal Iyer
10610dc2366fSVenugopal Iyer tx_cpu_init:
10620dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, mrp, NULL);
10630dc2366fSVenugopal Iyer
10640dc2366fSVenugopal Iyer /*
10650dc2366fSVenugopal Iyer * Copy the user specified CPUs to the effective CPUs
10660dc2366fSVenugopal Iyer */
10670dc2366fSVenugopal Iyer for (i = 0; i < mrp->mrp_ncpus; i++) {
10680dc2366fSVenugopal Iyer emrp->mrp_cpu[i] = mrp->mrp_cpu[i];
10690dc2366fSVenugopal Iyer }
10700dc2366fSVenugopal Iyer emrp->mrp_ncpus = mrp->mrp_ncpus;
10710dc2366fSVenugopal Iyer emrp->mrp_mask = mrp->mrp_mask;
10720dc2366fSVenugopal Iyer bzero(emrp->mrp_pool, MAXPATHLEN);
1073da14cebeSEric Cheng }
1074da14cebeSEric Cheng
1075da14cebeSEric Cheng /*
1076da14cebeSEric Cheng * mac_flow_cpu_init():
1077da14cebeSEric Cheng *
1078da14cebeSEric Cheng * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in
1079da14cebeSEric Cheng * the CPU binding information in srs_cpu for all Rx SRSes associated
1080da14cebeSEric Cheng * with a flent.
1081da14cebeSEric Cheng */
1082da14cebeSEric Cheng static void
mac_flow_cpu_init(flow_entry_t * flent,cpupart_t * cpupart)10830dc2366fSVenugopal Iyer mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart)
1084da14cebeSEric Cheng {
1085da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs;
1086da14cebeSEric Cheng processorid_t cpuid;
10873714f7beSPaul Winder int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0;
1088da14cebeSEric Cheng mac_cpus_t *srs_cpu;
10890dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props;
1090da14cebeSEric Cheng
10910dc2366fSVenugopal Iyer /*
10920dc2366fSVenugopal Iyer * The maximum number of CPUs available can either be
10930dc2366fSVenugopal Iyer * the number of CPUs in the pool or the number of CPUs
10940dc2366fSVenugopal Iyer * in the system.
10950dc2366fSVenugopal Iyer */
10960dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
10973714f7beSPaul Winder /*
10983714f7beSPaul Winder * We cannot exceed the hard limit imposed by data structures.
10993714f7beSPaul Winder * Leave space for polling CPU and the SRS worker thread when
11003714f7beSPaul Winder * "mac_latency_optimize" is not set.
11013714f7beSPaul Winder */
11023714f7beSPaul Winder maxcpus = MIN(maxcpus, MRP_NCPUS - 2);
11030dc2366fSVenugopal Iyer
11040dc2366fSVenugopal Iyer /*
11050dc2366fSVenugopal Iyer * Compute the number of soft rings needed on top for each Rx
11060dc2366fSVenugopal Iyer * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS
11070dc2366fSVenugopal Iyer * associated with h/w Rx rings. Soft ring count needed for
11080dc2366fSVenugopal Iyer * each h/w Rx SRS is computed and the same is applied to
11090dc2366fSVenugopal Iyer * software classified Rx SRS. The first Rx SRS in fe_rx_srs[]
11100dc2366fSVenugopal Iyer * is the software classified Rx SRS.
11110dc2366fSVenugopal Iyer */
11120dc2366fSVenugopal Iyer soft_ring_cnt = mac_compute_soft_ring_count(flent,
11130dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus);
11140dc2366fSVenugopal Iyer if (soft_ring_cnt == 0) {
1115da14cebeSEric Cheng /*
11160dc2366fSVenugopal Iyer * Even when soft_ring_cnt is 0, we still need
11170dc2366fSVenugopal Iyer * to create a soft ring for TCP, UDP and
11180dc2366fSVenugopal Iyer * OTHER. So set it to 1.
1119da14cebeSEric Cheng */
11200dc2366fSVenugopal Iyer soft_ring_cnt = 1;
11210dc2366fSVenugopal Iyer }
11223714f7beSPaul Winder
11233714f7beSPaul Winder emrp->mrp_ncpus = 0;
11243714f7beSPaul Winder for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt &&
11253714f7beSPaul Winder emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) {
11260dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[srs_cnt];
11270dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu;
11280dc2366fSVenugopal Iyer if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT)
11290dc2366fSVenugopal Iyer rx_srs->srs_fanout_state = SRS_FANOUT_REINIT;
11300dc2366fSVenugopal Iyer srs_cpu->mc_ncpus = soft_ring_cnt;
11310dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt;
11320dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
11330dc2366fSVenugopal Iyer for (j = 0; j < soft_ring_cnt; j++) {
11340dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart);
11350dc2366fSVenugopal Iyer srs_cpu->mc_cpus[j] = cpuid;
11360dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[j] = cpuid;
11370dc2366fSVenugopal Iyer }
11380dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart);
11390dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = cpuid;
11400dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ?
11410dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : -1);
11420dc2366fSVenugopal Iyer /* increment ncpus to account for polling cpu */
11430dc2366fSVenugopal Iyer srs_cpu->mc_ncpus++;
11440dc2366fSVenugopal Iyer srs_cpu->mc_cpus[j++] = cpuid;
11450dc2366fSVenugopal Iyer if (!mac_latency_optimize) {
11460dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart);
1147da14cebeSEric Cheng srs_cpu->mc_ncpus++;
1148da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid;
1149da14cebeSEric Cheng }
11500dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = cpuid;
11510dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
11520dc2366fSVenugopal Iyer
11533714f7beSPaul Winder /*
11543714f7beSPaul Winder * Copy fanout CPUs to fe_effective_props without duplicates.
11553714f7beSPaul Winder */
11563714f7beSPaul Winder for (i = 0; i < srs_cpu->mc_ncpus &&
11573714f7beSPaul Winder emrp->mrp_ncpus < MRP_NCPUS; i++) {
11583714f7beSPaul Winder for (j = 0; j < emrp->mrp_ncpus; j++) {
11593714f7beSPaul Winder if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i])
11603714f7beSPaul Winder break;
11613714f7beSPaul Winder }
11623714f7beSPaul Winder if (j == emrp->mrp_ncpus) {
11633714f7beSPaul Winder emrp->mrp_cpu[emrp->mrp_ncpus++] =
11643714f7beSPaul Winder srs_cpu->mc_cpus[i];
11653714f7beSPaul Winder }
11660dc2366fSVenugopal Iyer }
11670dc2366fSVenugopal Iyer }
11680dc2366fSVenugopal Iyer
11690dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, NULL, cpupart);
1170da14cebeSEric Cheng }
1171da14cebeSEric Cheng
1172da14cebeSEric Cheng /*
1173da14cebeSEric Cheng * DATAPATH SETUP ROUTINES
1174da14cebeSEric Cheng * (setup SRS and set/update FANOUT, B/W and PRIORITY)
1175da14cebeSEric Cheng */
1176da14cebeSEric Cheng
11770dc2366fSVenugopal Iyer /*
11780dc2366fSVenugopal Iyer * mac_srs_fanout_list_alloc:
11790dc2366fSVenugopal Iyer *
11800dc2366fSVenugopal Iyer * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of
11810dc2366fSVenugopal Iyer * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of
11820dc2366fSVenugopal Iyer * array space is needed to store Tx soft rings. Thus we allocate so
11830dc2366fSVenugopal Iyer * much array space for srs_tx_soft_rings.
11840dc2366fSVenugopal Iyer *
11850dc2366fSVenugopal Iyer * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth
11860dc2366fSVenugopal Iyer * of space to st_soft_rings. This array is used for quick access to
11870dc2366fSVenugopal Iyer * soft ring associated with a pseudo Tx ring based on the pseudo
11880dc2366fSVenugopal Iyer * ring's index (mr_index).
11890dc2366fSVenugopal Iyer */
1190da14cebeSEric Cheng static void
mac_srs_fanout_list_alloc(mac_soft_ring_set_t * mac_srs)1191da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs)
1192da14cebeSEric Cheng {
11930dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip;
11940dc2366fSVenugopal Iyer
11950dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) {
11960dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **)
11970dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) *
11980dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP);
119984de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
12000dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx;
12010dc2366fSVenugopal Iyer
12020dc2366fSVenugopal Iyer tx->st_soft_rings = (mac_soft_ring_t **)
12030dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) *
12040dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP);
12050dc2366fSVenugopal Iyer }
12060dc2366fSVenugopal Iyer } else {
12070dc2366fSVenugopal Iyer mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **)
12080dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
12090dc2366fSVenugopal Iyer KM_SLEEP);
12100dc2366fSVenugopal Iyer mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **)
12110dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
12120dc2366fSVenugopal Iyer KM_SLEEP);
12130dc2366fSVenugopal Iyer mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **)
12140dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT,
12150dc2366fSVenugopal Iyer KM_SLEEP);
12160dc2366fSVenugopal Iyer }
1217da14cebeSEric Cheng }
1218da14cebeSEric Cheng
1219da14cebeSEric Cheng static void
mac_srs_worker_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1220da14cebeSEric Cheng mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1221da14cebeSEric Cheng {
1222da14cebeSEric Cheng cpu_t *cp;
1223da14cebeSEric Cheng boolean_t clear = B_FALSE;
1224da14cebeSEric Cheng
1225da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock));
1226da14cebeSEric Cheng
1227da14cebeSEric Cheng if (!mac_srs_thread_bind)
1228da14cebeSEric Cheng return;
1229da14cebeSEric Cheng
1230da14cebeSEric Cheng cp = cpu_get(cpuid);
1231da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp))
1232da14cebeSEric Cheng return;
1233da14cebeSEric Cheng
1234da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
1235da14cebeSEric Cheng mac_srs->srs_state |= SRS_WORKER_BOUND;
1236da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid != -1)
1237da14cebeSEric Cheng clear = B_TRUE;
1238da14cebeSEric Cheng mac_srs->srs_worker_cpuid = cpuid;
1239da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
1240da14cebeSEric Cheng
1241da14cebeSEric Cheng if (clear)
1242da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker);
1243da14cebeSEric Cheng
1244da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_worker, cpuid);
1245da14cebeSEric Cheng DTRACE_PROBE1(worker__CPU, processorid_t, cpuid);
1246da14cebeSEric Cheng }
1247da14cebeSEric Cheng
1248da14cebeSEric Cheng static void
mac_srs_poll_bind(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)1249da14cebeSEric Cheng mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
1250da14cebeSEric Cheng {
1251da14cebeSEric Cheng cpu_t *cp;
1252da14cebeSEric Cheng boolean_t clear = B_FALSE;
1253da14cebeSEric Cheng
1254da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock));
1255da14cebeSEric Cheng
1256da14cebeSEric Cheng if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL)
1257da14cebeSEric Cheng return;
1258da14cebeSEric Cheng
1259da14cebeSEric Cheng cp = cpu_get(cpuid);
1260da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp))
1261da14cebeSEric Cheng return;
1262da14cebeSEric Cheng
1263da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
1264da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_BOUND;
1265da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid != -1)
1266da14cebeSEric Cheng clear = B_TRUE;
1267da14cebeSEric Cheng mac_srs->srs_poll_cpuid = cpuid;
1268da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
1269da14cebeSEric Cheng
1270da14cebeSEric Cheng if (clear)
1271da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr);
1272da14cebeSEric Cheng
1273da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_poll_thr, cpuid);
1274da14cebeSEric Cheng DTRACE_PROBE1(poll__CPU, processorid_t, cpuid);
1275da14cebeSEric Cheng }
1276da14cebeSEric Cheng
12770dc2366fSVenugopal Iyer /*
12780dc2366fSVenugopal Iyer * Re-target interrupt to the passed CPU. If re-target is successful,
12790dc2366fSVenugopal Iyer * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1.
12800dc2366fSVenugopal Iyer */
12810dc2366fSVenugopal Iyer void
mac_rx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs,processorid_t cpuid)12820dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid)
12830dc2366fSVenugopal Iyer {
12840dc2366fSVenugopal Iyer cpu_t *cp;
12850dc2366fSVenugopal Iyer mac_ring_t *ring = mac_srs->srs_ring;
12860dc2366fSVenugopal Iyer mac_intr_t *mintr = &ring->mr_info.mri_intr;
12870dc2366fSVenugopal Iyer flow_entry_t *flent = mac_srs->srs_flent;
12880dc2366fSVenugopal Iyer boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip);
12890dc2366fSVenugopal Iyer
12900dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock));
12910dc2366fSVenugopal Iyer
12920dc2366fSVenugopal Iyer /*
12930dc2366fSVenugopal Iyer * Don't re-target the interrupt for these cases:
12940dc2366fSVenugopal Iyer * 1) ring is NULL
12950dc2366fSVenugopal Iyer * 2) the interrupt is shared (mi_ddi_shared)
12960dc2366fSVenugopal Iyer * 3) ddi_handle is NULL and !primary
12970dc2366fSVenugopal Iyer * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2
12980dc2366fSVenugopal Iyer * Case 3 & 4 are because of mac_client_intr_cpu() routine.
12990dc2366fSVenugopal Iyer * This routine will re-target fixed interrupt for primary
13000dc2366fSVenugopal Iyer * mac client if the client has only one ring. In that
13010dc2366fSVenugopal Iyer * case, mc_rx_intr_cpu will already have the correct value.
13020dc2366fSVenugopal Iyer */
13030dc2366fSVenugopal Iyer if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 ||
13040dc2366fSVenugopal Iyer (mintr->mi_ddi_handle == NULL && !primary) || (primary &&
13050dc2366fSVenugopal Iyer mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) {
13060dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
13070dc2366fSVenugopal Iyer return;
13080dc2366fSVenugopal Iyer }
13090dc2366fSVenugopal Iyer
13100dc2366fSVenugopal Iyer if (mintr->mi_ddi_handle == NULL)
13110dc2366fSVenugopal Iyer return;
13120dc2366fSVenugopal Iyer
13130dc2366fSVenugopal Iyer cp = cpu_get(cpuid);
13140dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp))
13150dc2366fSVenugopal Iyer return;
13160dc2366fSVenugopal Iyer
1317657f87deSgongtian zhao - Sun Microsystems - Beijing China /* Drop the cpu_lock as set_intr_affinity() holds it */
13180dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
1319657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS)
13200dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid;
13210dc2366fSVenugopal Iyer else
13220dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1;
13230dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
13240dc2366fSVenugopal Iyer }
13250dc2366fSVenugopal Iyer
13260dc2366fSVenugopal Iyer /*
13270dc2366fSVenugopal Iyer * Re-target Tx interrupts
13280dc2366fSVenugopal Iyer */
13290dc2366fSVenugopal Iyer void
mac_tx_srs_retarget_intr(mac_soft_ring_set_t * mac_srs)13300dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs)
13310dc2366fSVenugopal Iyer {
13320dc2366fSVenugopal Iyer cpu_t *cp;
13330dc2366fSVenugopal Iyer mac_ring_t *ring;
13340dc2366fSVenugopal Iyer mac_intr_t *mintr;
13350dc2366fSVenugopal Iyer mac_soft_ring_t *sringp;
13360dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx;
13370dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu;
13380dc2366fSVenugopal Iyer processorid_t cpuid;
13390dc2366fSVenugopal Iyer int i;
13400dc2366fSVenugopal Iyer
13410dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock));
13420dc2366fSVenugopal Iyer
13430dc2366fSVenugopal Iyer srs_cpu = &mac_srs->srs_cpu;
13440dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_srs)) {
13450dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
13460dc2366fSVenugopal Iyer sringp = mac_srs->srs_tx_soft_rings[i];
13470dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2;
13480dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[i];
13490dc2366fSVenugopal Iyer cp = cpu_get(cpuid);
13500dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp) ||
13510dc2366fSVenugopal Iyer !MAC_RING_RETARGETABLE(ring)) {
13520dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1;
13530dc2366fSVenugopal Iyer continue;
13540dc2366fSVenugopal Iyer }
13550dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr;
13560dc2366fSVenugopal Iyer /*
1357657f87deSgongtian zhao - Sun Microsystems - Beijing China * Drop the cpu_lock as set_intr_affinity()
13580dc2366fSVenugopal Iyer * holds it
13590dc2366fSVenugopal Iyer */
13600dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
1361657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle,
13620dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS) {
13630dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = cpuid;
13640dc2366fSVenugopal Iyer } else {
13650dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1;
13660dc2366fSVenugopal Iyer }
13670dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
13680dc2366fSVenugopal Iyer }
13690dc2366fSVenugopal Iyer } else {
13700dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[0];
13710dc2366fSVenugopal Iyer cp = cpu_get(cpuid);
13720dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) {
13730dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1;
13740dc2366fSVenugopal Iyer return;
13750dc2366fSVenugopal Iyer }
13760dc2366fSVenugopal Iyer srs_tx = &mac_srs->srs_tx;
13770dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2;
13780dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring)) {
13790dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr;
13800dc2366fSVenugopal Iyer mutex_exit(&cpu_lock);
1381657f87deSgongtian zhao - Sun Microsystems - Beijing China if ((set_intr_affinity(mintr->mi_ddi_handle,
13820dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS)) {
13830dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = cpuid;
13840dc2366fSVenugopal Iyer } else {
13850dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1;
13860dc2366fSVenugopal Iyer }
13870dc2366fSVenugopal Iyer mutex_enter(&cpu_lock);
13880dc2366fSVenugopal Iyer }
13890dc2366fSVenugopal Iyer }
13900dc2366fSVenugopal Iyer }
13910dc2366fSVenugopal Iyer
1392da14cebeSEric Cheng /*
1393da14cebeSEric Cheng * When a CPU comes back online, bind the MAC kernel threads which
1394da14cebeSEric Cheng * were previously bound to that CPU, and had to be unbound because
1395da14cebeSEric Cheng * the CPU was going away.
1396da14cebeSEric Cheng *
1397da14cebeSEric Cheng * These functions are called with cpu_lock held and hence we can't
1398da14cebeSEric Cheng * cv_wait to grab the mac perimeter. Since these functions walk the soft
1399da14cebeSEric Cheng * ring list of an SRS without being in the perimeter, the list itself
1400da14cebeSEric Cheng * is protected by the SRS lock.
1401da14cebeSEric Cheng */
1402da14cebeSEric Cheng static void
mac_walk_srs_and_bind(int cpuid)1403da14cebeSEric Cheng mac_walk_srs_and_bind(int cpuid)
1404da14cebeSEric Cheng {
1405da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs;
1406da14cebeSEric Cheng mac_soft_ring_t *soft_ring;
1407da14cebeSEric Cheng
1408da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER);
1409da14cebeSEric Cheng
1410da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL)
1411da14cebeSEric Cheng goto done;
1412da14cebeSEric Cheng
1413da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
1414da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == -1 &&
1415da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save == cpuid) {
1416da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = -1;
1417da14cebeSEric Cheng mac_srs_worker_bind(mac_srs, cpuid);
1418da14cebeSEric Cheng }
1419da14cebeSEric Cheng
1420da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) {
1421da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == -1 &&
1422da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save == cpuid) {
1423da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = -1;
1424da14cebeSEric Cheng mac_srs_poll_bind(mac_srs, cpuid);
1425da14cebeSEric Cheng }
1426da14cebeSEric Cheng }
1427da14cebeSEric Cheng
1428da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */
1429da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
1430da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
1431da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) {
1432da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == -1 &&
1433da14cebeSEric Cheng soft_ring->s_ring_cpuid_save == cpuid) {
1434da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = -1;
1435da14cebeSEric Cheng (void) mac_soft_ring_bind(soft_ring, cpuid);
1436da14cebeSEric Cheng }
1437da14cebeSEric Cheng }
1438da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
1439da14cebeSEric Cheng }
1440da14cebeSEric Cheng done:
1441da14cebeSEric Cheng rw_exit(&mac_srs_g_lock);
1442da14cebeSEric Cheng }
1443da14cebeSEric Cheng
1444da14cebeSEric Cheng /*
1445da14cebeSEric Cheng * Change the priority of the SRS's poll and worker thread. Additionally,
1446da14cebeSEric Cheng * update the priority of the worker threads for the SRS's soft rings.
1447da14cebeSEric Cheng * Need to modify any associated squeue threads.
1448da14cebeSEric Cheng */
1449da14cebeSEric Cheng void
mac_update_srs_priority(mac_soft_ring_set_t * mac_srs,pri_t prival)1450da14cebeSEric Cheng mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival)
1451da14cebeSEric Cheng {
1452da14cebeSEric Cheng mac_soft_ring_t *ringp;
1453da14cebeSEric Cheng
1454da14cebeSEric Cheng mac_srs->srs_pri = prival;
1455da14cebeSEric Cheng thread_lock(mac_srs->srs_worker);
1456da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0);
1457da14cebeSEric Cheng thread_unlock(mac_srs->srs_worker);
1458da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) {
1459da14cebeSEric Cheng thread_lock(mac_srs->srs_poll_thr);
1460da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_poll_thr,
1461da14cebeSEric Cheng mac_srs->srs_pri, 0);
1462da14cebeSEric Cheng thread_unlock(mac_srs->srs_poll_thr);
1463da14cebeSEric Cheng }
1464da14cebeSEric Cheng if ((ringp = mac_srs->srs_soft_ring_head) == NULL)
1465da14cebeSEric Cheng return;
1466da14cebeSEric Cheng while (ringp != mac_srs->srs_soft_ring_tail) {
1467da14cebeSEric Cheng thread_lock(ringp->s_ring_worker);
1468da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker,
1469da14cebeSEric Cheng mac_srs->srs_pri, 0);
1470da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker);
1471da14cebeSEric Cheng ringp = ringp->s_ring_next;
1472da14cebeSEric Cheng }
1473da14cebeSEric Cheng ASSERT(ringp == mac_srs->srs_soft_ring_tail);
1474da14cebeSEric Cheng thread_lock(ringp->s_ring_worker);
1475da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0);
1476da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker);
1477da14cebeSEric Cheng }
1478da14cebeSEric Cheng
1479da14cebeSEric Cheng /*
1480da14cebeSEric Cheng * Change the receive bandwidth limit.
1481da14cebeSEric Cheng */
1482da14cebeSEric Cheng static void
mac_rx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1483da14cebeSEric Cheng mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1484da14cebeSEric Cheng {
1485da14cebeSEric Cheng mac_soft_ring_t *softring;
1486da14cebeSEric Cheng
1487da14cebeSEric Cheng mutex_enter(&srs->srs_lock);
1488da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock);
1489da14cebeSEric Cheng
1490da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1491da14cebeSEric Cheng /* Reset bandwidth limit */
1492da14cebeSEric Cheng if (srs->srs_type & SRST_BW_CONTROL) {
1493da14cebeSEric Cheng softring = srs->srs_soft_ring_head;
1494da14cebeSEric Cheng while (softring != NULL) {
1495da14cebeSEric Cheng softring->s_ring_type &= ~ST_RING_BW_CTL;
1496da14cebeSEric Cheng softring = softring->s_ring_next;
1497da14cebeSEric Cheng }
1498da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL;
1499da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain;
1500da14cebeSEric Cheng }
1501da14cebeSEric Cheng } else {
1502da14cebeSEric Cheng /* Set/Modify bandwidth limit */
1503da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1504da14cebeSEric Cheng /*
1505da14cebeSEric Cheng * Give twice the queuing capability before
1506da14cebeSEric Cheng * dropping packets. The unit is bytes/tick.
1507da14cebeSEric Cheng */
1508da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold =
1509da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1;
1510da14cebeSEric Cheng if (!(srs->srs_type & SRST_BW_CONTROL)) {
1511da14cebeSEric Cheng softring = srs->srs_soft_ring_head;
1512da14cebeSEric Cheng while (softring != NULL) {
1513da14cebeSEric Cheng softring->s_ring_type |= ST_RING_BW_CTL;
1514da14cebeSEric Cheng softring = softring->s_ring_next;
1515da14cebeSEric Cheng }
1516da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL;
1517da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain_bw;
1518da14cebeSEric Cheng }
1519da14cebeSEric Cheng }
1520da14cebeSEric Cheng done:
1521da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock);
1522da14cebeSEric Cheng mutex_exit(&srs->srs_lock);
1523da14cebeSEric Cheng }
1524da14cebeSEric Cheng
1525da14cebeSEric Cheng /* Change the transmit bandwidth limit */
1526da14cebeSEric Cheng static void
mac_tx_srs_update_bwlimit(mac_soft_ring_set_t * srs,mac_resource_props_t * mrp)1527da14cebeSEric Cheng mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp)
1528da14cebeSEric Cheng {
15290dc2366fSVenugopal Iyer uint32_t tx_mode, ring_info = 0;
1530f94ede51SEric Cheng mac_srs_tx_t *srs_tx = &srs->srs_tx;
1531f94ede51SEric Cheng mac_client_impl_t *mcip = srs->srs_mcip;
1532f94ede51SEric Cheng
1533f94ede51SEric Cheng /*
1534f94ede51SEric Cheng * We need to quiesce/restart the client here because mac_tx() and
1535f94ede51SEric Cheng * srs->srs_tx->st_func do not hold srs->srs_lock while accessing
1536f94ede51SEric Cheng * st_mode and related fields, which are modified by the code below.
1537f94ede51SEric Cheng */
15380dc2366fSVenugopal Iyer mac_tx_client_quiesce((mac_client_handle_t)mcip);
1539da14cebeSEric Cheng
1540da14cebeSEric Cheng mutex_enter(&srs->srs_lock);
1541da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock);
1542da14cebeSEric Cheng
1543da14cebeSEric Cheng tx_mode = srs_tx->st_mode;
1544da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) {
1545da14cebeSEric Cheng /* Reset bandwidth limit */
1546da14cebeSEric Cheng if (tx_mode == SRS_TX_BW) {
15470dc2366fSVenugopal Iyer if (srs_tx->st_arg2 != NULL)
15480dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(srs_tx->st_arg2);
1549da14cebeSEric Cheng if (mac_tx_serialize ||
15500dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) {
1551da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_SERIALIZE;
1552da14cebeSEric Cheng } else {
1553da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_DEFAULT;
1554da14cebeSEric Cheng }
1555da14cebeSEric Cheng } else if (tx_mode == SRS_TX_BW_FANOUT) {
1556da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_FANOUT;
15570dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_BW_AGGR) {
15580dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_AGGR;
1559da14cebeSEric Cheng }
1560da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL;
1561da14cebeSEric Cheng } else {
1562da14cebeSEric Cheng /* Set/Modify bandwidth limit */
1563da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
1564da14cebeSEric Cheng /*
1565da14cebeSEric Cheng * Give twice the queuing capability before
1566da14cebeSEric Cheng * dropping packets. The unit is bytes/tick.
1567da14cebeSEric Cheng */
1568da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold =
1569da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1;
1570da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL;
15710dc2366fSVenugopal Iyer if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT &&
15720dc2366fSVenugopal Iyer tx_mode != SRS_TX_BW_AGGR) {
1573da14cebeSEric Cheng if (tx_mode == SRS_TX_SERIALIZE ||
1574da14cebeSEric Cheng tx_mode == SRS_TX_DEFAULT) {
1575da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW;
1576da14cebeSEric Cheng } else if (tx_mode == SRS_TX_FANOUT) {
1577da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW_FANOUT;
15780dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_AGGR) {
15790dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_BW_AGGR;
1580da14cebeSEric Cheng } else {
1581da14cebeSEric Cheng ASSERT(0);
1582da14cebeSEric Cheng }
1583da14cebeSEric Cheng }
1584da14cebeSEric Cheng }
1585da14cebeSEric Cheng done:
1586da14cebeSEric Cheng srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode);
1587da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock);
1588da14cebeSEric Cheng mutex_exit(&srs->srs_lock);
1589f94ede51SEric Cheng
15900dc2366fSVenugopal Iyer mac_tx_client_restart((mac_client_handle_t)mcip);
1591da14cebeSEric Cheng }
1592da14cebeSEric Cheng
1593da14cebeSEric Cheng /*
1594da14cebeSEric Cheng * The uber function that deals with any update to bandwidth limits.
1595da14cebeSEric Cheng */
1596da14cebeSEric Cheng void
mac_srs_update_bwlimit(flow_entry_t * flent,mac_resource_props_t * mrp)1597da14cebeSEric Cheng mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp)
1598da14cebeSEric Cheng {
1599da14cebeSEric Cheng int count;
1600da14cebeSEric Cheng
1601da14cebeSEric Cheng for (count = 0; count < flent->fe_rx_srs_cnt; count++)
1602da14cebeSEric Cheng mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp);
1603da14cebeSEric Cheng mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp);
1604da14cebeSEric Cheng }
1605da14cebeSEric Cheng
1606da14cebeSEric Cheng /*
1607da14cebeSEric Cheng * When the first sub-flow is added to a link, we disable polling on the
160884de666eSRyan Zezeski * link and also modify the entry point to mac_rx_srs_subflow_process().
1609da14cebeSEric Cheng * (polling is disabled because with the subflow added, accounting
1610da14cebeSEric Cheng * for polling needs additional logic, it is assumed that when a subflow is
1611da14cebeSEric Cheng * added, we can take some hit as a result of disabling polling rather than
1612da14cebeSEric Cheng * adding more complexity - if this becomes a perf. issue we need to
1613da14cebeSEric Cheng * re-rvaluate this logic). When the last subflow is removed, we turn back
161484de666eSRyan Zezeski * polling and also reset the entry point to mac_rx_srs_process().
1615da14cebeSEric Cheng *
1616da14cebeSEric Cheng * In the future if there are multiple SRS, we can simply
1617da14cebeSEric Cheng * take one and give it to the flow rather than disabling polling and
1618da14cebeSEric Cheng * resetting the entry point.
1619da14cebeSEric Cheng */
1620da14cebeSEric Cheng void
mac_client_update_classifier(mac_client_impl_t * mcip,boolean_t enable)1621da14cebeSEric Cheng mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable)
1622da14cebeSEric Cheng {
1623da14cebeSEric Cheng flow_entry_t *flent = mcip->mci_flent;
1624da14cebeSEric Cheng int i;
1625da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip;
1626da14cebeSEric Cheng mac_rx_func_t rx_func;
1627da14cebeSEric Cheng uint_t rx_srs_cnt;
1628da14cebeSEric Cheng boolean_t enable_classifier;
1629da14cebeSEric Cheng
1630da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
1631da14cebeSEric Cheng
1632da14cebeSEric Cheng enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable;
1633da14cebeSEric Cheng
1634da14cebeSEric Cheng rx_func = enable_classifier ? mac_rx_srs_subflow_process :
1635da14cebeSEric Cheng mac_rx_srs_process;
1636da14cebeSEric Cheng
16374eaa4710SRishi Srivatsavai /* Tell mac_srs_poll_state_change to disable polling if necessary */
16384eaa4710SRishi Srivatsavai if (mip->mi_state_flags & MIS_POLL_DISABLE)
16394eaa4710SRishi Srivatsavai enable_classifier = B_TRUE;
16404eaa4710SRishi Srivatsavai
1641da14cebeSEric Cheng /*
1642da14cebeSEric Cheng * If receive function has already been configured correctly for
1643da14cebeSEric Cheng * current subflow configuration, do nothing.
1644da14cebeSEric Cheng */
1645da14cebeSEric Cheng if (flent->fe_cb_fn == (flow_fn_t)rx_func)
1646da14cebeSEric Cheng return;
1647da14cebeSEric Cheng
1648da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt;
1649da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) {
1650da14cebeSEric Cheng ASSERT(flent->fe_rx_srs[i] != NULL);
1651da14cebeSEric Cheng mac_srs_poll_state_change(flent->fe_rx_srs[i],
1652da14cebeSEric Cheng enable_classifier, rx_func);
1653da14cebeSEric Cheng }
1654da14cebeSEric Cheng
1655da14cebeSEric Cheng /*
1656da14cebeSEric Cheng * Change the S/W classifier so that we can land in the
1657da14cebeSEric Cheng * correct processing function with correct argument.
1658da14cebeSEric Cheng * If all subflows have been removed we can revert to
165984de666eSRyan Zezeski * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process().
1660da14cebeSEric Cheng */
1661da14cebeSEric Cheng mutex_enter(&flent->fe_lock);
1662da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)rx_func;
1663da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip;
1664da14cebeSEric Cheng flent->fe_cb_arg2 = flent->fe_rx_srs[0];
1665da14cebeSEric Cheng mutex_exit(&flent->fe_lock);
1666da14cebeSEric Cheng }
1667da14cebeSEric Cheng
1668da14cebeSEric Cheng static void
mac_srs_update_fanout_list(mac_soft_ring_set_t * mac_srs)1669da14cebeSEric Cheng mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs)
1670da14cebeSEric Cheng {
16710dc2366fSVenugopal Iyer int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0;
1672da14cebeSEric Cheng mac_soft_ring_t *softring;
1673da14cebeSEric Cheng
1674da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head;
1675da14cebeSEric Cheng if (softring == NULL) {
1676da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 0);
1677da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = 0;
1678da14cebeSEric Cheng mac_srs->srs_udp_ring_count = 0;
1679da14cebeSEric Cheng mac_srs->srs_oth_ring_count = 0;
16800dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = 0;
1681da14cebeSEric Cheng return;
1682da14cebeSEric Cheng }
1683da14cebeSEric Cheng
1684da14cebeSEric Cheng while (softring != NULL) {
16850dc2366fSVenugopal Iyer if (softring->s_ring_type & ST_RING_TCP) {
1686da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings[tcp_count++] = softring;
16870dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_UDP) {
1688da14cebeSEric Cheng mac_srs->srs_udp_soft_rings[udp_count++] = softring;
16890dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_OTH) {
1690da14cebeSEric Cheng mac_srs->srs_oth_soft_rings[oth_count++] = softring;
16910dc2366fSVenugopal Iyer } else {
16920dc2366fSVenugopal Iyer ASSERT(softring->s_ring_type & ST_RING_TX);
16930dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings[tx_count++] = softring;
16940dc2366fSVenugopal Iyer }
1695da14cebeSEric Cheng softring = softring->s_ring_next;
1696da14cebeSEric Cheng }
1697da14cebeSEric Cheng
1698da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count ==
16990dc2366fSVenugopal Iyer (tcp_count + udp_count + oth_count + tx_count));
1700da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = tcp_count;
1701da14cebeSEric Cheng mac_srs->srs_udp_ring_count = udp_count;
1702da14cebeSEric Cheng mac_srs->srs_oth_ring_count = oth_count;
17030dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = tx_count;
1704da14cebeSEric Cheng }
1705da14cebeSEric Cheng
1706da14cebeSEric Cheng void
mac_srs_create_proto_softrings(int id,uint16_t type,pri_t pri,mac_client_impl_t * mcip,mac_soft_ring_set_t * mac_srs,processorid_t cpuid,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,boolean_t set_bypass)17070dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri,
17080dc2366fSVenugopal Iyer mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs,
1709da14cebeSEric Cheng processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1,
1710da14cebeSEric Cheng mac_resource_handle_t x_arg2, boolean_t set_bypass)
1711da14cebeSEric Cheng {
1712da14cebeSEric Cheng mac_soft_ring_t *softring;
1713da14cebeSEric Cheng mac_rx_fifo_t mrf;
1714da14cebeSEric Cheng
1715da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t));
1716da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO;
1717da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll;
17182c465844SToomas Soome mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable;
17192c465844SToomas Soome mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable;
1720da14cebeSEric Cheng mrf.mrf_flow_priority = pri;
1721da14cebeSEric Cheng
1722da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
17230dc2366fSVenugopal Iyer (type|ST_RING_TCP), pri, mcip, mac_srs,
1724da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2);
1725da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL;
1726da14cebeSEric Cheng
1727da14cebeSEric Cheng /*
1728da14cebeSEric Cheng * TCP and UDP support DLS bypass. In addition TCP
1729da14cebeSEric Cheng * squeue can also poll their corresponding soft rings.
1730da14cebeSEric Cheng */
1731da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1732da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring,
1733da14cebeSEric Cheng mcip->mci_direct_rx_fn,
1734da14cebeSEric Cheng mcip->mci_direct_rx_arg);
1735da14cebeSEric Cheng
1736da14cebeSEric Cheng mrf.mrf_rx_arg = softring;
1737da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring;
1738da14cebeSEric Cheng
1739da14cebeSEric Cheng /*
1740da14cebeSEric Cheng * Make a call in IP to get a TCP squeue assigned to
1741da14cebeSEric Cheng * this softring to maintain full CPU locality through
1742da14cebeSEric Cheng * the stack and allow the squeue to be able to poll
1743da14cebeSEric Cheng * the softring so the flow control can be pushed
1744da14cebeSEric Cheng * all the way to H/W.
1745da14cebeSEric Cheng */
1746da14cebeSEric Cheng softring->s_ring_rx_arg2 =
1747da14cebeSEric Cheng mcip->mci_resource_add((void *)mcip->mci_resource_arg,
1748da14cebeSEric Cheng (mac_resource_t *)&mrf);
1749da14cebeSEric Cheng }
1750da14cebeSEric Cheng
1751da14cebeSEric Cheng /*
1752da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we
1753da14cebeSEric Cheng * don't make any ring addition callbacks for non-TCP
1754da14cebeSEric Cheng * rings. Now create the UDP softring and allow it to
1755da14cebeSEric Cheng * bypass the DLS layer.
1756da14cebeSEric Cheng */
1757da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
17580dc2366fSVenugopal Iyer (type|ST_RING_UDP), pri, mcip, mac_srs,
1759da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2);
1760da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL;
1761da14cebeSEric Cheng
1762da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) {
1763da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring,
1764da14cebeSEric Cheng mcip->mci_direct_rx_fn,
1765da14cebeSEric Cheng mcip->mci_direct_rx_arg);
1766da14cebeSEric Cheng }
1767da14cebeSEric Cheng
1768da14cebeSEric Cheng /* Create the Oth softrings which has to go through the DLS */
1769da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait,
17700dc2366fSVenugopal Iyer (type|ST_RING_OTH), pri, mcip, mac_srs,
1771da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2);
1772da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL;
1773da14cebeSEric Cheng }
1774da14cebeSEric Cheng
1775da14cebeSEric Cheng /*
1776da14cebeSEric Cheng * This routine associates a CPU or a set of CPU to process incoming
1777da14cebeSEric Cheng * traffic from a mac client. If multiple CPUs are specified, then
1778da14cebeSEric Cheng * so many soft rings are created with each soft ring worker thread
1779da14cebeSEric Cheng * bound to a CPU in the set. Each soft ring in turn will be
1780da14cebeSEric Cheng * associated with an squeue and the squeue will be moved to the
1781da14cebeSEric Cheng * same CPU as that of the soft ring's.
1782da14cebeSEric Cheng */
1783da14cebeSEric Cheng static void
mac_srs_fanout_modify(mac_client_impl_t * mcip,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs)17840dc2366fSVenugopal Iyer mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func,
17850dc2366fSVenugopal Iyer void *x_arg1, mac_resource_handle_t x_arg2,
17860dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs)
1787da14cebeSEric Cheng {
1788da14cebeSEric Cheng mac_soft_ring_t *softring;
1789ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0;
1790da14cebeSEric Cheng processorid_t cpuid = -1;
1791da14cebeSEric Cheng int i, srings_present, new_fanout_cnt;
1792da14cebeSEric Cheng mac_cpus_t *srs_cpu;
1793da14cebeSEric Cheng
1794da14cebeSEric Cheng /* fanout state is REINIT. Set it back to INIT */
1795da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT);
1796da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1797da14cebeSEric Cheng
1798da14cebeSEric Cheng /* how many are present right now */
1799da14cebeSEric Cheng srings_present = mac_rx_srs->srs_tcp_ring_count;
1800da14cebeSEric Cheng /* new request */
1801da14cebeSEric Cheng srs_cpu = &mac_rx_srs->srs_cpu;
18020dc2366fSVenugopal Iyer new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt;
1803da14cebeSEric Cheng
1804da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock);
1805da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1806da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL;
1807da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock);
1808da14cebeSEric Cheng
1809da14cebeSEric Cheng if (new_fanout_cnt > srings_present) {
1810da14cebeSEric Cheng /* soft rings increased */
1811da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock);
1812da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1813da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock);
1814da14cebeSEric Cheng
1815da14cebeSEric Cheng for (i = mac_rx_srs->srs_tcp_ring_count;
1816da14cebeSEric Cheng i < new_fanout_cnt; i++) {
1817da14cebeSEric Cheng /*
1818da14cebeSEric Cheng * Create the protocol softrings and set the
1819da14cebeSEric Cheng * DLS bypass where possible.
1820da14cebeSEric Cheng */
18210dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag,
1822da14cebeSEric Cheng mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
1823da14cebeSEric Cheng rx_func, x_arg1, x_arg2, B_TRUE);
1824da14cebeSEric Cheng }
1825da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs);
1826da14cebeSEric Cheng } else if (new_fanout_cnt < srings_present) {
1827da14cebeSEric Cheng /* soft rings decreased */
1828da14cebeSEric Cheng if (new_fanout_cnt == 1) {
1829da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock);
1830da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP;
1831da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO);
1832da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock);
1833da14cebeSEric Cheng }
1834da14cebeSEric Cheng /* Get rid of extra soft rings */
1835da14cebeSEric Cheng for (i = new_fanout_cnt;
1836da14cebeSEric Cheng i < mac_rx_srs->srs_tcp_ring_count; i++) {
1837da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i];
1838da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) {
1839da14cebeSEric Cheng mcip->mci_resource_remove(
1840da14cebeSEric Cheng (void *)mcip->mci_resource_arg,
1841da14cebeSEric Cheng softring->s_ring_rx_arg2);
1842da14cebeSEric Cheng }
1843da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs,
1844da14cebeSEric Cheng mac_rx_srs->srs_tcp_soft_rings[i]);
1845da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs,
1846da14cebeSEric Cheng mac_rx_srs->srs_udp_soft_rings[i]);
1847da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs,
1848da14cebeSEric Cheng mac_rx_srs->srs_oth_soft_rings[i]);
1849da14cebeSEric Cheng }
1850da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs);
1851da14cebeSEric Cheng }
1852da14cebeSEric Cheng
1853da14cebeSEric Cheng ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count);
1854da14cebeSEric Cheng mutex_enter(&cpu_lock);
1855da14cebeSEric Cheng for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) {
18560dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1857da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i],
1858da14cebeSEric Cheng cpuid);
1859da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i],
1860da14cebeSEric Cheng cpuid);
1861da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i],
1862da14cebeSEric Cheng cpuid);
1863da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i];
1864da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) {
1865da14cebeSEric Cheng mcip->mci_resource_bind((void *)mcip->mci_resource_arg,
1866da14cebeSEric Cheng softring->s_ring_rx_arg2, cpuid);
1867da14cebeSEric Cheng }
1868da14cebeSEric Cheng }
1869da14cebeSEric Cheng
18700dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
18710dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
18720dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1873da14cebeSEric Cheng /*
1874da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. Let's bind tx
1875da14cebeSEric Cheng * srs to the last cpu in mrp list.
1876da14cebeSEric Cheng */
18770dc2366fSVenugopal Iyer if (mac_tx_srs != NULL) {
1878da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
18790dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs);
1880da14cebeSEric Cheng }
1881da14cebeSEric Cheng mutex_exit(&cpu_lock);
1882da14cebeSEric Cheng }
1883da14cebeSEric Cheng
1884da14cebeSEric Cheng /*
1885da14cebeSEric Cheng * Bind SRS threads and soft rings to CPUs/create fanout list.
1886da14cebeSEric Cheng */
1887da14cebeSEric Cheng void
mac_srs_fanout_init(mac_client_impl_t * mcip,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_soft_ring_set_t * mac_rx_srs,mac_soft_ring_set_t * mac_tx_srs,cpupart_t * cpupart)18880dc2366fSVenugopal Iyer mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
18890dc2366fSVenugopal Iyer mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
18900dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs,
18910dc2366fSVenugopal Iyer cpupart_t *cpupart)
1892da14cebeSEric Cheng {
1893da14cebeSEric Cheng int i;
18940dc2366fSVenugopal Iyer processorid_t cpuid;
1895ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0;
1896da14cebeSEric Cheng int soft_ring_cnt;
1897da14cebeSEric Cheng mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu;
1898da14cebeSEric Cheng
1899da14cebeSEric Cheng /*
1900da14cebeSEric Cheng * Remove the no soft ring flag and we will adjust it
1901da14cebeSEric Cheng * appropriately further down.
1902da14cebeSEric Cheng */
1903da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock);
1904da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS;
1905da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock);
1906da14cebeSEric Cheng
1907da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_soft_ring_head == NULL);
1908da14cebeSEric Cheng
1909da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL)
1910da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL;
1911da14cebeSEric Cheng
1912da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT);
1913da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT;
1914da14cebeSEric Cheng /*
1915da14cebeSEric Cheng * Ring count can be 0 if no fanout is required and no cpu
1916da14cebeSEric Cheng * were specified. Leave the SRS worker and poll thread
1917da14cebeSEric Cheng * unbound
1918da14cebeSEric Cheng */
1919da14cebeSEric Cheng ASSERT(mrp != NULL);
19200dc2366fSVenugopal Iyer soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt;
1921da14cebeSEric Cheng
1922da14cebeSEric Cheng /* Step 1: bind cpu contains cpu list where threads need to bind */
1923da14cebeSEric Cheng if (soft_ring_cnt > 0) {
1924da14cebeSEric Cheng mutex_enter(&cpu_lock);
1925da14cebeSEric Cheng for (i = 0; i < soft_ring_cnt; i++) {
19260dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i];
1927da14cebeSEric Cheng /* Create the protocol softrings */
19280dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag,
19290dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
19300dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE);
1931da14cebeSEric Cheng }
19320dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid);
19330dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid);
19340dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu);
1935da14cebeSEric Cheng /*
1936da14cebeSEric Cheng * Bind Tx srs and soft ring threads too.
1937da14cebeSEric Cheng * Let's bind tx srs to the last cpu in
1938da14cebeSEric Cheng * mrp list.
1939da14cebeSEric Cheng */
1940da14cebeSEric Cheng if (mac_tx_srs == NULL) {
1941da14cebeSEric Cheng mutex_exit(&cpu_lock);
1942da14cebeSEric Cheng goto alldone;
1943da14cebeSEric Cheng }
1944da14cebeSEric Cheng
19450dc2366fSVenugopal Iyer BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp);
19460dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs);
1947da14cebeSEric Cheng mutex_exit(&cpu_lock);
1948da14cebeSEric Cheng } else {
1949da14cebeSEric Cheng mutex_enter(&cpu_lock);
1950da14cebeSEric Cheng /*
1951da14cebeSEric Cheng * For a subflow, mrp_workerid and mrp_pollid
1952da14cebeSEric Cheng * is not set.
1953da14cebeSEric Cheng */
19540dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid);
19550dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid);
1956da14cebeSEric Cheng mutex_exit(&cpu_lock);
1957da14cebeSEric Cheng goto no_softrings;
1958da14cebeSEric Cheng }
1959da14cebeSEric Cheng
1960da14cebeSEric Cheng alldone:
1961da14cebeSEric Cheng if (soft_ring_cnt > 1)
1962da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP;
1963da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs);
1964da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs);
1965da14cebeSEric Cheng return;
1966da14cebeSEric Cheng
1967da14cebeSEric Cheng no_softrings:
1968da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) {
1969da14cebeSEric Cheng mutex_enter(&cpu_lock);
19700dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart);
1971da14cebeSEric Cheng /* Create the protocol softrings */
19720dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(0, soft_ring_flag,
19730dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid,
19740dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE);
1975da14cebeSEric Cheng mutex_exit(&cpu_lock);
1976da14cebeSEric Cheng } else {
1977da14cebeSEric Cheng /*
1978da14cebeSEric Cheng * This is the case when there is no fanout which is
1979da14cebeSEric Cheng * true for subflows.
1980da14cebeSEric Cheng */
1981da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS;
1982da14cebeSEric Cheng }
1983da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs);
1984da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs);
1985da14cebeSEric Cheng }
1986da14cebeSEric Cheng
1987da14cebeSEric Cheng /*
1988da14cebeSEric Cheng * Calls mac_srs_fanout_init() or modify() depending upon whether
1989da14cebeSEric Cheng * the SRS is getting initialized or re-initialized.
1990da14cebeSEric Cheng */
1991da14cebeSEric Cheng void
mac_fanout_setup(mac_client_impl_t * mcip,flow_entry_t * flent,mac_resource_props_t * mrp,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,cpupart_t * cpupart)1992da14cebeSEric Cheng mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
1993da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1,
19940dc2366fSVenugopal Iyer mac_resource_handle_t x_arg2, cpupart_t *cpupart)
1995da14cebeSEric Cheng {
1996da14cebeSEric Cheng mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs;
1997da14cebeSEric Cheng int i, rx_srs_cnt;
1998da14cebeSEric Cheng
1999da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
200045948e49SRyan Zezeski
2001da14cebeSEric Cheng /*
200245948e49SRyan Zezeski * Aggr ports do not have SRSes. This function should never be
200345948e49SRyan Zezeski * called on an aggr port.
2004da14cebeSEric Cheng */
200545948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
2006da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[0];
200745948e49SRyan Zezeski
2008da14cebeSEric Cheng /*
2009da14cebeSEric Cheng * Set up the fanout on the tx side only once, with the
2010da14cebeSEric Cheng * first rx SRS. The CPU binding, fanout, and bandwidth
2011da14cebeSEric Cheng * criteria are common to both RX and TX, so
2012da14cebeSEric Cheng * initializing them along side avoids redundant code.
2013da14cebeSEric Cheng */
2014da14cebeSEric Cheng mac_tx_srs = flent->fe_tx_srs;
2015da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt;
2016da14cebeSEric Cheng
2017da14cebeSEric Cheng /* No fanout for subflows */
2018da14cebeSEric Cheng if (flent->fe_type & FLOW_USER) {
20190dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func,
20200dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
20210dc2366fSVenugopal Iyer cpupart);
2022da14cebeSEric Cheng return;
2023da14cebeSEric Cheng }
2024da14cebeSEric Cheng
20250dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_CPUS_USERSPEC)
20260dc2366fSVenugopal Iyer mac_flow_user_cpu_init(flent, mrp);
20270dc2366fSVenugopal Iyer else
20280dc2366fSVenugopal Iyer mac_flow_cpu_init(flent, cpupart);
20290dc2366fSVenugopal Iyer
20300dc2366fSVenugopal Iyer mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt;
2031da14cebeSEric Cheng
2032da14cebeSEric Cheng /*
2033da14cebeSEric Cheng * Set up fanout for both SW (0th SRS) and HW classified
2034da14cebeSEric Cheng * SRS (the rest of Rx SRSs in flent).
2035da14cebeSEric Cheng */
2036da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) {
2037da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[i];
2038da14cebeSEric Cheng if (i != 0)
2039da14cebeSEric Cheng mac_tx_srs = NULL;
2040da14cebeSEric Cheng switch (mac_rx_srs->srs_fanout_state) {
2041da14cebeSEric Cheng case SRS_FANOUT_UNINIT:
20420dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func,
20430dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs,
20440dc2366fSVenugopal Iyer cpupart);
2045da14cebeSEric Cheng break;
2046da14cebeSEric Cheng case SRS_FANOUT_INIT:
2047da14cebeSEric Cheng break;
2048da14cebeSEric Cheng case SRS_FANOUT_REINIT:
2049da14cebeSEric Cheng mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE);
20500dc2366fSVenugopal Iyer mac_srs_fanout_modify(mcip, rx_func, x_arg1,
20510dc2366fSVenugopal Iyer x_arg2, mac_rx_srs, mac_tx_srs);
2052da14cebeSEric Cheng mac_rx_srs_restart(mac_rx_srs);
2053da14cebeSEric Cheng break;
2054da14cebeSEric Cheng default:
2055da14cebeSEric Cheng VERIFY(mac_rx_srs->srs_fanout_state <=
2056da14cebeSEric Cheng SRS_FANOUT_REINIT);
2057da14cebeSEric Cheng break;
2058da14cebeSEric Cheng }
2059da14cebeSEric Cheng }
2060da14cebeSEric Cheng }
2061da14cebeSEric Cheng
2062da14cebeSEric Cheng /*
2063da14cebeSEric Cheng * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is
2064da14cebeSEric Cheng * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side
2065da14cebeSEric Cheng * processing is created.
2066da14cebeSEric Cheng *
2067da14cebeSEric Cheng * Details on Rx SRS:
2068da14cebeSEric Cheng * Create a SRS and also add the necessary soft rings for TCP and
2069da14cebeSEric Cheng * non-TCP based on fanout type and count specified.
2070da14cebeSEric Cheng *
2071da14cebeSEric Cheng * mac_soft_ring_fanout, mac_srs_fanout_modify (?),
2072da14cebeSEric Cheng * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need
2073da14cebeSEric Cheng * to be heavily modified.
2074da14cebeSEric Cheng *
2075da14cebeSEric Cheng * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear.
2076da14cebeSEric Cheng */
2077da14cebeSEric Cheng mac_soft_ring_set_t *
mac_srs_create(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t srs_type,mac_direct_rx_t rx_func,void * x_arg1,mac_resource_handle_t x_arg2,mac_ring_t * ring)2078da14cebeSEric Cheng mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type,
2079da14cebeSEric Cheng mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2,
2080da14cebeSEric Cheng mac_ring_t *ring)
2081da14cebeSEric Cheng {
208236f99a58SToomas Soome mac_soft_ring_set_t *mac_srs;
2083da14cebeSEric Cheng mac_srs_rx_t *srs_rx;
2084da14cebeSEric Cheng mac_srs_tx_t *srs_tx;
2085da14cebeSEric Cheng mac_bw_ctl_t *mac_bw;
2086da14cebeSEric Cheng mac_resource_props_t *mrp;
2087da14cebeSEric Cheng boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0);
2088da14cebeSEric Cheng
2089da14cebeSEric Cheng mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP);
2090da14cebeSEric Cheng bzero(mac_srs, sizeof (mac_soft_ring_set_t));
2091da14cebeSEric Cheng srs_rx = &mac_srs->srs_rx;
2092da14cebeSEric Cheng srs_tx = &mac_srs->srs_tx;
2093da14cebeSEric Cheng
2094da14cebeSEric Cheng mutex_enter(&flent->fe_lock);
2095da14cebeSEric Cheng
2096da14cebeSEric Cheng /*
2097da14cebeSEric Cheng * Get the bandwidth control structure from the flent. Get
2098da14cebeSEric Cheng * rid of any residual values in the control structure for
2099da14cebeSEric Cheng * the tx bw struct and also for the rx, if the rx srs is
2100da14cebeSEric Cheng * the 1st one being brought up (the rx bw ctl struct may
2101da14cebeSEric Cheng * be shared by multiple SRSs)
2102da14cebeSEric Cheng */
2103da14cebeSEric Cheng if (is_tx_srs) {
2104da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_tx_bw;
2105da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
2106da14cebeSEric Cheng flent->fe_tx_srs = mac_srs;
2107da14cebeSEric Cheng } else {
2108da14cebeSEric Cheng /*
2109da14cebeSEric Cheng * The bw counter (stored in the flent) is shared
2110da14cebeSEric Cheng * by SRS's within an rx group.
2111da14cebeSEric Cheng */
2112da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_rx_bw;
2113da14cebeSEric Cheng /* First rx SRS, clear the bw structure */
2114da14cebeSEric Cheng if (flent->fe_rx_srs_cnt == 0)
2115da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t));
211677e3ef6dSVenugopal Iyer
211777e3ef6dSVenugopal Iyer /*
211877e3ef6dSVenugopal Iyer * It is better to panic here rather than just assert because
211977e3ef6dSVenugopal Iyer * on a non-debug kernel we might end up courrupting memory
212077e3ef6dSVenugopal Iyer * and making it difficult to debug.
212177e3ef6dSVenugopal Iyer */
212277e3ef6dSVenugopal Iyer if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) {
212377e3ef6dSVenugopal Iyer panic("Array Overrun detected due to MAC client %p "
212477e3ef6dSVenugopal Iyer " having more rings than %d", (void *)mcip,
212577e3ef6dSVenugopal Iyer MAX_RINGS_PER_GROUP);
212677e3ef6dSVenugopal Iyer }
2127da14cebeSEric Cheng flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs;
2128da14cebeSEric Cheng flent->fe_rx_srs_cnt++;
2129da14cebeSEric Cheng }
2130da14cebeSEric Cheng mac_srs->srs_flent = flent;
2131da14cebeSEric Cheng mutex_exit(&flent->fe_lock);
2132da14cebeSEric Cheng
2133da14cebeSEric Cheng mac_srs->srs_state = 0;
2134da14cebeSEric Cheng mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS);
2135da14cebeSEric Cheng mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1;
2136da14cebeSEric Cheng mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1;
21370dc2366fSVenugopal Iyer mac_srs->srs_mcip = mcip;
2138da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_srs);
2139da14cebeSEric Cheng
2140da14cebeSEric Cheng /*
2141da14cebeSEric Cheng * For a flow we use the underlying MAC client's priority range with
2142da14cebeSEric Cheng * the priority value to find an absolute priority value. For a MAC
2143da14cebeSEric Cheng * client we use the MAC client's maximum priority as the value.
2144da14cebeSEric Cheng */
2145da14cebeSEric Cheng mrp = &flent->fe_effective_props;
2146da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_FLOW) != 0) {
2147da14cebeSEric Cheng mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri,
2148da14cebeSEric Cheng mcip->mci_max_pri, mrp->mrp_priority);
2149da14cebeSEric Cheng } else {
2150da14cebeSEric Cheng mac_srs->srs_pri = mcip->mci_max_pri;
2151da14cebeSEric Cheng }
2152da14cebeSEric Cheng /*
2153da14cebeSEric Cheng * We need to insert the SRS in the global list before
2154da14cebeSEric Cheng * binding the SRS and SR threads. Otherwise there is a
2155da14cebeSEric Cheng * is a small window where the cpu reconfig callbacks
2156da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail
2157da14cebeSEric Cheng * as there are bound threads.
2158da14cebeSEric Cheng */
2159da14cebeSEric Cheng mac_srs_add_glist(mac_srs);
2160da14cebeSEric Cheng
2161da14cebeSEric Cheng /* Initialize bw limit */
2162da14cebeSEric Cheng if ((mrp->mrp_mask & MRP_MAXBW) != 0) {
2163da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain_bw;
2164da14cebeSEric Cheng
2165da14cebeSEric Cheng mac_bw = mac_srs->srs_bw;
2166da14cebeSEric Cheng mutex_enter(&mac_bw->mac_bw_lock);
2167da14cebeSEric Cheng mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw);
2168da14cebeSEric Cheng
2169da14cebeSEric Cheng /*
2170da14cebeSEric Cheng * Give twice the queuing capability before
2171da14cebeSEric Cheng * dropping packets. The unit is bytes/tick.
2172da14cebeSEric Cheng */
2173da14cebeSEric Cheng mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1;
2174da14cebeSEric Cheng mutex_exit(&mac_bw->mac_bw_lock);
2175da14cebeSEric Cheng mac_srs->srs_type |= SRST_BW_CONTROL;
2176da14cebeSEric Cheng } else {
2177da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain;
2178da14cebeSEric Cheng }
2179da14cebeSEric Cheng
2180da14cebeSEric Cheng /*
2181da14cebeSEric Cheng * We use the following policy to control Receive
2182da14cebeSEric Cheng * Side Dynamic Polling:
2183da14cebeSEric Cheng * 1) We switch to poll mode anytime the processing thread causes
2184da14cebeSEric Cheng * a backlog to build up in SRS and its associated Soft Rings
2185da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0).
2186da14cebeSEric Cheng * 2) As long as the backlog stays under the low water mark
2187da14cebeSEric Cheng * (sr_lowat), we poll the H/W for more packets.
2188da14cebeSEric Cheng * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we
2189da14cebeSEric Cheng * stay in poll mode but don't poll the H/W for more packets.
2190da14cebeSEric Cheng * 4) Anytime in polling mode, if we poll the H/W for packets and
2191da14cebeSEric Cheng * find nothing plus we have an existing backlog
2192da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll
2193da14cebeSEric Cheng * the H/W for packets anymore (let the polling thread go to sleep).
219484de666eSRyan Zezeski * 5) Once the backlog is relieved (packets are processed) we reenable
2195da14cebeSEric Cheng * polling (by signalling the poll thread) only when the backlog
2196da14cebeSEric Cheng * dips below sr_poll_thres.
2197da14cebeSEric Cheng * 6) sr_hiwat is used exclusively when we are not polling capable
2198da14cebeSEric Cheng * and is used to decide when to drop packets so the SRS queue
2199da14cebeSEric Cheng * length doesn't grow infinitely.
2200da14cebeSEric Cheng */
2201da14cebeSEric Cheng if (!is_tx_srs) {
2202da14cebeSEric Cheng srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt;
2203da14cebeSEric Cheng /* Low water mark needs to be less than high water mark */
2204da14cebeSEric Cheng srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <=
2205da14cebeSEric Cheng mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt :
2206da14cebeSEric Cheng (mac_soft_ring_max_q_cnt >> 2);
2207da14cebeSEric Cheng /* Poll threshold need to be half of low water mark or less */
2208da14cebeSEric Cheng srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <=
2209da14cebeSEric Cheng (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres :
2210da14cebeSEric Cheng (srs_rx->sr_lowat >> 1);
2211da14cebeSEric Cheng if (mac_latency_optimize)
22123631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_LATENCY_OPT;
22133631b19bSRajagopal Kunhappan else
22143631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2215da14cebeSEric Cheng }
2216da14cebeSEric Cheng
2217cab61dfbSBryan Cantrill /*
2218cab61dfbSBryan Cantrill * Create the srs_worker with twice the stack of a normal kernel thread
2219cab61dfbSBryan Cantrill * to reduce the likelihood of stack overflows in receive-side
2220cab61dfbSBryan Cantrill * processing. (The larger stacks are not the only precaution taken
2221cab61dfbSBryan Cantrill * against stack overflows; see the use of mac_rx_srs_stack_needed
2222cab61dfbSBryan Cantrill * in mac_sched.c).
2223cab61dfbSBryan Cantrill */
2224cab61dfbSBryan Cantrill mac_srs->srs_worker = thread_create(NULL, default_stksize << 1,
2225da14cebeSEric Cheng mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri);
2226da14cebeSEric Cheng
2227da14cebeSEric Cheng if (is_tx_srs) {
2228da14cebeSEric Cheng /* Handle everything about Tx SRS and return */
2229da14cebeSEric Cheng mac_srs->srs_drain_func = mac_tx_srs_drain;
2230da14cebeSEric Cheng srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt;
2231da14cebeSEric Cheng srs_tx->st_hiwat =
2232da14cebeSEric Cheng (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ?
2233da14cebeSEric Cheng mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat;
2234da14cebeSEric Cheng srs_tx->st_arg1 = x_arg1;
2235da14cebeSEric Cheng srs_tx->st_arg2 = x_arg2;
22360dc2366fSVenugopal Iyer goto done;
2237da14cebeSEric Cheng }
2238da14cebeSEric Cheng
2239da14cebeSEric Cheng if ((srs_type & SRST_FLOW) != 0 ||
2240da14cebeSEric Cheng FLOW_TAB_EMPTY(mcip->mci_subflow_tab))
2241da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_process;
2242da14cebeSEric Cheng else
2243da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_subflow_process;
2244da14cebeSEric Cheng
2245da14cebeSEric Cheng srs_rx->sr_func = rx_func;
2246da14cebeSEric Cheng srs_rx->sr_arg1 = x_arg1;
2247da14cebeSEric Cheng srs_rx->sr_arg2 = x_arg2;
2248da14cebeSEric Cheng
2249da14cebeSEric Cheng if (ring != NULL) {
22500dc2366fSVenugopal Iyer uint_t ring_info;
22510dc2366fSVenugopal Iyer
2252da14cebeSEric Cheng /* Is the mac_srs created over the RX default group? */
2253da14cebeSEric Cheng if (ring->mr_gh == (mac_group_handle_t)
22540dc2366fSVenugopal Iyer MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) {
2255da14cebeSEric Cheng mac_srs->srs_type |= SRST_DEFAULT_GRP;
22560dc2366fSVenugopal Iyer }
2257da14cebeSEric Cheng mac_srs->srs_ring = ring;
2258da14cebeSEric Cheng ring->mr_srs = mac_srs;
2259da14cebeSEric Cheng ring->mr_classify_type = MAC_HW_CLASSIFIER;
2260da14cebeSEric Cheng ring->mr_flag |= MR_INCIPIENT;
2261da14cebeSEric Cheng
22624eaa4710SRishi Srivatsavai if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) &&
22634eaa4710SRishi Srivatsavai FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable)
2264da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB;
2265da14cebeSEric Cheng
2266da14cebeSEric Cheng mac_srs->srs_poll_thr = thread_create(NULL, 0,
2267da14cebeSEric Cheng mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN,
2268da14cebeSEric Cheng mac_srs->srs_pri);
2269ae6aa22aSVenugopal Iyer /*
2270ae6aa22aSVenugopal Iyer * Some drivers require serialization and don't send
2271ae6aa22aSVenugopal Iyer * packet chains in interrupt context. For such
227284de666eSRyan Zezeski * drivers, we should always queue in the soft ring
227384de666eSRyan Zezeski * so that we get a chance to switch into polling
2274ae6aa22aSVenugopal Iyer * mode under backlog.
2275ae6aa22aSVenugopal Iyer */
22760dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring);
22770dc2366fSVenugopal Iyer if (ring_info & MAC_RING_RX_ENQUEUE)
2278ae6aa22aSVenugopal Iyer mac_srs->srs_state |= SRS_SOFTRING_QUEUE;
2279da14cebeSEric Cheng }
22800dc2366fSVenugopal Iyer done:
22810dc2366fSVenugopal Iyer mac_srs_stat_create(mac_srs);
2282da14cebeSEric Cheng return (mac_srs);
2283da14cebeSEric Cheng }
2284da14cebeSEric Cheng
2285da14cebeSEric Cheng /*
2286da14cebeSEric Cheng * Figure out the number of soft rings required. Its dependant on
2287da14cebeSEric Cheng * if protocol fanout is required (for LINKs), global settings
2288da14cebeSEric Cheng * require us to do fanout for performance (based on mac_soft_ring_enable),
2289da14cebeSEric Cheng * or user has specifically requested fanout.
2290da14cebeSEric Cheng */
2291da14cebeSEric Cheng static uint32_t
mac_find_fanout(flow_entry_t * flent,uint32_t link_type)2292da14cebeSEric Cheng mac_find_fanout(flow_entry_t *flent, uint32_t link_type)
2293da14cebeSEric Cheng {
2294da14cebeSEric Cheng uint32_t fanout_type;
2295da14cebeSEric Cheng mac_resource_props_t *mrp = &flent->fe_effective_props;
2296da14cebeSEric Cheng
2297da14cebeSEric Cheng /* no fanout for subflows */
2298da14cebeSEric Cheng switch (link_type) {
2299da14cebeSEric Cheng case SRST_FLOW:
2300da14cebeSEric Cheng fanout_type = SRST_NO_SOFT_RINGS;
2301da14cebeSEric Cheng break;
2302da14cebeSEric Cheng case SRST_LINK:
2303da14cebeSEric Cheng fanout_type = SRST_FANOUT_PROTO;
2304da14cebeSEric Cheng break;
2305da14cebeSEric Cheng }
2306da14cebeSEric Cheng
2307da14cebeSEric Cheng /* A primary NIC/link is being plumbed */
2308da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) {
2309da14cebeSEric Cheng if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) {
2310da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP;
2311da14cebeSEric Cheng }
2312da14cebeSEric Cheng } else if (flent->fe_type & FLOW_VNIC) {
2313da14cebeSEric Cheng /* A VNIC is being created */
2314da14cebeSEric Cheng if (mrp != NULL && mrp->mrp_ncpus > 0) {
2315da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP;
2316da14cebeSEric Cheng }
2317da14cebeSEric Cheng }
2318da14cebeSEric Cheng
2319da14cebeSEric Cheng return (fanout_type);
2320da14cebeSEric Cheng }
2321da14cebeSEric Cheng
2322da14cebeSEric Cheng /*
2323da14cebeSEric Cheng * Change a group from h/w to s/w classification.
2324da14cebeSEric Cheng */
23250dc2366fSVenugopal Iyer void
mac_rx_switch_grp_to_sw(mac_group_t * group)2326da14cebeSEric Cheng mac_rx_switch_grp_to_sw(mac_group_t *group)
2327da14cebeSEric Cheng {
2328da14cebeSEric Cheng mac_ring_t *ring;
2329da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs;
2330da14cebeSEric Cheng
2331da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) {
2332da14cebeSEric Cheng if (ring->mr_classify_type == MAC_HW_CLASSIFIER) {
2333da14cebeSEric Cheng /*
2334da14cebeSEric Cheng * Remove the SRS associated with the HW ring.
2335da14cebeSEric Cheng * As a result, polling will be disabled.
2336da14cebeSEric Cheng */
2337da14cebeSEric Cheng mac_srs = ring->mr_srs;
2338da14cebeSEric Cheng ASSERT(mac_srs != NULL);
2339da14cebeSEric Cheng mac_rx_srs_remove(mac_srs);
2340da14cebeSEric Cheng ring->mr_srs = NULL;
2341da14cebeSEric Cheng }
2342da14cebeSEric Cheng
2343da14cebeSEric Cheng if (ring->mr_state != MR_INUSE)
2344da14cebeSEric Cheng (void) mac_start_ring(ring);
23450dc2366fSVenugopal Iyer
2346da14cebeSEric Cheng /*
2347da14cebeSEric Cheng * We need to perform SW classification
2348da14cebeSEric Cheng * for packets landing in these rings
2349da14cebeSEric Cheng */
2350da14cebeSEric Cheng ring->mr_flag = 0;
2351da14cebeSEric Cheng ring->mr_classify_type = MAC_SW_CLASSIFIER;
2352da14cebeSEric Cheng }
2353da14cebeSEric Cheng }
2354da14cebeSEric Cheng
2355da14cebeSEric Cheng /*
2356da14cebeSEric Cheng * Create the Rx SRS for S/W classifier and for each ring in the
2357da14cebeSEric Cheng * group (if exclusive group). Also create the Tx SRS.
2358da14cebeSEric Cheng */
2359da14cebeSEric Cheng void
mac_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2360da14cebeSEric Cheng mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
23610dc2366fSVenugopal Iyer uint32_t link_type)
23620dc2366fSVenugopal Iyer {
23630dc2366fSVenugopal Iyer cpupart_t *cpupart;
23640dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
23650dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip);
23660dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE;
23670dc2366fSVenugopal Iyer
23680dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mcip, flent, link_type);
23690dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mcip, flent, link_type);
23700dc2366fSVenugopal Iyer
237145948e49SRyan Zezeski /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */
237245948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
237345948e49SRyan Zezeski return;
237445948e49SRyan Zezeski
23750dc2366fSVenugopal Iyer pool_lock();
23760dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default);
23770dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip),
23780dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart);
23790dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp);
23800dc2366fSVenugopal Iyer pool_unlock();
23810dc2366fSVenugopal Iyer }
23820dc2366fSVenugopal Iyer
23830dc2366fSVenugopal Iyer /*
238484de666eSRyan Zezeski * Set up the Rx SRSes. If there is no group associated with the
238584de666eSRyan Zezeski * client, then only setup SW classification. If the client has
238684de666eSRyan Zezeski * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an
238784de666eSRyan Zezeski * SRS for each HW ring. If the client is sharing a group, then make
238884de666eSRyan Zezeski * sure to teardown the HW SRSes.
23890dc2366fSVenugopal Iyer */
23900dc2366fSVenugopal Iyer void
mac_rx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)23910dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
23920dc2366fSVenugopal Iyer uint32_t link_type)
2393da14cebeSEric Cheng {
2394da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip;
2395da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs;
239636f99a58SToomas Soome mac_ring_t *ring;
2397da14cebeSEric Cheng uint32_t fanout_type;
23980dc2366fSVenugopal Iyer mac_group_t *rx_group = flent->fe_rx_ring_group;
239984de666eSRyan Zezeski boolean_t no_unicast;
2400da14cebeSEric Cheng
240145948e49SRyan Zezeski /*
240245948e49SRyan Zezeski * If this is an an aggr port, then don't setup Rx SRS and Rx
240345948e49SRyan Zezeski * soft rings as they won't be used. However, we still need to
240445948e49SRyan Zezeski * start the rings to receive data on them.
240545948e49SRyan Zezeski */
240645948e49SRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) {
240745948e49SRyan Zezeski if (rx_group == NULL)
240845948e49SRyan Zezeski return;
240945948e49SRyan Zezeski
241045948e49SRyan Zezeski for (ring = rx_group->mrg_rings; ring != NULL;
241145948e49SRyan Zezeski ring = ring->mr_next) {
241245948e49SRyan Zezeski if (ring->mr_state != MR_INUSE)
241345948e49SRyan Zezeski (void) mac_start_ring(ring);
241445948e49SRyan Zezeski }
241545948e49SRyan Zezeski
241645948e49SRyan Zezeski return;
241745948e49SRyan Zezeski }
241845948e49SRyan Zezeski
241945948e49SRyan Zezeski /*
242045948e49SRyan Zezeski * Aggr ports should never have SRSes.
242145948e49SRyan Zezeski */
242245948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
242345948e49SRyan Zezeski
2424da14cebeSEric Cheng fanout_type = mac_find_fanout(flent, link_type);
242584de666eSRyan Zezeski no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0;
2426da14cebeSEric Cheng
242784de666eSRyan Zezeski /* Create the SRS for SW classification if none exists */
2428da14cebeSEric Cheng if (flent->fe_rx_srs[0] == NULL) {
2429da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt == 0);
2430da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type,
2431da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, NULL);
2432da14cebeSEric Cheng mutex_enter(&flent->fe_lock);
2433da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc;
2434da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip;
2435da14cebeSEric Cheng flent->fe_cb_arg2 = (void *)mac_srs;
2436da14cebeSEric Cheng mutex_exit(&flent->fe_lock);
2437da14cebeSEric Cheng }
2438da14cebeSEric Cheng
24390dc2366fSVenugopal Iyer if (rx_group == NULL)
2440da14cebeSEric Cheng return;
244184de666eSRyan Zezeski
2442da14cebeSEric Cheng /*
244384de666eSRyan Zezeski * If the group is marked RESERVED then setup an SRS and
244484de666eSRyan Zezeski * fanout for each HW ring.
2445da14cebeSEric Cheng */
24460dc2366fSVenugopal Iyer switch (rx_group->mrg_state) {
2447da14cebeSEric Cheng case MAC_GROUP_STATE_RESERVED:
24480dc2366fSVenugopal Iyer for (ring = rx_group->mrg_rings; ring != NULL;
2449da14cebeSEric Cheng ring = ring->mr_next) {
245084de666eSRyan Zezeski uint16_t vid = i_mac_flow_vid(mcip->mci_flent);
245184de666eSRyan Zezeski
2452da14cebeSEric Cheng switch (ring->mr_state) {
2453da14cebeSEric Cheng case MR_INUSE:
2454da14cebeSEric Cheng case MR_FREE:
2455da14cebeSEric Cheng if (ring->mr_srs != NULL)
2456da14cebeSEric Cheng break;
2457da14cebeSEric Cheng if (ring->mr_state != MR_INUSE)
2458da14cebeSEric Cheng (void) mac_start_ring(ring);
2459da14cebeSEric Cheng
24600dc2366fSVenugopal Iyer /*
246184de666eSRyan Zezeski * If a client requires SW VLAN
246284de666eSRyan Zezeski * filtering or has no unicast address
246384de666eSRyan Zezeski * then we don't create any HW ring
246484de666eSRyan Zezeski * SRSes.
24650dc2366fSVenugopal Iyer */
246684de666eSRyan Zezeski if ((!MAC_GROUP_HW_VLAN(rx_group) &&
246784de666eSRyan Zezeski vid != VLAN_ID_NONE) || no_unicast)
24680dc2366fSVenugopal Iyer break;
246984de666eSRyan Zezeski
247084de666eSRyan Zezeski /*
247184de666eSRyan Zezeski * When a client has exclusive use of
247284de666eSRyan Zezeski * a group, and that group's traffic
247384de666eSRyan Zezeski * is fully HW classified, we create
247484de666eSRyan Zezeski * an SRS for each HW ring in order to
247584de666eSRyan Zezeski * make use of dynamic polling of said
247684de666eSRyan Zezeski * HW rings.
247784de666eSRyan Zezeski */
2478da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent,
2479da14cebeSEric Cheng fanout_type | link_type,
2480da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, ring);
2481da14cebeSEric Cheng break;
2482da14cebeSEric Cheng default:
24830dc2366fSVenugopal Iyer cmn_err(CE_PANIC,
24840dc2366fSVenugopal Iyer "srs_setup: mcip = %p "
2485da14cebeSEric Cheng "trying to add UNKNOWN ring = %p\n",
2486da14cebeSEric Cheng (void *)mcip, (void *)ring);
2487da14cebeSEric Cheng break;
2488da14cebeSEric Cheng }
2489da14cebeSEric Cheng }
2490da14cebeSEric Cheng break;
2491da14cebeSEric Cheng case MAC_GROUP_STATE_SHARED:
2492da14cebeSEric Cheng /*
249384de666eSRyan Zezeski * When a group is shared by multiple clients, we must
249484de666eSRyan Zezeski * use SW classifiction to ensure packets are
249584de666eSRyan Zezeski * delivered to the correct client.
2496da14cebeSEric Cheng */
24970dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rx_group);
2498da14cebeSEric Cheng break;
2499da14cebeSEric Cheng default:
2500da14cebeSEric Cheng ASSERT(B_FALSE);
2501da14cebeSEric Cheng break;
2502da14cebeSEric Cheng }
2503da14cebeSEric Cheng }
2504da14cebeSEric Cheng
25050dc2366fSVenugopal Iyer /*
25060dc2366fSVenugopal Iyer * Set up the TX SRS.
25070dc2366fSVenugopal Iyer */
2508da14cebeSEric Cheng void
mac_tx_srs_group_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)25090dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2510da14cebeSEric Cheng uint32_t link_type)
25110dc2366fSVenugopal Iyer {
25120dc2366fSVenugopal Iyer /*
251345948e49SRyan Zezeski * If this is an exclusive client (e.g. an aggr port), then
251445948e49SRyan Zezeski * don't setup Tx SRS and Tx soft rings as they won't be used.
251545948e49SRyan Zezeski * However, we still need to start the rings to send data
251645948e49SRyan Zezeski * across them.
25170dc2366fSVenugopal Iyer */
25180dc2366fSVenugopal Iyer if (mcip->mci_state_flags & MCIS_EXCLUSIVE) {
251945948e49SRyan Zezeski mac_ring_t *ring;
252045948e49SRyan Zezeski mac_group_t *grp;
252145948e49SRyan Zezeski
25220dc2366fSVenugopal Iyer grp = (mac_group_t *)flent->fe_tx_ring_group;
252345948e49SRyan Zezeski
252445948e49SRyan Zezeski if (grp == NULL)
252545948e49SRyan Zezeski return;
252645948e49SRyan Zezeski
252745948e49SRyan Zezeski for (ring = grp->mrg_rings; ring != NULL;
252845948e49SRyan Zezeski ring = ring->mr_next) {
252945948e49SRyan Zezeski if (ring->mr_state != MR_INUSE)
25300dc2366fSVenugopal Iyer (void) mac_start_ring(ring);
25310dc2366fSVenugopal Iyer }
253245948e49SRyan Zezeski
25330dc2366fSVenugopal Iyer return;
25340dc2366fSVenugopal Iyer }
253545948e49SRyan Zezeski
253645948e49SRyan Zezeski /*
253745948e49SRyan Zezeski * Aggr ports should never have SRSes.
253845948e49SRyan Zezeski */
253945948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0);
254045948e49SRyan Zezeski
25410dc2366fSVenugopal Iyer if (flent->fe_tx_srs == NULL) {
25420dc2366fSVenugopal Iyer (void) mac_srs_create(mcip, flent, SRST_TX | link_type,
25430dc2366fSVenugopal Iyer NULL, mcip, NULL, NULL);
25440dc2366fSVenugopal Iyer }
254545948e49SRyan Zezeski
25460dc2366fSVenugopal Iyer mac_tx_srs_setup(mcip, flent);
25470dc2366fSVenugopal Iyer }
25480dc2366fSVenugopal Iyer
25490dc2366fSVenugopal Iyer /*
255084de666eSRyan Zezeski * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown
255184de666eSRyan Zezeski * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set
255284de666eSRyan Zezeski * when we wish to move a MAC client from one group to another. In
255384de666eSRyan Zezeski * that case, we need to release the current HW SRSes but keep the SW
255484de666eSRyan Zezeski * SRS for continued traffic classifiction.
25550dc2366fSVenugopal Iyer */
25560dc2366fSVenugopal Iyer void
mac_rx_srs_group_teardown(flow_entry_t * flent,boolean_t hwonly)25570dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly)
2558da14cebeSEric Cheng {
2559da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs;
2560da14cebeSEric Cheng int i;
25610dc2366fSVenugopal Iyer int count = flent->fe_rx_srs_cnt;
2562da14cebeSEric Cheng
25630dc2366fSVenugopal Iyer for (i = 0; i < count; i++) {
25640dc2366fSVenugopal Iyer if (i == 0 && hwonly)
25650dc2366fSVenugopal Iyer continue;
2566da14cebeSEric Cheng mac_srs = flent->fe_rx_srs[i];
2567da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED);
2568da14cebeSEric Cheng mac_srs_free(mac_srs);
2569da14cebeSEric Cheng flent->fe_rx_srs[i] = NULL;
25700dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt--;
2571da14cebeSEric Cheng }
257284de666eSRyan Zezeski
257384de666eSRyan Zezeski /*
257484de666eSRyan Zezeski * If we are only tearing down the HW SRSes then there must be
257584de666eSRyan Zezeski * one SRS left for SW classification. Otherwise we are tearing
257684de666eSRyan Zezeski * down both HW and SW and there should be no SRSes left.
257784de666eSRyan Zezeski */
257884de666eSRyan Zezeski if (hwonly)
257984de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 1);
258084de666eSRyan Zezeski else
258184de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 0);
25820dc2366fSVenugopal Iyer }
25830dc2366fSVenugopal Iyer
25840dc2366fSVenugopal Iyer /*
25850dc2366fSVenugopal Iyer * Remove the TX SRS.
25860dc2366fSVenugopal Iyer */
25870dc2366fSVenugopal Iyer void
mac_tx_srs_group_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)25880dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
25890dc2366fSVenugopal Iyer uint32_t link_type)
25900dc2366fSVenugopal Iyer {
25910dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs;
25920dc2366fSVenugopal Iyer mac_srs_tx_t *tx;
25930dc2366fSVenugopal Iyer
25940dc2366fSVenugopal Iyer if ((tx_srs = flent->fe_tx_srs) == NULL)
25950dc2366fSVenugopal Iyer return;
2596da14cebeSEric Cheng
2597da14cebeSEric Cheng tx = &tx_srs->srs_tx;
2598da14cebeSEric Cheng switch (link_type) {
2599da14cebeSEric Cheng case SRST_FLOW:
2600da14cebeSEric Cheng /*
2601da14cebeSEric Cheng * For flows, we need to work with passed
2602da14cebeSEric Cheng * flent to find the Rx/Tx SRS.
2603da14cebeSEric Cheng */
2604da14cebeSEric Cheng mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED);
2605da14cebeSEric Cheng break;
2606da14cebeSEric Cheng case SRST_LINK:
26070dc2366fSVenugopal Iyer mac_tx_client_condemn((mac_client_handle_t)mcip);
2608da14cebeSEric Cheng if (tx->st_arg2 != NULL) {
2609da14cebeSEric Cheng ASSERT(tx_srs->srs_type & SRST_TX);
26100dc2366fSVenugopal Iyer /*
26110dc2366fSVenugopal Iyer * The ring itself will be stopped when
26120dc2366fSVenugopal Iyer * we release the group or in the
26130dc2366fSVenugopal Iyer * mac_datapath_teardown (for the default
26140dc2366fSVenugopal Iyer * group)
26150dc2366fSVenugopal Iyer */
26160dc2366fSVenugopal Iyer tx->st_arg2 = NULL;
2617da14cebeSEric Cheng }
2618da14cebeSEric Cheng break;
2619da14cebeSEric Cheng default:
2620da14cebeSEric Cheng ASSERT(B_FALSE);
2621da14cebeSEric Cheng break;
2622da14cebeSEric Cheng }
2623da14cebeSEric Cheng mac_srs_free(tx_srs);
2624da14cebeSEric Cheng flent->fe_tx_srs = NULL;
2625da14cebeSEric Cheng }
2626da14cebeSEric Cheng
2627da14cebeSEric Cheng /*
26280dc2366fSVenugopal Iyer * This is the group state machine.
26290dc2366fSVenugopal Iyer *
26300dc2366fSVenugopal Iyer * The state of an Rx group is given by
2631da14cebeSEric Cheng * the following table. The default group and its rings are started in
2632da14cebeSEric Cheng * mac_start itself and the default group stays in SHARED state until
2633da14cebeSEric Cheng * mac_stop at which time the group and rings are stopped and and it
2634da14cebeSEric Cheng * reverts to the Registered state.
2635da14cebeSEric Cheng *
2636da14cebeSEric Cheng * Typically this function is called on a group after adding or removing a
2637da14cebeSEric Cheng * client from it, to find out what should be the new state of the group.
2638da14cebeSEric Cheng * If the new state is RESERVED, then the client that owns this group
2639da14cebeSEric Cheng * exclusively is also returned. Note that adding or removing a client from
2640da14cebeSEric Cheng * a group could also impact the default group and the caller needs to
2641da14cebeSEric Cheng * evaluate the effect on the default group.
2642da14cebeSEric Cheng *
2643da14cebeSEric Cheng * Group type # of clients mi_nactiveclients Group State
2644da14cebeSEric Cheng * in the group
2645da14cebeSEric Cheng *
2646da14cebeSEric Cheng * Non-default 0 N.A. REGISTERED
2647da14cebeSEric Cheng * Non-default 1 N.A. RESERVED
2648da14cebeSEric Cheng *
2649da14cebeSEric Cheng * Default 0 N.A. SHARED
2650da14cebeSEric Cheng * Default 1 1 RESERVED
2651da14cebeSEric Cheng * Default 1 > 1 SHARED
2652da14cebeSEric Cheng * Default > 1 N.A. SHARED
26530dc2366fSVenugopal Iyer *
26540dc2366fSVenugopal Iyer * For a TX group, the following is the state table.
26550dc2366fSVenugopal Iyer *
26560dc2366fSVenugopal Iyer * Group type # of clients Group State
26570dc2366fSVenugopal Iyer * in the group
26580dc2366fSVenugopal Iyer *
26590dc2366fSVenugopal Iyer * Non-default 0 REGISTERED
26600dc2366fSVenugopal Iyer * Non-default 1 RESERVED
26610dc2366fSVenugopal Iyer *
26620dc2366fSVenugopal Iyer * Default 0 REGISTERED
26630dc2366fSVenugopal Iyer * Default 1 RESERVED
26640dc2366fSVenugopal Iyer * Default > 1 SHARED
2665da14cebeSEric Cheng */
2666da14cebeSEric Cheng mac_group_state_t
mac_group_next_state(mac_group_t * grp,mac_client_impl_t ** group_only_mcip,mac_group_t * defgrp,boolean_t rx_group)26670dc2366fSVenugopal Iyer mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip,
26680dc2366fSVenugopal Iyer mac_group_t *defgrp, boolean_t rx_group)
2669da14cebeSEric Cheng {
2670da14cebeSEric Cheng mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh;
2671da14cebeSEric Cheng
2672da14cebeSEric Cheng *group_only_mcip = NULL;
2673da14cebeSEric Cheng
2674da14cebeSEric Cheng /* Non-default group */
2675da14cebeSEric Cheng
26760dc2366fSVenugopal Iyer if (grp != defgrp) {
26770dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp))
2678da14cebeSEric Cheng return (MAC_GROUP_STATE_REGISTERED);
2679da14cebeSEric Cheng
26800dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
2681da14cebeSEric Cheng if (*group_only_mcip != NULL)
2682da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED);
2683da14cebeSEric Cheng
2684da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED);
2685da14cebeSEric Cheng }
2686da14cebeSEric Cheng
2687da14cebeSEric Cheng /* Default group */
2688da14cebeSEric Cheng
26890dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) {
26900dc2366fSVenugopal Iyer if (rx_group)
26910dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED);
26920dc2366fSVenugopal Iyer else
26930dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_REGISTERED);
26940dc2366fSVenugopal Iyer }
26950dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp);
26960dc2366fSVenugopal Iyer if (*group_only_mcip == NULL)
26970dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED);
26980dc2366fSVenugopal Iyer
26990dc2366fSVenugopal Iyer if (rx_group && mip->mi_nactiveclients != 1)
2700da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED);
2701da14cebeSEric Cheng
2702da14cebeSEric Cheng ASSERT(*group_only_mcip != NULL);
2703da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED);
2704da14cebeSEric Cheng }
2705da14cebeSEric Cheng
2706da14cebeSEric Cheng /*
2707da14cebeSEric Cheng * OVERVIEW NOTES FOR DATAPATH
2708da14cebeSEric Cheng * ===========================
2709da14cebeSEric Cheng *
2710da14cebeSEric Cheng * Create an SRS and setup the corresponding flow function and args.
2711da14cebeSEric Cheng * Add a classification rule for the flow specified by 'flent' and program
2712da14cebeSEric Cheng * the hardware classifier when applicable.
2713da14cebeSEric Cheng *
2714da14cebeSEric Cheng * Rx ring assignment, SRS, polling and B/W enforcement
2715da14cebeSEric Cheng * ----------------------------------------------------
2716da14cebeSEric Cheng *
2717da14cebeSEric Cheng * We try to use H/W classification on NIC and assign traffic to a
2718da14cebeSEric Cheng * MAC address to a particular Rx ring. There is a 1-1 mapping
2719da14cebeSEric Cheng * between a SRS and a Rx ring. The SRS (short for soft ring set)
2720da14cebeSEric Cheng * dynamically switches the underlying Rx ring between interrupt
2721da14cebeSEric Cheng * and polling mode and enforces any specified B/W control.
2722da14cebeSEric Cheng *
2723da14cebeSEric Cheng * There is always a SRS created and tied to each H/W and S/W rule.
2724da14cebeSEric Cheng * Whenever we create a H/W rule, we always add the the same rule to
2725da14cebeSEric Cheng * S/W classifier and tie a SRS to it.
2726da14cebeSEric Cheng *
2727da14cebeSEric Cheng * In case a B/W control is specified, its broken into bytes
2728da14cebeSEric Cheng * per ticks and as soon as the quota for a tick is exhausted,
2729da14cebeSEric Cheng * the underlying Rx ring is forced into poll mode for remianing
2730da14cebeSEric Cheng * tick. The SRS poll thread only polls for bytes that are
2731da14cebeSEric Cheng * allowed to come in the SRS. We typically let 4x the configured
2732da14cebeSEric Cheng * B/W worth of packets to come in the SRS (to prevent unnecessary
2733da14cebeSEric Cheng * drops due to bursts) but only process the specified amount.
2734da14cebeSEric Cheng *
2735da14cebeSEric Cheng * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more
2736da14cebeSEric Cheng * Rx rings (and corresponding SRSs) assigned to it. The SRS
2737da14cebeSEric Cheng * in turn can have softrings to do protocol level fanout or
2738da14cebeSEric Cheng * softrings to do S/W based fanout or both. In case the NIC
2739da14cebeSEric Cheng * has no Rx rings, we do S/W classification to respective SRS.
2740da14cebeSEric Cheng * The S/W classification rule is always setup and ready. This
2741da14cebeSEric Cheng * allows the MAC layer to reassign Rx rings whenever needed
2742da14cebeSEric Cheng * but packets still continue to flow via the default path and
2743da14cebeSEric Cheng * getting S/W classified to correct SRS.
2744da14cebeSEric Cheng *
2745da14cebeSEric Cheng * In other cases where a NIC or VNIC is plumbed, our goal is use
2746da14cebeSEric Cheng * H/W classifier and get two Rx ring assigned for the Link. One
2747da14cebeSEric Cheng * for TCP and one for UDP|SCTP. The respective SRS still do the
2748da14cebeSEric Cheng * polling on the Rx ring. For Link that is plumbed for IP, there
2749da14cebeSEric Cheng * is a TCP squeue which also does polling and can control the
2750da14cebeSEric Cheng * the Rx ring directly (where SRS is just pass through). For
2751da14cebeSEric Cheng * the following cases, the SRS does the polling underneath.
2752da14cebeSEric Cheng * 1) non IP based Links (Links which are not plumbed via ifconfig)
2753da14cebeSEric Cheng * and paths which have no IP squeues (UDP & SCTP)
2754da14cebeSEric Cheng * 2) If B/W control is specified on the Link
2755da14cebeSEric Cheng * 3) If S/W fanout is secified
2756da14cebeSEric Cheng *
2757da14cebeSEric Cheng * Note1: As of current implementation, we try to assign only 1 Rx
2758da14cebeSEric Cheng * ring per Link and more than 1 Rx ring for primary Link for
2759da14cebeSEric Cheng * H/W based fanout. We always create following softrings per SRS:
2760da14cebeSEric Cheng * 1) TCP softring which is polled by TCP squeue where possible
2761da14cebeSEric Cheng * (and also bypasses DLS)
2762da14cebeSEric Cheng * 2) UDP/SCTP based which bypasses DLS
2763da14cebeSEric Cheng * 3) OTH softring which goes via DLS (currently deal with IPv6
2764da14cebeSEric Cheng * and non TCP/UDP/SCTP for IPv4 packets).
2765da14cebeSEric Cheng *
2766da14cebeSEric Cheng * It is necessary to create 3 softrings since SRS has to poll
2767da14cebeSEric Cheng * the single Rx ring underneath and enforce any link level B/W
2768da14cebeSEric Cheng * control (we can't switch the Rx ring in poll mode just based
2769da14cebeSEric Cheng * on TCP squeue if the same Rx ring is sharing UDP and other
2770da14cebeSEric Cheng * traffic as well). Once polling is done and any Link level B/W
2771da14cebeSEric Cheng * control is specified, the packets are assigned to respective
2772da14cebeSEric Cheng * softring based on protocol. Since TCP has IP based squeue
2773da14cebeSEric Cheng * which benefits by polling, we separate TCP packets into
2774da14cebeSEric Cheng * its own softring which can be polled by IP squeue. We need
2775da14cebeSEric Cheng * to separate out UDP/SCTP to UDP softring since it can bypass
2776da14cebeSEric Cheng * the DLS layer which has heavy performance advanatges and we
2777da14cebeSEric Cheng * need a softring (OTH) for rest.
2778da14cebeSEric Cheng *
2779da14cebeSEric Cheng * ToDo: The 3 softrings for protocol are needed only till we can
2780da14cebeSEric Cheng * get rid of DLS from datapath, make IPv4 and IPv6 paths
2781da14cebeSEric Cheng * symmetric (deal with mac_header_info for v6 and polling for
2782da14cebeSEric Cheng * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues
2783da14cebeSEric Cheng * are generic), and bring SAP based classification to MAC layer
2784da14cebeSEric Cheng *
2785da14cebeSEric Cheng * H/W and S/W based fanout and multiple Rx rings per Link
2786da14cebeSEric Cheng * -------------------------------------------------------
2787da14cebeSEric Cheng *
2788da14cebeSEric Cheng * In case, fanout is requested (or determined automatically based
2789da14cebeSEric Cheng * on Link speed and processor speed), we try to assign multiple
2790da14cebeSEric Cheng * Rx rings per Link with their respective SRS. In this case
2791da14cebeSEric Cheng * the NIC should be capable of fanning out incoming packets between
2792da14cebeSEric Cheng * the assigned Rx rings (H/W based fanout). All the SRS
2793da14cebeSEric Cheng * individually switch their Rx ring between interrupt and polling
2794da14cebeSEric Cheng * mode but share a common B/W control counter in case of Link
2795da14cebeSEric Cheng * level B/W is specified.
2796da14cebeSEric Cheng *
2797da14cebeSEric Cheng * If S/W based fanout is specified in lieu of H/W based fanout,
2798da14cebeSEric Cheng * the Link SRS creates the specified number of softrings for
2799da14cebeSEric Cheng * each protocol (TCP, UDP, OTH). Incoming packets are fanned
2800da14cebeSEric Cheng * out to the correct softring based on their protocol and
2801da14cebeSEric Cheng * protocol specific hash function.
2802da14cebeSEric Cheng *
2803da14cebeSEric Cheng * Primary and non primary MAC clients
2804da14cebeSEric Cheng * -----------------------------------
2805da14cebeSEric Cheng *
2806da14cebeSEric Cheng * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links
2807da14cebeSEric Cheng * and are a Layer 2 construct.
2808da14cebeSEric Cheng *
2809da14cebeSEric Cheng * Primary NIC:
2810da14cebeSEric Cheng * The Link that owns the primary MAC address and typically
2811da14cebeSEric Cheng * is used as the data NIC in non virtualized cases. As such
2812da14cebeSEric Cheng * H/W resources are preferntially given to primary NIC. As
2813da14cebeSEric Cheng * far as code is concerned, there is no difference in the
2814da14cebeSEric Cheng * primary NIC vs VNICs. They are all treated as Links.
2815da14cebeSEric Cheng * At the very first call to mac_unicast_add() we program the S/W
2816da14cebeSEric Cheng * classifier for the primary MAC address, get a soft ring set
2817da14cebeSEric Cheng * (and soft rings based on 'ip_soft_ring_cnt')
2818da14cebeSEric Cheng * and a Rx ring assigned for polling to get enabled.
2819da14cebeSEric Cheng * When IP get plumbed and negotiates polling, we can
2820da14cebeSEric Cheng * let squeue do the polling on TCP softring.
2821da14cebeSEric Cheng *
2822da14cebeSEric Cheng * VNICs:
2823da14cebeSEric Cheng * Same as any other Link. As long as the H/W resource assignments
2824da14cebeSEric Cheng * are equal, the data path and setup for all Links is same.
2825da14cebeSEric Cheng *
2826da14cebeSEric Cheng * Flows:
2827da14cebeSEric Cheng * Can be configured on Links. They have their own SRS and the
2828da14cebeSEric Cheng * S/W classifier is programmed appropriately based on the flow.
2829da14cebeSEric Cheng * The flows typically deal with layer 3 and above and
2830da14cebeSEric Cheng * creates a soft ring set specific to the flow. The receive
2831da14cebeSEric Cheng * side function is switched from mac_rx_srs_process to
2832da14cebeSEric Cheng * mac_rx_srs_subflow_process which first tries to assign the
2833da14cebeSEric Cheng * packet to appropriate flow SRS and failing which assigns it
2834da14cebeSEric Cheng * to link SRS. This allows us to avoid the layered approach
2835da14cebeSEric Cheng * which gets complex.
2836da14cebeSEric Cheng *
2837da14cebeSEric Cheng * By the time mac_datapath_setup() completes, we already have the
2838da14cebeSEric Cheng * soft rings set, Rx rings, soft rings, etc figured out and both H/W
2839da14cebeSEric Cheng * and S/W classifiers programmed. IP is not plumbed yet (and might
2840da14cebeSEric Cheng * never be for Virtual Machines guest OS path). When IP is plumbed
2841da14cebeSEric Cheng * (for both NIC and VNIC), we do a capability negotiation for polling
2842da14cebeSEric Cheng * and upcall functions etc.
2843da14cebeSEric Cheng *
2844da14cebeSEric Cheng * Rx ring Assignement NOTES
2845da14cebeSEric Cheng * -------------------------
2846da14cebeSEric Cheng *
2847da14cebeSEric Cheng * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings
2848da14cebeSEric Cheng * as NIC with a single default ring), we assign the only ring to
28490dc2366fSVenugopal Iyer * primary Link. The primary Link SRS can do polling on it as long as
28500dc2366fSVenugopal Iyer * it is the only link in use and we compare the MAC address for unicast
28510dc2366fSVenugopal Iyer * packets before accepting an incoming packet (there is no need for S/W
28520dc2366fSVenugopal Iyer * classification in this case). We disable polling on the only ring the
28530dc2366fSVenugopal Iyer * moment 2nd link gets created (the polling remains enabled even though
28540dc2366fSVenugopal Iyer * there are broadcast and * multicast flows created).
2855da14cebeSEric Cheng *
2856da14cebeSEric Cheng * If the NIC has more than 1 Rx ring, we assign the default ring (the
2857da14cebeSEric Cheng * 1st ring) to deal with broadcast, multicast and traffic for other
2858da14cebeSEric Cheng * NICs which needs S/W classification. We assign the primary mac
2859da14cebeSEric Cheng * addresses to another ring by specifiying a classification rule for
2860da14cebeSEric Cheng * primary unicast MAC address to the selected ring. The primary Link
2861da14cebeSEric Cheng * (and its SRS) can continue to poll the assigned Rx ring at all times
2862da14cebeSEric Cheng * independantly.
2863da14cebeSEric Cheng *
2864da14cebeSEric Cheng * Note: In future, if no fanout is specified, we try to assign 2 Rx
2865da14cebeSEric Cheng * rings for the primary Link with the primary MAC address + TCP going
2866da14cebeSEric Cheng * to one ring and primary MAC address + UDP|SCTP going to other ring.
2867da14cebeSEric Cheng * Any remaining traffic for primary MAC address can go to the default
2868da14cebeSEric Cheng * Rx ring and get S/W classified. This way the respective SRSs don't
2869da14cebeSEric Cheng * need to do proto fanout and don't need to have softrings at all and
2870da14cebeSEric Cheng * can poll their respective Rx rings.
2871da14cebeSEric Cheng *
2872da14cebeSEric Cheng * As an optimization, when a new NIC or VNIC is created, we can get
2873da14cebeSEric Cheng * only one Rx ring and make it a TCP specific Rx ring and use the
2874da14cebeSEric Cheng * H/W default Rx ring for the rest (this Rx ring is never polled).
28750dc2366fSVenugopal Iyer *
28760dc2366fSVenugopal Iyer * For clients that don't have MAC address, but want to receive and
28770dc2366fSVenugopal Iyer * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath.
28780dc2366fSVenugopal Iyer * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we
28790dc2366fSVenugopal Iyer * always give the default group and use software classification (i.e.
28800dc2366fSVenugopal Iyer * even if this is the only client in the default group, we will
28810dc2366fSVenugopal Iyer * leave group as shared).
2882da14cebeSEric Cheng */
288384de666eSRyan Zezeski
2884da14cebeSEric Cheng int
mac_datapath_setup(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)2885da14cebeSEric Cheng mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent,
2886da14cebeSEric Cheng uint32_t link_type)
2887da14cebeSEric Cheng {
2888da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip;
28890dc2366fSVenugopal Iyer mac_group_t *rgroup = NULL;
28900dc2366fSVenugopal Iyer mac_group_t *tgroup = NULL;
28910dc2366fSVenugopal Iyer mac_group_t *default_rgroup;
28920dc2366fSVenugopal Iyer mac_group_t *default_tgroup;
2893da14cebeSEric Cheng int err;
289484de666eSRyan Zezeski uint16_t vid;
289536f99a58SToomas Soome uint8_t *mac_addr;
2896da14cebeSEric Cheng mac_group_state_t next_state;
2897da14cebeSEric Cheng mac_client_impl_t *group_only_mcip;
28980dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
28990dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip);
29000dc2366fSVenugopal Iyer boolean_t rxhw;
29010dc2366fSVenugopal Iyer boolean_t txhw;
29020dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE;
29030dc2366fSVenugopal Iyer cpupart_t *cpupart;
29040dc2366fSVenugopal Iyer boolean_t no_unicast;
29050dc2366fSVenugopal Iyer boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC;
29060dc2366fSVenugopal Iyer mac_client_impl_t *reloc_pmcip = NULL;
290784de666eSRyan Zezeski boolean_t use_hw;
2908da14cebeSEric Cheng
2909da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
2910da14cebeSEric Cheng
2911da14cebeSEric Cheng switch (link_type) {
2912da14cebeSEric Cheng case SRST_FLOW:
29130dc2366fSVenugopal Iyer mac_srs_group_setup(mcip, flent, link_type);
2914da14cebeSEric Cheng return (0);
2915da14cebeSEric Cheng
2916da14cebeSEric Cheng case SRST_LINK:
29170dc2366fSVenugopal Iyer no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR;
2918da14cebeSEric Cheng mac_addr = flent->fe_flow_desc.fd_dst_mac;
2919da14cebeSEric Cheng
29200dc2366fSVenugopal Iyer /* Default RX group */
29210dc2366fSVenugopal Iyer default_rgroup = MAC_DEFAULT_RX_GROUP(mip);
2922da14cebeSEric Cheng
29230dc2366fSVenugopal Iyer /* Default TX group */
29240dc2366fSVenugopal Iyer default_tgroup = MAC_DEFAULT_TX_GROUP(mip);
29250dc2366fSVenugopal Iyer
29260dc2366fSVenugopal Iyer if (no_unicast) {
29270dc2366fSVenugopal Iyer rgroup = default_rgroup;
29280dc2366fSVenugopal Iyer tgroup = default_tgroup;
29290dc2366fSVenugopal Iyer goto grp_found;
2930da14cebeSEric Cheng }
29310dc2366fSVenugopal Iyer rxhw = (mrp->mrp_mask & MRP_RX_RINGS) &&
29320dc2366fSVenugopal Iyer (mrp->mrp_nrxrings > 0 ||
29330dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_RXRINGS_UNSPEC));
29340dc2366fSVenugopal Iyer txhw = (mrp->mrp_mask & MRP_TX_RINGS) &&
29350dc2366fSVenugopal Iyer (mrp->mrp_ntxrings > 0 ||
29360dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_TXRINGS_UNSPEC));
2937da14cebeSEric Cheng
29380dc2366fSVenugopal Iyer /*
293984de666eSRyan Zezeski * All the rings initially belong to the default group
294084de666eSRyan Zezeski * under dynamic grouping. The primary client uses the
294184de666eSRyan Zezeski * default group when it is the only client. The
294284de666eSRyan Zezeski * default group is also used as the destination for
294384de666eSRyan Zezeski * all multicast and broadcast traffic of all clients.
294484de666eSRyan Zezeski * Therefore, the primary client loses its ability to
294584de666eSRyan Zezeski * poll the softrings on addition of a second client.
294684de666eSRyan Zezeski * To avoid a performance penalty, MAC will move the
294784de666eSRyan Zezeski * primary client to a dedicated group when it can.
294884de666eSRyan Zezeski *
294984de666eSRyan Zezeski * When using static grouping, the primary client
295084de666eSRyan Zezeski * begins life on a non-default group. There is
295184de666eSRyan Zezeski * no moving needed upon addition of a second client.
29520dc2366fSVenugopal Iyer */
29530dc2366fSVenugopal Iyer if (!isprimary && mip->mi_nactiveclients == 2 &&
29540dc2366fSVenugopal Iyer (group_only_mcip = mac_primary_client_handle(mip)) !=
29550dc2366fSVenugopal Iyer NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
29560dc2366fSVenugopal Iyer reloc_pmcip = mac_check_primary_relocation(
29570dc2366fSVenugopal Iyer group_only_mcip, rxhw);
29580dc2366fSVenugopal Iyer }
295984de666eSRyan Zezeski
29600dc2366fSVenugopal Iyer /*
29610dc2366fSVenugopal Iyer * Check to see if we can get an exclusive group for
29620dc2366fSVenugopal Iyer * this mac address or if there already exists a
29630dc2366fSVenugopal Iyer * group that has this mac address (case of VLANs).
29640dc2366fSVenugopal Iyer * If no groups are available, use the default group.
29650dc2366fSVenugopal Iyer */
29660dc2366fSVenugopal Iyer rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE);
29670dc2366fSVenugopal Iyer if (rgroup == NULL && rxhw) {
29680dc2366fSVenugopal Iyer err = ENOSPC;
29690dc2366fSVenugopal Iyer goto setup_failed;
29700dc2366fSVenugopal Iyer } else if (rgroup == NULL) {
29710dc2366fSVenugopal Iyer rgroup = default_rgroup;
29720dc2366fSVenugopal Iyer }
297384de666eSRyan Zezeski
297484de666eSRyan Zezeski /*
297584de666eSRyan Zezeski * If we are adding a second client to a
297684de666eSRyan Zezeski * non-default group then we need to move the
297784de666eSRyan Zezeski * existing client to the default group and
297884de666eSRyan Zezeski * add the new client to the default group as
297984de666eSRyan Zezeski * well.
298084de666eSRyan Zezeski */
298184de666eSRyan Zezeski if (rgroup != default_rgroup &&
298284de666eSRyan Zezeski rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) {
298384de666eSRyan Zezeski group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup);
298484de666eSRyan Zezeski err = mac_rx_switch_group(group_only_mcip, rgroup,
298584de666eSRyan Zezeski default_rgroup);
298684de666eSRyan Zezeski
298784de666eSRyan Zezeski if (err != 0)
298884de666eSRyan Zezeski goto setup_failed;
298984de666eSRyan Zezeski
299084de666eSRyan Zezeski rgroup = default_rgroup;
299184de666eSRyan Zezeski }
299284de666eSRyan Zezeski
29930dc2366fSVenugopal Iyer /*
29940dc2366fSVenugopal Iyer * Check to see if we can get an exclusive group for
29950dc2366fSVenugopal Iyer * this mac client. If no groups are available, use
29960dc2366fSVenugopal Iyer * the default group.
29970dc2366fSVenugopal Iyer */
29980dc2366fSVenugopal Iyer tgroup = mac_reserve_tx_group(mcip, B_FALSE);
29990dc2366fSVenugopal Iyer if (tgroup == NULL && txhw) {
30000dc2366fSVenugopal Iyer if (rgroup != NULL && rgroup != default_rgroup)
30010dc2366fSVenugopal Iyer mac_release_rx_group(mcip, rgroup);
30020dc2366fSVenugopal Iyer err = ENOSPC;
30030dc2366fSVenugopal Iyer goto setup_failed;
30040dc2366fSVenugopal Iyer } else if (tgroup == NULL) {
30050dc2366fSVenugopal Iyer tgroup = default_tgroup;
3006da14cebeSEric Cheng }
3007da14cebeSEric Cheng
3008da14cebeSEric Cheng /*
3009da14cebeSEric Cheng * Some NICs don't support any Rx rings, so there may not
3010da14cebeSEric Cheng * even be a default group.
3011da14cebeSEric Cheng */
30120dc2366fSVenugopal Iyer grp_found:
30130dc2366fSVenugopal Iyer if (rgroup != NULL) {
30140dc2366fSVenugopal Iyer if (rgroup != default_rgroup &&
30150dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(rgroup) &&
301636f99a58SToomas Soome (rxhw || mcip->mci_share != 0)) {
30170dc2366fSVenugopal Iyer MAC_RX_GRP_RESERVED(mip);
30180dc2366fSVenugopal Iyer if (mip->mi_rx_group_type ==
30190dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) {
30200dc2366fSVenugopal Iyer MAC_RX_RING_RESERVED(mip,
30210dc2366fSVenugopal Iyer rgroup->mrg_cur_count);
30220dc2366fSVenugopal Iyer }
30230dc2366fSVenugopal Iyer }
302484de666eSRyan Zezeski
30250dc2366fSVenugopal Iyer flent->fe_rx_ring_group = rgroup;
3026da14cebeSEric Cheng /*
302784de666eSRyan Zezeski * Add the client to the group and update the
302884de666eSRyan Zezeski * group's state. If rgroup != default_group
302984de666eSRyan Zezeski * then the rgroup should only ever have one
303084de666eSRyan Zezeski * client and be in the RESERVED state. But no
303184de666eSRyan Zezeski * matter what, the default_rgroup will enter
303284de666eSRyan Zezeski * the SHARED state since it has to receive
303384de666eSRyan Zezeski * all broadcast and multicast traffic. This
303484de666eSRyan Zezeski * case is handled later in the function.
3035da14cebeSEric Cheng */
30360dc2366fSVenugopal Iyer mac_group_add_client(rgroup, mcip);
30370dc2366fSVenugopal Iyer next_state = mac_group_next_state(rgroup,
30380dc2366fSVenugopal Iyer &group_only_mcip, default_rgroup, B_TRUE);
30390dc2366fSVenugopal Iyer mac_set_group_state(rgroup, next_state);
3040da14cebeSEric Cheng }
3041da14cebeSEric Cheng
30420dc2366fSVenugopal Iyer if (tgroup != NULL) {
30430dc2366fSVenugopal Iyer if (tgroup != default_tgroup &&
30440dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(tgroup) &&
304536f99a58SToomas Soome (txhw || mcip->mci_share != 0)) {
30460dc2366fSVenugopal Iyer MAC_TX_GRP_RESERVED(mip);
30470dc2366fSVenugopal Iyer if (mip->mi_tx_group_type ==
30480dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) {
30490dc2366fSVenugopal Iyer MAC_TX_RING_RESERVED(mip,
30500dc2366fSVenugopal Iyer tgroup->mrg_cur_count);
30510dc2366fSVenugopal Iyer }
30520dc2366fSVenugopal Iyer }
30530dc2366fSVenugopal Iyer flent->fe_tx_ring_group = tgroup;
30540dc2366fSVenugopal Iyer mac_group_add_client(tgroup, mcip);
30550dc2366fSVenugopal Iyer next_state = mac_group_next_state(tgroup,
30560dc2366fSVenugopal Iyer &group_only_mcip, default_tgroup, B_FALSE);
30570dc2366fSVenugopal Iyer tgroup->mrg_state = next_state;
30580dc2366fSVenugopal Iyer }
3059da14cebeSEric Cheng
30600dc2366fSVenugopal Iyer /* We are setting up minimal datapath only */
306184de666eSRyan Zezeski if (no_unicast) {
306284de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type);
30630dc2366fSVenugopal Iyer break;
306484de666eSRyan Zezeski }
306584de666eSRyan Zezeski
306684de666eSRyan Zezeski /* Program software classification. */
3067da14cebeSEric Cheng if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0)
3068da14cebeSEric Cheng goto setup_failed;
3069da14cebeSEric Cheng
307084de666eSRyan Zezeski /* Program hardware classification. */
307184de666eSRyan Zezeski vid = i_mac_flow_vid(flent);
307284de666eSRyan Zezeski use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0;
307384de666eSRyan Zezeski err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw);
307484de666eSRyan Zezeski
307584de666eSRyan Zezeski if (err != 0)
3076da14cebeSEric Cheng goto setup_failed;
307784de666eSRyan Zezeski
3078da14cebeSEric Cheng mcip->mci_unicast = mac_find_macaddr(mip, mac_addr);
307984de666eSRyan Zezeski VERIFY3P(mcip->mci_unicast, !=, NULL);
308084de666eSRyan Zezeski
308184de666eSRyan Zezeski /*
308284de666eSRyan Zezeski * Setup the Rx and Tx SRSes. If the client has a
308384de666eSRyan Zezeski * reserved group, then mac_srs_group_setup() creates
308484de666eSRyan Zezeski * the required SRSes for the HW rings. If we have a
308584de666eSRyan Zezeski * shared group, mac_srs_group_setup() dismantles the
308684de666eSRyan Zezeski * HW SRSes of the previously exclusive group.
308784de666eSRyan Zezeski */
308884de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type);
308984de666eSRyan Zezeski
3090797f979dSCody Peter Mello /* (Re)init the v6 token & local addr used by link protection */
3091797f979dSCody Peter Mello mac_protect_update_mac_token(mcip);
3092da14cebeSEric Cheng break;
3093da14cebeSEric Cheng
3094da14cebeSEric Cheng default:
3095da14cebeSEric Cheng ASSERT(B_FALSE);
3096da14cebeSEric Cheng break;
3097da14cebeSEric Cheng }
3098da14cebeSEric Cheng
3099da14cebeSEric Cheng /*
3100da14cebeSEric Cheng * All broadcast and multicast traffic is received only on the default
3101da14cebeSEric Cheng * group. If we have setup the datapath for a non-default group above
3102da14cebeSEric Cheng * then move the default group to shared state to allow distribution of
3103da14cebeSEric Cheng * incoming broadcast traffic to the other groups and dismantle the
3104da14cebeSEric Cheng * SRSes over the default group.
3105da14cebeSEric Cheng */
31060dc2366fSVenugopal Iyer if (rgroup != NULL) {
31070dc2366fSVenugopal Iyer if (rgroup != default_rgroup) {
31080dc2366fSVenugopal Iyer if (default_rgroup->mrg_state ==
3109da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED) {
31100dc2366fSVenugopal Iyer group_only_mcip = MAC_GROUP_ONLY_CLIENT(
31110dc2366fSVenugopal Iyer default_rgroup);
3112da14cebeSEric Cheng ASSERT(group_only_mcip != NULL &&
3113da14cebeSEric Cheng mip->mi_nactiveclients > 1);
3114da14cebeSEric Cheng
31150dc2366fSVenugopal Iyer mac_set_group_state(default_rgroup,
3116da14cebeSEric Cheng MAC_GROUP_STATE_SHARED);
31170dc2366fSVenugopal Iyer mac_rx_srs_group_setup(group_only_mcip,
31180dc2366fSVenugopal Iyer group_only_mcip->mci_flent, SRST_LINK);
31190dc2366fSVenugopal Iyer pool_lock();
31200dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default);
31210dc2366fSVenugopal Iyer mac_fanout_setup(group_only_mcip,
3122da14cebeSEric Cheng group_only_mcip->mci_flent,
31230dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(group_only_mcip),
31240dc2366fSVenugopal Iyer mac_rx_deliver, group_only_mcip, NULL,
31250dc2366fSVenugopal Iyer cpupart);
31260dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart,
31270dc2366fSVenugopal Iyer mrp, emrp);
31280dc2366fSVenugopal Iyer pool_unlock();
3129da14cebeSEric Cheng }
31300dc2366fSVenugopal Iyer ASSERT(default_rgroup->mrg_state ==
3131da14cebeSEric Cheng MAC_GROUP_STATE_SHARED);
3132da14cebeSEric Cheng }
313384de666eSRyan Zezeski
3134da14cebeSEric Cheng /*
313584de666eSRyan Zezeski * A VLAN MAC client on a reserved group still
313684de666eSRyan Zezeski * requires SW classification if the MAC doesn't
313784de666eSRyan Zezeski * provide VLAN HW filtering.
313884de666eSRyan Zezeski *
313984de666eSRyan Zezeski * Clients with no unicast address also require SW
314084de666eSRyan Zezeski * classification.
3141da14cebeSEric Cheng */
31420dc2366fSVenugopal Iyer if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED &&
314384de666eSRyan Zezeski ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) ||
314484de666eSRyan Zezeski no_unicast)) {
31450dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rgroup);
3146da14cebeSEric Cheng }
314784de666eSRyan Zezeski
3148da14cebeSEric Cheng }
314984de666eSRyan Zezeski
31500dc2366fSVenugopal Iyer mac_set_rings_effective(mcip);
3151da14cebeSEric Cheng return (0);
3152da14cebeSEric Cheng
3153da14cebeSEric Cheng setup_failed:
31540dc2366fSVenugopal Iyer /* Switch the primary back to default group */
31550dc2366fSVenugopal Iyer if (reloc_pmcip != NULL) {
31560dc2366fSVenugopal Iyer (void) mac_rx_switch_group(reloc_pmcip,
31570dc2366fSVenugopal Iyer reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup);
31580dc2366fSVenugopal Iyer }
3159da14cebeSEric Cheng mac_datapath_teardown(mcip, flent, link_type);
3160da14cebeSEric Cheng return (err);
3161da14cebeSEric Cheng }
3162da14cebeSEric Cheng
3163da14cebeSEric Cheng void
mac_datapath_teardown(mac_client_impl_t * mcip,flow_entry_t * flent,uint32_t link_type)3164da14cebeSEric Cheng mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent,
3165da14cebeSEric Cheng uint32_t link_type)
3166da14cebeSEric Cheng {
3167da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip;
3168da14cebeSEric Cheng mac_group_t *group = NULL;
3169da14cebeSEric Cheng mac_client_impl_t *grp_only_mcip;
3170da14cebeSEric Cheng flow_entry_t *group_only_flent;
3171da14cebeSEric Cheng mac_group_t *default_group;
3172da14cebeSEric Cheng boolean_t check_default_group = B_FALSE;
3173da14cebeSEric Cheng mac_group_state_t next_state;
31740dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip);
317584de666eSRyan Zezeski uint16_t vid;
3176da14cebeSEric Cheng
3177da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip));
3178da14cebeSEric Cheng
3179da14cebeSEric Cheng switch (link_type) {
3180da14cebeSEric Cheng case SRST_FLOW:
31810dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE);
31820dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW);
3183da14cebeSEric Cheng return;
3184da14cebeSEric Cheng
3185da14cebeSEric Cheng case SRST_LINK:
3186da14cebeSEric Cheng /* Stop sending packets */
3187da14cebeSEric Cheng mac_tx_client_block(mcip);
318884de666eSRyan Zezeski group = flent->fe_rx_ring_group;
318984de666eSRyan Zezeski vid = i_mac_flow_vid(flent);
3190da14cebeSEric Cheng
319184de666eSRyan Zezeski /*
319284de666eSRyan Zezeski * Stop the packet flow from the hardware by disabling
319384de666eSRyan Zezeski * any hardware filters assigned to this client.
319484de666eSRyan Zezeski */
3195da14cebeSEric Cheng if (mcip->mci_unicast != NULL) {
3196da14cebeSEric Cheng int err;
319784de666eSRyan Zezeski
319884de666eSRyan Zezeski err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid);
319984de666eSRyan Zezeski
3200da14cebeSEric Cheng if (err != 0) {
320184de666eSRyan Zezeski cmn_err(CE_WARN, "%s: failed to remove a MAC HW"
320284de666eSRyan Zezeski " filters because of error 0x%x",
3203da14cebeSEric Cheng mip->mi_name, err);
3204da14cebeSEric Cheng }
320584de666eSRyan Zezeski
3206da14cebeSEric Cheng mcip->mci_unicast = NULL;
3207da14cebeSEric Cheng }
3208da14cebeSEric Cheng
3209da14cebeSEric Cheng /* Stop the packets coming from the S/W classifier */
3210da14cebeSEric Cheng mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE);
3211da14cebeSEric Cheng mac_flow_wait(flent, FLOW_DRIVER_UPCALL);
3212da14cebeSEric Cheng
321345948e49SRyan Zezeski /* Quiesce and destroy all the SRSes. */
32140dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE);
32150dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_LINK);
32160dc2366fSVenugopal Iyer
321745948e49SRyan Zezeski ASSERT3P(mcip->mci_flent, ==, flent);
321845948e49SRyan Zezeski ASSERT3P(flent->fe_next, ==, NULL);
3219da14cebeSEric Cheng
3220da14cebeSEric Cheng /*
3221da14cebeSEric Cheng * Release our hold on the group as well. We need
3222da14cebeSEric Cheng * to check if the shared group has only one client
3223da14cebeSEric Cheng * left who can use it exclusively. Also, if we
3224da14cebeSEric Cheng * were the last client, release the group.
3225da14cebeSEric Cheng */
32260dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip);
3227da14cebeSEric Cheng if (group != NULL) {
32280dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip);
32290dc2366fSVenugopal Iyer next_state = mac_group_next_state(group,
32300dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE);
323184de666eSRyan Zezeski
3232da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) {
3233da14cebeSEric Cheng /*
3234da14cebeSEric Cheng * Only one client left on this RX group.
3235da14cebeSEric Cheng */
323684de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL);
32370dc2366fSVenugopal Iyer mac_set_group_state(group,
3238da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED);
3239da14cebeSEric Cheng group_only_flent = grp_only_mcip->mci_flent;
3240da14cebeSEric Cheng
3241da14cebeSEric Cheng /*
3242da14cebeSEric Cheng * The only remaining client has exclusive
3243da14cebeSEric Cheng * access on the group. Allow it to
3244da14cebeSEric Cheng * dynamically poll the H/W rings etc.
3245da14cebeSEric Cheng */
32460dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip,
32470dc2366fSVenugopal Iyer group_only_flent, SRST_LINK);
32480dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip,
32490dc2366fSVenugopal Iyer group_only_flent,
32500dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip),
32510dc2366fSVenugopal Iyer mac_rx_deliver, grp_only_mcip, NULL, NULL);
3252da14cebeSEric Cheng mac_rx_group_unmark(group, MR_INCIPIENT);
32530dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip);
3254da14cebeSEric Cheng } else if (next_state == MAC_GROUP_STATE_REGISTERED) {
3255da14cebeSEric Cheng /*
3256da14cebeSEric Cheng * This is a non-default group being freed up.
3257da14cebeSEric Cheng * We need to reevaluate the default group
3258da14cebeSEric Cheng * to see if the primary client can get
3259da14cebeSEric Cheng * exclusive access to the default group.
3260da14cebeSEric Cheng */
326184de666eSRyan Zezeski VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip));
32620dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) {
32630dc2366fSVenugopal Iyer MAC_RX_GRP_RELEASED(mip);
32640dc2366fSVenugopal Iyer if (mip->mi_rx_group_type ==
32650dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) {
32660dc2366fSVenugopal Iyer MAC_RX_RING_RELEASED(mip,
32670dc2366fSVenugopal Iyer group->mrg_cur_count);
32680dc2366fSVenugopal Iyer }
32690dc2366fSVenugopal Iyer }
3270da14cebeSEric Cheng mac_release_rx_group(mcip, group);
32710dc2366fSVenugopal Iyer mac_set_group_state(group,
3272da14cebeSEric Cheng MAC_GROUP_STATE_REGISTERED);
3273da14cebeSEric Cheng check_default_group = B_TRUE;
3274da14cebeSEric Cheng } else {
327584de666eSRyan Zezeski VERIFY3S(next_state, ==,
327684de666eSRyan Zezeski MAC_GROUP_STATE_SHARED);
32770dc2366fSVenugopal Iyer mac_set_group_state(group,
3278da14cebeSEric Cheng MAC_GROUP_STATE_SHARED);
3279da14cebeSEric Cheng mac_rx_group_unmark(group, MR_CONDEMNED);
3280da14cebeSEric Cheng }
3281da14cebeSEric Cheng flent->fe_rx_ring_group = NULL;
3282da14cebeSEric Cheng }
32830dc2366fSVenugopal Iyer /*
32840dc2366fSVenugopal Iyer * Remove the client from the TX group. Additionally, if
32850dc2366fSVenugopal Iyer * this a non-default group, then we also need to release
32860dc2366fSVenugopal Iyer * the group.
32870dc2366fSVenugopal Iyer */
32880dc2366fSVenugopal Iyer group = flent->fe_tx_ring_group;
32890dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_TX_GROUP(mip);
32900dc2366fSVenugopal Iyer if (group != NULL) {
32910dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip);
32920dc2366fSVenugopal Iyer next_state = mac_group_next_state(group,
32930dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_FALSE);
32940dc2366fSVenugopal Iyer if (next_state == MAC_GROUP_STATE_REGISTERED) {
32950dc2366fSVenugopal Iyer if (group != default_group) {
32960dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_TX_RINGS) {
32970dc2366fSVenugopal Iyer MAC_TX_GRP_RELEASED(mip);
32980dc2366fSVenugopal Iyer if (mip->mi_tx_group_type ==
32990dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) {
33000dc2366fSVenugopal Iyer MAC_TX_RING_RELEASED(
33010dc2366fSVenugopal Iyer mip, group->
33020dc2366fSVenugopal Iyer mrg_cur_count);
33030dc2366fSVenugopal Iyer }
33040dc2366fSVenugopal Iyer }
33050dc2366fSVenugopal Iyer mac_release_tx_group(mcip, group);
33060dc2366fSVenugopal Iyer /*
33070dc2366fSVenugopal Iyer * If the default group is reserved,
33080dc2366fSVenugopal Iyer * then we need to set the effective
33090dc2366fSVenugopal Iyer * rings as we would have given
33100dc2366fSVenugopal Iyer * back some rings when the group
33110dc2366fSVenugopal Iyer * was released
33120dc2366fSVenugopal Iyer */
33130dc2366fSVenugopal Iyer if (mip->mi_tx_group_type ==
33140dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC &&
33150dc2366fSVenugopal Iyer default_group->mrg_state ==
33160dc2366fSVenugopal Iyer MAC_GROUP_STATE_RESERVED) {
33170dc2366fSVenugopal Iyer grp_only_mcip =
33180dc2366fSVenugopal Iyer MAC_GROUP_ONLY_CLIENT
33190dc2366fSVenugopal Iyer (default_group);
33200dc2366fSVenugopal Iyer mac_set_rings_effective(
33210dc2366fSVenugopal Iyer grp_only_mcip);
33220dc2366fSVenugopal Iyer }
33230dc2366fSVenugopal Iyer } else {
33240dc2366fSVenugopal Iyer mac_ring_t *ring;
33250dc2366fSVenugopal Iyer int cnt;
33260dc2366fSVenugopal Iyer int ringcnt;
33270dc2366fSVenugopal Iyer
33280dc2366fSVenugopal Iyer /*
33290dc2366fSVenugopal Iyer * Stop all the rings except the
33300dc2366fSVenugopal Iyer * default ring.
33310dc2366fSVenugopal Iyer */
33320dc2366fSVenugopal Iyer ringcnt = group->mrg_cur_count;
33330dc2366fSVenugopal Iyer ring = group->mrg_rings;
33340dc2366fSVenugopal Iyer for (cnt = 0; cnt < ringcnt; cnt++) {
33350dc2366fSVenugopal Iyer if (ring->mr_state ==
33360dc2366fSVenugopal Iyer MR_INUSE && ring !=
33370dc2366fSVenugopal Iyer (mac_ring_t *)
33380dc2366fSVenugopal Iyer mip->mi_default_tx_ring) {
33390dc2366fSVenugopal Iyer mac_stop_ring(ring);
33400dc2366fSVenugopal Iyer ring->mr_flag = 0;
33410dc2366fSVenugopal Iyer }
33420dc2366fSVenugopal Iyer ring = ring->mr_next;
33430dc2366fSVenugopal Iyer }
33440dc2366fSVenugopal Iyer }
33450dc2366fSVenugopal Iyer } else if (next_state == MAC_GROUP_STATE_RESERVED) {
33460dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip);
33470dc2366fSVenugopal Iyer }
33480dc2366fSVenugopal Iyer flent->fe_tx_ring_group = NULL;
33490dc2366fSVenugopal Iyer group->mrg_state = next_state;
33500dc2366fSVenugopal Iyer }
3351da14cebeSEric Cheng break;
3352da14cebeSEric Cheng default:
3353da14cebeSEric Cheng ASSERT(B_FALSE);
3354da14cebeSEric Cheng break;
3355da14cebeSEric Cheng }
3356da14cebeSEric Cheng
3357da14cebeSEric Cheng /*
3358da14cebeSEric Cheng * The mac client using the default group gets exclusive access to the
3359da14cebeSEric Cheng * default group if and only if it is the sole client on the entire
3360da14cebeSEric Cheng * mip. If so set the group state to reserved, and set up the SRSes
3361da14cebeSEric Cheng * over the default group.
3362da14cebeSEric Cheng */
3363da14cebeSEric Cheng if (check_default_group) {
33640dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip);
336584de666eSRyan Zezeski VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED);
33660dc2366fSVenugopal Iyer next_state = mac_group_next_state(default_group,
33670dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE);
3368da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) {
336984de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL);
337084de666eSRyan Zezeski VERIFY3U(mip->mi_nactiveclients, ==, 1);
33710dc2366fSVenugopal Iyer mac_set_group_state(default_group,
3372da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED);
33730dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip,
33740dc2366fSVenugopal Iyer grp_only_mcip->mci_flent, SRST_LINK);
33750dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip,
3376da14cebeSEric Cheng grp_only_mcip->mci_flent,
33770dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver,
33780dc2366fSVenugopal Iyer grp_only_mcip, NULL, NULL);
3379ae6aa22aSVenugopal Iyer mac_rx_group_unmark(default_group, MR_INCIPIENT);
33800dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip);
3381da14cebeSEric Cheng }
3382da14cebeSEric Cheng }
33830dc2366fSVenugopal Iyer
33840dc2366fSVenugopal Iyer /*
33850dc2366fSVenugopal Iyer * If the primary is the only one left and the MAC supports
33860dc2366fSVenugopal Iyer * dynamic grouping, we need to see if the primary needs to
33870dc2366fSVenugopal Iyer * be moved to the default group so that it can use all the
33880dc2366fSVenugopal Iyer * H/W rings.
33890dc2366fSVenugopal Iyer */
33900dc2366fSVenugopal Iyer if (!(flent->fe_type & FLOW_PRIMARY_MAC) &&
33910dc2366fSVenugopal Iyer mip->mi_nactiveclients == 1 &&
33920dc2366fSVenugopal Iyer mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) {
33930dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip);
33940dc2366fSVenugopal Iyer grp_only_mcip = mac_primary_client_handle(mip);
33950dc2366fSVenugopal Iyer if (grp_only_mcip == NULL)
33960dc2366fSVenugopal Iyer return;
33970dc2366fSVenugopal Iyer group_only_flent = grp_only_mcip->mci_flent;
33980dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(grp_only_mcip);
33990dc2366fSVenugopal Iyer /*
34000dc2366fSVenugopal Iyer * If the primary has an explicit property set, leave it
34010dc2366fSVenugopal Iyer * alone.
34020dc2366fSVenugopal Iyer */
34030dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS)
34040dc2366fSVenugopal Iyer return;
34050dc2366fSVenugopal Iyer /*
34060dc2366fSVenugopal Iyer * Switch the primary to the default group.
34070dc2366fSVenugopal Iyer */
34080dc2366fSVenugopal Iyer (void) mac_rx_switch_group(grp_only_mcip,
34090dc2366fSVenugopal Iyer group_only_flent->fe_rx_ring_group, default_group);
34100dc2366fSVenugopal Iyer }
3411da14cebeSEric Cheng }
3412da14cebeSEric Cheng
3413da14cebeSEric Cheng /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */
3414da14cebeSEric Cheng
3415da14cebeSEric Cheng static void
mac_srs_fanout_list_free(mac_soft_ring_set_t * mac_srs)3416da14cebeSEric Cheng mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs)
3417da14cebeSEric Cheng {
34180dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) {
34190dc2366fSVenugopal Iyer mac_srs_tx_t *tx;
34200dc2366fSVenugopal Iyer
34210dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tcp_soft_rings == NULL);
34220dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_udp_soft_rings == NULL);
34230dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_oth_soft_rings == NULL);
34240dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings != NULL);
34250dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_tx_soft_rings,
34260dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
34270dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = NULL;
34280dc2366fSVenugopal Iyer tx = &mac_srs->srs_tx;
34290dc2366fSVenugopal Iyer if (tx->st_soft_rings != NULL) {
34300dc2366fSVenugopal Iyer kmem_free(tx->st_soft_rings,
34310dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP);
34320dc2366fSVenugopal Iyer }
34330dc2366fSVenugopal Iyer } else {
34340dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings == NULL);
34350dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tcp_soft_rings != NULL);
34360dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_tcp_soft_rings,
34370dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
34380dc2366fSVenugopal Iyer mac_srs->srs_tcp_soft_rings = NULL;
34390dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_udp_soft_rings != NULL);
34400dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_udp_soft_rings,
34410dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
34420dc2366fSVenugopal Iyer mac_srs->srs_udp_soft_rings = NULL;
34430dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_oth_soft_rings != NULL);
34440dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_oth_soft_rings,
34450dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT);
34460dc2366fSVenugopal Iyer mac_srs->srs_oth_soft_rings = NULL;
34470dc2366fSVenugopal Iyer }
3448da14cebeSEric Cheng }
3449da14cebeSEric Cheng
3450da14cebeSEric Cheng /*
3451da14cebeSEric Cheng * An RX SRS is attached to at most one mac_ring.
3452da14cebeSEric Cheng * A TX SRS has no rings.
3453da14cebeSEric Cheng */
3454da14cebeSEric Cheng static void
mac_srs_ring_free(mac_soft_ring_set_t * mac_srs)3455da14cebeSEric Cheng mac_srs_ring_free(mac_soft_ring_set_t *mac_srs)
3456da14cebeSEric Cheng {
3457da14cebeSEric Cheng mac_client_impl_t *mcip;
3458da14cebeSEric Cheng mac_ring_t *ring;
3459da14cebeSEric Cheng flow_entry_t *flent;
3460da14cebeSEric Cheng
3461da14cebeSEric Cheng ring = mac_srs->srs_ring;
3462da14cebeSEric Cheng if (mac_srs->srs_type & SRST_TX) {
3463da14cebeSEric Cheng ASSERT(ring == NULL);
3464da14cebeSEric Cheng return;
3465da14cebeSEric Cheng }
3466da14cebeSEric Cheng
3467da14cebeSEric Cheng if (ring == NULL)
3468da14cebeSEric Cheng return;
3469da14cebeSEric Cheng
3470da14cebeSEric Cheng /*
3471da14cebeSEric Cheng * Broadcast flows don't have a client impl association, but they
3472da14cebeSEric Cheng * use only soft rings.
3473da14cebeSEric Cheng */
3474da14cebeSEric Cheng flent = mac_srs->srs_flent;
3475da14cebeSEric Cheng mcip = flent->fe_mcip;
3476da14cebeSEric Cheng ASSERT(mcip != NULL);
3477da14cebeSEric Cheng
3478da14cebeSEric Cheng ring->mr_classify_type = MAC_NO_CLASSIFIER;
3479da14cebeSEric Cheng ring->mr_srs = NULL;
3480da14cebeSEric Cheng }
3481da14cebeSEric Cheng
3482da14cebeSEric Cheng /*
3483da14cebeSEric Cheng * Physical unlink and free of the data structures happen below. This is
3484da14cebeSEric Cheng * driven from mac_flow_destroy(), on the last refrele of a flow.
3485da14cebeSEric Cheng *
3486da14cebeSEric Cheng * Assumes Rx srs is 1-1 mapped with an ring.
3487da14cebeSEric Cheng */
3488da14cebeSEric Cheng void
mac_srs_free(mac_soft_ring_set_t * mac_srs)3489da14cebeSEric Cheng mac_srs_free(mac_soft_ring_set_t *mac_srs)
3490da14cebeSEric Cheng {
3491da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == NULL ||
3492da14cebeSEric Cheng MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3493da14cebeSEric Cheng ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE |
3494da14cebeSEric Cheng SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE));
3495da14cebeSEric Cheng
3496c61a1653SRyan Zezeski mac_drop_chain(mac_srs->srs_first, "SRS free");
3497da14cebeSEric Cheng mac_srs_ring_free(mac_srs);
34980dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_srs);
3499da14cebeSEric Cheng mac_srs_fanout_list_free(mac_srs);
3500da14cebeSEric Cheng
3501da14cebeSEric Cheng mac_srs->srs_bw = NULL;
35020dc2366fSVenugopal Iyer mac_srs_stat_delete(mac_srs);
3503da14cebeSEric Cheng kmem_cache_free(mac_srs_cache, mac_srs);
3504da14cebeSEric Cheng }
3505da14cebeSEric Cheng
3506da14cebeSEric Cheng static void
mac_srs_soft_rings_quiesce(mac_soft_ring_set_t * mac_srs,uint_t s_ring_flag)3507da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag)
3508da14cebeSEric Cheng {
3509da14cebeSEric Cheng mac_soft_ring_t *softring;
3510da14cebeSEric Cheng
3511da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3512da14cebeSEric Cheng
3513da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_srs, s_ring_flag);
3514da14cebeSEric Cheng if (s_ring_flag == S_RING_CONDEMNED) {
3515da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count !=
3516da14cebeSEric Cheng mac_srs->srs_soft_ring_count)
3517da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3518da14cebeSEric Cheng } else {
3519da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count !=
3520da14cebeSEric Cheng mac_srs->srs_soft_ring_count)
3521da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3522da14cebeSEric Cheng }
3523da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3524da14cebeSEric Cheng
3525da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3526da906b4eSRyan Zezeski softring = softring->s_ring_next) {
3527da14cebeSEric Cheng (void) untimeout(softring->s_ring_tid);
3528da906b4eSRyan Zezeski softring->s_ring_tid = NULL;
3529da906b4eSRyan Zezeski }
3530da14cebeSEric Cheng
3531da14cebeSEric Cheng (void) untimeout(mac_srs->srs_tid);
3532da906b4eSRyan Zezeski mac_srs->srs_tid = NULL;
3533da14cebeSEric Cheng
3534da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3535da14cebeSEric Cheng }
3536da14cebeSEric Cheng
3537da14cebeSEric Cheng /*
3538da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the
3539da14cebeSEric Cheng * background. At this point upcalls from the driver (both hardware classified
3540da14cebeSEric Cheng * and software classified) have been cut off. We now need to quiesce the
3541da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as
3542da14cebeSEric Cheng * the master controller. The steps involved are described below in the function
3543da14cebeSEric Cheng */
3544da14cebeSEric Cheng void
mac_srs_worker_quiesce(mac_soft_ring_set_t * mac_srs)3545da14cebeSEric Cheng mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs)
3546da14cebeSEric Cheng {
3547da14cebeSEric Cheng uint_t s_ring_flag;
3548da14cebeSEric Cheng uint_t srs_poll_wait_flag;
3549da14cebeSEric Cheng
3550da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3551da14cebeSEric Cheng ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE));
3552da14cebeSEric Cheng
3553da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) {
3554da14cebeSEric Cheng s_ring_flag = S_RING_CONDEMNED;
3555da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_EXITED;
3556da14cebeSEric Cheng } else {
3557da14cebeSEric Cheng s_ring_flag = S_RING_QUIESCE;
3558da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_QUIESCED;
3559da14cebeSEric Cheng }
3560da14cebeSEric Cheng
3561da14cebeSEric Cheng /*
3562da14cebeSEric Cheng * In the case of Rx SRS wait till the poll thread is done.
3563da14cebeSEric Cheng */
3564da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) == 0 &&
3565da14cebeSEric Cheng mac_srs->srs_poll_thr != NULL) {
3566da14cebeSEric Cheng while (!(mac_srs->srs_state & srs_poll_wait_flag))
3567da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3568da14cebeSEric Cheng
3569da14cebeSEric Cheng /*
3570da14cebeSEric Cheng * Turn off polling as part of the quiesce operation.
3571da14cebeSEric Cheng */
3572da14cebeSEric Cheng MAC_SRS_POLLING_OFF(mac_srs);
3573da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS);
3574da14cebeSEric Cheng }
3575da14cebeSEric Cheng
3576da14cebeSEric Cheng /*
3577da14cebeSEric Cheng * Then signal the soft ring worker threads to quiesce or quit
3578da14cebeSEric Cheng * as needed and then wait till that happens.
3579da14cebeSEric Cheng */
3580da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag);
3581da14cebeSEric Cheng
3582da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED)
3583da14cebeSEric Cheng mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE);
3584da14cebeSEric Cheng else
3585da14cebeSEric Cheng mac_srs->srs_state |= SRS_QUIESCE_DONE;
3586da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv);
3587da14cebeSEric Cheng }
3588da14cebeSEric Cheng
3589da14cebeSEric Cheng /*
3590da14cebeSEric Cheng * Signal an SRS to start a temporary quiesce, or permanent removal, or restart
3591da14cebeSEric Cheng * a quiesced SRS by setting the appropriate flags and signaling the SRS worker
3592da14cebeSEric Cheng * or poll thread. This function is internal to the quiescing logic and is
3593da14cebeSEric Cheng * called internally from the SRS quiesce or flow quiesce or client quiesce
3594da14cebeSEric Cheng * higher level functions.
3595da14cebeSEric Cheng */
3596da14cebeSEric Cheng void
mac_srs_signal(mac_soft_ring_set_t * mac_srs,uint_t srs_flag)3597da14cebeSEric Cheng mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag)
3598da14cebeSEric Cheng {
3599da14cebeSEric Cheng mac_ring_t *ring;
3600da14cebeSEric Cheng
3601da14cebeSEric Cheng ring = mac_srs->srs_ring;
3602da14cebeSEric Cheng ASSERT(ring == NULL || ring->mr_refcnt == 0);
3603da14cebeSEric Cheng
3604da14cebeSEric Cheng if (srs_flag == SRS_CONDEMNED) {
3605da14cebeSEric Cheng /*
3606da14cebeSEric Cheng * The SRS is going away. We need to unbind the SRS and SR
3607da14cebeSEric Cheng * threads before removing from the global SRS list. Otherwise
3608da14cebeSEric Cheng * there is a small window where the cpu reconfig callbacks
3609da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail since
3610da14cebeSEric Cheng * there are still bound threads.
3611da14cebeSEric Cheng */
3612da14cebeSEric Cheng mac_srs_threads_unbind(mac_srs);
3613da14cebeSEric Cheng mac_srs_remove_glist(mac_srs);
3614da14cebeSEric Cheng }
3615da14cebeSEric Cheng /*
3616da14cebeSEric Cheng * Wakeup the SRS worker and poll threads.
3617da14cebeSEric Cheng */
3618da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3619da14cebeSEric Cheng mac_srs->srs_state |= srs_flag;
3620da14cebeSEric Cheng cv_signal(&mac_srs->srs_async);
3621da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv);
3622da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3623da14cebeSEric Cheng }
3624da14cebeSEric Cheng
3625da14cebeSEric Cheng /*
3626da14cebeSEric Cheng * In the Rx side, the quiescing is done bottom up. After the Rx upcalls
3627da14cebeSEric Cheng * from the driver are done, then the Rx SRS is quiesced and only then can
3628da14cebeSEric Cheng * we signal the soft rings. Thus this function can't be called arbitrarily
3629da14cebeSEric Cheng * without satisfying the prerequisites. On the Tx side, the threads from
3630da14cebeSEric Cheng * top need to quiesced, then the Tx SRS and only then can we signal the
3631da14cebeSEric Cheng * Tx soft rings.
3632da14cebeSEric Cheng */
3633da14cebeSEric Cheng static void
mac_srs_soft_rings_signal(mac_soft_ring_set_t * mac_srs,uint_t sr_flag)3634da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag)
3635da14cebeSEric Cheng {
3636da14cebeSEric Cheng mac_soft_ring_t *softring;
3637da14cebeSEric Cheng
3638da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3639da14cebeSEric Cheng softring = softring->s_ring_next)
3640da14cebeSEric Cheng mac_soft_ring_signal(softring, sr_flag);
3641da14cebeSEric Cheng }
3642da14cebeSEric Cheng
3643da14cebeSEric Cheng /*
3644da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the
3645da14cebeSEric Cheng * background. At this point the SRS is quiesced and we need to restart the
3646da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as
3647da14cebeSEric Cheng * the master controller. The steps involved are described below in the function
3648da14cebeSEric Cheng */
3649da14cebeSEric Cheng void
mac_srs_worker_restart(mac_soft_ring_set_t * mac_srs)3650da14cebeSEric Cheng mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs)
3651da14cebeSEric Cheng {
3652da14cebeSEric Cheng boolean_t iam_rx_srs;
3653da14cebeSEric Cheng mac_soft_ring_t *softring;
3654da14cebeSEric Cheng
3655da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock));
3656da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) != 0) {
3657da14cebeSEric Cheng iam_rx_srs = B_FALSE;
3658da14cebeSEric Cheng ASSERT((mac_srs->srs_state &
3659da14cebeSEric Cheng (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3660da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE));
3661da14cebeSEric Cheng } else {
3662da14cebeSEric Cheng iam_rx_srs = B_TRUE;
3663da14cebeSEric Cheng ASSERT((mac_srs->srs_state &
3664da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)) ==
3665da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE));
3666da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) {
3667da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) ==
3668da14cebeSEric Cheng SRS_POLL_THR_QUIESCED);
3669da14cebeSEric Cheng }
3670da14cebeSEric Cheng }
3671da14cebeSEric Cheng
3672da14cebeSEric Cheng /*
3673da14cebeSEric Cheng * Signal any quiesced soft ring workers to restart and wait for the
3674da14cebeSEric Cheng * soft ring down count to come down to zero.
3675da14cebeSEric Cheng */
3676da14cebeSEric Cheng if (mac_srs->srs_soft_ring_quiesced_count != 0) {
3677da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL;
3678da14cebeSEric Cheng softring = softring->s_ring_next) {
3679da14cebeSEric Cheng if (!(softring->s_ring_state & S_RING_QUIESCE))
3680da14cebeSEric Cheng continue;
3681da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_RESTART);
3682da14cebeSEric Cheng }
3683da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 0)
3684da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3685da14cebeSEric Cheng }
3686da14cebeSEric Cheng
3687da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART);
3688da14cebeSEric Cheng if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) {
3689da14cebeSEric Cheng /*
3690da14cebeSEric Cheng * Signal the poll thread and ask it to restart. Wait till it
3691da14cebeSEric Cheng * actually restarts and the SRS_POLL_THR_QUIESCED flag gets
3692da14cebeSEric Cheng * cleared.
3693da14cebeSEric Cheng */
3694da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_THR_RESTART;
3695da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv);
3696da14cebeSEric Cheng while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED)
3697da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3698da14cebeSEric Cheng ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART));
3699da14cebeSEric Cheng }
3700da14cebeSEric Cheng /* Wake up any waiter waiting for the restart to complete */
3701da14cebeSEric Cheng mac_srs->srs_state |= SRS_RESTART_DONE;
3702da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv);
3703da14cebeSEric Cheng }
3704da14cebeSEric Cheng
3705da14cebeSEric Cheng static void
mac_srs_worker_unbind(mac_soft_ring_set_t * mac_srs)3706da14cebeSEric Cheng mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs)
3707da14cebeSEric Cheng {
3708da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3709da14cebeSEric Cheng if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) {
3710da14cebeSEric Cheng ASSERT(mac_srs->srs_worker_cpuid == -1);
3711da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3712da14cebeSEric Cheng return;
3713da14cebeSEric Cheng }
3714da14cebeSEric Cheng
3715da14cebeSEric Cheng mac_srs->srs_worker_cpuid = -1;
3716da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_WORKER_BOUND;
3717da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker);
3718da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3719da14cebeSEric Cheng }
3720da14cebeSEric Cheng
3721da14cebeSEric Cheng static void
mac_srs_poll_unbind(mac_soft_ring_set_t * mac_srs)3722da14cebeSEric Cheng mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs)
3723da14cebeSEric Cheng {
3724da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3725da14cebeSEric Cheng if (mac_srs->srs_poll_thr == NULL ||
3726da14cebeSEric Cheng (mac_srs->srs_state & SRS_POLL_BOUND) == 0) {
3727da14cebeSEric Cheng ASSERT(mac_srs->srs_poll_cpuid == -1);
3728da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3729da14cebeSEric Cheng return;
3730da14cebeSEric Cheng }
3731da14cebeSEric Cheng
3732da14cebeSEric Cheng mac_srs->srs_poll_cpuid = -1;
3733da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLL_BOUND;
3734da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr);
3735da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3736da14cebeSEric Cheng }
3737da14cebeSEric Cheng
3738da14cebeSEric Cheng static void
mac_srs_threads_unbind(mac_soft_ring_set_t * mac_srs)3739da14cebeSEric Cheng mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs)
3740da14cebeSEric Cheng {
3741da14cebeSEric Cheng mac_soft_ring_t *soft_ring;
3742da14cebeSEric Cheng
3743da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip));
3744da14cebeSEric Cheng
3745da14cebeSEric Cheng mutex_enter(&cpu_lock);
3746da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs);
3747da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX))
3748da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs);
3749da14cebeSEric Cheng
3750da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3751da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) {
3752da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring);
3753da14cebeSEric Cheng }
3754da14cebeSEric Cheng mutex_exit(&cpu_lock);
3755da14cebeSEric Cheng }
3756da14cebeSEric Cheng
3757da14cebeSEric Cheng /*
3758da14cebeSEric Cheng * When a CPU is going away, unbind all MAC threads which are bound
3759da14cebeSEric Cheng * to that CPU. The affinity of the thread to the CPU is saved to allow
3760da14cebeSEric Cheng * the thread to be rebound to the CPU if it comes back online.
3761da14cebeSEric Cheng */
3762da14cebeSEric Cheng static void
mac_walk_srs_and_unbind(int cpuid)3763da14cebeSEric Cheng mac_walk_srs_and_unbind(int cpuid)
3764da14cebeSEric Cheng {
3765da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs;
3766da14cebeSEric Cheng mac_soft_ring_t *soft_ring;
3767da14cebeSEric Cheng
3768da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER);
3769da14cebeSEric Cheng
3770da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL)
3771da14cebeSEric Cheng goto done;
3772da14cebeSEric Cheng
3773da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) {
3774da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == cpuid) {
3775da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = cpuid;
3776da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs);
3777da14cebeSEric Cheng }
3778da14cebeSEric Cheng
3779da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) {
3780da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == cpuid) {
3781da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = cpuid;
3782da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs);
3783da14cebeSEric Cheng }
3784da14cebeSEric Cheng }
3785da14cebeSEric Cheng
3786da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */
3787da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3788da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL;
3789da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) {
3790da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == cpuid) {
3791da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = cpuid;
3792da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring);
3793da14cebeSEric Cheng }
3794da14cebeSEric Cheng }
3795da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3796da14cebeSEric Cheng }
3797da14cebeSEric Cheng done:
3798da14cebeSEric Cheng rw_exit(&mac_srs_g_lock);
3799da14cebeSEric Cheng }
3800da14cebeSEric Cheng
3801da14cebeSEric Cheng /* TX SETUP and TEARDOWN ROUTINES */
3802da14cebeSEric Cheng
3803da14cebeSEric Cheng /*
3804da14cebeSEric Cheng * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring()
3805da14cebeSEric Cheng * handle the case where the number of rings is one. I.e. there is
3806da14cebeSEric Cheng * a ring pointed to by mac_srs->srs_tx_arg2.
3807da14cebeSEric Cheng */
3808da14cebeSEric Cheng void
mac_tx_srs_add_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3809da14cebeSEric Cheng mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3810da14cebeSEric Cheng {
3811da14cebeSEric Cheng mac_client_impl_t *mcip = mac_srs->srs_mcip;
3812da14cebeSEric Cheng mac_soft_ring_t *soft_ring;
38130dc2366fSVenugopal Iyer int count = mac_srs->srs_tx_ring_count;
38140dc2366fSVenugopal Iyer uint32_t soft_ring_type = ST_RING_TX;
38150dc2366fSVenugopal Iyer uint_t ring_info;
3816da14cebeSEric Cheng
3817da14cebeSEric Cheng ASSERT(mac_srs->srs_state & SRS_QUIESCE);
38180dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring);
38190dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE))
38200dc2366fSVenugopal Iyer soft_ring_type |= ST_RING_WORKER_ONLY;
38210dc2366fSVenugopal Iyer soft_ring = mac_soft_ring_create(count, 0,
38220dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, mcip, mac_srs, -1,
3823da14cebeSEric Cheng NULL, mcip, (mac_resource_handle_t)tx_ring);
38240dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count++;
38250dc2366fSVenugopal Iyer mac_srs_update_fanout_list(mac_srs);
3826da14cebeSEric Cheng /*
3827da14cebeSEric Cheng * put this soft ring in quiesce mode too so when we restart
3828da14cebeSEric Cheng * all soft rings in the srs are in the same state.
3829da14cebeSEric Cheng */
3830da14cebeSEric Cheng mac_soft_ring_signal(soft_ring, S_RING_QUIESCE);
3831da14cebeSEric Cheng }
3832da14cebeSEric Cheng
3833da14cebeSEric Cheng static void
mac_soft_ring_remove(mac_soft_ring_set_t * mac_srs,mac_soft_ring_t * softring)3834da14cebeSEric Cheng mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring)
3835da14cebeSEric Cheng {
3836da14cebeSEric Cheng int sringcnt;
3837da14cebeSEric Cheng
3838da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
3839da14cebeSEric Cheng sringcnt = mac_srs->srs_soft_ring_count;
3840da14cebeSEric Cheng ASSERT(sringcnt > 0);
3841da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_CONDEMNED);
3842da14cebeSEric Cheng
3843da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_condemned_count == 0);
3844da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 1)
3845da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock);
3846da14cebeSEric Cheng
3847da14cebeSEric Cheng if (softring == mac_srs->srs_soft_ring_head) {
3848da14cebeSEric Cheng mac_srs->srs_soft_ring_head = softring->s_ring_next;
3849da14cebeSEric Cheng if (mac_srs->srs_soft_ring_head != NULL) {
3850da14cebeSEric Cheng mac_srs->srs_soft_ring_head->s_ring_prev = NULL;
3851da14cebeSEric Cheng } else {
3852da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL;
3853da14cebeSEric Cheng }
3854da14cebeSEric Cheng } else {
3855da14cebeSEric Cheng softring->s_ring_prev->s_ring_next =
3856da14cebeSEric Cheng softring->s_ring_next;
3857da14cebeSEric Cheng if (softring->s_ring_next != NULL) {
3858da14cebeSEric Cheng softring->s_ring_next->s_ring_prev =
3859da14cebeSEric Cheng softring->s_ring_prev;
3860da14cebeSEric Cheng } else {
3861da14cebeSEric Cheng mac_srs->srs_soft_ring_tail =
3862da14cebeSEric Cheng softring->s_ring_prev;
3863da14cebeSEric Cheng }
3864da14cebeSEric Cheng }
3865da14cebeSEric Cheng mac_srs->srs_soft_ring_count--;
3866da14cebeSEric Cheng
3867da14cebeSEric Cheng mac_srs->srs_soft_ring_condemned_count--;
3868da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
3869da14cebeSEric Cheng
38700dc2366fSVenugopal Iyer mac_soft_ring_free(softring);
3871da14cebeSEric Cheng }
3872da14cebeSEric Cheng
3873da14cebeSEric Cheng void
mac_tx_srs_del_ring(mac_soft_ring_set_t * mac_srs,mac_ring_t * tx_ring)3874da14cebeSEric Cheng mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring)
3875da14cebeSEric Cheng {
3876da14cebeSEric Cheng int i;
3877da14cebeSEric Cheng mac_soft_ring_t *soft_ring, *remove_sring;
38780dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip;
3879da14cebeSEric Cheng
3880da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock);
38810dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) {
38820dc2366fSVenugopal Iyer soft_ring = mac_srs->srs_tx_soft_rings[i];
3883da14cebeSEric Cheng if (soft_ring->s_ring_tx_arg2 == tx_ring)
3884da14cebeSEric Cheng break;
3885da14cebeSEric Cheng }
3886da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock);
38870dc2366fSVenugopal Iyer ASSERT(i < mac_srs->srs_tx_ring_count);
3888da14cebeSEric Cheng remove_sring = soft_ring;
38890dc2366fSVenugopal Iyer /*
38900dc2366fSVenugopal Iyer * In the case of aggr, the soft ring associated with a Tx ring
38910dc2366fSVenugopal Iyer * is also stored in st_soft_rings[] array. That entry should
38920dc2366fSVenugopal Iyer * be removed.
38930dc2366fSVenugopal Iyer */
389484de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) {
38950dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx;
38960dc2366fSVenugopal Iyer
38970dc2366fSVenugopal Iyer ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring);
38980dc2366fSVenugopal Iyer tx->st_soft_rings[tx_ring->mr_index] = NULL;
38990dc2366fSVenugopal Iyer }
3900da14cebeSEric Cheng mac_soft_ring_remove(mac_srs, remove_sring);
3901da14cebeSEric Cheng mac_srs_update_fanout_list(mac_srs);
3902da14cebeSEric Cheng }
3903da14cebeSEric Cheng
3904da14cebeSEric Cheng /*
3905da14cebeSEric Cheng * mac_tx_srs_setup():
3906da14cebeSEric Cheng * Used to setup Tx rings. If no free Tx ring is available, then default
3907da14cebeSEric Cheng * Tx ring is used.
3908da14cebeSEric Cheng */
3909da14cebeSEric Cheng void
mac_tx_srs_setup(mac_client_impl_t * mcip,flow_entry_t * flent)39100dc2366fSVenugopal Iyer mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent)
3911da14cebeSEric Cheng {
39120dc2366fSVenugopal Iyer mac_impl_t *mip = mcip->mci_mip;
39130dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs;
39140dc2366fSVenugopal Iyer int i;
39150dc2366fSVenugopal Iyer int tx_ring_count = 0;
39160dc2366fSVenugopal Iyer uint32_t soft_ring_type;
39170dc2366fSVenugopal Iyer mac_group_t *grp = NULL;
39180dc2366fSVenugopal Iyer mac_ring_t *ring;
39190dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &tx_srs->srs_tx;
39200dc2366fSVenugopal Iyer boolean_t is_aggr;
39210dc2366fSVenugopal Iyer uint_t ring_info = 0;
39220dc2366fSVenugopal Iyer
392384de666eSRyan Zezeski is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0;
39240dc2366fSVenugopal Iyer grp = flent->fe_tx_ring_group;
39250dc2366fSVenugopal Iyer if (grp == NULL) {
39260dc2366fSVenugopal Iyer ring = (mac_ring_t *)mip->mi_default_tx_ring;
39270dc2366fSVenugopal Iyer goto no_group;
39280dc2366fSVenugopal Iyer }
39290dc2366fSVenugopal Iyer tx_ring_count = grp->mrg_cur_count;
39300dc2366fSVenugopal Iyer ring = grp->mrg_rings;
3931da14cebeSEric Cheng /*
3932da14cebeSEric Cheng * An attempt is made to reserve 'tx_ring_count' number
3933da14cebeSEric Cheng * of Tx rings. If tx_ring_count is 0, default Tx ring
3934da14cebeSEric Cheng * is used. If it is 1, an attempt is made to reserve one
3935da14cebeSEric Cheng * Tx ring. In both the cases, the ring information is
3936da14cebeSEric Cheng * stored in Tx SRS. If multiple Tx rings are specified,
3937da14cebeSEric Cheng * then each Tx ring will have a Tx-side soft ring. All
3938da14cebeSEric Cheng * these soft rings will be hang off Tx SRS.
3939da14cebeSEric Cheng */
39400dc2366fSVenugopal Iyer switch (grp->mrg_state) {
39410dc2366fSVenugopal Iyer case MAC_GROUP_STATE_SHARED:
39420dc2366fSVenugopal Iyer case MAC_GROUP_STATE_RESERVED:
39430dc2366fSVenugopal Iyer if (tx_ring_count <= 1 && !is_aggr) {
39440dc2366fSVenugopal Iyer no_group:
39450dc2366fSVenugopal Iyer if (ring != NULL &&
39460dc2366fSVenugopal Iyer ring->mr_state != MR_INUSE) {
39470dc2366fSVenugopal Iyer (void) mac_start_ring(ring);
39480dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(
39490dc2366fSVenugopal Iyer (mac_ring_handle_t)ring);
39500dc2366fSVenugopal Iyer }
39510dc2366fSVenugopal Iyer tx->st_arg2 = (void *)ring;
39520dc2366fSVenugopal Iyer mac_tx_srs_stat_recreate(tx_srs, B_FALSE);
39530dc2366fSVenugopal Iyer if (tx_srs->srs_type & SRST_BW_CONTROL) {
39540dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_BW;
39550dc2366fSVenugopal Iyer } else if (mac_tx_serialize ||
39560dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) {
39570dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_SERIALIZE;
39580dc2366fSVenugopal Iyer } else {
39590dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_DEFAULT;
39600dc2366fSVenugopal Iyer }
3961da14cebeSEric Cheng break;
396263f531d1SSriharsha Basavapatna }
39630dc2366fSVenugopal Iyer soft_ring_type = ST_RING_TX;
39640dc2366fSVenugopal Iyer if (tx_srs->srs_type & SRST_BW_CONTROL) {
39650dc2366fSVenugopal Iyer tx->st_mode = is_aggr ?
39660dc2366fSVenugopal Iyer SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT;
39670dc2366fSVenugopal Iyer } else {
39680dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? SRS_TX_AGGR :
39690dc2366fSVenugopal Iyer SRS_TX_FANOUT;
39700dc2366fSVenugopal Iyer }
39710dc2366fSVenugopal Iyer for (i = 0; i < tx_ring_count; i++) {
39720dc2366fSVenugopal Iyer ASSERT(ring != NULL);
39730dc2366fSVenugopal Iyer switch (ring->mr_state) {
39740dc2366fSVenugopal Iyer case MR_INUSE:
39750dc2366fSVenugopal Iyer case MR_FREE:
39760dc2366fSVenugopal Iyer ASSERT(ring->mr_srs == NULL);
39770dc2366fSVenugopal Iyer
39780dc2366fSVenugopal Iyer if (ring->mr_state != MR_INUSE)
39790dc2366fSVenugopal Iyer (void) mac_start_ring(ring);
39800dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(
39810dc2366fSVenugopal Iyer (mac_ring_handle_t)ring);
39820dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info &
39830dc2366fSVenugopal Iyer MAC_RING_TX_SERIALIZE)) {
39840dc2366fSVenugopal Iyer soft_ring_type |=
39850dc2366fSVenugopal Iyer ST_RING_WORKER_ONLY;
39860dc2366fSVenugopal Iyer }
39870dc2366fSVenugopal Iyer (void) mac_soft_ring_create(i, 0,
39880dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri,
39890dc2366fSVenugopal Iyer mcip, tx_srs, -1, NULL, mcip,
39900dc2366fSVenugopal Iyer (mac_resource_handle_t)ring);
39910dc2366fSVenugopal Iyer break;
39920dc2366fSVenugopal Iyer default:
39930dc2366fSVenugopal Iyer cmn_err(CE_PANIC,
39940dc2366fSVenugopal Iyer "srs_setup: mcip = %p "
39950dc2366fSVenugopal Iyer "trying to add UNKNOWN ring = %p\n",
39960dc2366fSVenugopal Iyer (void *)mcip, (void *)ring);
39970dc2366fSVenugopal Iyer break;
39980dc2366fSVenugopal Iyer }
39990dc2366fSVenugopal Iyer ring = ring->mr_next;
40000dc2366fSVenugopal Iyer }
40010dc2366fSVenugopal Iyer mac_srs_update_fanout_list(tx_srs);
40020dc2366fSVenugopal Iyer break;
40030dc2366fSVenugopal Iyer default:
40040dc2366fSVenugopal Iyer ASSERT(B_FALSE);
40050dc2366fSVenugopal Iyer break;
4006da14cebeSEric Cheng }
4007da14cebeSEric Cheng tx->st_func = mac_tx_get_func(tx->st_mode);
40080dc2366fSVenugopal Iyer if (is_aggr) {
40090dc2366fSVenugopal Iyer VERIFY(i_mac_capab_get((mac_handle_t)mip,
40100dc2366fSVenugopal Iyer MAC_CAPAB_AGGR, &tx->st_capab_aggr));
4011da14cebeSEric Cheng }
40120dc2366fSVenugopal Iyer DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs,
40130dc2366fSVenugopal Iyer int, tx->st_mode, int, tx_srs->srs_tx_ring_count);
4014da14cebeSEric Cheng }
4015da14cebeSEric Cheng
401608ac1c49SNicolas Droux /*
401708ac1c49SNicolas Droux * Update the fanout of a client if its recorded link speed doesn't match
401808ac1c49SNicolas Droux * its current link speed.
401908ac1c49SNicolas Droux */
402008ac1c49SNicolas Droux void
mac_fanout_recompute_client(mac_client_impl_t * mcip,cpupart_t * cpupart)40210dc2366fSVenugopal Iyer mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart)
402208ac1c49SNicolas Droux {
402308ac1c49SNicolas Droux uint64_t link_speed;
402408ac1c49SNicolas Droux mac_resource_props_t *mcip_mrp;
40250dc2366fSVenugopal Iyer flow_entry_t *flent = mcip->mci_flent;
40260dc2366fSVenugopal Iyer mac_soft_ring_set_t *rx_srs;
40270dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu;
40280dc2366fSVenugopal Iyer int soft_ring_count, maxcpus;
402908ac1c49SNicolas Droux
403008ac1c49SNicolas Droux ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip));
403108ac1c49SNicolas Droux
403208ac1c49SNicolas Droux link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip,
403308ac1c49SNicolas Droux MAC_STAT_IFSPEED);
403408ac1c49SNicolas Droux
403508ac1c49SNicolas Droux if ((link_speed != 0) &&
403608ac1c49SNicolas Droux (link_speed != mcip->mci_flent->fe_nic_speed)) {
403708ac1c49SNicolas Droux mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
40380dc2366fSVenugopal Iyer /*
40390dc2366fSVenugopal Iyer * Before calling mac_fanout_setup(), check to see if
40400dc2366fSVenugopal Iyer * the SRSes already have the right number of soft
40410dc2366fSVenugopal Iyer * rings. mac_fanout_setup() is a heavy duty operation
40420dc2366fSVenugopal Iyer * where new cpu bindings are done for SRS and soft
40430dc2366fSVenugopal Iyer * ring threads and interrupts re-targeted.
40440dc2366fSVenugopal Iyer */
40450dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus;
40460dc2366fSVenugopal Iyer soft_ring_count = mac_compute_soft_ring_count(flent,
40470dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus);
40480dc2366fSVenugopal Iyer /*
40490dc2366fSVenugopal Iyer * If soft_ring_count returned by
40500dc2366fSVenugopal Iyer * mac_compute_soft_ring_count() is 0, bump it
40510dc2366fSVenugopal Iyer * up by 1 because we always have atleast one
40520dc2366fSVenugopal Iyer * TCP, UDP, and OTH soft ring associated with
40530dc2366fSVenugopal Iyer * an SRS.
40540dc2366fSVenugopal Iyer */
40550dc2366fSVenugopal Iyer soft_ring_count = (soft_ring_count == 0) ?
40560dc2366fSVenugopal Iyer 1 : soft_ring_count;
40570dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[0];
40580dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu;
40590dc2366fSVenugopal Iyer if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) {
40600dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, mcip_mrp,
40610dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart);
40620dc2366fSVenugopal Iyer }
406308ac1c49SNicolas Droux }
406408ac1c49SNicolas Droux }
406508ac1c49SNicolas Droux
4066da14cebeSEric Cheng /*
406745948e49SRyan Zezeski * Walk through the list of MAC clients for the MAC.
406845948e49SRyan Zezeski * For each active MAC client, recompute the number of soft rings
4069da14cebeSEric Cheng * associated with every client, only if current speed is different
4070da14cebeSEric Cheng * from the speed that was previously used for soft ring computation.
4071da14cebeSEric Cheng * If the cable is disconnected whlie the NIC is started, we would get
4072da14cebeSEric Cheng * notification with speed set to 0. We do not recompute in that case.
4073da14cebeSEric Cheng */
4074da14cebeSEric Cheng void
mac_fanout_recompute(mac_impl_t * mip)4075da14cebeSEric Cheng mac_fanout_recompute(mac_impl_t *mip)
4076da14cebeSEric Cheng {
4077da14cebeSEric Cheng mac_client_impl_t *mcip;
40780dc2366fSVenugopal Iyer cpupart_t *cpupart;
40790dc2366fSVenugopal Iyer boolean_t use_default;
40800dc2366fSVenugopal Iyer mac_resource_props_t *mrp, *emrp;
408108ac1c49SNicolas Droux
4082da14cebeSEric Cheng i_mac_perim_enter(mip);
40832c4ec682SEric Cheng if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 ||
40842c4ec682SEric Cheng mip->mi_linkstate != LINK_STATE_UP) {
4085da14cebeSEric Cheng i_mac_perim_exit(mip);
4086da14cebeSEric Cheng return;
4087da14cebeSEric Cheng }
4088da14cebeSEric Cheng
4089da14cebeSEric Cheng for (mcip = mip->mi_clients_list; mcip != NULL;
4090da14cebeSEric Cheng mcip = mcip->mci_client_next) {
409145948e49SRyan Zezeski /* Aggr port clients don't have SRSes. */
409245948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0)
409345948e49SRyan Zezeski continue;
409445948e49SRyan Zezeski
409508ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 ||
409608ac1c49SNicolas Droux !MCIP_DATAPATH_SETUP(mcip))
4097da14cebeSEric Cheng continue;
40980dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(mcip);
40990dc2366fSVenugopal Iyer emrp = MCIP_EFFECTIVE_PROPS(mcip);
41000dc2366fSVenugopal Iyer use_default = B_FALSE;
41010dc2366fSVenugopal Iyer pool_lock();
41020dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default);
41030dc2366fSVenugopal Iyer mac_fanout_recompute_client(mcip, cpupart);
41040dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp);
41050dc2366fSVenugopal Iyer pool_unlock();
4106da14cebeSEric Cheng }
410745948e49SRyan Zezeski
4108da14cebeSEric Cheng i_mac_perim_exit(mip);
4109da14cebeSEric Cheng }
41104eaa4710SRishi Srivatsavai
41114eaa4710SRishi Srivatsavai /*
41124eaa4710SRishi Srivatsavai * Given a MAC, change the polling state for all its MAC clients. 'enable' is
41134eaa4710SRishi Srivatsavai * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by
41144eaa4710SRishi Srivatsavai * default.
41154eaa4710SRishi Srivatsavai */
41164eaa4710SRishi Srivatsavai void
mac_poll_state_change(mac_handle_t mh,boolean_t enable)41174eaa4710SRishi Srivatsavai mac_poll_state_change(mac_handle_t mh, boolean_t enable)
41184eaa4710SRishi Srivatsavai {
41194eaa4710SRishi Srivatsavai mac_impl_t *mip = (mac_impl_t *)mh;
41204eaa4710SRishi Srivatsavai mac_client_impl_t *mcip;
41214eaa4710SRishi Srivatsavai
41224eaa4710SRishi Srivatsavai i_mac_perim_enter(mip);
41234eaa4710SRishi Srivatsavai if (enable)
41244eaa4710SRishi Srivatsavai mip->mi_state_flags &= ~MIS_POLL_DISABLE;
41254eaa4710SRishi Srivatsavai else
41264eaa4710SRishi Srivatsavai mip->mi_state_flags |= MIS_POLL_DISABLE;
41274eaa4710SRishi Srivatsavai for (mcip = mip->mi_clients_list; mcip != NULL;
41284eaa4710SRishi Srivatsavai mcip = mcip->mci_client_next)
41294eaa4710SRishi Srivatsavai mac_client_update_classifier(mcip, B_TRUE);
41304eaa4710SRishi Srivatsavai i_mac_perim_exit(mip);
41314eaa4710SRishi Srivatsavai }
4132