1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_MAC_SOFT_RING_H
28 #define	_SYS_MAC_SOFT_RING_H
29 
30 #ifdef	__cplusplus
31 extern "C" {
32 #endif
33 
34 #include <sys/types.h>
35 #include <sys/cpuvar.h>
36 #include <sys/cpupart.h>
37 #include <sys/processor.h>
38 #include <sys/stream.h>
39 #include <sys/squeue.h>
40 #include <sys/dlpi.h>
41 #include <sys/mac_impl.h>
42 #include <sys/mac_stat.h>
43 
44 #define	S_RING_NAMELEN 64
45 
46 #define	MAX_SR_FANOUT	24
47 
48 extern boolean_t mac_soft_ring_enable;
49 extern boolean_t mac_latency_optimize;
50 
51 typedef struct mac_soft_ring_s mac_soft_ring_t;
52 typedef struct mac_soft_ring_set_s mac_soft_ring_set_t;
53 
54 typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *);
55 typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *,
56     uintptr_t, uint16_t, mblk_t **);
57 
58 
59 /* Tx notify callback */
60 typedef struct mac_tx_notify_cb_s {
61 	mac_cb_t		mtnf_link;	/* Linked list of callbacks */
62 	mac_tx_notify_t		mtnf_fn;	/* The callback function */
63 	void			*mtnf_arg;	/* Callback function argument */
64 } mac_tx_notify_cb_t;
65 
66 struct mac_soft_ring_s {
67 	/* Keep the most used members 64bytes cache aligned */
68 	kmutex_t	s_ring_lock;	/* lock before using any member */
69 	uint16_t	s_ring_type;	/* processing model of the sq */
70 	uint16_t	s_ring_state;	/* state flags and message count */
71 	int		s_ring_count;	/* # of mblocks in mac_soft_ring */
72 	size_t		s_ring_size;	/* Size of data queued */
73 	mblk_t		*s_ring_first;	/* first mblk chain or NULL */
74 	mblk_t		*s_ring_last;	/* last mblk chain or NULL */
75 
76 	mac_direct_rx_t	s_ring_rx_func;
77 	void		*s_ring_rx_arg1;
78 	mac_resource_handle_t  s_ring_rx_arg2;
79 
80 	/*
81 	 * Threshold after which packets get dropped.
82 	 * Is always greater than s_ring_tx_hiwat
83 	 */
84 	int		s_ring_tx_max_q_cnt;
85 	/* # of mblocks after which to apply flow control */
86 	int		s_ring_tx_hiwat;
87 	/* # of mblocks after which to relieve flow control */
88 	int		s_ring_tx_lowat;
89 	boolean_t	s_ring_tx_woken_up;
90 	uint32_t	s_ring_hiwat_cnt;	/* times blocked for Tx descs */
91 
92 	void		*s_ring_tx_arg1;
93 	void		*s_ring_tx_arg2;
94 
95 	/* Tx notify callback */
96 	mac_cb_info_t	s_ring_notify_cb_info;		/* cb list info */
97 	mac_cb_t	*s_ring_notify_cb_list;		/* The cb list */
98 
99 	clock_t		s_ring_awaken;	/* time async thread was awakened */
100 
101 	kthread_t	*s_ring_run;	/* Current thread processing sq */
102 	processorid_t	s_ring_cpuid;	/* processor to bind to */
103 	processorid_t	s_ring_cpuid_save;	/* saved cpuid during offline */
104 	kcondvar_t	s_ring_async;	/* async thread blocks on */
105 	clock_t		s_ring_wait;	/* lbolts to wait after a fill() */
106 	timeout_id_t	s_ring_tid;	/* timer id of pending timeout() */
107 	kthread_t	*s_ring_worker;	/* kernel thread id */
108 	char		s_ring_name[S_RING_NAMELEN + 1];
109 	uint32_t	s_ring_total_inpkt;
110 	uint32_t	s_ring_total_rbytes;
111 	uint32_t	s_ring_drops;
112 	struct mac_client_impl_s *s_ring_mcip;
113 	kstat_t		*s_ring_ksp;
114 
115 	/* Teardown, poll disable control ops */
116 	kcondvar_t	s_ring_client_cv; /* Client wait for control op */
117 
118 	mac_soft_ring_set_t *s_ring_set;   /* The SRS this ring belongs to */
119 	mac_soft_ring_t	*s_ring_next;
120 	mac_soft_ring_t	*s_ring_prev;
121 	mac_soft_ring_drain_func_t s_ring_drain_func;
122 
123 	mac_tx_stats_t	s_st_stat;
124 };
125 
126 typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t);
127 
128 /* Transmit side Soft Ring Set */
129 typedef struct mac_srs_tx_s {
130 	/* Members for Tx size processing */
131 	uint32_t	st_mode;
132 	mac_tx_func_t	st_func;
133 	void		*st_arg1;
134 	void		*st_arg2;
135 	mac_group_t	*st_group;	/* TX group for share */
136 	boolean_t	st_woken_up;
137 
138 	/*
139 	 * st_max_q_cnt is the queue depth threshold to limit
140 	 * outstanding packets on the Tx SRS. Once the limit
141 	 * is reached, Tx SRS will drop packets until the
142 	 * limit goes below the threshold.
143 	 */
144 	uint32_t	st_max_q_cnt;	/* max. outstanding packets */
145 	/*
146 	 * st_hiwat is used Tx serializer and bandwidth mode.
147 	 * This is the queue depth threshold upto which
148 	 * packets will get buffered with no flow-control
149 	 * back pressure applied to the caller. Once this
150 	 * threshold is reached, back pressure will be
151 	 * applied to the caller of mac_tx() (mac_tx() starts
152 	 * returning a cookie to indicate a blocked SRS).
153 	 * st_hiwat should always be lesser than or equal to
154 	 * st_max_q_cnt.
155 	 */
156 	uint32_t	st_hiwat;	/* mblk cnt to apply flow control */
157 	uint32_t	st_lowat;	/* mblk cnt to relieve flow control */
158 	uint32_t	st_hiwat_cnt; /* times blocked for Tx descs */
159 	mac_tx_stats_t	st_stat;
160 	mac_capab_aggr_t	st_capab_aggr;
161 	/*
162 	 * st_soft_rings is used as an array to store aggr Tx soft
163 	 * rings. When aggr_find_tx_ring() returns a pseudo ring,
164 	 * the associated soft ring has to be found. st_soft_rings
165 	 * array stores the soft ring associated with a pseudo Tx
166 	 * ring and it can be accessed using the pseudo ring
167 	 * index (mr_index). Note that the ring index is unique
168 	 * for each ring in a group.
169 	 */
170 	mac_soft_ring_t **st_soft_rings;
171 } mac_srs_tx_t;
172 
173 /* Receive side Soft Ring Set */
174 typedef struct mac_srs_rx_s {
175 	/*
176 	 * Upcall Function for fanout, Rx processing etc. Perhaps
177 	 * the same 3 members below can be used for Tx
178 	 * processing, but looking around, mac_rx_func_t has
179 	 * proliferated too much into various files at different
180 	 * places. I am leaving the consolidation battle for
181 	 * another day.
182 	 */
183 	mac_direct_rx_t		sr_func;	/* srs_lock */
184 	void			*sr_arg1;	/* srs_lock */
185 	mac_resource_handle_t 	sr_arg2;	/* srs_lock */
186 	mac_rx_func_t		sr_lower_proc;	/* Atomically changed */
187 	uint32_t		sr_poll_pkt_cnt;
188 	uint32_t		sr_poll_thres;
189 
190 	/* mblk cnt to apply flow control */
191 	uint32_t		sr_hiwat;
192 	/* mblk cnt to relieve flow control */
193 	uint32_t		sr_lowat;
194 	mac_rx_stats_t		sr_stat;
195 
196 	/* Times polling was enabled */
197 	uint32_t		sr_poll_on;
198 	/* Times polling was enabled by worker thread */
199 	uint32_t		sr_worker_poll_on;
200 	/* Times polling was disabled */
201 	uint32_t		sr_poll_off;
202 	/* Poll thread signalled count */
203 	uint32_t		sr_poll_thr_sig;
204 	/* Poll thread busy */
205 	uint32_t		sr_poll_thr_busy;
206 	/* SRS drains, stays in poll mode but doesn't poll */
207 	uint32_t		sr_poll_drain_no_poll;
208 	/*
209 	 * SRS has nothing to do and no packets in H/W but
210 	 * there is a backlog in softrings. SRS stays in
211 	 * poll mode but doesn't do polling.
212 	 */
213 	uint32_t		sr_poll_no_poll;
214 	/* Active polling restarted */
215 	uint32_t		sr_below_hiwat;
216 	/* Found packets in last poll so try and poll again */
217 	uint32_t		sr_poll_again;
218 	/*
219 	 * Packets in queue but poll thread not allowed to process so
220 	 * signal the worker thread.
221 	 */
222 	uint32_t		sr_poll_sig_worker;
223 	/*
224 	 * Poll thread has nothing to do and H/W has nothing so
225 	 * reenable the interrupts.
226 	 */
227 	uint32_t		sr_poll_intr_enable;
228 	/*
229 	 * Poll thread has nothing to do and worker thread was already
230 	 * running so it can decide to reenable interrupt or poll again.
231 	 */
232 	uint32_t		sr_poll_goto_sleep;
233 	/* Worker thread goes back to draining the queue */
234 	uint32_t		sr_drain_again;
235 	/* More Packets in queue so signal the poll thread to drain */
236 	uint32_t		sr_drain_poll_sig;
237 	/* More Packets in queue so signal the worker thread to drain */
238 	uint32_t		sr_drain_worker_sig;
239 	/* Poll thread is already running so worker has nothing to do */
240 	uint32_t		sr_drain_poll_running;
241 	/* We have packets already queued so keep polling */
242 	uint32_t		sr_drain_keep_polling;
243 	/* Drain is done and interrupts are reenabled */
244 	uint32_t		sr_drain_finish_intr;
245 	/* Polling thread needs to schedule worker wakeup */
246 	uint32_t		sr_poll_worker_wakeup;
247 } mac_srs_rx_t;
248 
249 /*
250  * mac_soft_ring_set_s:
251  * This is used both for Tx and Rx side. The srs_type identifies Rx or
252  * Tx type.
253  *
254  * Note that the structure is carefully crafted, with Rx elements coming
255  * first followed by Tx specific members. Future additions to this
256  * structure should follow the same guidelines.
257  *
258  * Rx-side notes:
259  * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow
260  * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On
261  * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode
262  * (if poll capable) and feeds the packets to soft_ring_list via choosen
263  * fanout type (specified by srs_type). In poll mode, the poll thread which is
264  * also a pointer can pick up the packets and feed them to various
265  * soft_ring_list.
266  *
267  * The srs_type can either be protocol based or fanout based where fanout itelf
268  * can be various types
269  *
270  * The polling works by turning off interrupts as soon as a packets
271  * are queued on the soft ring set. Once the backlog is clear and poll
272  * thread return empty handed i.e. Rx ring doesn't have anything, the
273  * interrupt is turned back on. For this purpose we keep a separate
274  * srs_poll_pkt_cnt counter which tracks the packets queued between SRS
275  * and the soft rings as well. The counter is incremented when packets
276  * are queued and decremented when SRS processes them (in case it has
277  * no soft rings) or the soft ring process them. Its important that
278  * in case SRS has softrings, the decrement doesn't happen till the
279  * packet is processed by the soft rings since it takes very little time
280  * for SRS to queue packet from SRS to soft rings and it will keep
281  * bringing more packets in the system faster than soft rings can
282  * process them.
283  *
284  * Tx side notes:
285  * The srs structure acts as a serializer with a worker thread. The
286  * default behavior of srs though is to act as a pass-thru. The queues
287  * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx
288  * descriptors or to enforce bandwidth limits.
289  *
290  * When multiple Tx rings are present, the SRS state will be set to
291  * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process()
292  * function will be fanned out to one of the Tx side soft rings based on
293  * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will
294  * be associated with a distinct h/w Tx ring.
295  */
296 
297 struct mac_soft_ring_set_s {
298 	/*
299 	 * Common elements, common to both Rx and Tx SRS type.
300 	 * The following block of fields are protected by srs_lock
301 	 */
302 	kmutex_t	srs_lock;
303 	uint32_t	srs_type;
304 	uint32_t	srs_state;	/* state flags */
305 	uint32_t	srs_count;
306 	mblk_t		*srs_first;	/* first mblk chain or NULL */
307 	mblk_t		*srs_last;	/* last mblk chain or NULL */
308 	kcondvar_t	srs_async;	/* cv for worker thread */
309 	kcondvar_t	srs_cv;		/* cv for poll thread */
310 	kcondvar_t	srs_quiesce_done_cv;	/* cv for removal */
311 	timeout_id_t	srs_tid;	/* timeout id for pending timeout */
312 
313 	/*
314 	 * List of soft rings & processing function.
315 	 * The following block is protected by Rx quiescence.
316 	 * i.e. they can be changed only after quiescing the SRS
317 	 * Protected by srs_lock.
318 	 */
319 	mac_soft_ring_t	*srs_soft_ring_head;
320 	mac_soft_ring_t	*srs_soft_ring_tail;
321 	int		srs_soft_ring_count;
322 	int		srs_soft_ring_quiesced_count;
323 	int		srs_soft_ring_condemned_count;
324 	mac_soft_ring_t	**srs_tcp_soft_rings;
325 	int		srs_tcp_ring_count;
326 	mac_soft_ring_t	**srs_udp_soft_rings;
327 	int		srs_udp_ring_count;
328 	mac_soft_ring_t	**srs_oth_soft_rings;
329 	int		srs_oth_ring_count;
330 	/*
331 	 * srs_tx_soft_rings is used by tx_srs in
332 	 * when operating in multi tx ring mode.
333 	 */
334 	mac_soft_ring_t	**srs_tx_soft_rings;
335 	int		srs_tx_ring_count;
336 
337 	/*
338 	 * Bandwidth control related members.
339 	 * They are common to both Rx- and Tx-side.
340 	 * Following protected by srs_lock
341 	 */
342 	mac_bw_ctl_t	*srs_bw;
343 	size_t		srs_size;	/* Size of packets queued in bytes */
344 	pri_t		srs_pri;
345 
346 	mac_soft_ring_set_t	*srs_next;	/* mac_srs_g_lock */
347 	mac_soft_ring_set_t	*srs_prev;	/* mac_srs_g_lock */
348 
349 	/* Attribute specific drain func (BW ctl vs non-BW ctl)	*/
350 	mac_srs_drain_proc_t	srs_drain_func;	/* Write once (WO) */
351 
352 	/*
353 	 * If the associated ring is exclusively used by a mac client, e.g.,
354 	 * an aggregation, this fields is used to keep a reference to the
355 	 * MAC client's pseudo ring.
356 	 */
357 	mac_resource_handle_t	srs_mrh;
358 	/*
359 	 * The following blocks are write once (WO) and valid for the life
360 	 * of the SRS
361 	 */
362 	struct mac_client_impl_s *srs_mcip;	/* back ptr to mac client */
363 	void			*srs_flent;	/* back ptr to flent */
364 	mac_ring_t		*srs_ring;	/*  Ring Descriptor */
365 
366 	/* Teardown, disable control ops */
367 	kcondvar_t	srs_client_cv;	/* Client wait for the control op */
368 
369 	kthread_t	*srs_worker;	/* WO, worker thread */
370 	kthread_t	*srs_poll_thr;	/* WO, poll thread */
371 
372 	uint_t		srs_ind;	/* Round Robin indx for picking up SR */
373 	processorid_t	srs_worker_cpuid;	/* processor to bind to */
374 	processorid_t	srs_worker_cpuid_save;	/* saved cpuid during offline */
375 	processorid_t	srs_poll_cpuid;		/* processor to bind to */
376 	processorid_t	srs_poll_cpuid_save;	/* saved cpuid during offline */
377 	uint_t		srs_fanout_state;
378 	mac_cpus_t	srs_cpu;
379 
380 	mac_srs_rx_t	srs_rx;
381 	mac_srs_tx_t	srs_tx;
382 	kstat_t		*srs_ksp;
383 };
384 
385 /*
386  * type flags - combination allowed to process and drain the queue
387  */
388 #define	ST_RING_WORKER_ONLY  	0x0001	/* Worker thread only */
389 #define	ST_RING_ANY		0x0002	/* Any thread can process the queue */
390 #define	ST_RING_TCP		0x0004
391 #define	ST_RING_UDP		0x0008
392 #define	ST_RING_OTH		0x0010
393 
394 #define	ST_RING_BW_CTL		0x0020
395 #define	ST_RING_TX		0x0040
396 
397 /*
398  * State flags.
399  */
400 #define	S_RING_PROC		0x0001	/* being processed */
401 #define	S_RING_BOUND		0x0002	/* Worker thread is bound to a cpu */
402 #define	S_RING_BLOCK		0x0004	/* No Tx descs */
403 #define	S_RING_TX_HIWAT		0x0008	/* Tx high watermark reached */
404 
405 #define	S_RING_WAKEUP_CLIENT	0x0010	/* flow ctrl, client wakeup needed */
406 #define	S_RING_BLANK		0x0020	/* Has been put into polling mode */
407 #define	S_RING_CLIENT_WAIT	0x0040	/* Client waiting for control op */
408 
409 #define	S_RING_CONDEMNED	0x0100	/* Being torn down */
410 #define	S_RING_CONDEMNED_DONE	0x0200	/* Being torn down */
411 #define	S_RING_QUIESCE		0x0400	/* No traffic flow, transient flag */
412 #define	S_RING_QUIESCE_DONE	0x0800	/* No traffic flow, transient flag */
413 
414 #define	S_RING_RESTART		0x1000	/* Go back to normal traffic flow */
415 #define	S_RING_ENQUEUED		0x2000	/* Pkts enqueued in Tx soft ring */
416 
417 /*
418  * arguments for processors to bind to
419  */
420 #define	S_RING_BIND_NONE	-1
421 
422 /*
423  * defines for srs_type - identifies a link or a sub-flow
424  * and other static characteristics of a SRS like a tx
425  * srs, tcp only srs, etc.
426  */
427 #define	SRST_LINK		0x00000001
428 #define	SRST_FLOW		0x00000002
429 #define	SRST_NO_SOFT_RINGS	0x00000004
430 #define	SRST_TCP_ONLY		0x00000008
431 
432 #define	SRST_FANOUT_PROTO	0x00000010
433 #define	SRST_FANOUT_SRC_IP	0x00000020
434 #define	SRST_FANOUT_OTH		0x00000040
435 #define	SRST_DEFAULT_GRP	0x00000080
436 
437 #define	SRST_TX			0x00000100
438 #define	SRST_BW_CONTROL		0x00000200
439 #define	SRST_DIRECT_POLL	0x00000400
440 
441 #define	SRST_DLS_BYPASS		0x00001000
442 #define	SRST_CLIENT_POLL_ENABLED 0x00002000
443 
444 /*
445  * soft ring set flags. These bits are dynamic in nature and get
446  * applied to srs_state. They reflect the state of SRS at any
447  * point of time
448  */
449 #define	SRS_BLANK		0x00000001
450 #define	SRS_WORKER_BOUND	0x00000002
451 #define	SRS_POLL_BOUND		0x00000004
452 #define	SRS_POLLING_CAPAB	0x00000008
453 
454 #define	SRS_PROC		0x00000010
455 #define	SRS_GET_PKTS		0x00000020
456 #define	SRS_POLLING		0x00000040
457 #define	SRS_BW_ENFORCED		0x00000080
458 
459 #define	SRS_WORKER		0x00000100
460 #define	SRS_ENQUEUED		0x00000200
461 #define	SRS_ANY_PROCESS		0x00000400
462 #define	SRS_PROC_FAST		0x00000800
463 
464 #define	SRS_POLL_PROC		0x00001000
465 #define	SRS_TX_BLOCKED		0x00002000	/* out of Tx descs */
466 #define	SRS_TX_HIWAT		0x00004000	/* Tx count exceeds hiwat */
467 #define	SRS_TX_WAKEUP_CLIENT	0x00008000	/* Flow-ctl: wakeup client */
468 
469 #define	SRS_CLIENT_PROC		0x00010000
470 #define	SRS_CLIENT_WAIT		0x00020000
471 #define	SRS_QUIESCE		0x00040000
472 #define	SRS_QUIESCE_DONE	0x00080000
473 
474 #define	SRS_CONDEMNED		0x00100000
475 #define	SRS_CONDEMNED_DONE	0x00200000
476 #define	SRS_POLL_THR_QUIESCED	0x00400000
477 #define	SRS_RESTART		0x00800000
478 
479 #define	SRS_RESTART_DONE	0x01000000
480 #define	SRS_POLL_THR_RESTART	0x02000000
481 #define	SRS_IN_GLIST		0x04000000
482 #define	SRS_POLL_THR_EXITED	0x08000000
483 
484 #define	SRS_QUIESCE_PERM	0x10000000
485 #define	SRS_LATENCY_OPT		0x20000000
486 #define	SRS_SOFTRING_QUEUE	0x40000000
487 
488 #define	SRS_QUIESCED(srs)	(srs->srs_state & SRS_QUIESCE_DONE)
489 
490 /*
491  * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be
492  * able to be restarted.
493  */
494 #define	SRS_QUIESCED_PERMANENT(srs)	(srs->srs_state & SRS_QUIESCE_PERM)
495 
496 /*
497  * soft ring set (SRS) Tx modes
498  */
499 typedef enum {
500 	SRS_TX_DEFAULT = 0,
501 	SRS_TX_SERIALIZE,
502 	SRS_TX_FANOUT,
503 	SRS_TX_BW,
504 	SRS_TX_BW_FANOUT,
505 	SRS_TX_AGGR,
506 	SRS_TX_BW_AGGR
507 } mac_tx_srs_mode_t;
508 
509 /*
510  * SRS fanout states
511  */
512 typedef enum {
513 	SRS_FANOUT_UNINIT = 0,
514 	SRS_FANOUT_INIT,
515 	SRS_FANOUT_REINIT
516 } mac_srs_fanout_state_t;
517 
518 /*
519  * Structure for dls statistics
520  */
521 struct dls_kstats {
522 	kstat_named_t	dlss_soft_ring_pkt_drop;
523 };
524 
525 extern struct dls_kstats dls_kstat;
526 
527 #define	DLS_BUMP_STAT(x, y)	(dls_kstat.x.value.ui32 += y)
528 
529 /* Turn dynamic polling off */
530 #define	MAC_SRS_POLLING_OFF(mac_srs) {					\
531 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
532 	if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \
533 	    (SRS_POLLING_CAPAB|SRS_POLLING)) {				\
534 		(mac_srs)->srs_state &= ~SRS_POLLING;			\
535 		(void) mac_hwring_enable_intr((mac_ring_handle_t)	\
536 		    (mac_srs)->srs_ring);				\
537 		(mac_srs)->srs_rx.sr_poll_off++;			\
538 	}								\
539 }
540 
541 #define	MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz)	{	\
542 	mblk_t 		*tmp;		       			\
543 	boolean_t	bw_ctl = B_FALSE;			\
544 								\
545 	ASSERT((head) != NULL);					\
546 	cnt = 0;						\
547 	sz = 0;							\
548 	if ((mac_srs)->srs_type & SRST_BW_CONTROL)		\
549 		bw_ctl = B_TRUE;				\
550 	tmp = tail = (head);					\
551 	if ((head)->b_next == NULL) {				\
552 		cnt = 1;					\
553 		if (bw_ctl)					\
554 			sz += msgdsize(head);			\
555 	} else {						\
556 		while (tmp != NULL) {				\
557 			tail = tmp;				\
558 			cnt++;					\
559 			if (bw_ctl)				\
560 				sz += msgdsize(tmp);		\
561 			tmp = tmp->b_next;			\
562 		}						\
563 	}							\
564 }
565 
566 /*
567  * Decrement the cumulative packet count in SRS and its
568  * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check
569  * if if the interface was left in a polling mode and no one
570  * is really processing the queue (to get the interface out
571  * of poll mode). If no one is processing the queue, then
572  * acquire the PROC and signal the poll thread to check the
573  * interface for packets and get the interface back to interrupt
574  * mode if nothing is found.
575  */
576 #define	MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) {		        \
577 	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
578 	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
579 									\
580 	srs_rx->sr_poll_pkt_cnt -= cnt;					\
581 	if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) && 	\
582 		(((mac_srs)->srs_state &				\
583 		(SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING))	\
584 	{								\
585 		(mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS);	\
586 		cv_signal(&(mac_srs)->srs_cv); 				\
587 		srs_rx->sr_below_hiwat++;				\
588 	}								\
589 }
590 
591 /*
592  * The following two macros are used to update the inbound packet and byte.
593  * count. The packet and byte count reflect the packets and bytes that are
594  * taken out of the SRS's queue, i.e. indicating they are being delivered.
595  * The srs_count and srs_size are updated in different locations as the
596  * srs_size is also used to take into account any bandwidth limits. The
597  * srs_size is updated only when a soft ring, if any, sends a packet up,
598  * as opposed to updating it when the SRS sends a packet to the SR, i.e.
599  * the srs_size reflects the packets in the SRS and SRs. These
600  * macros decrement the srs_size and srs_count and also increment the
601  * ipackets and ibytes stats resp.
602  *
603  * xxx-venu These are done under srs_lock, for now we still update
604  * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if
605  * just updating them would be accurate enough.
606  *
607  * If we are updating these for a sub-flow SRS, then we need to also
608  * updated it's MAC client bandwidth info, if the MAC client is also
609  * bandwidth regulated.
610  */
611 #define	MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) {				\
612 	if ((srs)->srs_type & SRST_BW_CONTROL) {			\
613 		mutex_enter(&(srs)->srs_bw->mac_bw_lock);		\
614 		(srs)->srs_bw->mac_bw_sz -= (sz);			\
615 		(srs)->srs_bw->mac_bw_used += (sz);			\
616 		mutex_exit(&(srs)->srs_bw->mac_bw_lock);		\
617 	}								\
618 }
619 
620 #define	MAC_TX_UPDATE_BW_INFO(srs, sz) {				\
621 	(srs)->srs_bw->mac_bw_sz -= (sz);				\
622 	(srs)->srs_bw->mac_bw_used += (sz);				\
623 }
624 
625 #define	MAC_TX_SOFT_RINGS(mac_srs) ((mac_srs)->srs_tx_ring_count >= 1)
626 
627 /* Soft ring flags for teardown */
628 #define	SRS_POLL_THR_OWNER	(SRS_PROC | SRS_POLLING | SRS_GET_PKTS)
629 #define	SRS_PAUSE		(SRS_CONDEMNED | SRS_QUIESCE)
630 #define	S_RING_PAUSE		(S_RING_CONDEMNED | S_RING_QUIESCE)
631 
632 /* Soft rings */
633 extern void mac_soft_ring_init(void);
634 extern void mac_soft_ring_finish(void);
635 extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *,
636     mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t,
637     cpupart_t *);
638 
639 extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *);
640 extern void mac_soft_ring_blank(void *, time_t, uint_t, int);
641 extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, int);
642 extern void mac_soft_ring_destroy(mac_soft_ring_t *);
643 extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *);
644 
645 /* Rx SRS */
646 extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *,
647     flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t,
648     mac_ring_t *);
649 extern void mac_srs_free(mac_soft_ring_set_t *);
650 extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t);
651 extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t);
652 extern void mac_rx_srs_retarget_intr(mac_soft_ring_set_t *, processorid_t);
653 extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *);
654 
655 extern void mac_srs_change_upcall(void *, mac_direct_rx_t, void *);
656 extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *);
657 extern void mac_srs_client_poll_enable(struct mac_client_impl_s *,
658     mac_soft_ring_set_t *);
659 extern void mac_srs_client_poll_disable(struct mac_client_impl_s *,
660     mac_soft_ring_set_t *);
661 extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *,
662     mac_soft_ring_set_t *);
663 extern void mac_srs_client_poll_restart(struct mac_client_impl_s *,
664     mac_soft_ring_set_t *);
665 extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
666 extern void mac_rx_srs_restart(mac_soft_ring_set_t *);
667 extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *,
668     boolean_t);
669 extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
670 
671 /* Tx SRS, Tx softring */
672 extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t);
673 extern void mac_tx_srs_setup(struct mac_client_impl_s *, flow_entry_t *);
674 extern mac_tx_func_t mac_tx_get_func(uint32_t);
675 extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *,
676     mac_tx_stats_t *);
677 extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *);
678 extern mac_soft_ring_t *mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *,
679     mac_ring_t *);
680 extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *);
681 extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *);
682 extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *,
683     uint16_t, mblk_t **);
684 
685 /* Subflow specific stuff */
686 extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *,
687     mac_resource_props_t *, int, int, mac_direct_rx_t);
688 extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *);
689 extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *);
690 extern void mac_srs_update_drv(struct mac_client_impl_s *);
691 extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t);
692 extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t);
693 
694 extern void mac_soft_ring_intr_enable(void *);
695 extern boolean_t mac_soft_ring_intr_disable(void *);
696 extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, uint16_t,
697     pri_t, mac_client_impl_t *, mac_soft_ring_set_t *,
698     processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t);
699 extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t);
700 	extern void mac_soft_ring_unbind(mac_soft_ring_t *);
701 extern void mac_soft_ring_free(mac_soft_ring_t *);
702 extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t);
703 extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *,
704     mblk_t *, mblk_t *, int, size_t);
705 extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *,
706     mblk_t *, uint16_t, mblk_t **);
707 extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *);
708 extern void mac_srs_worker_restart(mac_soft_ring_set_t *);
709 extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *,
710     mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t);
711 
712 extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t);
713 extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t);
714 extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *,
715     boolean_t);
716 extern void mac_srs_worker(mac_soft_ring_set_t *);
717 extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *);
718 extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t);
719 
720 extern void mac_tx_srs_restart(mac_soft_ring_set_t *);
721 extern void mac_rx_srs_remove(mac_soft_ring_set_t *);
722 
723 #ifdef	__cplusplus
724 }
725 #endif
726 
727 #endif	/* _SYS_MAC_SOFT_RING_H */
728