1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2017 Joyent, Inc.
26 */
27
28#ifndef	_SYS_MAC_SOFT_RING_H
29#define	_SYS_MAC_SOFT_RING_H
30
31#ifdef	__cplusplus
32extern "C" {
33#endif
34
35#include <sys/types.h>
36#include <sys/cpuvar.h>
37#include <sys/cpupart.h>
38#include <sys/processor.h>
39#include <sys/stream.h>
40#include <sys/squeue.h>
41#include <sys/dlpi.h>
42#include <sys/mac_impl.h>
43#include <sys/mac_stat.h>
44
45#define	S_RING_NAMELEN 64
46
47#define	MAX_SR_FANOUT	24
48
49extern boolean_t mac_soft_ring_enable;
50extern boolean_t mac_latency_optimize;
51
52typedef struct mac_soft_ring_s mac_soft_ring_t;
53typedef struct mac_soft_ring_set_s mac_soft_ring_set_t;
54
55typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *);
56typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *,
57    uintptr_t, uint16_t, mblk_t **);
58
59
60/* Tx notify callback */
61typedef struct mac_tx_notify_cb_s {
62	mac_cb_t		mtnf_link;	/* Linked list of callbacks */
63	mac_tx_notify_t		mtnf_fn;	/* The callback function */
64	void			*mtnf_arg;	/* Callback function argument */
65} mac_tx_notify_cb_t;
66
67struct mac_soft_ring_s {
68	/* Keep the most used members 64bytes cache aligned */
69	kmutex_t	s_ring_lock;	/* lock before using any member */
70	uint16_t	s_ring_type;	/* processing model of the sq */
71	uint16_t	s_ring_state;	/* state flags and message count */
72	int		s_ring_count;	/* # of mblocks in mac_soft_ring */
73	size_t		s_ring_size;	/* Size of data queued */
74	mblk_t		*s_ring_first;	/* first mblk chain or NULL */
75	mblk_t		*s_ring_last;	/* last mblk chain or NULL */
76
77	mac_direct_rx_t	s_ring_rx_func;
78	void		*s_ring_rx_arg1;
79	mac_resource_handle_t  s_ring_rx_arg2;
80
81	/*
82	 * Threshold after which packets get dropped.
83	 * Is always greater than s_ring_tx_hiwat
84	 */
85	int		s_ring_tx_max_q_cnt;
86	/* # of mblocks after which to apply flow control */
87	int		s_ring_tx_hiwat;
88	/* # of mblocks after which to relieve flow control */
89	int		s_ring_tx_lowat;
90	boolean_t	s_ring_tx_woken_up;
91	uint32_t	s_ring_hiwat_cnt;	/* times blocked for Tx descs */
92
93	void		*s_ring_tx_arg1;
94	void		*s_ring_tx_arg2;
95
96	/* Tx notify callback */
97	mac_cb_info_t	s_ring_notify_cb_info;		/* cb list info */
98	mac_cb_t	*s_ring_notify_cb_list;		/* The cb list */
99
100	clock_t		s_ring_awaken;	/* time async thread was awakened */
101
102	kthread_t	*s_ring_run;	/* Current thread processing sq */
103	processorid_t	s_ring_cpuid;	/* processor to bind to */
104	processorid_t	s_ring_cpuid_save;	/* saved cpuid during offline */
105	kcondvar_t	s_ring_async;	/* async thread blocks on */
106	clock_t		s_ring_wait;	/* lbolts to wait after a fill() */
107	timeout_id_t	s_ring_tid;	/* timer id of pending timeout() */
108	kthread_t	*s_ring_worker;	/* kernel thread id */
109	char		s_ring_name[S_RING_NAMELEN + 1];
110	uint32_t	s_ring_total_inpkt;
111	uint32_t	s_ring_total_rbytes;
112	uint32_t	s_ring_drops;
113	struct mac_client_impl_s *s_ring_mcip;
114	kstat_t		*s_ring_ksp;
115
116	/* Teardown, poll disable control ops */
117	kcondvar_t	s_ring_client_cv; /* Client wait for control op */
118
119	mac_soft_ring_set_t *s_ring_set;   /* The SRS this ring belongs to */
120	mac_soft_ring_t	*s_ring_next;
121	mac_soft_ring_t	*s_ring_prev;
122	mac_soft_ring_drain_func_t s_ring_drain_func;
123
124	mac_tx_stats_t	s_st_stat;
125};
126
127typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t);
128
129/* Transmit side Soft Ring Set */
130typedef struct mac_srs_tx_s {
131	/* Members for Tx size processing */
132	uint32_t	st_mode;
133	mac_tx_func_t	st_func;
134	void		*st_arg1;
135	void		*st_arg2;
136	mac_group_t	*st_group;	/* TX group for share */
137	boolean_t	st_woken_up;
138
139	/*
140	 * st_max_q_cnt is the queue depth threshold to limit
141	 * outstanding packets on the Tx SRS. Once the limit
142	 * is reached, Tx SRS will drop packets until the
143	 * limit goes below the threshold.
144	 */
145	uint32_t	st_max_q_cnt;	/* max. outstanding packets */
146	/*
147	 * st_hiwat is used Tx serializer and bandwidth mode.
148	 * This is the queue depth threshold upto which
149	 * packets will get buffered with no flow-control
150	 * back pressure applied to the caller. Once this
151	 * threshold is reached, back pressure will be
152	 * applied to the caller of mac_tx() (mac_tx() starts
153	 * returning a cookie to indicate a blocked SRS).
154	 * st_hiwat should always be lesser than or equal to
155	 * st_max_q_cnt.
156	 */
157	uint32_t	st_hiwat;	/* mblk cnt to apply flow control */
158	uint32_t	st_lowat;	/* mblk cnt to relieve flow control */
159	uint32_t	st_hiwat_cnt; /* times blocked for Tx descs */
160	mac_tx_stats_t	st_stat;
161	mac_capab_aggr_t	st_capab_aggr;
162	/*
163	 * st_soft_rings is used as an array to store aggr Tx soft
164	 * rings. When aggr_find_tx_ring() returns a pseudo ring,
165	 * the associated soft ring has to be found. st_soft_rings
166	 * array stores the soft ring associated with a pseudo Tx
167	 * ring and it can be accessed using the pseudo ring
168	 * index (mr_index). Note that the ring index is unique
169	 * for each ring in a group.
170	 */
171	mac_soft_ring_t **st_soft_rings;
172} mac_srs_tx_t;
173
174/* Receive side Soft Ring Set */
175typedef struct mac_srs_rx_s {
176	/*
177	 * Upcall Function for fanout, Rx processing etc. Perhaps
178	 * the same 3 members below can be used for Tx
179	 * processing, but looking around, mac_rx_func_t has
180	 * proliferated too much into various files at different
181	 * places. I am leaving the consolidation battle for
182	 * another day.
183	 */
184	mac_direct_rx_t		sr_func;	/* srs_lock */
185	void			*sr_arg1;	/* srs_lock */
186	mac_resource_handle_t	sr_arg2;	/* srs_lock */
187	mac_rx_func_t		sr_lower_proc;	/* Atomically changed */
188	uint32_t		sr_poll_pkt_cnt;
189	uint32_t		sr_poll_thres;
190
191	/* mblk cnt to apply flow control */
192	uint32_t		sr_hiwat;
193	/* mblk cnt to relieve flow control */
194	uint32_t		sr_lowat;
195	mac_rx_stats_t		sr_stat;
196
197	/* Times polling was enabled */
198	uint32_t		sr_poll_on;
199	/* Times polling was enabled by worker thread */
200	uint32_t		sr_worker_poll_on;
201	/* Times polling was disabled */
202	uint32_t		sr_poll_off;
203	/* Poll thread signalled count */
204	uint32_t		sr_poll_thr_sig;
205	/* Poll thread busy */
206	uint32_t		sr_poll_thr_busy;
207	/* SRS drains, stays in poll mode but doesn't poll */
208	uint32_t		sr_poll_drain_no_poll;
209	/*
210	 * SRS has nothing to do and no packets in H/W but
211	 * there is a backlog in softrings. SRS stays in
212	 * poll mode but doesn't do polling.
213	 */
214	uint32_t		sr_poll_no_poll;
215	/* Active polling restarted */
216	uint32_t		sr_below_hiwat;
217	/* Found packets in last poll so try and poll again */
218	uint32_t		sr_poll_again;
219	/*
220	 * Packets in queue but poll thread not allowed to process so
221	 * signal the worker thread.
222	 */
223	uint32_t		sr_poll_sig_worker;
224	/*
225	 * Poll thread has nothing to do and H/W has nothing so
226	 * reenable the interrupts.
227	 */
228	uint32_t		sr_poll_intr_enable;
229	/*
230	 * Poll thread has nothing to do and worker thread was already
231	 * running so it can decide to reenable interrupt or poll again.
232	 */
233	uint32_t		sr_poll_goto_sleep;
234	/* Worker thread goes back to draining the queue */
235	uint32_t		sr_drain_again;
236	/* More Packets in queue so signal the poll thread to drain */
237	uint32_t		sr_drain_poll_sig;
238	/* More Packets in queue so signal the worker thread to drain */
239	uint32_t		sr_drain_worker_sig;
240	/* Poll thread is already running so worker has nothing to do */
241	uint32_t		sr_drain_poll_running;
242	/* We have packets already queued so keep polling */
243	uint32_t		sr_drain_keep_polling;
244	/* Drain is done and interrupts are reenabled */
245	uint32_t		sr_drain_finish_intr;
246	/* Polling thread needs to schedule worker wakeup */
247	uint32_t		sr_poll_worker_wakeup;
248} mac_srs_rx_t;
249
250/*
251 * mac_soft_ring_set_s:
252 * This is used both for Tx and Rx side. The srs_type identifies Rx or
253 * Tx type.
254 *
255 * Note that the structure is carefully crafted, with Rx elements coming
256 * first followed by Tx specific members. Future additions to this
257 * structure should follow the same guidelines.
258 *
259 * Rx-side notes:
260 * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow
261 * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On
262 * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode
263 * (if poll capable) and feeds the packets to soft_ring_list via choosen
264 * fanout type (specified by srs_type). In poll mode, the poll thread which is
265 * also a pointer can pick up the packets and feed them to various
266 * soft_ring_list.
267 *
268 * The srs_type can either be protocol based or fanout based where fanout itelf
269 * can be various types
270 *
271 * The polling works by turning off interrupts as soon as a packets
272 * are queued on the soft ring set. Once the backlog is clear and poll
273 * thread return empty handed i.e. Rx ring doesn't have anything, the
274 * interrupt is turned back on. For this purpose we keep a separate
275 * srs_poll_pkt_cnt counter which tracks the packets queued between SRS
276 * and the soft rings as well. The counter is incremented when packets
277 * are queued and decremented when SRS processes them (in case it has
278 * no soft rings) or the soft ring process them. Its important that
279 * in case SRS has softrings, the decrement doesn't happen till the
280 * packet is processed by the soft rings since it takes very little time
281 * for SRS to queue packet from SRS to soft rings and it will keep
282 * bringing more packets in the system faster than soft rings can
283 * process them.
284 *
285 * Tx side notes:
286 * The srs structure acts as a serializer with a worker thread. The
287 * default behavior of srs though is to act as a pass-thru. The queues
288 * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx
289 * descriptors or to enforce bandwidth limits.
290 *
291 * When multiple Tx rings are present, the SRS state will be set to
292 * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process()
293 * function will be fanned out to one of the Tx side soft rings based on
294 * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will
295 * be associated with a distinct h/w Tx ring.
296 */
297
298struct mac_soft_ring_set_s {
299	/*
300	 * Common elements, common to both Rx and Tx SRS type.
301	 * The following block of fields are protected by srs_lock
302	 */
303	kmutex_t	srs_lock;
304	uint32_t	srs_type;
305	uint32_t	srs_state;	/* state flags */
306	uint32_t	srs_count;
307	mblk_t		*srs_first;	/* first mblk chain or NULL */
308	mblk_t		*srs_last;	/* last mblk chain or NULL */
309	kcondvar_t	srs_async;	/* cv for worker thread */
310	kcondvar_t	srs_cv;		/* cv for poll thread */
311	kcondvar_t	srs_quiesce_done_cv;	/* cv for removal */
312	timeout_id_t	srs_tid;	/* timeout id for pending timeout */
313
314	/*
315	 * List of soft rings & processing function.
316	 * The following block is protected by Rx quiescence.
317	 * i.e. they can be changed only after quiescing the SRS
318	 * Protected by srs_lock.
319	 */
320	mac_soft_ring_t	*srs_soft_ring_head;
321	mac_soft_ring_t	*srs_soft_ring_tail;
322	int		srs_soft_ring_count;
323	int		srs_soft_ring_quiesced_count;
324	int		srs_soft_ring_condemned_count;
325	mac_soft_ring_t	**srs_tcp_soft_rings;
326	int		srs_tcp_ring_count;
327	mac_soft_ring_t	**srs_udp_soft_rings;
328	int		srs_udp_ring_count;
329	mac_soft_ring_t	**srs_oth_soft_rings;
330	int		srs_oth_ring_count;
331	/*
332	 * srs_tx_soft_rings is used by tx_srs in
333	 * when operating in multi tx ring mode.
334	 */
335	mac_soft_ring_t	**srs_tx_soft_rings;
336	int		srs_tx_ring_count;
337
338	/*
339	 * Bandwidth control related members.
340	 * They are common to both Rx- and Tx-side.
341	 * Following protected by srs_lock
342	 */
343	mac_bw_ctl_t	*srs_bw;
344	size_t		srs_size;	/* Size of packets queued in bytes */
345	pri_t		srs_pri;
346
347	mac_soft_ring_set_t	*srs_next;	/* mac_srs_g_lock */
348	mac_soft_ring_set_t	*srs_prev;	/* mac_srs_g_lock */
349
350	/* Attribute specific drain func (BW ctl vs non-BW ctl)	*/
351	mac_srs_drain_proc_t	srs_drain_func;	/* Write once (WO) */
352
353	/*
354	 * If the associated ring is exclusively used by a mac client, e.g.,
355	 * an aggregation, this fields is used to keep a reference to the
356	 * MAC client's pseudo ring.
357	 */
358	mac_resource_handle_t	srs_mrh;
359	/*
360	 * The following blocks are write once (WO) and valid for the life
361	 * of the SRS
362	 */
363	struct mac_client_impl_s *srs_mcip;	/* back ptr to mac client */
364	void			*srs_flent;	/* back ptr to flent */
365	mac_ring_t		*srs_ring;	/*  Ring Descriptor */
366
367	/* Teardown, disable control ops */
368	kcondvar_t	srs_client_cv;	/* Client wait for the control op */
369
370	kthread_t	*srs_worker;	/* WO, worker thread */
371	kthread_t	*srs_poll_thr;	/* WO, poll thread */
372
373	uint_t		srs_ind;	/* Round Robin indx for picking up SR */
374	processorid_t	srs_worker_cpuid;	/* processor to bind to */
375	processorid_t	srs_worker_cpuid_save;	/* saved cpuid during offline */
376	processorid_t	srs_poll_cpuid;		/* processor to bind to */
377	processorid_t	srs_poll_cpuid_save;	/* saved cpuid during offline */
378	uint_t		srs_fanout_state;
379	mac_cpus_t	srs_cpu;
380
381	mac_srs_rx_t	srs_rx;
382	mac_srs_tx_t	srs_tx;
383	kstat_t		*srs_ksp;
384};
385
386/*
387 * type flags - combination allowed to process and drain the queue
388 */
389#define	ST_RING_WORKER_ONLY	0x0001	/* Worker thread only */
390#define	ST_RING_ANY		0x0002	/* Any thread can process the queue */
391#define	ST_RING_TCP		0x0004
392#define	ST_RING_UDP		0x0008
393#define	ST_RING_OTH		0x0010
394
395#define	ST_RING_BW_CTL		0x0020
396#define	ST_RING_TX		0x0040
397
398/*
399 * State flags.
400 */
401#define	S_RING_PROC		0x0001	/* being processed */
402#define	S_RING_BOUND		0x0002	/* Worker thread is bound to a cpu */
403#define	S_RING_BLOCK		0x0004	/* No Tx descs */
404#define	S_RING_TX_HIWAT		0x0008	/* Tx high watermark reached */
405
406#define	S_RING_WAKEUP_CLIENT	0x0010	/* flow ctrl, client wakeup needed */
407#define	S_RING_BLANK		0x0020	/* Has been put into polling mode */
408#define	S_RING_CLIENT_WAIT	0x0040	/* Client waiting for control op */
409
410#define	S_RING_CONDEMNED	0x0100	/* Being torn down */
411#define	S_RING_CONDEMNED_DONE	0x0200	/* Being torn down */
412#define	S_RING_QUIESCE		0x0400	/* No traffic flow, transient flag */
413#define	S_RING_QUIESCE_DONE	0x0800	/* No traffic flow, transient flag */
414
415#define	S_RING_RESTART		0x1000	/* Go back to normal traffic flow */
416#define	S_RING_ENQUEUED		0x2000	/* Pkts enqueued in Tx soft ring */
417
418/*
419 * arguments for processors to bind to
420 */
421#define	S_RING_BIND_NONE	-1
422
423/*
424 * defines for srs_type - identifies a link or a sub-flow
425 * and other static characteristics of a SRS like a tx
426 * srs, tcp only srs, etc.
427 */
428#define	SRST_LINK		0x00000001
429#define	SRST_FLOW		0x00000002
430#define	SRST_NO_SOFT_RINGS	0x00000004
431#define	SRST_TCP_ONLY		0x00000008
432
433#define	SRST_FANOUT_PROTO	0x00000010
434#define	SRST_FANOUT_SRC_IP	0x00000020
435#define	SRST_FANOUT_OTH		0x00000040
436#define	SRST_DEFAULT_GRP	0x00000080
437
438#define	SRST_TX			0x00000100
439#define	SRST_BW_CONTROL		0x00000200
440#define	SRST_DIRECT_POLL	0x00000400
441
442#define	SRST_DLS_BYPASS		0x00001000
443#define	SRST_CLIENT_POLL_ENABLED 0x00002000
444
445/*
446 * soft ring set flags. These bits are dynamic in nature and get
447 * applied to srs_state. They reflect the state of SRS at any
448 * point of time
449 */
450#define	SRS_BLANK		0x00000001
451#define	SRS_WORKER_BOUND	0x00000002
452#define	SRS_POLL_BOUND		0x00000004
453#define	SRS_POLLING_CAPAB	0x00000008
454
455#define	SRS_PROC		0x00000010
456#define	SRS_GET_PKTS		0x00000020
457#define	SRS_POLLING		0x00000040
458#define	SRS_BW_ENFORCED		0x00000080
459
460#define	SRS_WORKER		0x00000100
461#define	SRS_ENQUEUED		0x00000200
462#define	SRS_ANY_PROCESS		0x00000400
463#define	SRS_PROC_FAST		0x00000800
464
465#define	SRS_POLL_PROC		0x00001000
466#define	SRS_TX_BLOCKED		0x00002000	/* out of Tx descs */
467#define	SRS_TX_HIWAT		0x00004000	/* Tx count exceeds hiwat */
468#define	SRS_TX_WAKEUP_CLIENT	0x00008000	/* Flow-ctl: wakeup client */
469
470#define	SRS_CLIENT_PROC		0x00010000
471#define	SRS_CLIENT_WAIT		0x00020000
472#define	SRS_QUIESCE		0x00040000
473#define	SRS_QUIESCE_DONE	0x00080000
474
475#define	SRS_CONDEMNED		0x00100000
476#define	SRS_CONDEMNED_DONE	0x00200000
477#define	SRS_POLL_THR_QUIESCED	0x00400000
478#define	SRS_RESTART		0x00800000
479
480#define	SRS_RESTART_DONE	0x01000000
481#define	SRS_POLL_THR_RESTART	0x02000000
482#define	SRS_IN_GLIST		0x04000000
483#define	SRS_POLL_THR_EXITED	0x08000000
484
485#define	SRS_QUIESCE_PERM	0x10000000
486#define	SRS_LATENCY_OPT		0x20000000
487#define	SRS_SOFTRING_QUEUE	0x40000000
488
489#define	SRS_QUIESCED(srs)	(srs->srs_state & SRS_QUIESCE_DONE)
490
491/*
492 * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be
493 * able to be restarted.
494 */
495#define	SRS_QUIESCED_PERMANENT(srs)	(srs->srs_state & SRS_QUIESCE_PERM)
496
497/*
498 * soft ring set (SRS) Tx modes
499 */
500typedef enum {
501	SRS_TX_DEFAULT = 0,
502	SRS_TX_SERIALIZE,
503	SRS_TX_FANOUT,
504	SRS_TX_BW,
505	SRS_TX_BW_FANOUT,
506	SRS_TX_AGGR,
507	SRS_TX_BW_AGGR
508} mac_tx_srs_mode_t;
509
510/*
511 * SRS fanout states
512 */
513typedef enum {
514	SRS_FANOUT_UNINIT = 0,
515	SRS_FANOUT_INIT,
516	SRS_FANOUT_REINIT
517} mac_srs_fanout_state_t;
518
519/*
520 * Structure for dls statistics
521 */
522struct dls_kstats {
523	kstat_named_t	dlss_soft_ring_pkt_drop;
524};
525
526extern struct dls_kstats dls_kstat;
527
528#define	DLS_BUMP_STAT(x, y)	(dls_kstat.x.value.ui32 += y)
529
530/* Turn dynamic polling off */
531#define	MAC_SRS_POLLING_OFF(mac_srs) {					\
532	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
533	if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \
534	    (SRS_POLLING_CAPAB|SRS_POLLING)) {				\
535		(mac_srs)->srs_state &= ~SRS_POLLING;			\
536		(void) mac_hwring_enable_intr((mac_ring_handle_t)	\
537		    (mac_srs)->srs_ring);				\
538		(mac_srs)->srs_rx.sr_poll_off++;			\
539	}								\
540}
541
542#define	MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz)	{	\
543	mblk_t		*tmp;					\
544	boolean_t	bw_ctl = B_FALSE;			\
545								\
546	ASSERT((head) != NULL);					\
547	cnt = 0;						\
548	sz = 0;							\
549	if ((mac_srs)->srs_type & SRST_BW_CONTROL)		\
550		bw_ctl = B_TRUE;				\
551	tmp = tail = (head);					\
552	if ((head)->b_next == NULL) {				\
553		cnt = 1;					\
554		if (bw_ctl)					\
555			sz += msgdsize(head);			\
556	} else {						\
557		while (tmp != NULL) {				\
558			tail = tmp;				\
559			cnt++;					\
560			if (bw_ctl)				\
561				sz += msgdsize(tmp);		\
562			tmp = tmp->b_next;			\
563		}						\
564	}							\
565}
566
567/*
568 * Decrement the cumulative packet count in SRS and its
569 * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check
570 * if if the interface was left in a polling mode and no one
571 * is really processing the queue (to get the interface out
572 * of poll mode). If no one is processing the queue, then
573 * acquire the PROC and signal the poll thread to check the
574 * interface for packets and get the interface back to interrupt
575 * mode if nothing is found.
576 */
577#define	MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) {		        \
578	mac_srs_rx_t	*srs_rx = &(mac_srs)->srs_rx;			\
579	ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));			\
580									\
581	srs_rx->sr_poll_pkt_cnt -= cnt;					\
582	if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) &&	\
583	    (((mac_srs)->srs_state &					\
584	    (SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING))	\
585	{								\
586		(mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS);	\
587		cv_signal(&(mac_srs)->srs_cv);				\
588		srs_rx->sr_below_hiwat++;				\
589	}								\
590}
591
592/*
593 * The following two macros are used to update the inbound packet and byte.
594 * count. The packet and byte count reflect the packets and bytes that are
595 * taken out of the SRS's queue, i.e. indicating they are being delivered.
596 * The srs_count and srs_size are updated in different locations as the
597 * srs_size is also used to take into account any bandwidth limits. The
598 * srs_size is updated only when a soft ring, if any, sends a packet up,
599 * as opposed to updating it when the SRS sends a packet to the SR, i.e.
600 * the srs_size reflects the packets in the SRS and SRs. These
601 * macros decrement the srs_size and srs_count and also increment the
602 * ipackets and ibytes stats resp.
603 *
604 * xxx-venu These are done under srs_lock, for now we still update
605 * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if
606 * just updating them would be accurate enough.
607 *
608 * If we are updating these for a sub-flow SRS, then we need to also
609 * updated it's MAC client bandwidth info, if the MAC client is also
610 * bandwidth regulated.
611 */
612#define	MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) {				\
613	if ((srs)->srs_type & SRST_BW_CONTROL) {			\
614		mutex_enter(&(srs)->srs_bw->mac_bw_lock);		\
615		(srs)->srs_bw->mac_bw_sz -= (sz);			\
616		(srs)->srs_bw->mac_bw_used += (sz);			\
617		mutex_exit(&(srs)->srs_bw->mac_bw_lock);		\
618	}								\
619}
620
621#define	MAC_TX_UPDATE_BW_INFO(srs, sz) {				\
622	(srs)->srs_bw->mac_bw_sz -= (sz);				\
623	(srs)->srs_bw->mac_bw_used += (sz);				\
624}
625
626#define	MAC_TX_SOFT_RINGS(mac_srs) ((mac_srs)->srs_tx_ring_count >= 1)
627
628/* Soft ring flags for teardown */
629#define	SRS_POLL_THR_OWNER	(SRS_PROC | SRS_POLLING | SRS_GET_PKTS)
630#define	SRS_PAUSE		(SRS_CONDEMNED | SRS_QUIESCE)
631#define	S_RING_PAUSE		(S_RING_CONDEMNED | S_RING_QUIESCE)
632
633/* Soft rings */
634extern void mac_soft_ring_init(void);
635extern void mac_soft_ring_finish(void);
636extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *,
637    mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t,
638    cpupart_t *);
639
640extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *);
641extern void mac_soft_ring_blank(void *, time_t, uint_t, int);
642extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, size_t);
643extern void mac_soft_ring_destroy(mac_soft_ring_t *);
644extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *);
645
646/* Rx SRS */
647extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *,
648    flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t,
649    mac_ring_t *);
650extern void mac_srs_free(mac_soft_ring_set_t *);
651extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t);
652extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t);
653extern void mac_rx_srs_retarget_intr(mac_soft_ring_set_t *, processorid_t);
654extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *);
655
656extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *);
657extern void mac_srs_client_poll_enable(struct mac_client_impl_s *,
658    mac_soft_ring_set_t *);
659extern void mac_srs_client_poll_disable(struct mac_client_impl_s *,
660    mac_soft_ring_set_t *);
661extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *,
662    mac_soft_ring_set_t *);
663extern void mac_srs_client_poll_restart(struct mac_client_impl_s *,
664    mac_soft_ring_set_t *);
665extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
666extern void mac_rx_srs_restart(mac_soft_ring_set_t *);
667extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *,
668    boolean_t);
669extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
670
671/* Tx SRS, Tx softring */
672extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t);
673extern void mac_tx_srs_setup(struct mac_client_impl_s *, flow_entry_t *);
674extern mac_tx_func_t mac_tx_get_func(uint32_t);
675extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *,
676    mac_tx_stats_t *);
677extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *);
678extern mac_soft_ring_t *mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *,
679    mac_ring_t *);
680extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *);
681extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *);
682extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *,
683    uint16_t, mblk_t **);
684
685/* Subflow specific stuff */
686extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *,
687    mac_resource_props_t *, int, int, mac_direct_rx_t);
688extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *);
689extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *);
690extern void mac_srs_update_drv(struct mac_client_impl_s *);
691extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t);
692extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t);
693
694extern int mac_soft_ring_intr_enable(void *);
695extern boolean_t mac_soft_ring_intr_disable(void *);
696extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, uint16_t,
697    pri_t, mac_client_impl_t *, mac_soft_ring_set_t *,
698    processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t);
699extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t);
700	extern void mac_soft_ring_unbind(mac_soft_ring_t *);
701extern void mac_soft_ring_free(mac_soft_ring_t *);
702extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t);
703extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *,
704    mblk_t *, mblk_t *, int, size_t);
705extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *,
706    mblk_t *, uint16_t, mblk_t **);
707extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *);
708extern void mac_srs_worker_restart(mac_soft_ring_set_t *);
709extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *,
710    mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t);
711
712extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t);
713extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t);
714extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *,
715    boolean_t);
716extern void mac_srs_worker(mac_soft_ring_set_t *);
717extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *);
718extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t);
719
720extern void mac_tx_srs_restart(mac_soft_ring_set_t *);
721extern void mac_rx_srs_remove(mac_soft_ring_set_t *);
722
723#ifdef	__cplusplus
724}
725#endif
726
727#endif	/* _SYS_MAC_SOFT_RING_H */
728