1/*
2 * Copyright (c) 2008-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 *    this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 *    this list of conditions and the following disclaimer in the documentation
12 *    and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#ifndef	_SYS_SFXGE_H
32#define	_SYS_SFXGE_H
33
34#ifdef	__cplusplus
35extern "C" {
36#endif
37
38#include <sys/types.h>
39#include <sys/ddi.h>
40#include <sys/sunddi.h>
41#include <sys/stream.h>
42#include <sys/ethernet.h>
43#include <sys/cpuvar.h>
44
45#include <sys/mac.h>
46#include <sys/mac_ether.h>
47#include <sys/mac_provider.h>
48
49#include "sfxge_ioc.h"
50#include "sfxge_debug.h"
51
52#include "efx.h"
53#include "efx_regs.h"
54
55#ifdef	_KERNEL
56
57#define	SFXGE_DRIVER_NAME "sfxge"
58
59#define	SFXGE_CPU_CACHE_SIZE	64
60
61typedef struct sfxge_s	sfxge_t;
62
63typedef enum sfxge_intr_state_e {
64	SFXGE_INTR_UNINITIALIZED = 0,
65	SFXGE_INTR_INITIALIZED,
66	SFXGE_INTR_TESTING,
67	SFXGE_INTR_STARTED
68} sfxge_intr_state_t;
69
70typedef struct sfxge_intr_s {
71	ddi_intr_handle_t	*si_table;
72	int			si_table_size;
73	int			si_nalloc;
74	int			si_type;
75	int			si_cap;
76	efsys_mem_t		si_mem;
77	uint64_t		si_mask;
78	sfxge_intr_state_t	si_state;
79	uint32_t		si_zero_count;
80	int			si_intr_pri;
81} sfxge_intr_t;
82
83typedef enum sfxge_promisc_type_e {
84	SFXGE_PROMISC_OFF = 0,
85	SFXGE_PROMISC_ALL_MULTI,
86	SFXGE_PROMISC_ALL_PHYS
87} sfxge_promisc_type_t;
88
89typedef enum sfxge_link_duplex_e {
90	SFXGE_LINK_DUPLEX_UNKNOWN = 0,
91	SFXGE_LINK_DUPLEX_HALF,
92	SFXGE_LINK_DUPLEX_FULL
93} sfxge_link_duplex_t;
94
95typedef enum sfxge_unicst_type_e {
96	SFXGE_UNICST_BIA = 0,
97	SFXGE_UNICST_LAA,
98	SFXGE_UNICST_NTYPES
99} sfxge_unicst_type_t;
100
101typedef struct sfxge_phy_s {
102		kstat_t			*sp_ksp;
103		kstat_named_t		*sp_stat;
104		uint32_t		*sp_statbuf;
105		efsys_mem_t		sp_mem;
106} sfxge_phy_t;
107
108typedef enum sfxge_mac_state_e {
109	SFXGE_MAC_UNINITIALIZED = 0,
110	SFXGE_MAC_INITIALIZED,
111	SFXGE_MAC_STARTED
112} sfxge_mac_state_t;
113
114typedef struct sfxge_mac_s {
115	sfxge_t			*sm_sp;
116	efsys_mem_t		sm_mem;
117	kstat_t			*sm_ksp;
118	kstat_named_t		*sm_stat;
119	uint8_t			sm_bia[ETHERADDRL];
120	uint8_t			sm_laa[ETHERADDRL];
121	boolean_t		sm_laa_valid;
122	unsigned int		sm_fcntl;
123	sfxge_promisc_type_t	sm_promisc;
124	uint8_t			sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX *
125	    ETHERADDRL]; /* List of multicast addresses to filter on */
126	int			sm_mcast_count;
127	clock_t			sm_lbolt;
128	kmutex_t		sm_lock;
129	efx_link_mode_t		sm_link_mode;
130	unsigned int		sm_link_speed;
131	sfxge_link_duplex_t	sm_link_duplex;
132	boolean_t		sm_link_up;
133	boolean_t		sm_link_poll_reqd;
134	kcondvar_t		sm_link_poll_kv;
135	boolean_t		sm_mac_stats_timer_reqd;
136	boolean_t		sm_mac_stats_pend;
137	ddi_taskq_t		*sm_tqp;
138	sfxge_mac_state_t	sm_state;
139	sfxge_phy_t		sm_phy;
140	uint32_t		sm_phy_cap_to_set;
141	uint32_t		sm_phy_cap_to_unset;
142} sfxge_mac_t;
143
144typedef enum sfxge_mon_state_e {
145	SFXGE_MON_UNINITIALIZED = 0,
146	SFXGE_MON_INITIALIZED,
147	SFXGE_MON_STARTED
148} sfxge_mon_state_t;
149
150typedef struct sfxge_mon_s {
151	sfxge_t			*sm_sp;
152	efx_mon_type_t		sm_type;
153	unsigned int		sm_devid;
154	kstat_t			*sm_ksp;
155	kstat_named_t		*sm_stat;
156	efx_mon_stat_value_t	*sm_statbuf;
157	kmutex_t		sm_lock;
158	sfxge_mon_state_t	sm_state;
159	efsys_mem_t		sm_mem;
160	int			sm_polling;
161} sfxge_mon_t;
162
163typedef enum sfxge_sram_state_e {
164	SFXGE_SRAM_UNINITIALIZED = 0,
165	SFXGE_SRAM_INITIALIZED,
166	SFXGE_SRAM_STARTED
167} sfxge_sram_state_t;
168
169typedef struct sfxge_sram_s {
170	sfxge_t			*ss_sp;
171	kmutex_t		ss_lock;
172	struct map		*ss_buf_tbl_map;
173	unsigned int		ss_count;
174	sfxge_sram_state_t	ss_state;
175} sfxge_sram_t;
176
177typedef enum sfxge_mcdi_state_e {
178	SFXGE_MCDI_UNINITIALIZED = 0,
179	SFXGE_MCDI_INITIALIZED,
180	SFXGE_MCDI_BUSY,
181	SFXGE_MCDI_COMPLETED
182} sfxge_mcdi_state_t;
183
184typedef struct sfxge_mcdi_s {
185	sfxge_t			*sm_sp;
186	kmutex_t		sm_lock;
187	sfxge_mcdi_state_t	sm_state;
188	efx_mcdi_transport_t	sm_emt;
189	efsys_mem_t		sm_mem;
190	kcondvar_t		sm_kv;		/* MCDI poll complete */
191} sfxge_mcdi_t;
192
193#define	SFXGE_NEVS			4096
194#define	SFXGE_RX_NDESCS			1024
195#define	SFXGE_TX_NDESCS			1024
196#define	SFXGE_TX_NLABELS		EFX_EV_TX_NLABELS
197
198#define	SFXGE_DEFAULT_RXQ_SIZE		1024
199#define	SFXGE_DEFAULT_MODERATION	30
200
201typedef enum sfxge_evq_state_e {
202	SFXGE_EVQ_UNINITIALIZED = 0,
203	SFXGE_EVQ_INITIALIZED,
204	SFXGE_EVQ_STARTING,
205	SFXGE_EVQ_STARTED
206} sfxge_evq_state_t;
207
208#define	SFXGE_EV_BATCH	(SFXGE_NEVS / 4)
209
210typedef struct sfxge_txq_s	sfxge_txq_t;
211
212typedef struct sfxge_evq_s {
213	union {
214		struct {
215			sfxge_t			*__se_sp;
216			unsigned int		__se_index;
217			efsys_mem_t		__se_mem;
218			unsigned int		__se_id;
219			kstat_t			*__se_ksp;
220			kstat_named_t		*__se_stat;
221			efx_ev_callbacks_t	__se_eec;
222			sfxge_evq_state_t	__se_state;
223			boolean_t		__se_exception;
224		} __se_s1;
225		uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
226	} __se_u1;
227	union {
228		struct {
229			kmutex_t		__se_lock;
230			kcondvar_t		__se_init_kv;
231			efx_evq_t		*__se_eep;
232			unsigned int		__se_count;
233			unsigned int		__se_rx;
234			unsigned int		__se_tx;
235			sfxge_txq_t		*__se_stp;
236			sfxge_txq_t		**__se_stpp;
237			processorid_t		__se_cpu_id;
238			uint16_t		__se_ev_batch;
239		} __se_s2;
240		uint8_t	__se_pad[SFXGE_CPU_CACHE_SIZE];
241	} __se_u2;
242	union {
243		struct {
244			sfxge_txq_t	*__se_label_stp[SFXGE_TX_NLABELS];
245		} __se_s3;
246		uint8_t	__se_pad[SFXGE_CPU_CACHE_SIZE * 4];
247	} __se_u3;
248} sfxge_evq_t;
249
250#define	se_sp		__se_u1.__se_s1.__se_sp
251#define	se_index	__se_u1.__se_s1.__se_index
252#define	se_mem		__se_u1.__se_s1.__se_mem
253#define	se_id		__se_u1.__se_s1.__se_id
254#define	se_ksp		__se_u1.__se_s1.__se_ksp
255#define	se_stat		__se_u1.__se_s1.__se_stat
256#define	se_eec		__se_u1.__se_s1.__se_eec
257#define	se_state	__se_u1.__se_s1.__se_state
258#define	se_exception	__se_u1.__se_s1.__se_exception
259
260#define	se_lock		__se_u2.__se_s2.__se_lock
261#define	se_init_kv	__se_u2.__se_s2.__se_init_kv
262#define	se_eep		__se_u2.__se_s2.__se_eep
263#define	se_count	__se_u2.__se_s2.__se_count
264#define	se_rx		__se_u2.__se_s2.__se_rx
265#define	se_tx		__se_u2.__se_s2.__se_tx
266#define	se_stp		__se_u2.__se_s2.__se_stp
267#define	se_stpp		__se_u2.__se_s2.__se_stpp
268#define	se_cpu_id	__se_u2.__se_s2.__se_cpu_id
269#define	se_ev_batch	__se_u2.__se_s2.__se_ev_batch
270
271#define	se_label_stp	__se_u3.__se_s3.__se_label_stp
272
273
274#define	SFXGE_MAGIC_RESERVED	0x8000
275
276#define	SFXGE_MAGIC_DMAQ_LABEL_WIDTH  5
277#define	SFXGE_MAGIC_DMAQ_LABEL_MASK   ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)
278
279#define	SFXGE_MAGIC_RX_QFLUSH_DONE					\
280	(SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
281
282#define	SFXGE_MAGIC_RX_QFLUSH_FAILED					\
283	(SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
284
285#define	SFXGE_MAGIC_RX_QFPP_TRIM					\
286	(SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
287
288#define	SFXGE_MAGIC_TX_QFLUSH_DONE					\
289	(SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))
290
291typedef struct sfxge_rxq_s		sfxge_rxq_t;
292
293#define	SFXGE_ETHERTYPE_LOOPBACK	0x9000	/* Xerox loopback */
294
295typedef struct sfxge_rx_packet_s	sfxge_rx_packet_t;
296
297struct sfxge_rx_packet_s {
298	union {
299		struct {
300			frtn_t			__srp_free;
301			uint16_t		__srp_flags;
302			uint16_t		__srp_size;
303			mblk_t			*__srp_mp;
304			struct ether_header	*__srp_etherhp;
305			struct ip		*__srp_iphp;
306			struct tcphdr		*__srp_thp;
307			size_t			__srp_off;
308		} __srp_s1;
309		uint8_t	__srp_pad[SFXGE_CPU_CACHE_SIZE];
310	} __srp_u1;
311	union {
312		struct {
313			sfxge_rxq_t		*__srp_srp;
314			ddi_dma_handle_t	__srp_dma_handle;
315			ddi_acc_handle_t	__srp_acc_handle;
316			unsigned char		*__srp_base;
317			size_t			__srp_mblksize;
318			uint64_t		__srp_addr;
319			boolean_t		__srp_recycle;
320			caddr_t			__srp_putp;
321		} __srp_s2;
322		uint8_t	__srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
323	} __srp_u2;
324};
325
326#define	srp_free	__srp_u1.__srp_s1.__srp_free
327#define	srp_flags	__srp_u1.__srp_s1.__srp_flags
328#define	srp_size	__srp_u1.__srp_s1.__srp_size
329#define	srp_mp		__srp_u1.__srp_s1.__srp_mp
330#define	srp_etherhp	__srp_u1.__srp_s1.__srp_etherhp
331#define	srp_iphp	__srp_u1.__srp_s1.__srp_iphp
332#define	srp_thp		__srp_u1.__srp_s1.__srp_thp
333#define	srp_off		__srp_u1.__srp_s1.__srp_off
334
335#define	srp_srp		__srp_u2.__srp_s2.__srp_srp
336#define	srp_dma_handle	__srp_u2.__srp_s2.__srp_dma_handle
337#define	srp_acc_handle	__srp_u2.__srp_s2.__srp_acc_handle
338#define	srp_base	__srp_u2.__srp_s2.__srp_base
339#define	srp_mblksize	__srp_u2.__srp_s2.__srp_mblksize
340#define	srp_addr	__srp_u2.__srp_s2.__srp_addr
341#define	srp_recycle	__srp_u2.__srp_s2.__srp_recycle
342#define	srp_putp	__srp_u2.__srp_s2.__srp_putp
343
344#define	SFXGE_RX_FPP_NSLOTS	8
345#define	SFXGE_RX_FPP_MASK	(SFXGE_RX_FPP_NSLOTS - 1)
346
347/* Free packet pool putlist (dynamically allocated) */
348typedef struct sfxge_rx_fpp_putlist_s {
349	kmutex_t		srfpl_lock;
350	unsigned int		srfpl_count;
351	mblk_t			*srfpl_putp;
352	mblk_t			**srfpl_putpp;
353} sfxge_rx_fpp_putlist_t;
354
355/* Free packet pool */
356typedef struct sfxge_rx_fpp_s {
357	caddr_t		srfpp_putp;
358	unsigned int	srfpp_loaned;
359	mblk_t		*srfpp_get;
360	unsigned int	srfpp_count;
361	unsigned int	srfpp_min;
362	/* Low water mark: Don't trim to below this */
363	unsigned int    srfpp_lowat;
364} sfxge_rx_fpp_t;
365
366typedef struct sfxge_rx_flow_s	sfxge_rx_flow_t;
367
368struct sfxge_rx_flow_s {
369	uint32_t		srf_tag;
370	/* in-order segment count */
371	unsigned int		srf_count;
372	uint16_t		srf_tci;
373	uint32_t		srf_saddr;
374	uint32_t		srf_daddr;
375	uint16_t		srf_sport;
376	uint16_t		srf_dport;
377	/* sequence number */
378	uint32_t		srf_seq;
379	clock_t			srf_lbolt;
380	mblk_t			*srf_mp;
381	mblk_t			**srf_mpp;
382	struct ether_header	*srf_etherhp;
383	struct ip		*srf_iphp;
384	struct tcphdr		*srf_first_thp;
385	struct tcphdr		*srf_last_thp;
386	size_t			srf_len;
387	sfxge_rx_flow_t		*srf_next;
388};
389
390#define	SFXGE_MAX_FLOW		1024
391#define	SFXGE_SLOW_START	20
392
393typedef enum sfxge_flush_state_e {
394	SFXGE_FLUSH_INACTIVE = 0,
395	SFXGE_FLUSH_DONE,
396	SFXGE_FLUSH_PENDING,
397	SFXGE_FLUSH_FAILED
398} sfxge_flush_state_t;
399
400typedef enum sfxge_rxq_state_e {
401	SFXGE_RXQ_UNINITIALIZED = 0,
402	SFXGE_RXQ_INITIALIZED,
403	SFXGE_RXQ_STARTED
404} sfxge_rxq_state_t;
405
406
407#define	SFXGE_RX_BATCH	128
408#define	SFXGE_RX_NSTATS	8 /* note that *esballoc share one kstat */
409
410struct sfxge_rxq_s {
411	union {
412		struct {
413			sfxge_t				*__sr_sp;
414			unsigned int			__sr_index;
415			efsys_mem_t			__sr_mem;
416			unsigned int			__sr_id;
417			unsigned int			__sr_lowat;
418			unsigned int			__sr_hiwat;
419			volatile timeout_id_t		__sr_tid;
420			sfxge_rxq_state_t		__sr_state;
421		} __sr_s1;
422		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
423	} __sr_u1;
424	union {
425		struct {
426			sfxge_rx_packet_t		**__sr_srpp;
427			unsigned int			__sr_added;
428			unsigned int			__sr_pushed;
429			unsigned int			__sr_pending;
430			unsigned int			__sr_completed;
431			unsigned int			__sr_loopback;
432			mblk_t   			*__sr_mp;
433			mblk_t   			**__sr_mpp;
434			sfxge_rx_flow_t			*__sr_flow;
435			sfxge_rx_flow_t			*__sr_srfp;
436			sfxge_rx_flow_t			**__sr_srfpp;
437			clock_t				__sr_rto;
438		} __sr_s2;
439		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
440	} __sr_u2;
441	union {
442		struct {
443			sfxge_rx_fpp_t			__sr_fpp;
444			efx_rxq_t			*__sr_erp;
445			volatile sfxge_flush_state_t	__sr_flush;
446			kcondvar_t			__sr_flush_kv;
447			kstat_t				*__sr_ksp;
448		} __sr_s3;
449		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE];
450	} __sr_u3;
451	struct {
452		/* NB must match SFXGE_RX_NSTATS */
453		uint32_t    srk_rx_pkt_mem_limit;
454		uint32_t    srk_kcache_alloc_nomem;
455		uint32_t    srk_dma_alloc_nomem;
456		uint32_t    srk_dma_alloc_fail;
457		uint32_t    srk_dma_bind_nomem;
458		uint32_t    srk_dma_bind_fail;
459		uint32_t    srk_desballoc_fail;
460		uint32_t    srk_rxq_empty_discard;
461	} sr_kstat;
462};
463
464#define	sr_sp		__sr_u1.__sr_s1.__sr_sp
465#define	sr_index	__sr_u1.__sr_s1.__sr_index
466#define	sr_mem		__sr_u1.__sr_s1.__sr_mem
467#define	sr_id		__sr_u1.__sr_s1.__sr_id
468#define	sr_mrh		__sr_u1.__sr_s1.__sr_mrh
469#define	sr_lowat	__sr_u1.__sr_s1.__sr_lowat
470#define	sr_hiwat	__sr_u1.__sr_s1.__sr_hiwat
471#define	sr_tid		__sr_u1.__sr_s1.__sr_tid
472#define	sr_state	__sr_u1.__sr_s1.__sr_state
473
474#define	sr_srpp		__sr_u2.__sr_s2.__sr_srpp
475#define	sr_added	__sr_u2.__sr_s2.__sr_added
476#define	sr_pushed	__sr_u2.__sr_s2.__sr_pushed
477#define	sr_pending	__sr_u2.__sr_s2.__sr_pending
478#define	sr_completed	__sr_u2.__sr_s2.__sr_completed
479#define	sr_loopback	__sr_u2.__sr_s2.__sr_loopback
480#define	sr_mp		__sr_u2.__sr_s2.__sr_mp
481#define	sr_mpp		__sr_u2.__sr_s2.__sr_mpp
482#define	sr_flow		__sr_u2.__sr_s2.__sr_flow
483#define	sr_srfp		__sr_u2.__sr_s2.__sr_srfp
484#define	sr_srfpp	__sr_u2.__sr_s2.__sr_srfpp
485#define	sr_rto		__sr_u2.__sr_s2.__sr_rto
486
487#define	sr_fpp		__sr_u3.__sr_s3.__sr_fpp
488#define	sr_erp		__sr_u3.__sr_s3.__sr_erp
489#define	sr_flush	__sr_u3.__sr_s3.__sr_flush
490#define	sr_flush_kv	__sr_u3.__sr_s3.__sr_flush_kv
491#define	sr_ksp		__sr_u3.__sr_s3.__sr_ksp
492
493typedef struct sfxge_tx_packet_s	sfxge_tx_packet_t;
494
495/* Packet type from parsing transmit packet */
496typedef enum sfxge_packet_type_e {
497	SFXGE_PACKET_TYPE_UNKNOWN = 0,
498	SFXGE_PACKET_TYPE_IPV4_TCP,
499	SFXGE_PACKET_TYPE_IPV4_UDP,
500	SFXGE_PACKET_TYPE_IPV4_SCTP,
501	SFXGE_PACKET_TYPE_IPV4_OTHER,
502	SFXGE_PACKET_NTYPES
503} sfxge_packet_type_t;
504
505struct sfxge_tx_packet_s {
506	sfxge_tx_packet_t	*stp_next;
507	mblk_t			*stp_mp;
508	struct ether_header	*stp_etherhp;
509	struct ip 		*stp_iphp;
510	struct tcphdr		*stp_thp;
511	size_t			stp_off;
512	size_t			stp_size;
513	size_t			stp_mss;
514	uint32_t		stp_dpl_put_len;
515};
516
517#define	SFXGE_TX_FPP_MAX	64
518
519typedef struct sfxge_tx_fpp_s {
520	sfxge_tx_packet_t	*stf_stpp;
521	unsigned int		stf_count;
522} sfxge_tx_fpp_t;
523
524typedef struct sfxge_tx_mapping_s	sfxge_tx_mapping_t;
525
526#define	SFXGE_TX_MAPPING_NADDR	(((1 << 16) >> 12) + 2)
527
528struct sfxge_tx_mapping_s {
529	sfxge_tx_mapping_t	*stm_next;
530	sfxge_t			*stm_sp;
531	mblk_t			*stm_mp;
532	ddi_dma_handle_t	stm_dma_handle;
533	caddr_t			stm_base;
534	size_t			stm_size;
535	size_t			stm_off;
536	uint64_t		stm_addr[SFXGE_TX_MAPPING_NADDR];
537};
538
539typedef struct sfxge_tx_fmp_s {
540	sfxge_tx_mapping_t	*stf_stmp;
541	unsigned int		stf_count;
542} sfxge_tx_fmp_t;
543
544typedef struct sfxge_tx_buffer_s	sfxge_tx_buffer_t;
545
546struct sfxge_tx_buffer_s {
547	sfxge_tx_buffer_t	*stb_next;
548	size_t			stb_off;
549	efsys_mem_t		stb_esm;
550};
551
552#define	SFXGE_TX_BUFFER_SIZE	0x400
553#define	SFXGE_TX_HEADER_SIZE	0x100
554#define	SFXGE_TX_COPY_THRESHOLD	0x200
555
556typedef struct sfxge_tx_fbp_s {
557	sfxge_tx_buffer_t	*stf_stbp;
558	unsigned int		stf_count;
559} sfxge_tx_fbp_t;
560
561typedef struct sfxge_tx_dpl_s {
562	uintptr_t		std_put;
563	sfxge_tx_packet_t	*std_get;
564	sfxge_tx_packet_t	**std_getp;
565	unsigned int		std_count; /* only get list count */
566	unsigned int		get_pkt_limit;
567	unsigned int		put_pkt_limit;
568	unsigned int		get_full_count;
569	unsigned int		put_full_count;
570} sfxge_tx_dpl_t;
571
572typedef enum sfxge_txq_state_e {
573	SFXGE_TXQ_UNINITIALIZED = 0,
574	SFXGE_TXQ_INITIALIZED,
575	SFXGE_TXQ_STARTED,
576	SFXGE_TXQ_FLUSH_PENDING,
577	SFXGE_TXQ_FLUSH_DONE,
578	SFXGE_TXQ_FLUSH_FAILED
579} sfxge_txq_state_t;
580
581typedef enum sfxge_txq_type_e {
582	SFXGE_TXQ_NON_CKSUM = 0,
583	SFXGE_TXQ_IP_CKSUM,
584	SFXGE_TXQ_IP_TCP_UDP_CKSUM,
585	SFXGE_TXQ_NTYPES
586} sfxge_txq_type_t;
587
588#define	SFXGE_TXQ_UNBLOCK_LEVEL1	(EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
589#define	SFXGE_TXQ_UNBLOCK_LEVEL2	0
590#define	SFXGE_TXQ_NOT_BLOCKED		-1
591
592#define	SFXGE_TX_BATCH	64
593
594struct sfxge_txq_s {
595	union {
596		struct {
597			sfxge_t				*__st_sp;
598			unsigned int			__st_index;
599			unsigned int			__st_label;
600			sfxge_txq_type_t		__st_type;
601			unsigned int			__st_evq;
602			efsys_mem_t			__st_mem;
603			unsigned int			__st_id;
604			kstat_t				*__st_ksp;
605			kstat_named_t			*__st_stat;
606			sfxge_txq_state_t		__st_state;
607		} __st_s1;
608		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE * 2];
609	} __st_u1;
610	union {
611		struct {
612			sfxge_tx_dpl_t			__st_dpl;
613		} __st_s2;
614		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE];
615	} __st_u2;
616	union {
617		struct {
618			kmutex_t			__st_lock;
619			/* mapping pool - sfxge_tx_mapping_t */
620			sfxge_tx_fmp_t			__st_fmp;
621			/* buffer pool - sfxge_tx_buffer_t */
622			sfxge_tx_fbp_t			__st_fbp;
623			/* packet pool - sfxge_tx_packet_t */
624			sfxge_tx_fpp_t			__st_fpp;
625			efx_buffer_t			*__st_eb;
626			unsigned int			__st_n;
627			efx_txq_t			*__st_etp;
628			sfxge_tx_mapping_t		**__st_stmp;
629			sfxge_tx_buffer_t		**__st_stbp;
630			mblk_t				**__st_mp;
631			unsigned int			__st_added;
632			unsigned int			__st_reaped;
633			int				__st_unblock;
634		} __st_s3;
635		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE * 3];
636	} __st_u3;
637	union {
638		struct {
639			sfxge_txq_t			*__st_next;
640			unsigned int			__st_pending;
641			unsigned int			__st_completed;
642
643		} __st_s4;
644		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE];
645	} __st_u4;
646};
647
648#define	st_sp		__st_u1.__st_s1.__st_sp
649#define	st_index	__st_u1.__st_s1.__st_index
650#define	st_label	__st_u1.__st_s1.__st_label
651#define	st_type		__st_u1.__st_s1.__st_type
652#define	st_evq		__st_u1.__st_s1.__st_evq
653#define	st_mem		__st_u1.__st_s1.__st_mem
654#define	st_id		__st_u1.__st_s1.__st_id
655#define	st_ksp		__st_u1.__st_s1.__st_ksp
656#define	st_stat		__st_u1.__st_s1.__st_stat
657#define	st_state	__st_u1.__st_s1.__st_state
658
659#define	st_dpl		__st_u2.__st_s2.__st_dpl
660
661#define	st_lock		__st_u3.__st_s3.__st_lock
662#define	st_fmp		__st_u3.__st_s3.__st_fmp
663#define	st_fbp		__st_u3.__st_s3.__st_fbp
664#define	st_fpp		__st_u3.__st_s3.__st_fpp
665#define	st_eb		__st_u3.__st_s3.__st_eb
666#define	st_n		__st_u3.__st_s3.__st_n
667#define	st_etp		__st_u3.__st_s3.__st_etp
668#define	st_stmp		__st_u3.__st_s3.__st_stmp
669#define	st_stbp		__st_u3.__st_s3.__st_stbp
670#define	st_mp		__st_u3.__st_s3.__st_mp
671#define	st_added	__st_u3.__st_s3.__st_added
672#define	st_reaped	__st_u3.__st_s3.__st_reaped
673#define	st_unblock	__st_u3.__st_s3.__st_unblock
674
675#define	st_next		__st_u4.__st_s4.__st_next
676#define	st_pending	__st_u4.__st_s4.__st_pending
677#define	st_completed	__st_u4.__st_s4.__st_completed
678
679typedef enum sfxge_rx_scale_state_e {
680	SFXGE_RX_SCALE_UNINITIALIZED = 0,
681	SFXGE_RX_SCALE_INITIALIZED,
682	SFXGE_RX_SCALE_STARTED
683} sfxge_rx_scale_state_t;
684
685#define	SFXGE_RX_SCALE_MAX	EFX_RSS_TBL_SIZE
686
687typedef struct sfxge_rx_scale_s {
688	kmutex_t		srs_lock;
689	unsigned int		*srs_cpu;
690	unsigned int		srs_tbl[SFXGE_RX_SCALE_MAX];
691	unsigned int		srs_count;
692	kstat_t			*srs_ksp;
693	sfxge_rx_scale_state_t	srs_state;
694} sfxge_rx_scale_t;
695
696
697typedef enum sfxge_rx_coalesce_mode_e {
698	SFXGE_RX_COALESCE_OFF = 0,
699	SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
700	SFXGE_RX_COALESCE_ALLOW_PUSH = 2
701} sfxge_rx_coalesce_mode_t;
702
703typedef enum sfxge_vpd_type_e {
704	SFXGE_VPD_ID = 0,
705	SFXGE_VPD_PN = 1,
706	SFXGE_VPD_SN = 2,
707	SFXGE_VPD_EC = 3,
708	SFXGE_VPD_MN = 4,
709	SFXGE_VPD_VD = 5,
710	SFXGE_VPD_VE = 6,
711	SFXGE_VPD_MAX = 7,
712} sfxge_vpd_type_t;
713
714typedef struct sfxge_vpd_kstat_s {
715	kstat_t		*svk_ksp;
716	kstat_named_t	svk_stat[SFXGE_VPD_MAX];
717	efx_vpd_value_t	*svk_vv;
718} sfxge_vpd_kstat_t;
719
720typedef struct sfxge_cfg_kstat_s {
721	struct {
722		kstat_named_t	sck_mac;
723		kstat_named_t	sck_version;
724	} kstat;
725	struct {
726		char		sck_mac[64 + 1];
727	} buf;
728} sfxge_cfg_kstat_t;
729
730typedef enum sfxge_state_e {
731	SFXGE_UNINITIALIZED = 0,
732	SFXGE_INITIALIZED,
733	SFXGE_REGISTERED,
734	SFXGE_STARTING,
735	SFXGE_STARTED,
736	SFXGE_STOPPING
737} sfxge_state_t;
738
739typedef enum sfxge_hw_err_e {
740	SFXGE_HW_OK = 0,
741	SFXGE_HW_ERR,
742} sfxge_hw_err_t;
743
744typedef enum sfxge_action_on_hw_err_e {
745	SFXGE_RECOVER = 0,
746	SFXGE_INVISIBLE = 1,
747	SFXGE_LEAVE_DEAD = 2,
748} sfxge_action_on_hw_err_t;
749
750typedef char *sfxge_mac_priv_prop_t;
751
752#define	SFXGE_TOEPLITZ_KEY_LEN 40
753
754struct sfxge_s {
755	kmutex_t			s_state_lock;
756	sfxge_state_t			s_state;
757	dev_info_t			*s_dip;
758	ddi_taskq_t			*s_tqp;
759	ddi_acc_handle_t		s_pci_handle;
760	uint16_t			s_pci_venid;
761	uint16_t			s_pci_devid;
762#if EFSYS_OPT_MCDI_LOGGING
763	unsigned int			s_bus_addr;
764#endif
765	efx_family_t			s_family;
766	unsigned int			s_pcie_nlanes;
767	unsigned int 			s_pcie_linkspeed;
768	kmutex_t			s_nic_lock;
769	efsys_bar_t			s_bar;
770	sfxge_intr_t			s_intr;
771	sfxge_mac_t			s_mac;
772	sfxge_mon_t			s_mon;
773	sfxge_sram_t			s_sram;
774	sfxge_mcdi_t			s_mcdi;
775	kmem_cache_t			*s_eq0c; /* eventQ 0 */
776	kmem_cache_t			*s_eqXc; /* all other eventQs */
777	sfxge_evq_t			*s_sep[SFXGE_RX_SCALE_MAX];
778	unsigned int			s_ev_moderation;
779	kmem_cache_t			*s_rqc;
780	sfxge_rxq_t			*s_srp[SFXGE_RX_SCALE_MAX];
781	sfxge_rx_scale_t		s_rx_scale;
782	size_t				s_rx_prefix_size;
783	size_t				s_rx_buffer_size;
784	size_t				s_rx_buffer_align;
785	sfxge_rx_coalesce_mode_t	s_rx_coalesce_mode;
786	int64_t				s_rx_pkt_mem_max;
787	volatile uint64_t		s_rx_pkt_mem_alloc;
788	kmem_cache_t			*s_rpc;
789	kmem_cache_t			*s_tqc;
790	unsigned int			s_tx_scale_base[SFXGE_TXQ_NTYPES];
791	unsigned int			s_tx_scale_max[SFXGE_TXQ_NTYPES];
792	int 				s_tx_qcount;
793	sfxge_txq_t			*s_stp[SFXGE_RX_SCALE_MAX *
794	    SFXGE_TXQ_NTYPES]; /* Sparse array */
795	kmem_cache_t			*s_tpc;
796	int				s_tx_flush_pending;
797	kmutex_t			s_tx_flush_lock;
798	kcondvar_t			s_tx_flush_kv;
799	kmem_cache_t			*s_tbc;
800	kmem_cache_t			*s_tmc;
801	efx_nic_t			*s_enp;
802	sfxge_vpd_kstat_t		s_vpd_kstat;
803	sfxge_cfg_kstat_t		s_cfg_kstat;
804	kstat_t				*s_cfg_ksp;
805	size_t				s_mtu;
806	int				s_rxq_poll_usec;
807	mac_callbacks_t			s_mc;
808	mac_handle_t			s_mh;
809	sfxge_mac_priv_prop_t		*s_mac_priv_props;
810	int				s_mac_priv_props_alloc;
811	volatile uint32_t		s_nested_restarts;
812	uint32_t			s_num_restarts;
813	uint32_t			s_num_restarts_hw_err;
814	sfxge_hw_err_t			s_hw_err;
815	sfxge_action_on_hw_err_t	s_action_on_hw_err;
816	uint16_t			s_rxq_size;
817	uint16_t			s_evq0_size;
818	uint16_t			s_evqX_size;
819#if EFSYS_OPT_MCDI_LOGGING
820	int				s_mcdi_logging;
821#endif
822	const uint32_t			*s_toeplitz_cache;
823};
824
825typedef struct sfxge_dma_buffer_attr_s {
826	dev_info_t		*sdba_dip;
827	ddi_dma_attr_t		*sdba_dattrp;
828	int			(*sdba_callback) (caddr_t);
829	size_t			sdba_length;
830	uint_t			sdba_memflags;
831	ddi_device_acc_attr_t	*sdba_devaccp;
832	uint_t			sdba_bindflags;
833	int			sdba_maxcookies;
834	boolean_t		sdba_zeroinit;
835} sfxge_dma_buffer_attr_t;
836
837extern const char		sfxge_ident[];
838extern uint8_t			sfxge_brdcst[];
839
840extern kmutex_t			sfxge_global_lock;
841
842extern unsigned int		*sfxge_cpu;
843
844extern int			sfxge_start(sfxge_t *, boolean_t);
845extern void			sfxge_stop(sfxge_t *);
846extern void			sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
847extern int			sfxge_restart_dispatch(sfxge_t *, uint_t,
848    sfxge_hw_err_t, const char *, uint32_t);
849
850extern void			sfxge_gld_link_update(sfxge_t *);
851extern void			sfxge_gld_mtu_update(sfxge_t *);
852extern void			sfxge_gld_rx_post(sfxge_t *, unsigned int,
853    mblk_t *);
854extern void			sfxge_gld_rx_push(sfxge_t *);
855extern int			sfxge_gld_register(sfxge_t *);
856extern int			sfxge_gld_unregister(sfxge_t *);
857
858extern int			sfxge_dma_buffer_create(efsys_mem_t *,
859    const sfxge_dma_buffer_attr_t *);
860extern void			sfxge_dma_buffer_destroy(efsys_mem_t *);
861
862extern int			sfxge_intr_init(sfxge_t *);
863extern int			sfxge_intr_start(sfxge_t *);
864extern void			sfxge_intr_stop(sfxge_t *);
865extern void			sfxge_intr_fini(sfxge_t *);
866extern void			sfxge_intr_fatal(sfxge_t *);
867
868extern int			sfxge_ev_init(sfxge_t *);
869extern int			sfxge_ev_start(sfxge_t *);
870extern void			sfxge_ev_moderation_get(sfxge_t *,
871    unsigned int *);
872extern int			sfxge_ev_moderation_set(sfxge_t *,
873    unsigned int);
874extern int			sfxge_ev_qmoderate(sfxge_t *, unsigned int,
875    unsigned int);
876extern int			sfxge_ev_qpoll(sfxge_t *, unsigned int);
877extern int			sfxge_ev_qprime(sfxge_t *, unsigned int);
878extern void			sfxge_ev_stop(sfxge_t *);
879extern void			sfxge_ev_fini(sfxge_t *);
880extern int			sfxge_ev_txlabel_alloc(sfxge_t *sp,
881    unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
882extern int			sfxge_ev_txlabel_free(sfxge_t *sp,
883    unsigned int evq, sfxge_txq_t *stp, unsigned int label);
884
885extern int			sfxge_mon_init(sfxge_t *);
886extern int			sfxge_mon_start(sfxge_t *);
887extern void			sfxge_mon_stop(sfxge_t *);
888extern void			sfxge_mon_fini(sfxge_t *);
889
890extern int			sfxge_mac_init(sfxge_t *);
891extern int			sfxge_mac_start(sfxge_t *, boolean_t);
892extern void			sfxge_mac_stat_get(sfxge_t *, unsigned int,
893    uint64_t *);
894extern void			sfxge_mac_link_check(sfxge_t *, boolean_t *);
895extern void			sfxge_mac_link_speed_get(sfxge_t *,
896    unsigned int *);
897extern void			sfxge_mac_link_duplex_get(sfxge_t *,
898    sfxge_link_duplex_t *);
899extern void			sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
900extern int			sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
901extern int			sfxge_mac_unicst_get(sfxge_t *,
902    sfxge_unicst_type_t, uint8_t *);
903extern int			sfxge_mac_unicst_set(sfxge_t *,
904    uint8_t *);
905extern int			sfxge_mac_promisc_set(sfxge_t *,
906    sfxge_promisc_type_t);
907extern int			sfxge_mac_multicst_add(sfxge_t *,
908    uint8_t const *addr);
909extern int			sfxge_mac_multicst_remove(sfxge_t *,
910    uint8_t const *addr);
911extern void			sfxge_mac_stop(sfxge_t *);
912extern void			sfxge_mac_fini(sfxge_t *);
913extern void			sfxge_mac_link_update(sfxge_t *sp,
914    efx_link_mode_t mode);
915
916extern int			sfxge_mcdi_init(sfxge_t *sp);
917extern void			sfxge_mcdi_fini(sfxge_t *sp);
918extern int			sfxge_mcdi_ioctl(sfxge_t *sp,
919    sfxge_mcdi_ioc_t *smip);
920extern int			sfxge_mcdi2_ioctl(sfxge_t *sp,
921    sfxge_mcdi2_ioc_t *smip);
922
923extern int			sfxge_phy_init(sfxge_t *);
924extern void			sfxge_phy_link_mode_get(sfxge_t *,
925    efx_link_mode_t *);
926extern void			sfxge_phy_fini(sfxge_t *);
927extern int			sfxge_phy_kstat_init(sfxge_t *sp);
928extern void			sfxge_phy_kstat_fini(sfxge_t *sp);
929extern uint8_t			sfxge_phy_lp_cap_test(sfxge_t *sp,
930    uint32_t field);
931extern int			sfxge_phy_cap_apply(sfxge_t *sp,
932    boolean_t use_default);
933extern uint8_t			sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
934    uint32_t field, boolean_t *mutablep);
935extern int			sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
936    int set);
937
938extern int			sfxge_rx_init(sfxge_t *);
939extern int			sfxge_rx_start(sfxge_t *);
940extern void			sfxge_rx_coalesce_mode_get(sfxge_t *,
941    sfxge_rx_coalesce_mode_t *);
942extern int			sfxge_rx_coalesce_mode_set(sfxge_t *,
943    sfxge_rx_coalesce_mode_t);
944extern unsigned int		sfxge_rx_scale_prop_get(sfxge_t *);
945extern void			sfxge_rx_scale_update(void *);
946extern int			sfxge_rx_scale_count_get(sfxge_t *,
947    unsigned int *);
948extern int			sfxge_rx_scale_count_set(sfxge_t *,
949    unsigned int);
950extern void			sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
951extern void			sfxge_rx_qflush_done(sfxge_rxq_t *);
952extern void			sfxge_rx_qflush_failed(sfxge_rxq_t *);
953extern void			sfxge_rx_qfpp_trim(sfxge_rxq_t *);
954extern void			sfxge_rx_stop(sfxge_t *);
955extern unsigned int 		sfxge_rx_loaned(sfxge_t *);
956extern void			sfxge_rx_fini(sfxge_t *);
957
958extern int			sfxge_tx_init(sfxge_t *);
959extern int			sfxge_tx_start(sfxge_t *);
960extern int			sfxge_tx_packet_add(sfxge_t *, mblk_t *);
961extern void			sfxge_tx_qcomplete(sfxge_txq_t *);
962extern void			sfxge_tx_qflush_done(sfxge_txq_t *);
963extern void			sfxge_tx_stop(sfxge_t *);
964extern void			sfxge_tx_fini(sfxge_t *);
965extern void			sfxge_tx_qdpl_flush(sfxge_txq_t *stp);
966
967extern void			sfxge_sram_init(sfxge_t *);
968extern int			sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
969    uint32_t *);
970extern int			sfxge_sram_start(sfxge_t *);
971extern int			sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
972    efsys_mem_t *, size_t);
973extern void			sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
974    size_t);
975extern void			sfxge_sram_stop(sfxge_t *);
976extern void			sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
977    size_t);
978extern void			sfxge_sram_fini(sfxge_t *);
979
980extern sfxge_packet_type_t	sfxge_pkthdr_parse(mblk_t *,
981    struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
982    uint16_t *, uint16_t *);
983
984extern int sfxge_toeplitz_hash_init(sfxge_t *);
985extern void sfxge_toeplitz_hash_fini(sfxge_t *);
986extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
987    uint8_t *, uint16_t, uint8_t *, uint16_t);
988
989/*
990 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
991 * To compute the same hash value as Siena/Huntington hardware, the inputs
992 * must be in big endian (network) byte order.
993 */
994#define	SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)	\
995	do { \
996		(_hash) = sfxge_toeplitz_hash(_sp, \
997					sizeof (struct in_addr), \
998					(uint8_t *)(_raddr), \
999					(_rport), \
1000					(uint8_t *)(_laddr), \
1001					(_lport)); \
1002		_NOTE(CONSTANTCONDITION) \
1003	} while (B_FALSE)
1004
1005/*
1006 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
1007 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
1008 * For other IPv4 non-TCP packets, use zero for the port numbers.
1009 */
1010#define	SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)	\
1011	SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))
1012
1013
1014extern int		sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);
1015
1016extern int		sfxge_pci_init(sfxge_t *);
1017extern void		sfxge_pcie_check_link(sfxge_t *, unsigned int,
1018    unsigned int);
1019extern void		sfxge_pci_fini(sfxge_t *);
1020
1021extern int		sfxge_bar_init(sfxge_t *);
1022extern void		sfxge_bar_fini(sfxge_t *);
1023
1024extern int		sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);
1025
1026
1027#endif /* _KERNEL */
1028
1029#ifdef	__cplusplus
1030}
1031#endif
1032
1033#endif	/* _SYS_SFXGE_H */
1034