xref: /illumos-gate/usr/src/uts/common/io/i40e/i40e_sw.h (revision 63783c5c)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
14  * Copyright 2019 Joyent, Inc.
15  * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
16  * Copyright 2020 Ryan Zezeski
17  * Copyright 2020 RackTop Systems, Inc.
18  */
19 
20 /*
21  * Please see i40e_main.c for an introduction to the device driver, its layout,
22  * and more.
23  */
24 
25 #ifndef	_I40E_SW_H
26 #define	_I40E_SW_H
27 
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31 
32 #include <sys/types.h>
33 #include <sys/conf.h>
34 #include <sys/debug.h>
35 #include <sys/stropts.h>
36 #include <sys/stream.h>
37 #include <sys/strsun.h>
38 #include <sys/strlog.h>
39 #include <sys/kmem.h>
40 #include <sys/stat.h>
41 #include <sys/kstat.h>
42 #include <sys/modctl.h>
43 #include <sys/errno.h>
44 #include <sys/dlpi.h>
45 #include <sys/mac_provider.h>
46 #include <sys/mac_ether.h>
47 #include <sys/vlan.h>
48 #include <sys/ddi.h>
49 #include <sys/sunddi.h>
50 #include <sys/pci.h>
51 #include <sys/pcie.h>
52 #include <sys/sdt.h>
53 #include <sys/ethernet.h>
54 #include <sys/pattr.h>
55 #include <sys/strsubr.h>
56 #include <sys/netlb.h>
57 #include <sys/random.h>
58 #include <inet/common.h>
59 #include <inet/tcp.h>
60 #include <inet/ip.h>
61 #include <inet/mi.h>
62 #include <inet/nd.h>
63 #include <netinet/udp.h>
64 #include <netinet/sctp.h>
65 #include <sys/bitmap.h>
66 #include <sys/cpuvar.h>
67 #include <sys/ddifm.h>
68 #include <sys/fm/protocol.h>
69 #include <sys/fm/util.h>
70 #include <sys/disp.h>
71 #include <sys/fm/io/ddi.h>
72 #include <sys/list.h>
73 #include <sys/debug.h>
74 #include <sys/sdt.h>
75 #include <sys/ddi_ufm.h>
76 #include "i40e_type.h"
77 #include "i40e_osdep.h"
78 #include "i40e_prototype.h"
79 #include "i40e_xregs.h"
80 
81 #define	I40E_MODULE_NAME "i40e"
82 
83 #define	I40E_ADAPTER_REGSET	1
84 
85 /*
86  * Configuration constants. Note that the hardware defines a minimum bound of 32
87  * descriptors and requires that the programming of the descriptor lengths be
88  * aligned in units of 32 descriptors.
89  */
90 #define	I40E_MIN_TX_RING_SIZE	64
91 #define	I40E_MAX_TX_RING_SIZE	4096
92 #define	I40E_DEF_TX_RING_SIZE	1024
93 
94 /*
95  * Place an artificial limit on the max number of groups. The X710
96  * series supports up to 384 VSIs to be partitioned across PFs as the
97  * driver sees fit. But until we support more interrupts this seems
98  * like a good place to start.
99  */
100 #define	I40E_MIN_NUM_RX_GROUPS	1
101 #define	I40E_MAX_NUM_RX_GROUPS	32
102 #define	I40E_DEF_NUM_RX_GROUPS	16
103 
104 #define	I40E_MIN_RX_RING_SIZE	64
105 #define	I40E_MAX_RX_RING_SIZE	4096
106 #define	I40E_DEF_RX_RING_SIZE	1024
107 
108 #define	I40E_DESC_ALIGN		32
109 
110 /*
111  * Sizes used for asynchronous processing of the adminq. We allocate a fixed
112  * size buffer for each instance of the device during attach time, rather than
113  * allocating and freeing one during interrupt processing.
114  *
115  * We also define the descriptor size of the admin queue here.
116  */
117 #define	I40E_ADMINQ_BUFSZ	4096
118 #define	I40E_MAX_ADMINQ_SIZE	1024
119 #define	I40E_DEF_ADMINQ_SIZE	256
120 
121 /*
122  * Note, while the min and maximum values are based upon the sizing of the ring
123  * itself, the default is taken from ixgbe without much thought. It's basically
124  * been cargo culted. See i40e_transceiver.c for a bit more information.
125  */
126 #define	I40E_MIN_RX_LIMIT_PER_INTR	16
127 #define	I40E_MAX_RX_LIMIT_PER_INTR	4096
128 #define	I40E_DEF_RX_LIMIT_PER_INTR	256
129 
130 /*
131  * Valid MTU ranges. Note that the XL710's maximum payload is actually 9728.
132  * However, we need to adjust for the ETHERFCSL (4 bytes) and the Ethernet VLAN
133  * header size (18 bytes) to get the actual maximum frame we can use. If
134  * different adapters end up with different sizes, we should make this value a
135  * bit more dynamic.
136  */
137 #define	I40E_MAX_MTU	9706
138 #define	I40E_MIN_MTU	ETHERMIN
139 #define	I40E_DEF_MTU	ETHERMTU
140 
141 /*
142  * Interrupt throttling related values. Interrupt throttling values are defined
143  * in two microsecond increments. Note that a value of zero basically says do no
144  * ITR activity. A helpful way to think about these is that setting the ITR to a
145  * value will allow a certain number of interrupts per second.
146  *
147  * Our default values for RX allow 20k interrupts per second while our default
148  * values for TX allow for 5k interrupts per second. For other class interrupts,
149  * we limit ourselves to a rate of 2k/s.
150  */
151 #define	I40E_MIN_ITR		0x0000
152 #define	I40E_MAX_ITR		0x0FF0
153 #define	I40E_DEF_RX_ITR		0x0019
154 #define	I40E_DEF_TX_ITR		0x0064
155 #define	I40E_DEF_OTHER_ITR	0x00FA
156 
157 /*
158  * Indexes into the three ITR registers that we have.
159  */
160 typedef enum i40e_itr_index {
161 	I40E_ITR_INDEX_RX	= 0x0,
162 	I40E_ITR_INDEX_TX	= 0x1,
163 	I40E_ITR_INDEX_OTHER	= 0x2,
164 	I40E_ITR_INDEX_NONE	= 0x3
165 } i40e_itr_index_t;
166 
167 /*
168  * The hardware claims to support LSO up to 256 KB, but due to the limitations
169  * imposed by the IP header for non-jumbo frames, we cap it at 64 KB.
170  */
171 #define	I40E_LSO_MAXLEN	(64 * 1024)
172 
173 #define	I40E_CYCLIC_PERIOD NANOSEC	/* 1 second */
174 #define	I40E_DRAIN_RX_WAIT	(500 * MILLISEC)	/* In us */
175 
176 /*
177  * All the other queue types for are defined by the common code. However, this
178  * is the constant to indicate that it's terminated.
179  */
180 #define	I40E_QUEUE_TYPE_EOL	0x7FF
181 
182 /*
183  * See the comments in i40e_transceiver.c as to the purpose of this value and
184  * how it's used to ensure that the IP header is eventually aligned when it's
185  * received by the OS.
186  */
187 #define	I40E_BUF_IPHDR_ALIGNMENT	2
188 
189 /*
190  * The XL710 controller has a total of eight buffers available for the
191  * transmission of any single frame. This is defined in 8.4.1 - Transmit
192  * Packet in System Memory.
193  */
194 #define	I40E_TX_MAX_COOKIE	8
195 
196 /*
197  * An LSO frame can be as large as 64KB, so we allow a DMA bind to span more
198  * cookies than a non-LSO frame.  The key here to is to select a value such
199  * that once the HW has chunked up the LSO frame into MSS-sized segments that no
200  * single segment spans more than 8 cookies (see comments for
201  * I40E_TX_MAX_COOKIE)
202  */
203 #define	I40E_TX_LSO_MAX_COOKIE	32
204 
205 /*
206  * Sizing to determine the amount of available descriptors at which we'll
207  * consider ourselves blocked. Also, when we have these available, we'll then
208  * consider ourselves available to transmit to MAC again. Strictly speaking, the
209  * MAX is based on the ring size. The default sizing is based on ixgbe.
210  */
211 #define	I40E_MIN_TX_BLOCK_THRESH	I40E_TX_MAX_COOKIE
212 #define	I40E_DEF_TX_BLOCK_THRESH	I40E_MIN_TX_BLOCK_THRESH
213 
214 /*
215  * Sizing for DMA thresholds. These are used to indicate whether or not we
216  * should perform a bcopy or a DMA binding of a given message block. The range
217  * allows for setting things such that we'll always do a bcopy (a high value) or
218  * always perform a DMA binding (a low value).
219  */
220 #define	I40E_MIN_RX_DMA_THRESH		0
221 #define	I40E_DEF_RX_DMA_THRESH		256
222 #define	I40E_MAX_RX_DMA_THRESH		INT32_MAX
223 
224 #define	I40E_MIN_TX_DMA_THRESH		0
225 #define	I40E_DEF_TX_DMA_THRESH		256
226 #define	I40E_MAX_TX_DMA_THRESH		INT32_MAX
227 
228 /*
229  * The max size of each individual tx buffer is 16KB - 1.
230  * See table 8-17
231  */
232 #define	I40E_MAX_TX_BUFSZ		0x0000000000003FFFull
233 
234 /*
235  * Resource sizing counts. There are various aspects of hardware where we may
236  * have some variable number of elements that we need to handle. Such as the
237  * hardware capabilities and switch capacities. We cannot know a priori how many
238  * elements to do, so instead we take a starting guess and then will grow it up
239  * to an upper bound on a number of elements, to limit memory consumption in
240  * case of a hardware bug.
241  */
242 #define	I40E_HW_CAP_DEFAULT	40
243 #define	I40E_SWITCH_CAP_DEFAULT	25
244 
245 /*
246  * Host Memory Context related constants.
247  */
248 #define	I40E_HMC_RX_CTX_UNIT		128
249 #define	I40E_HMC_RX_DBUFF_MIN		1024
250 #define	I40E_HMC_RX_DBUFF_MAX		(16 * 1024 - 128)
251 #define	I40E_HMC_RX_DTYPE_NOSPLIT	0
252 #define	I40E_HMC_RX_DSIZE_32BYTE	1
253 #define	I40E_HMC_RX_CRCSTRIP_ENABLE	1
254 #define	I40E_HMC_RX_FC_DISABLE		0
255 #define	I40E_HMC_RX_L2TAGORDER		1
256 #define	I40E_HMC_RX_HDRSPLIT_DISABLE	0
257 #define	I40E_HMC_RX_INVLAN_DONTSTRIP	0
258 #define	I40E_HMC_RX_TPH_DISABLE		0
259 #define	I40E_HMC_RX_LOWRXQ_NOINTR	0
260 #define	I40E_HMC_RX_PREFENA		1
261 
262 #define	I40E_HMC_TX_CTX_UNIT		128
263 #define	I40E_HMC_TX_NEW_CONTEXT		1
264 #define	I40E_HMC_TX_FC_DISABLE		0
265 #define	I40E_HMC_TX_TS_DISABLE		0
266 #define	I40E_HMC_TX_FD_DISABLE		0
267 #define	I40E_HMC_TX_ALT_VLAN_DISABLE	0
268 #define	I40E_HMC_TX_WB_ENABLE		1
269 #define	I40E_HMC_TX_TPH_DISABLE		0
270 
271 /*
272  * This defines the error mask that we care about from rx descriptors. Currently
273  * we're only concerned with the general errors and oversize errors.
274  */
275 #define	I40E_RX_ERR_BITS	((1 << I40E_RX_DESC_ERROR_RXE_SHIFT) | \
276 	(1 << I40E_RX_DESC_ERROR_OVERSIZE_SHIFT))
277 
278 /*
279  * Property sizing macros for firmware versions, etc. They need to be large
280  * enough to hold 32-bit quantities transformed to strings as %d.%d or %x.
281  */
282 #define	I40E_DDI_PROP_LEN	64
283 
284 #define	I40E_GROUP_NOMSIX	1
285 #define	I40E_TRQPAIR_NOMSIX	1
286 
287 /*
288  * It seems reasonable to cast this to void because the only reason that we
289  * should be getting a DDI_FAILURE is due to the fact that we specify addresses
290  * out of range. Because we specify no offset or address, it shouldn't happen.
291  */
292 #ifdef	DEBUG
293 #define	I40E_DMA_SYNC(handle, flag)	ASSERT0(ddi_dma_sync( \
294 					    (handle)->dmab_dma_handle, 0, 0, \
295 					    (flag)))
296 #else	/* !DEBUG */
297 #define	I40E_DMA_SYNC(handle, flag)	((void) ddi_dma_sync( \
298 					    (handle)->dmab_dma_handle, 0, 0, \
299 					    (flag)))
300 #endif	/* DEBUG */
301 
302 /*
303  * Constants related to ring startup and teardown. These refer to the amount of
304  * time that we're willing to wait for a ring to spin up and spin down.
305  */
306 #define	I40E_RING_WAIT_NTRIES	10
307 #define	I40E_RING_WAIT_PAUSE	10	/* ms */
308 #define	I40E_RING_ENABLE_GAP	50	/* ms */
309 
310 /*
311  * Printed Board Assembly (PBA) length. These are derived from Table 6-2.
312  */
313 #define	I40E_PBANUM_LENGTH	12
314 #define	I40E_PBANUM_STRLEN	13
315 
316 /*
317  * Define the maximum number of queues for a traffic class. These values come
318  * from the 'Number and offset of queue pairs per TCs' section of the 'Add VSI
319  * Command Buffer' table. For the 710 controller family this is table 7-62
320  * (r2.5) and for the 722 this is table 38-216 (r2.0).
321  */
322 #define	I40E_710_MAX_TC_QUEUES	64
323 #define	I40E_722_MAX_TC_QUEUES	128
324 
325 /*
326  * Define the size of the HLUT table size. The HLUT table can either be 128 or
327  * 512 bytes. We always set the table size to be 512 bytes in i40e_chip_start().
328  * Note, this should not be confused with the common code's macro
329  * I40E_HASH_LUT_SIZE_512 which is the bit pattern needed to tell the card to
330  * use a 512 byte HLUT.
331  */
332 #define	I40E_HLUT_TABLE_SIZE	512
333 
334 /*
335  * Bit flags for attach_progress
336  */
337 typedef enum i40e_attach_state {
338 	I40E_ATTACH_PCI_CONFIG	= 0x0001,	/* PCI config setup */
339 	I40E_ATTACH_REGS_MAP	= 0x0002,	/* Registers mapped */
340 	I40E_ATTACH_PROPS	= 0x0004,	/* Properties initialized */
341 	I40E_ATTACH_ALLOC_INTR	= 0x0008,	/* Interrupts allocated */
342 	I40E_ATTACH_ALLOC_RINGSLOCKS	= 0x0010, /* Rings & locks allocated */
343 	I40E_ATTACH_ADD_INTR	= 0x0020,	/* Intr handlers added */
344 	I40E_ATTACH_COMMON_CODE	= 0x0040,	/* Intel code initialized */
345 	I40E_ATTACH_INIT	= 0x0080,	/* Device initialized */
346 	I40E_ATTACH_STATS	= 0x0200,	/* Kstats created */
347 	I40E_ATTACH_MAC		= 0x0800,	/* MAC registered */
348 	I40E_ATTACH_ENABLE_INTR	= 0x1000,	/* DDI interrupts enabled */
349 	I40E_ATTACH_FM_INIT	= 0x2000,	/* FMA initialized */
350 	I40E_ATTACH_LINK_TIMER	= 0x4000,	/* link check timer */
351 	I40E_ATTACH_UFM_INIT	= 0x8000,	/* DDI UFM initialized */
352 } i40e_attach_state_t;
353 
354 
355 /*
356  * State flags that what's going on in in the device. Some of these state flags
357  * indicate some aspirational work that needs to happen in the driver.
358  *
359  * I40E_UNKNOWN:	The device has yet to be started.
360  * I40E_INITIALIZED:	The device has been fully attached.
361  * I40E_STARTED:	The device has come out of the GLDV3 start routine.
362  * I40E_SUSPENDED:	The device is suspended and I/O among other things
363  *			should not occur. This happens because of an actual
364  *			DDI_SUSPEND or interrupt adjustments.
365  * I40E_STALL:		The tx stall detection logic has found a stall.
366  * I40E_OVERTEMP:	The device has encountered a temperature alarm.
367  * I40E_INTR_ADJUST:	Our interrupts are being manipulated and therefore we
368  *			shouldn't be manipulating their state.
369  * I40E_ERROR:		We've detected an FM error and degraded the device.
370  */
371 typedef enum i40e_state {
372 	I40E_UNKNOWN		= 0x00,
373 	I40E_INITIALIZED	= 0x01,
374 	I40E_STARTED		= 0x02,
375 	I40E_SUSPENDED		= 0x04,
376 	I40E_STALL		= 0x08,
377 	I40E_OVERTEMP		= 0x20,
378 	I40E_INTR_ADJUST	= 0x40,
379 	I40E_ERROR		= 0x80
380 } i40e_state_t;
381 
382 
383 /*
384  * Definitions for common Intel things that we use and some slightly more usable
385  * names.
386  */
387 typedef struct i40e_hw i40e_hw_t;
388 typedef struct i40e_aqc_switch_resource_alloc_element_resp i40e_switch_rsrc_t;
389 
390 /*
391  * Handles and addresses of DMA buffers.
392  */
393 typedef struct i40e_dma_buffer {
394 	caddr_t		dmab_address;		/* Virtual address */
395 	uint64_t	dmab_dma_address;	/* DMA (Hardware) address */
396 	ddi_acc_handle_t dmab_acc_handle;	/* Data access handle */
397 	ddi_dma_handle_t dmab_dma_handle;	/* DMA handle */
398 	size_t		dmab_size;		/* Buffer size */
399 	size_t		dmab_len;		/* Data length in the buffer */
400 } i40e_dma_buffer_t;
401 
402 /*
403  * RX Control Block
404  */
405 typedef struct i40e_rx_control_block {
406 	mblk_t			*rcb_mp;
407 	uint32_t		rcb_ref;
408 	i40e_dma_buffer_t	rcb_dma;
409 	frtn_t			rcb_free_rtn;
410 	struct i40e_rx_data	*rcb_rxd;
411 } i40e_rx_control_block_t;
412 
413 typedef enum {
414 	I40E_TX_NONE,
415 	I40E_TX_COPY,
416 	I40E_TX_DMA,
417 	I40E_TX_DESC,
418 } i40e_tx_type_t;
419 
420 typedef struct i40e_tx_desc i40e_tx_desc_t;
421 typedef struct i40e_tx_context_desc i40e_tx_context_desc_t;
422 typedef union i40e_32byte_rx_desc i40e_rx_desc_t;
423 
424 struct i40e_dma_bind_info {
425 	caddr_t dbi_paddr;
426 	size_t dbi_len;
427 };
428 
429 typedef struct i40e_tx_control_block {
430 	struct i40e_tx_control_block	*tcb_next;
431 	mblk_t				*tcb_mp;
432 	i40e_tx_type_t			tcb_type;
433 	ddi_dma_handle_t		tcb_dma_handle;
434 	ddi_dma_handle_t		tcb_lso_dma_handle;
435 	i40e_dma_buffer_t		tcb_dma;
436 	struct i40e_dma_bind_info	*tcb_bind_info;
437 	uint_t				tcb_bind_ncookies;
438 	boolean_t			tcb_used_lso;
439 } i40e_tx_control_block_t;
440 
441 /*
442  * Receive ring data (used below).
443  */
444 typedef struct i40e_rx_data {
445 	struct i40e	*rxd_i40e;
446 
447 	/*
448 	 * RX descriptor ring definitions
449 	 */
450 	i40e_dma_buffer_t rxd_desc_area;	/* DMA buffer of rx desc ring */
451 	i40e_rx_desc_t *rxd_desc_ring;		/* Rx desc ring */
452 	uint32_t rxd_desc_next;			/* Index of next rx desc */
453 
454 	/*
455 	 * RX control block list definitions
456 	 */
457 	kmutex_t		rxd_free_lock;	/* Lock to protect free data */
458 	i40e_rx_control_block_t	*rxd_rcb_area;	/* Array of control blocks */
459 	i40e_rx_control_block_t	**rxd_work_list; /* Work list of rcbs */
460 	i40e_rx_control_block_t	**rxd_free_list; /* Free list of rcbs */
461 	uint32_t		rxd_rcb_free;	/* Number of free rcbs */
462 
463 	/*
464 	 * RX software ring settings
465 	 */
466 	uint32_t	rxd_ring_size;		/* Rx descriptor ring size */
467 	uint32_t	rxd_free_list_size;	/* Rx free list size */
468 
469 	/*
470 	 * RX outstanding data. This is used to keep track of outstanding loaned
471 	 * descriptors after we've shut down receiving information. Note these
472 	 * are protected by the i40e_t`i40e_rx_pending_lock.
473 	 */
474 	uint32_t	rxd_rcb_pending;
475 	boolean_t	rxd_shutdown;
476 } i40e_rx_data_t;
477 
478 /*
479  * Structures for unicast and multicast addresses. Note that we keep the VSI id
480  * around for unicast addresses, since they may belong to different VSIs.
481  * However, since all multicast addresses belong to the default VSI, we don't
482  * duplicate that information.
483  */
484 typedef struct i40e_uaddr {
485 	uint8_t iua_mac[ETHERADDRL];
486 	int	iua_vsi;
487 } i40e_uaddr_t;
488 
489 typedef struct i40e_maddr {
490 	uint8_t ima_mac[ETHERADDRL];
491 } i40e_maddr_t;
492 
493 /*
494  * Collection of RX statistics on a given queue.
495  */
496 typedef struct i40e_rxq_stat {
497 	/*
498 	 * The i40e hardware does not maintain statistics on a per-ring basis,
499 	 * only on a per-PF and per-VSI level. As such, to satisfy the GLDv3, we
500 	 * need to maintain our own stats for packets and bytes.
501 	 */
502 	kstat_named_t	irxs_bytes;	/* Bytes in on queue */
503 	kstat_named_t	irxs_packets;	/* Packets in on queue */
504 
505 	/*
506 	 * The following set of stats cover non-checksum data path issues.
507 	 */
508 	kstat_named_t	irxs_rx_desc_error;	/* Error bit set on desc */
509 	kstat_named_t	irxs_rx_copy_nomem;	/* allocb failure for copy */
510 	kstat_named_t	irxs_rx_intr_limit;	/* Hit i40e_rx_limit_per_intr */
511 	kstat_named_t	irxs_rx_bind_norcb;	/* No replacement rcb free */
512 	kstat_named_t	irxs_rx_bind_nomp;	/* No mblk_t in bind rcb */
513 
514 	/*
515 	 * The following set of statistics covers rx checksum related activity.
516 	 * These are all primarily set in i40e_rx_hcksum. If rx checksum
517 	 * activity is disabled, then these should all be zero.
518 	 */
519 	kstat_named_t	irxs_hck_v4hdrok;	/* Valid IPv4 Header */
520 	kstat_named_t	irxs_hck_l4hdrok;	/* Valid L4 Header */
521 	kstat_named_t	irxs_hck_unknown;	/* !pinfo.known */
522 	kstat_named_t	irxs_hck_nol3l4p;	/* Missing L3L4P bit in desc */
523 	kstat_named_t	irxs_hck_iperr;		/* IPE error bit set */
524 	kstat_named_t	irxs_hck_eiperr;	/* EIPE error bit set */
525 	kstat_named_t	irxs_hck_l4err;		/* L4E error bit set */
526 	kstat_named_t	irxs_hck_v6skip;	/* IPv6 case hw fails on */
527 	kstat_named_t	irxs_hck_set;		/* Total times we set cksum */
528 	kstat_named_t	irxs_hck_miss;		/* Times with zero cksum bits */
529 } i40e_rxq_stat_t;
530 
531 /*
532  * Collection of TX Statistics on a given queue
533  */
534 typedef struct i40e_txq_stat {
535 	kstat_named_t	itxs_bytes;		/* Bytes out on queue */
536 	kstat_named_t	itxs_packets;		/* Packets out on queue */
537 	kstat_named_t	itxs_descriptors;	/* Descriptors issued */
538 	kstat_named_t	itxs_recycled;		/* Descriptors reclaimed */
539 	kstat_named_t	itxs_force_copy;	/* non-TSO force copy */
540 	kstat_named_t	itxs_tso_force_copy;	/* TSO force copy */
541 	/*
542 	 * Various failure conditions.
543 	 */
544 	kstat_named_t	itxs_hck_meoifail;	/* ether offload failures */
545 	kstat_named_t	itxs_hck_nol2info;	/* Missing l2 info */
546 	kstat_named_t	itxs_hck_nol3info;	/* Missing l3 info */
547 	kstat_named_t	itxs_hck_nol4info;	/* Missing l4 info */
548 	kstat_named_t	itxs_hck_badl3;		/* Not IPv4/IPv6 */
549 	kstat_named_t	itxs_hck_badl4;		/* Bad L4 Paylaod */
550 	kstat_named_t	itxs_lso_nohck;		/* Missing offloads for LSO */
551 	kstat_named_t	itxs_bind_fails;	/* DMA bind failures */
552 	kstat_named_t	itxs_tx_short;		/* Tx chain too short */
553 
554 	kstat_named_t	itxs_err_notcb;		/* No tcb's available */
555 	kstat_named_t	itxs_err_nodescs;	/* No tcb's available */
556 	kstat_named_t	itxs_err_context;	/* Total context failures */
557 
558 	kstat_named_t	itxs_num_unblocked;	/* Number of MAC unblocks */
559 } i40e_txq_stat_t;
560 
561 /*
562  * An instance of an XL710 transmit/receive queue pair. This currently
563  * represents a combination of both a transmit and receive ring, though they
564  * should really be split apart into separate logical structures. Unfortunately,
565  * during initial work we mistakenly joined them together.
566  */
567 typedef struct i40e_trqpair {
568 	struct i40e *itrq_i40e;
569 
570 	/* interrupt control structures */
571 	kmutex_t itrq_intr_lock;
572 	kcondvar_t itrq_intr_cv;
573 	boolean_t itrq_intr_busy;	/* Busy processing interrupt */
574 	boolean_t itrq_intr_quiesce;	/* Interrupt quiesced */
575 
576 	hrtime_t irtq_time_stopped;	/* Time when ring was stopped */
577 
578 	/* Receive-side structures. */
579 	kmutex_t itrq_rx_lock;
580 	mac_ring_handle_t itrq_macrxring; /* Receive ring handle. */
581 	i40e_rx_data_t *itrq_rxdata;	/* Receive ring rx data. */
582 	uint64_t itrq_rxgen;		/* Generation number for mac/GLDv3. */
583 	uint32_t itrq_index;		/* Queue index in the PF */
584 	uint32_t itrq_rx_intrvec;	/* Receive interrupt vector. */
585 	boolean_t itrq_intr_poll;	/* True when polling */
586 
587 	/* Receive-side stats. */
588 	i40e_rxq_stat_t	itrq_rxstat;
589 	kstat_t	*itrq_rxkstat;
590 
591 	/* Transmit-side structures. */
592 	kmutex_t itrq_tx_lock;
593 	kcondvar_t itrq_tx_cv;
594 	uint_t itrq_tx_active;		/* No. of active i40e_ring_tx()'s */
595 	boolean_t itrq_tx_quiesce;	/* Tx is quiesced */
596 	mac_ring_handle_t itrq_mactxring; /* Transmit ring handle. */
597 	uint32_t itrq_tx_intrvec;	/* Transmit interrupt vector. */
598 	boolean_t itrq_tx_blocked;	/* Does MAC think we're blocked? */
599 
600 	/*
601 	 * TX data sizing
602 	 */
603 	uint32_t		itrq_tx_ring_size;
604 	uint32_t		itrq_tx_free_list_size;
605 
606 	/*
607 	 * TX descriptor ring data
608 	 */
609 	i40e_dma_buffer_t	itrq_desc_area;	/* DMA buffer of tx desc ring */
610 	i40e_tx_desc_t		*itrq_desc_ring; /* TX Desc ring */
611 	volatile uint32_t	*itrq_desc_wbhead; /* TX write-back index */
612 	uint32_t		itrq_desc_head;	/* Last index hw freed */
613 	uint32_t		itrq_desc_tail;	/* Index of next free desc */
614 	uint32_t		itrq_desc_free;	/* Number of free descriptors */
615 
616 	/*
617 	 * TX control block (tcb) data
618 	 */
619 	kmutex_t		itrq_tcb_lock;
620 	i40e_tx_control_block_t	*itrq_tcb_area;	/* Array of control blocks */
621 	i40e_tx_control_block_t	**itrq_tcb_work_list;	/* In use tcb */
622 	i40e_tx_control_block_t	**itrq_tcb_free_list;	/* Available tcb */
623 	uint32_t		itrq_tcb_free;	/* Count of free tcb */
624 
625 	/* Transmit-side stats. */
626 	i40e_txq_stat_t		itrq_txstat;
627 	kstat_t			*itrq_txkstat;
628 
629 } i40e_trqpair_t;
630 
631 /*
632  * VSI statistics.
633  *
634  * This mirrors the i40e_eth_stats structure but transforms it into a kstat.
635  * Note that the stock statistic structure also includes entries for tx
636  * discards. However, this is not actually implemented for the VSI (see Table
637  * 7-221), hence why we don't include the member which would always have a value
638  * of zero. This choice was made to minimize confusion to someone looking at
639  * these, as a value of zero does not necessarily equate to the fact that it's
640  * not implemented.
641  */
642 typedef struct i40e_vsi_stats {
643 	uint64_t ivs_rx_bytes;			/* gorc */
644 	uint64_t ivs_rx_unicast;		/* uprc */
645 	uint64_t ivs_rx_multicast;		/* mprc */
646 	uint64_t ivs_rx_broadcast;		/* bprc */
647 	uint64_t ivs_rx_discards;		/* rdpc */
648 	uint64_t ivs_rx_unknown_protocol;	/* rupp */
649 	uint64_t ivs_tx_bytes;			/* gotc */
650 	uint64_t ivs_tx_unicast;		/* uptc */
651 	uint64_t ivs_tx_multicast;		/* mptc */
652 	uint64_t ivs_tx_broadcast;		/* bptc */
653 	uint64_t ivs_tx_errors;			/* tepc */
654 } i40e_vsi_stats_t;
655 
656 typedef struct i40e_vsi_kstats {
657 	kstat_named_t	ivk_rx_bytes;
658 	kstat_named_t	ivk_rx_unicast;
659 	kstat_named_t	ivk_rx_multicast;
660 	kstat_named_t	ivk_rx_broadcast;
661 	kstat_named_t	ivk_rx_discards;
662 	kstat_named_t	ivk_rx_unknown_protocol;
663 	kstat_named_t	ivk_tx_bytes;
664 	kstat_named_t	ivk_tx_unicast;
665 	kstat_named_t	ivk_tx_multicast;
666 	kstat_named_t	ivk_tx_broadcast;
667 	kstat_named_t	ivk_tx_errors;
668 } i40e_vsi_kstats_t;
669 
670 /*
671  * For pf statistics, we opt not to use the standard statistics as defined by
672  * the Intel common code. This also currently combines statistics that are
673  * global across the entire device.
674  */
675 typedef struct i40e_pf_stats {
676 	uint64_t ips_rx_bytes;			/* gorc */
677 	uint64_t ips_rx_unicast;		/* uprc */
678 	uint64_t ips_rx_multicast;		/* mprc */
679 	uint64_t ips_rx_broadcast;		/* bprc */
680 	uint64_t ips_tx_bytes;			/* gotc */
681 	uint64_t ips_tx_unicast;		/* uptc */
682 	uint64_t ips_tx_multicast;		/* mptc */
683 	uint64_t ips_tx_broadcast;		/* bptc */
684 
685 	uint64_t ips_rx_size_64;		/* prc64 */
686 	uint64_t ips_rx_size_127;		/* prc127 */
687 	uint64_t ips_rx_size_255;		/* prc255 */
688 	uint64_t ips_rx_size_511;		/* prc511 */
689 	uint64_t ips_rx_size_1023;		/* prc1023 */
690 	uint64_t ips_rx_size_1522;		/* prc1522 */
691 	uint64_t ips_rx_size_9522;		/* prc9522 */
692 
693 	uint64_t ips_tx_size_64;		/* ptc64 */
694 	uint64_t ips_tx_size_127;		/* ptc127 */
695 	uint64_t ips_tx_size_255;		/* ptc255 */
696 	uint64_t ips_tx_size_511;		/* ptc511 */
697 	uint64_t ips_tx_size_1023;		/* ptc1023 */
698 	uint64_t ips_tx_size_1522;		/* ptc1522 */
699 	uint64_t ips_tx_size_9522;		/* ptc9522 */
700 
701 	uint64_t ips_link_xon_rx;		/* lxonrxc */
702 	uint64_t ips_link_xoff_rx;		/* lxoffrxc */
703 	uint64_t ips_link_xon_tx;		/* lxontxc */
704 	uint64_t ips_link_xoff_tx;		/* lxofftxc */
705 	uint64_t ips_priority_xon_rx[8];	/* pxonrxc[8] */
706 	uint64_t ips_priority_xoff_rx[8];	/* pxoffrxc[8] */
707 	uint64_t ips_priority_xon_tx[8];	/* pxontxc[8] */
708 	uint64_t ips_priority_xoff_tx[8];	/* pxofftxc[8] */
709 	uint64_t ips_priority_xon_2_xoff[8];	/* rxon2offcnt[8] */
710 
711 	uint64_t ips_crc_errors;		/* crcerrs */
712 	uint64_t ips_illegal_bytes;		/* illerrc */
713 	uint64_t ips_mac_local_faults;		/* mlfc */
714 	uint64_t ips_mac_remote_faults;		/* mrfc */
715 	uint64_t ips_rx_length_errors;		/* rlec */
716 	uint64_t ips_rx_undersize;		/* ruc */
717 	uint64_t ips_rx_fragments;		/* rfc */
718 	uint64_t ips_rx_oversize;		/* roc */
719 	uint64_t ips_rx_jabber;			/* rjc */
720 	uint64_t ips_rx_discards;		/* rdpc */
721 	uint64_t ips_rx_vm_discards;		/* ldpc */
722 	uint64_t ips_rx_short_discards;		/* mspdc */
723 	uint64_t ips_tx_dropped_link_down;	/* tdold */
724 	uint64_t ips_rx_unknown_protocol;	/* rupp */
725 	uint64_t ips_rx_err1;			/* rxerr1 */
726 	uint64_t ips_rx_err2;			/* rxerr2 */
727 } i40e_pf_stats_t;
728 
729 typedef struct i40e_pf_kstats {
730 	kstat_named_t ipk_rx_bytes;		/* gorc */
731 	kstat_named_t ipk_rx_unicast;		/* uprc */
732 	kstat_named_t ipk_rx_multicast;		/* mprc */
733 	kstat_named_t ipk_rx_broadcast;		/* bprc */
734 	kstat_named_t ipk_tx_bytes;		/* gotc */
735 	kstat_named_t ipk_tx_unicast;		/* uptc */
736 	kstat_named_t ipk_tx_multicast;		/* mptc */
737 	kstat_named_t ipk_tx_broadcast;		/* bptc */
738 
739 	kstat_named_t ipk_rx_size_64;		/* prc64 */
740 	kstat_named_t ipk_rx_size_127;		/* prc127 */
741 	kstat_named_t ipk_rx_size_255;		/* prc255 */
742 	kstat_named_t ipk_rx_size_511;		/* prc511 */
743 	kstat_named_t ipk_rx_size_1023;		/* prc1023 */
744 	kstat_named_t ipk_rx_size_1522;		/* prc1522 */
745 	kstat_named_t ipk_rx_size_9522;		/* prc9522 */
746 
747 	kstat_named_t ipk_tx_size_64;		/* ptc64 */
748 	kstat_named_t ipk_tx_size_127;		/* ptc127 */
749 	kstat_named_t ipk_tx_size_255;		/* ptc255 */
750 	kstat_named_t ipk_tx_size_511;		/* ptc511 */
751 	kstat_named_t ipk_tx_size_1023;		/* ptc1023 */
752 	kstat_named_t ipk_tx_size_1522;		/* ptc1522 */
753 	kstat_named_t ipk_tx_size_9522;		/* ptc9522 */
754 
755 	kstat_named_t ipk_link_xon_rx;		/* lxonrxc */
756 	kstat_named_t ipk_link_xoff_rx;		/* lxoffrxc */
757 	kstat_named_t ipk_link_xon_tx;		/* lxontxc */
758 	kstat_named_t ipk_link_xoff_tx;		/* lxofftxc */
759 	kstat_named_t ipk_priority_xon_rx[8];	/* pxonrxc[8] */
760 	kstat_named_t ipk_priority_xoff_rx[8];	/* pxoffrxc[8] */
761 	kstat_named_t ipk_priority_xon_tx[8];	/* pxontxc[8] */
762 	kstat_named_t ipk_priority_xoff_tx[8];	/* pxofftxc[8] */
763 	kstat_named_t ipk_priority_xon_2_xoff[8];	/* rxon2offcnt[8] */
764 
765 	kstat_named_t ipk_crc_errors;		/* crcerrs */
766 	kstat_named_t ipk_illegal_bytes;	/* illerrc */
767 	kstat_named_t ipk_mac_local_faults;	/* mlfc */
768 	kstat_named_t ipk_mac_remote_faults;	/* mrfc */
769 	kstat_named_t ipk_rx_length_errors;	/* rlec */
770 	kstat_named_t ipk_rx_undersize;		/* ruc */
771 	kstat_named_t ipk_rx_fragments;		/* rfc */
772 	kstat_named_t ipk_rx_oversize;		/* roc */
773 	kstat_named_t ipk_rx_jabber;		/* rjc */
774 	kstat_named_t ipk_rx_discards;		/* rdpc */
775 	kstat_named_t ipk_rx_vm_discards;	/* ldpc */
776 	kstat_named_t ipk_rx_short_discards;	/* mspdc */
777 	kstat_named_t ipk_tx_dropped_link_down;	/* tdold */
778 	kstat_named_t ipk_rx_unknown_protocol;	/* rupp */
779 	kstat_named_t ipk_rx_err1;		/* rxerr1 */
780 	kstat_named_t ipk_rx_err2;		/* rxerr2 */
781 } i40e_pf_kstats_t;
782 
783 /*
784  * Resources that are pooled and specific to a given i40e_t.
785  */
786 typedef struct i40e_func_rsrc {
787 	uint_t	ifr_nrx_queue;
788 	uint_t	ifr_nrx_queue_used;
789 	uint_t	ifr_ntx_queue;
790 	uint_t	ifr_trx_queue_used;
791 	uint_t	ifr_nvsis;
792 	uint_t	ifr_nvsis_used;
793 	uint_t	ifr_nmacfilt;
794 	uint_t	ifr_nmacfilt_used;
795 	uint_t	ifr_nmcastfilt;
796 	uint_t	ifr_nmcastfilt_used;
797 } i40e_func_rsrc_t;
798 
799 typedef struct i40e_vsi {
800 	uint16_t		iv_seid;
801 	uint16_t		iv_number;
802 	kstat_t			*iv_kstats;
803 	i40e_vsi_stats_t	iv_stats;
804 	uint16_t		iv_stats_id;
805 } i40e_vsi_t;
806 
807 /*
808  * While irg_index and irg_grp_hdl aren't used anywhere, they are
809  * still useful for debugging.
810  */
811 typedef struct i40e_rx_group {
812 	uint32_t		irg_index;    /* index in i40e_rx_groups[] */
813 	uint16_t		irg_vsi_seid; /* SEID of VSI for this group */
814 	mac_group_handle_t	irg_grp_hdl;  /* handle to mac_group_t */
815 	struct i40e		*irg_i40e;    /* ref to i40e_t */
816 } i40e_rx_group_t;
817 
818 /*
819  * Main i40e per-instance state.
820  */
821 typedef struct i40e {
822 	list_node_t	i40e_glink;		/* Global list link */
823 	list_node_t	i40e_dlink;		/* Device list link */
824 	kmutex_t	i40e_general_lock;	/* General device lock */
825 
826 	/*
827 	 * General Data and management
828 	 */
829 	dev_info_t	*i40e_dip;
830 	int		i40e_instance;
831 	int		i40e_fm_capabilities;
832 	uint_t		i40e_state;
833 	i40e_attach_state_t i40e_attach_progress;
834 	mac_handle_t	i40e_mac_hdl;
835 	ddi_periodic_t	i40e_periodic_id;
836 
837 	/*
838 	 * Pointers to common code data structures and memory for the common
839 	 * code.
840 	 */
841 	struct i40e_hw				i40e_hw_space;
842 	struct i40e_osdep			i40e_osdep_space;
843 	struct i40e_aq_get_phy_abilities_resp	i40e_phy;
844 	void					*i40e_aqbuf;
845 
846 #define	I40E_DEF_VSI_IDX	0
847 #define	I40E_DEF_VSI(i40e)	((i40e)->i40e_vsis[I40E_DEF_VSI_IDX])
848 #define	I40E_DEF_VSI_SEID(i40e)	(I40E_DEF_VSI(i40e).iv_seid)
849 
850 	/*
851 	 * Device state, switch information, and resources.
852 	 */
853 	i40e_vsi_t		i40e_vsis[I40E_MAX_NUM_RX_GROUPS];
854 	uint16_t		i40e_mac_seid;	 /* SEID of physical MAC */
855 	uint16_t		i40e_veb_seid;	 /* switch atop MAC (SEID) */
856 	uint16_t		i40e_vsi_avail;	 /* VSIs avail to this PF */
857 	uint16_t		i40e_vsi_used;	 /* VSIs used by this PF */
858 	struct i40e_device	*i40e_device;
859 	i40e_func_rsrc_t	i40e_resources;
860 	uint16_t		i40e_switch_rsrc_alloc;
861 	uint16_t		i40e_switch_rsrc_actual;
862 	i40e_switch_rsrc_t	*i40e_switch_rsrcs;
863 	i40e_uaddr_t		*i40e_uaddrs;
864 	i40e_maddr_t		*i40e_maddrs;
865 	int			i40e_mcast_promisc_count;
866 	boolean_t		i40e_promisc_on;
867 	link_state_t		i40e_link_state;
868 	uint32_t		i40e_link_speed;	/* In Mbps */
869 	link_duplex_t		i40e_link_duplex;
870 	link_fec_t		i40e_fec_requested;
871 	uint_t			i40e_sdu;
872 	uint_t			i40e_frame_max;
873 
874 	/*
875 	 * Transmit and receive information, tunables, and MAC info.
876 	 */
877 	i40e_trqpair_t	*i40e_trqpairs;
878 	boolean_t	i40e_mr_enable;
879 	uint_t		i40e_num_trqpairs; /* total TRQPs (per PF) */
880 	uint_t		i40e_num_trqpairs_per_vsi; /* TRQPs per VSI */
881 	uint_t		i40e_other_itr;
882 
883 	i40e_rx_group_t	*i40e_rx_groups;
884 	uint_t		i40e_num_rx_groups;
885 	int		i40e_num_rx_descs;
886 	uint32_t	i40e_rx_ring_size;
887 	uint32_t	i40e_rx_buf_size;
888 	boolean_t	i40e_rx_hcksum_enable;
889 	uint32_t	i40e_rx_dma_min;
890 	uint32_t	i40e_rx_limit_per_intr;
891 	uint_t		i40e_rx_itr;
892 
893 	int		i40e_num_tx_descs;
894 	uint32_t	i40e_tx_ring_size;
895 	uint32_t	i40e_tx_buf_size;
896 	uint32_t	i40e_tx_block_thresh;
897 	boolean_t	i40e_tx_hcksum_enable;
898 	boolean_t	i40e_tx_lso_enable;
899 	uint32_t	i40e_tx_dma_min;
900 	uint_t		i40e_tx_itr;
901 
902 	/*
903 	 * Interrupt state
904 	 */
905 	uint_t		i40e_intr_pri;
906 	uint_t		i40e_intr_force;
907 	uint_t		i40e_intr_type;
908 	int		i40e_intr_cap;
909 	uint32_t	i40e_intr_count;
910 	uint32_t	i40e_intr_count_max;
911 	uint32_t	i40e_intr_count_min;
912 	size_t		i40e_intr_size;
913 	ddi_intr_handle_t *i40e_intr_handles;
914 	ddi_cb_handle_t	i40e_callback_handle;
915 
916 	/*
917 	 * DMA attributes. See i40e_transceiver.c for why we have copies of them
918 	 * in the i40e_t.
919 	 */
920 	ddi_dma_attr_t		i40e_static_dma_attr;
921 	ddi_dma_attr_t		i40e_txbind_dma_attr;
922 	ddi_dma_attr_t		i40e_txbind_lso_dma_attr;
923 	ddi_device_acc_attr_t	i40e_desc_acc_attr;
924 	ddi_device_acc_attr_t	i40e_buf_acc_attr;
925 
926 	/*
927 	 * The following two fields are used to protect and keep track of
928 	 * outstanding, loaned buffers to MAC. If we have these, we can't
929 	 * detach as we have active DMA memory outstanding.
930 	 */
931 	kmutex_t	i40e_rx_pending_lock;
932 	kcondvar_t	i40e_rx_pending_cv;
933 	uint32_t	i40e_rx_pending;
934 
935 	/*
936 	 * PF statistics and VSI statistics.
937 	 */
938 	kmutex_t		i40e_stat_lock;
939 	kstat_t			*i40e_pf_kstat;
940 	i40e_pf_stats_t		i40e_pf_stat;
941 
942 	/*
943 	 * Misc. stats and counters that should maybe one day be kstats.
944 	 */
945 	uint64_t	i40e_s_link_status_errs;
946 	uint32_t	i40e_s_link_status_lasterr;
947 
948 	/*
949 	 * LED information. Note this state is only modified in
950 	 * i40e_gld_set_led() which is protected by MAC's serializer lock.
951 	 */
952 	uint32_t	i40e_led_status;
953 	boolean_t	i40e_led_saved;
954 
955 	/* DDI UFM handle */
956 	ddi_ufm_handle_t	*i40e_ufmh;
957 } i40e_t;
958 
959 /*
960  * The i40e_device represents a PCI device which encapsulates multiple physical
961  * functions which are represented as an i40e_t. This is used to track the use
962  * of pooled resources throughout all of the various devices.
963  */
964 typedef struct i40e_device {
965 	list_node_t	id_link;
966 	dev_info_t	*id_parent;
967 	uint_t		id_pci_bus;
968 	uint_t		id_pci_device;
969 	uint_t		id_nfuncs;	/* Total number of functions */
970 	uint_t		id_nreg;	/* Total number present */
971 	list_t		id_i40e_list;	/* List of i40e_t's registered */
972 	i40e_switch_rsrc_t	*id_rsrcs; /* Switch resources for this PF */
973 	uint_t		id_rsrcs_alloc;	/* Total allocated resources */
974 	uint_t		id_rsrcs_act;	/* Actual number of resources */
975 } i40e_device_t;
976 
977 /* Values for the interrupt forcing on the NIC. */
978 #define	I40E_INTR_NONE			0
979 #define	I40E_INTR_MSIX			1
980 #define	I40E_INTR_MSI			2
981 #define	I40E_INTR_LEGACY		3
982 
983 /* Hint that we don't want to do any polling... */
984 #define	I40E_POLL_NULL			-1
985 
986 /*
987  * Logging functions.
988  */
989 /*PRINTFLIKE2*/
990 extern void i40e_error(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
991 /*PRINTFLIKE2*/
992 extern void i40e_notice(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
993 /*PRINTFLIKE2*/
994 extern void i40e_log(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
995 
996 /*
997  * General link handling functions.
998  */
999 extern void i40e_link_check(i40e_t *);
1000 extern void i40e_update_mtu(i40e_t *);
1001 
1002 /*
1003  * FMA functions.
1004  */
1005 extern int i40e_check_acc_handle(ddi_acc_handle_t);
1006 extern int i40e_check_dma_handle(ddi_dma_handle_t);
1007 extern void i40e_fm_ereport(i40e_t *, char *);
1008 
1009 /*
1010  * Interrupt handlers and interrupt handler setup.
1011  */
1012 extern void i40e_intr_chip_init(i40e_t *);
1013 extern void i40e_intr_chip_fini(i40e_t *);
1014 extern uint_t i40e_intr_msix(void *, void *);
1015 extern uint_t i40e_intr_msi(void *, void *);
1016 extern uint_t i40e_intr_legacy(void *, void *);
1017 extern void i40e_intr_io_enable_all(i40e_t *);
1018 extern void i40e_intr_io_disable_all(i40e_t *);
1019 extern void i40e_intr_io_clear_cause(i40e_t *);
1020 extern void i40e_intr_rx_queue_disable(i40e_trqpair_t *);
1021 extern void i40e_intr_rx_queue_enable(i40e_trqpair_t *);
1022 extern void i40e_intr_set_itr(i40e_t *, i40e_itr_index_t, uint_t);
1023 extern void i40e_intr_quiesce(i40e_trqpair_t *);
1024 
1025 /*
1026  * Receive-side functions
1027  */
1028 extern mblk_t *i40e_ring_rx(i40e_trqpair_t *, int);
1029 extern mblk_t *i40e_ring_rx_poll(void *, int);
1030 extern void i40e_rx_recycle(caddr_t);
1031 extern boolean_t i40e_ring_tx_quiesce(i40e_trqpair_t *);
1032 
1033 /*
1034  * Transmit-side functions
1035  */
1036 mblk_t *i40e_ring_tx(void *, mblk_t *);
1037 extern void i40e_tx_recycle_ring(i40e_trqpair_t *);
1038 extern void i40e_tx_cleanup_ring(i40e_trqpair_t *);
1039 
1040 /*
1041  * Statistics functions.
1042  */
1043 extern boolean_t i40e_stats_init(i40e_t *);
1044 extern void i40e_stats_fini(i40e_t *);
1045 extern boolean_t i40e_stat_vsi_init(i40e_t *, uint_t);
1046 extern void i40e_stat_vsi_fini(i40e_t *, uint_t);
1047 extern boolean_t i40e_stats_trqpair_init(i40e_trqpair_t *);
1048 extern void i40e_stats_trqpair_fini(i40e_trqpair_t *);
1049 extern int i40e_m_stat(void *, uint_t, uint64_t *);
1050 extern int i40e_rx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);
1051 extern int i40e_tx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);
1052 extern mac_ether_media_t i40e_link_to_media(i40e_t *);
1053 
1054 /*
1055  * MAC/GLDv3 functions, and functions called by MAC/GLDv3 support code.
1056  */
1057 extern boolean_t i40e_register_mac(i40e_t *);
1058 extern boolean_t i40e_start(i40e_t *);
1059 extern void i40e_stop(i40e_t *);
1060 extern int i40e_setup_ring(i40e_trqpair_t *);
1061 extern boolean_t i40e_shutdown_ring(i40e_trqpair_t *);
1062 
1063 /*
1064  * DMA & buffer functions and attributes
1065  */
1066 extern void i40e_init_dma_attrs(i40e_t *, boolean_t);
1067 extern boolean_t i40e_alloc_ring_mem(i40e_trqpair_t *);
1068 extern void i40e_free_ring_mem(i40e_trqpair_t *, boolean_t);
1069 
1070 #ifdef __cplusplus
1071 }
1072 #endif
1073 
1074 #endif /* _I40E_SW_H */
1075