xref: /illumos-gate/usr/src/uts/common/io/ena/ena_hw.h (revision 6f443ebc)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2021 Oxide Computer Company
14  */
15 
16 /*
17  * This file declares all constants and structures dealing with the
18  * physical ENA device. It is based on the ena_com code of the public
19  * Linux and FreeBSD drivers. While this file is based on the common
20  * code it doesn't share the same type names. Where it is useful, a
21  * "common" reference is added to include the name of the type as
22  * defined in the common code.
23  *
24  * The Linux driver defines enq_admin_aq_entry as the top-level type
25  * for admin command descriptors. From this type you can access the
26  * common bits shared by every descriptor (ena_admin_aq_common_desc)
27  * as well as the control buffer (ena_admin_ctrl_buff_info) which is
28  * present for _some_ commands. Other than that, this top-level type
29  * treats the rest of the data as an opaque array of unsigned 32-bit
30  * integers. Then, for each individual command, the Linux driver
31  * defines a dedicated type, each of which contains the following:
32  *
33  * 1. The common descriptor: ena_admin_aq_common_desc.
34  *
35  * 2. The optional control buffer desc: ena_admin_ctrl_buff_info.
36  *
37  * 3. The command-specific data.
38  *
39  * 4. Optional padding to make sure all commands are 64 bytes in size.
40  *
41  * Furthermore, there may be further common types for commands which
42  * are made up of several sub-commands, e.g. the get/set feature
43  * commands.
44  *
45  * Finally, when a command is passed to the common function for
46  * executing commands (ena_com_execute_admin_command()), it is cast as
47  * a pointer to the top-level type: ena_admin_aq_entry.
48  *
49  * This works for the Linux driver just fine, but it causes lots of
50  * repetition in the structure definitions and also means there is no
51  * easy way to determine all valid commands. This ENA driver has
52  * turned the Linux approach inside out -- the top-level type is a
53  * union of all possible commands: enahw_cmd_desc_t. Each command may
54  * then further sub-type via unions to represent its sub-commands.
55  * This same treatment was given to the response descriptor:
56  * enahw_resp_desc_t.
57  *
58  * What is the point of knowing all this? Well, when referencing the
59  * common type in the comment above the enahw_ type, you need to keep
60  * in mind that the Linux/common type will include all the common
61  * descriptor bits, whereas these types do not.
62  *
63  * The common code DOES NOT pack any of these structures, and thus
64  * neither do we. That means these structures all rely on natural
65  * compiler alignment, just as the common code does. In ena.c you will
66  * find CTASSERTs for many of these structures, to verify they are of
67  * the expected size.
68  */
69 
70 #ifndef	_ENA_HW_H
71 #define	_ENA_HW_H
72 
73 #include <sys/ddi.h>
74 #include <sys/sunddi.h>
75 #include <sys/types.h>
76 #include <sys/debug.h>
77 #include <sys/ethernet.h>
78 
79 /*
80  * The common code sets the upper limit of I/O queues to 128. In this
81  * case a "queue" is a SQ+CQ pair that forms a logical queue or ring
82  * for sending or receiving packets. Thus, at maximum, we may expect
83  * 128 Tx rings, and 128 Rx rings; though, practically speaking, the
84  * number of rings will often be limited by number of CPUs or
85  * available interrupts.
86  *
87  * common: ENA_MAX_NUM_IO_QUEUES
88  */
89 #define	ENAHW_MAX_NUM_IO_QUEUES	128
90 
91 /*
92  * Generate a 32-bit bitmask where the bits between high (inclusive)
93  * and low (inclusive) are set to 1.
94  */
95 #define	GENMASK(h, l)	(((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
96 
97 /*
98  * Generate a 64-bit bitmask where bit b is set to 1.
99  */
100 #define	BIT(b)	(1UL << (b))
101 
102 #define	ENAHW_DMA_ADMINQ_ALIGNMENT	8
103 
104 #define	ENAHW_ADMIN_CQ_DESC_BUF_ALIGNMENT	8
105 #define	ENAHW_ADMIN_SQ_DESC_BUF_ALIGNMENT	8
106 #define	ENAHW_AENQ_DESC_BUF_ALIGNMENT		8
107 #define	ENAHW_HOST_INFO_ALIGNMENT		8
108 #define	ENAHW_HOST_INFO_ALLOC_SZ		4096
109 #define	ENAHW_IO_CQ_DESC_BUF_ALIGNMENT		4096
110 #define	ENAHW_IO_SQ_DESC_BUF_ALIGNMENT		8
111 
112 /*
113  * BAR0 register offsets.
114  *
115  * Any register not defined in the common code was marked as a gap,
116  * using the hex address of the register as suffix. The idea is to
117  * make it clear where the gaps are and allow the
118  * ena_hw_update_reg_cache() function to display any bits stored in
119  * these gaps in case they turn out to be interesting later.
120  */
121 #define	ENAHW_REG_VERSION		0x0
122 #define	ENAHW_REG_CONTROLLER_VERSION	0x4
123 #define	ENAHW_REG_CAPS			0x8
124 #define	ENAHW_REG_CAPS_EXT		0xc
125 #define	ENAHW_REG_ASQ_BASE_LO		0x10
126 #define	ENAHW_REG_ASQ_BASE_HI		0x14
127 #define	ENAHW_REG_ASQ_CAPS		0x18
128 #define	ENAHW_REG_GAP_1C		0x1c
129 #define	ENAHW_REG_ACQ_BASE_LO		0x20
130 #define	ENAHW_REG_ACQ_BASE_HI		0x24
131 #define	ENAHW_REG_ACQ_CAPS		0x28
132 #define	ENAHW_REG_ASQ_DB		0x2c
133 #define	ENAHW_REG_ACQ_TAIL		0x30
134 #define	ENAHW_REG_AENQ_CAPS		0x34
135 #define	ENAHW_REG_AENQ_BASE_LO		0x38
136 #define	ENAHW_REG_AENQ_BASE_HI		0x3c
137 #define	ENAHW_REG_AENQ_HEAD_DB		0x40
138 #define	ENAHW_REG_AENQ_TAIL		0x44
139 #define	ENAHW_REG_GAP_48		0x48
140 #define	ENAHW_REG_INTERRUPT_MASK	0x4c
141 #define	ENAHW_REG_GAP_50		0x50
142 #define	ENAHW_REG_DEV_CTL		0x54
143 #define	ENAHW_REG_DEV_STS		0x58
144 #define	ENAHW_REG_MMIO_REG_READ		0x5c
145 #define	ENAHW_REG_MMIO_RESP_LO		0x60
146 #define	ENAHW_REG_MMIO_RESP_HI		0x64
147 #define	ENAHW_REG_RSS_IND_ENTRY_UPDATE	0x68
148 #define	ENAHW_NUM_REGS		((ENAHW_REG_RSS_IND_ENTRY_UPDATE / 4) + 1)
149 
150 /*
151  * Device Version (Register 0x0)
152  */
153 #define	ENAHW_DEV_MINOR_VSN_MASK	0xff
154 #define	ENAHW_DEV_MAJOR_VSN_SHIFT	8
155 #define	ENAHW_DEV_MAJOR_VSN_MASK	0xff00
156 
157 #define	ENAHW_DEV_MAJOR_VSN(vsn)					\
158 	(((vsn) & ENAHW_DEV_MAJOR_VSN_MASK) >> ENAHW_DEV_MAJOR_VSN_SHIFT)
159 #define	ENAHW_DEV_MINOR_VSN(vsn)		\
160 	((vsn) & ENAHW_DEV_MINOR_VSN_MASK)
161 
162 /*
163  * Controller Version (Register 0x4)
164  */
165 #define	ENAHW_CTRL_SUBMINOR_VSN_MASK	0xff
166 #define	ENAHW_CTRL_MINOR_VSN_SHIFT	8
167 #define	ENAHW_CTRL_MINOR_VSN_MASK	0xff00
168 #define	ENAHW_CTRL_MAJOR_VSN_SHIFT	16
169 #define	ENAHW_CTRL_MAJOR_VSN_MASK	0xff0000
170 #define	ENAHW_CTRL_IMPL_ID_SHIFT	24
171 #define	ENAHW_CTRL_IMPL_ID_MASK		0xff000000
172 
173 #define	ENAHW_CTRL_MAJOR_VSN(vsn)				\
174 	(((vsn) & ENAHW_CTRL_MAJOR_VSN_MASK) >> ENAHW_CTRL_MAJOR_VSN_SHIFT)
175 #define	ENAHW_CTRL_MINOR_VSN(vsn)				\
176 	(((vsn) & ENAHW_CTRL_MINOR_VSN_MASK) >> ENAHW_CTRL_MINOR_VSN_SHIFT)
177 #define	ENAHW_CTRL_SUBMINOR_VSN(vsn)	\
178 	((vsn) & ENAHW_CTRL_SUBMINOR_VSN_MASK)
179 #define	ENAHW_CTRL_IMPL_ID(vsn)				\
180 	(((vsn) & ENAHW_CTRL_IMPL_ID_MASK) >> ENAHW_CTRL_IMPL_ID_SHIFT)
181 
182 /*
183  * Device Caps (Register 0x8)
184  */
185 #define	ENAHW_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK	0x1
186 #define	ENAHW_CAPS_RESET_TIMEOUT_SHIFT			1
187 #define	ENAHW_CAPS_RESET_TIMEOUT_MASK			0x3e
188 #define	ENAHW_CAPS_RESET_TIMEOUT(v)		    \
189 	(((v) & ENAHW_CAPS_RESET_TIMEOUT_MASK) >>   \
190 	    ENAHW_CAPS_RESET_TIMEOUT_SHIFT)
191 #define	ENAHW_CAPS_DMA_ADDR_WIDTH_SHIFT			8
192 #define	ENAHW_CAPS_DMA_ADDR_WIDTH_MASK			0xff00
193 #define	ENAHW_CAPS_DMA_ADDR_WIDTH(v)		     \
194 	(((v) & ENAHW_CAPS_DMA_ADDR_WIDTH_MASK) >>   \
195 	    ENAHW_CAPS_DMA_ADDR_WIDTH_SHIFT)
196 #define	ENAHW_CAPS_ADMIN_CMD_TIMEOUT_SHIFT		16
197 #define	ENAHW_CAPS_ADMIN_CMD_TIMEOUT_MASK		0xf0000
198 #define	ENAHW_CAPS_ADMIN_CMD_TIMEOUT(v)			\
199 	(((v) & ENAHW_CAPS_ADMIN_CMD_TIMEOUT_MASK) >>	\
200 	    ENAHW_CAPS_ADMIN_CMD_TIMEOUT_SHIFT)
201 
202 enum enahw_reset_reason_types {
203 	ENAHW_RESET_NORMAL			= 0,
204 	ENAHW_RESET_KEEP_ALIVE_TO		= 1,
205 	ENAHW_RESET_ADMIN_TO			= 2,
206 	ENAHW_RESET_MISS_TX_CMPL		= 3,
207 	ENAHW_RESET_INV_RX_REQ_ID		= 4,
208 	ENAHW_RESET_INV_TX_REQ_ID		= 5,
209 	ENAHW_RESET_TOO_MANY_RX_DESCS		= 6,
210 	ENAHW_RESET_INIT_ERR			= 7,
211 	ENAHW_RESET_DRIVER_INVALID_STATE	= 8,
212 	ENAHW_RESET_OS_TRIGGER			= 9,
213 	ENAHW_RESET_OS_NETDEV_WD		= 10,
214 	ENAHW_RESET_SHUTDOWN			= 11,
215 	ENAHW_RESET_USER_TRIGGER		= 12,
216 	ENAHW_RESET_GENERIC			= 13,
217 	ENAHW_RESET_MISS_INTERRUPT		= 14,
218 	ENAHW_RESET_LAST,
219 };
220 
221 /*
222  * Admin Submission Queue Caps (Register 0x18)
223  */
224 #define	ENAHW_ASQ_CAPS_DEPTH_MASK		0xffff
225 #define	ENAHW_ASQ_CAPS_ENTRY_SIZE_SHIFT		16
226 #define	ENAHW_ASQ_CAPS_ENTRY_SIZE_MASK		0xffff0000
227 
228 #define	ENAHW_ASQ_CAPS_DEPTH(x)	((x) & ENAHW_ASQ_CAPS_DEPTH_MASK)
229 
230 #define	ENAHW_ASQ_CAPS_ENTRY_SIZE(x)			\
231 	(((x) << ENAHW_ASQ_CAPS_ENTRY_SIZE_SHIFT) &	\
232 	    ENAHW_ASQ_CAPS_ENTRY_SIZE_MASK)
233 
234 /*
235  * Admin Completion Queue Caps (Register 0x28)
236  */
237 #define	ENAHW_ACQ_CAPS_DEPTH_MASK	0xffff
238 #define	ENAHW_ACQ_CAPS_ENTRY_SIZE_SHIFT	16
239 #define	ENAHW_ACQ_CAPS_ENTRY_SIZE_MASK	0xffff0000
240 
241 #define	ENAHW_ACQ_CAPS_DEPTH(x)	((x) & ENAHW_ACQ_CAPS_DEPTH_MASK)
242 
243 #define	ENAHW_ACQ_CAPS_ENTRY_SIZE(x)			\
244 	(((x) << ENAHW_ACQ_CAPS_ENTRY_SIZE_SHIFT) &	\
245 	    ENAHW_ACQ_CAPS_ENTRY_SIZE_MASK)
246 
247 /*
248  * Asynchronous Event Notification Queue Caps (Register 0x34)
249  */
250 #define	ENAHW_AENQ_CAPS_DEPTH_MASK		0xffff
251 #define	ENAHW_AENQ_CAPS_ENTRY_SIZE_SHIFT	16
252 #define	ENAHW_AENQ_CAPS_ENTRY_SIZE_MASK		0xffff0000
253 
254 #define	ENAHW_AENQ_CAPS_DEPTH(x) ((x) & ENAHW_AENQ_CAPS_DEPTH_MASK)
255 
256 #define	ENAHW_AENQ_CAPS_ENTRY_SIZE(x)		     \
257 	(((x) << ENAHW_AENQ_CAPS_ENTRY_SIZE_SHIFT) & \
258 	    ENAHW_AENQ_CAPS_ENTRY_SIZE_MASK)
259 
260 /*
261  * Interrupt Mask (Register 0x4c)
262  */
263 #define	ENAHW_INTR_UNMASK	0x0
264 #define	ENAHW_INTR_MASK		0x1
265 
266 /*
267  * Device Control (Register 0x54)
268  */
269 #define	ENAHW_DEV_CTL_DEV_RESET_MASK		0x1
270 #define	ENAHW_DEV_CTL_AQ_RESTART_SHIFT		1
271 #define	ENAHW_DEV_CTL_AQ_RESTART_MASK		0x2
272 #define	ENAHW_DEV_CTL_QUIESCENT_SHIFT		2
273 #define	ENAHW_DEV_CTL_QUIESCENT_MASK		0x4
274 #define	ENAHW_DEV_CTL_IO_RESUME_SHIFT		3
275 #define	ENAHW_DEV_CTL_IO_RESUME_MASK		0x8
276 #define	ENAHW_DEV_CTL_RESET_REASON_SHIFT	28
277 #define	ENAHW_DEV_CTL_RESET_REASON_MASK		0xf0000000
278 
279 /*
280  * Device Status (Register 0x58)
281  */
282 #define	ENAHW_DEV_STS_READY_MASK			0x1
283 #define	ENAHW_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT	1
284 #define	ENAHW_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK	0x2
285 #define	ENAHW_DEV_STS_AQ_RESTART_FINISHED_SHIFT		2
286 #define	ENAHW_DEV_STS_AQ_RESTART_FINISHED_MASK		0x4
287 #define	ENAHW_DEV_STS_RESET_IN_PROGRESS_SHIFT		3
288 #define	ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK		0x8
289 #define	ENAHW_DEV_STS_RESET_FINISHED_SHIFT		4
290 #define	ENAHW_DEV_STS_RESET_FINISHED_MASK		0x10
291 #define	ENAHW_DEV_STS_FATAL_ERROR_SHIFT			5
292 #define	ENAHW_DEV_STS_FATAL_ERROR_MASK			0x20
293 #define	ENAHW_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT	6
294 #define	ENAHW_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK	0x40
295 #define	ENAHW_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT	7
296 #define	ENAHW_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK	0x80
297 
298 /* common: ena_admin_aenq_common_desc */
299 typedef struct enahw_aenq_desc {
300 	uint16_t	ead_group;
301 	uint16_t	ead_syndrome;
302 	uint8_t		ead_flags;
303 	uint8_t		ead_rsvd1[3];
304 	uint32_t	ead_ts_low;
305 	uint32_t	ead_ts_high;
306 
307 	union {
308 		uint32_t	raw[12];
309 
310 		struct {
311 			uint32_t flags;
312 		} link_change;
313 
314 		struct {
315 			uint32_t rx_drops_low;
316 			uint32_t rx_drops_high;
317 			uint32_t tx_drops_low;
318 			uint32_t tx_drops_high;
319 		} keep_alive;
320 	} ead_payload;
321 } enahw_aenq_desc_t;
322 
323 #define	ENAHW_AENQ_DESC_PHASE_MASK	BIT(0)
324 
325 #define	ENAHW_AENQ_DESC_PHASE(desc)		\
326 	((desc)->ead_flags & ENAHW_AENQ_DESC_PHASE_MASK)
327 
328 #define	ENAHW_AENQ_LINK_CHANGE_LINK_STATUS_MASK	BIT(0)
329 
330 /*
331  * Asynchronous Event Notification Queue groups.
332  *
333  * Note: These values represent the bit position of each feature as
334  * returned by ENAHW_FEAT_AENQ_CONFIG. We encode them this way so that
335  * they can double as an index into the AENQ handlers array.
336  *
337  * common: ena_admin_aenq_group
338  */
339 typedef enum enahw_aenq_groups {
340 	ENAHW_AENQ_GROUP_LINK_CHANGE		= 0,
341 	ENAHW_AENQ_GROUP_FATAL_ERROR		= 1,
342 	ENAHW_AENQ_GROUP_WARNING		= 2,
343 	ENAHW_AENQ_GROUP_NOTIFICATION		= 3,
344 	ENAHW_AENQ_GROUP_KEEP_ALIVE		= 4,
345 	ENAHW_AENQ_GROUP_REFRESH_CAPABILITIES	= 5,
346 	ENAHW_AENQ_GROUPS_ARR_NUM		= 6,
347 } enahw_aenq_groups_t;
348 
349 /*
350  * The reason for ENAHW_AENQ_GROUP_NOFIFICATION.
351  *
352  * common: ena_admin_aenq_notification_syndrome
353  */
354 typedef enum enahw_aenq_syndrome {
355 	ENAHW_AENQ_SYNDROME_UPDATE_HINTS	= 2,
356 } enahw_aenq_syndrome_t;
357 
358 /*
359  * ENA devices use a 48-bit memory space.
360  *
361  * common: ena_common_mem_addr
362  */
363 typedef struct enahw_addr {
364 	uint32_t	ea_low;
365 	uint16_t	ea_high;
366 	uint16_t	ea_rsvd; /* must be zero */
367 } enahw_addr_t;
368 
369 /* common: ena_admin_ctrl_buff_info */
370 struct enahw_ctrl_buff {
371 	uint32_t	ecb_length;
372 	enahw_addr_t	ecb_addr;
373 };
374 
375 /* common: ena_admin_get_set_feature_common_desc */
376 struct enahw_feat_common {
377 	/*
378 	 * 1:0 Select which value you want.
379 	 *
380 	 *	0x1 = Current value.
381 	 *	0x3 = Default value.
382 	 *
383 	 *	Note: Linux seems to set this to 0 to get the value,
384 	 *	not sure if that's a bug or just another way to get the
385 	 *	current value.
386 	 *
387 	 * 7:3 Reserved.
388 	 */
389 	uint8_t	efc_flags;
390 
391 	/* An id from enahw_feature_id_t. */
392 	uint8_t	efc_id;
393 
394 	/*
395 	 * Each feature is versioned, allowing upgrades to the feature
396 	 * set without breaking backwards compatibility. The driver
397 	 * uses this field to specify which version it supports
398 	 * (starting from zero). Linux doesn't document this very well
399 	 * and sets this value to 0 for most features. We define a set
400 	 * of macros, underneath the enahw_feature_id_t type, clearly
401 	 * documenting the version we support for each feature.
402 	 */
403 	uint8_t	efc_version;
404 	uint8_t	efc_rsvd;
405 };
406 
407 /* common: ena_admin_get_feat_cmd */
408 typedef struct enahw_cmd_get_feat {
409 	struct enahw_ctrl_buff		ecgf_ctrl_buf;
410 	struct enahw_feat_common	ecgf_comm;
411 	uint32_t			egcf_unused[11];
412 } enahw_cmd_get_feat_t;
413 
414 /*
415  * N.B. Linux sets efc_flags to 0 (via memset) when reading the
416  * current value, but the comments say it should be 0x1. We follow the
417  * comments.
418  */
419 #define	ENAHW_GET_FEAT_FLAGS_GET_CURR_VAL(desc)		\
420 	((desc)->ecgf_comm.efc_flags) |= 0x1
421 #define	ENAHW_GET_FEAT_FLAGS_GET_DEF_VAL(desc)		\
422 	((desc)->ecgf_comm.efc_flags) |= 0x3
423 
424 /*
425  * Set the MTU of the device. This value does not include the L2
426  * headers or trailers, only the payload.
427  *
428  * common: ena_admin_set_feature_mtu_desc
429  */
430 typedef struct enahw_feat_mtu {
431 	uint32_t efm_mtu;
432 } enahw_feat_mtu_t;
433 
434 /* common: ena_admin_set_feature_host_attr_desc */
435 typedef struct enahw_feat_host_attr {
436 	enahw_addr_t	efha_os_addr;
437 	enahw_addr_t	efha_debug_addr;
438 	uint32_t	efha_debug_sz;
439 } enahw_feat_host_attr_t;
440 
441 /*
442  * ENAHW_FEAT_AENQ_CONFIG
443  *
444  * common: ena_admin_feature_aenq_desc
445  */
446 typedef struct enahw_feat_aenq {
447 	/* Bitmask of AENQ groups this device supports. */
448 	uint32_t efa_supported_groups;
449 
450 	/* Bitmask of AENQ groups currently enabled. */
451 	uint32_t efa_enabled_groups;
452 } enahw_feat_aenq_t;
453 
454 /* common: ena_admin_set_feat_cmd */
455 typedef struct enahw_cmd_set_feat {
456 	struct enahw_ctrl_buff		ecsf_ctrl_buf;
457 	struct enahw_feat_common	ecsf_comm;
458 
459 	union {
460 		uint32_t			ecsf_raw[11];
461 		enahw_feat_host_attr_t		ecsf_host_attr;
462 		enahw_feat_mtu_t		ecsf_mtu;
463 		enahw_feat_aenq_t		ecsf_aenq;
464 	} ecsf_feat;
465 } enahw_cmd_set_feat_t;
466 
467 /*
468  * Used to populate the host information buffer which the Nitro
469  * hypervisor supposedly uses for display, debugging, and possibly
470  * other purposes.
471  *
472  * common: ena_admin_host_info
473  */
474 typedef struct enahw_host_info {
475 	uint32_t	ehi_os_type;
476 	uint8_t		ehi_os_dist_str[128];
477 	uint32_t	ehi_os_dist;
478 	uint8_t		ehi_kernel_ver_str[32];
479 	uint32_t	ehi_kernel_ver;
480 	uint32_t	ehi_driver_ver;
481 	uint32_t	ehi_supported_net_features[2];
482 	uint16_t	ehi_ena_spec_version;
483 	uint16_t	ehi_bdf;
484 	uint16_t	ehi_num_cpus;
485 	uint16_t	ehi_rsvd;
486 	uint32_t	ehi_driver_supported_features;
487 } enahw_host_info_t;
488 
489 #define	ENAHW_HOST_INFO_MAJOR_MASK				GENMASK(7, 0)
490 #define	ENAHW_HOST_INFO_MINOR_SHIFT				8
491 #define	ENAHW_HOST_INFO_MINOR_MASK				GENMASK(15, 8)
492 #define	ENAHW_HOST_INFO_SUB_MINOR_SHIFT				16
493 #define	ENAHW_HOST_INFO_SUB_MINOR_MASK				GENMASK(23, 16)
494 #define	ENAHW_HOST_INFO_SPEC_MAJOR_SHIFT			8
495 #define	ENAHW_HOST_INFO_MODULE_TYPE_SHIFT			24
496 #define	ENAHW_HOST_INFO_MODULE_TYPE_MASK			GENMASK(31, 24)
497 #define	ENAHW_HOST_INFO_FUNCTION_MASK				GENMASK(2, 0)
498 #define	ENAHW_HOST_INFO_DEVICE_SHIFT				3
499 #define	ENAHW_HOST_INFO_DEVICE_MASK				GENMASK(7, 3)
500 #define	ENAHW_HOST_INFO_BUS_SHIFT				8
501 #define	ENAHW_HOST_INFO_BUS_MASK				GENMASK(15, 8)
502 #define	ENAHW_HOST_INFO_RX_OFFSET_SHIFT				1
503 #define	ENAHW_HOST_INFO_RX_OFFSET_MASK				BIT(1)
504 #define	ENAHW_HOST_INFO_INTERRUPT_MODERATION_SHIFT		2
505 #define	ENAHW_HOST_INFO_INTERRUPT_MODERATION_MASK		BIT(2)
506 #define	ENAHW_HOST_INFO_RX_BUF_MIRRORING_SHIFT			3
507 #define	ENAHW_HOST_INFO_RX_BUF_MIRRORING_MASK			BIT(3)
508 #define	ENAHW_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT	4
509 #define	ENAHW_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK	BIT(4)
510 
511 /* common: ena_admin_os_type */
512 enum enahw_os_type {
513 	ENAHW_OS_LINUX		= 1,
514 	ENAHW_OS_WIN		= 2,
515 	ENAHW_OS_DPDK		= 3,
516 	ENAHW_OS_FREEBSD	= 4,
517 	ENAHW_OS_IPXE		= 5,
518 	ENAHW_OS_ESXI		= 6,
519 	ENAHW_OS_MACOS		= 7,
520 	ENAHW_OS_GROUPS_NUM	= 7,
521 };
522 
523 /*
524  * Create I/O Completion Queue
525  *
526  * A completion queue is where the device writes responses to I/O
527  * requests. The admin completion queue must be created before such a
528  * command can be issued, see ena_admin_cq_init().
529  *
530  * common: ena_admin_aq_create_cq_cmd
531  */
532 typedef struct enahw_cmd_create_cq {
533 	/*
534 	 * 7-6	reserved
535 	 *
536 	 * 5	interrupt mode: when set the device sends an interrupt
537 	 *	for each completion, otherwise the driver must poll
538 	 *	the queue.
539 	 *
540 	 * 4-0	reserved
541 	 */
542 	uint8_t		ecq_caps_1;
543 
544 	/*
545 	 * 7-5	reserved
546 	 *
547 	 * 4-0	CQ entry size (in words): the size of a single CQ entry
548 	 *	in multiples of 32-bit words.
549 	 *
550 	 *	NOTE: According to the common code the "valid" values
551 	 *	are 4 or 8 -- this is incorrect. The valid values are
552 	 *	2 and 4. The common code does have an "extended" Rx
553 	 *	completion descriptor, ena_eth_io_rx_cdesc_ext, that
554 	 *	is 32 bytes and thus would use a value of 8, but it is
555 	 *	not used by the Linux or FreeBSD drivers, so we do not
556 	 *	bother with it.
557 	 *
558 	 *	Type			Bytes		Value
559 	 *	enahw_tx_cdesc_t	8		2
560 	 *	enahw_rx_cdesc_t	16		4
561 	 */
562 	uint8_t		ecq_caps_2;
563 
564 	/* The number of CQ entries, must be a power of 2. */
565 	uint16_t	ecq_num_descs;
566 
567 	/* The MSI-X vector assigned to this CQ. */
568 	uint32_t	ecq_msix_vector;
569 
570 	/*
571 	 * The CQ's physical base address. The CQ memory must be
572 	 * physically contiguous.
573 	 */
574 	enahw_addr_t	ecq_addr;
575 } enahw_cmd_create_cq_t;
576 
577 #define	ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_SHIFT	5
578 #define	ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_MASK		(BIT(5))
579 #define	ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS_MASK		(GENMASK(4, 0))
580 
581 #define	ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLE(cmd)	\
582 	((cmd)->ecq_caps_1 |= ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_MASK)
583 
584 #define	ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS(cmd, val)		\
585 	(((cmd)->ecq_caps_2) |=					\
586 	    ((val) & ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS_MASK))
587 
588 /*
589  * Destroy Completion Queue
590  *
591  * common: ena_admin_aq_destroy_cq_cmd
592  */
593 typedef struct enahw_cmd_destroy_cq {
594 	uint16_t	edcq_idx;
595 	uint16_t	edcq_rsvd;
596 } enahw_cmd_destroy_cq_t;
597 
598 /*
599  * common: ena_admin_aq_create_sq_cmd
600  */
601 typedef struct enahw_cmd_create_sq {
602 	/*
603 	 * 7-5	direction: 0x1 = Tx, 0x2 = Rx
604 	 * 4-0	reserved
605 	 */
606 	uint8_t		ecsq_dir;
607 	uint8_t		ecsq_rsvd1;
608 
609 	/*
610 	 * 7	reserved
611 	 *
612 	 * 6-4	completion policy: How are completion events generated.
613 	 *
614 	 *    See enahw_completion_policy_type_t for a description of
615 	 *    the various values.
616 	 *
617 	 * 3-0	placement policy: Where the descriptor ring and
618 	 *			  headers reside.
619 	 *
620 	 *    See enahw_placement_policy_t for a description of the
621 	 *    various values.
622 	 */
623 	uint8_t		ecsq_caps_2;
624 
625 	/*
626 	 * 7-1	reserved
627 	 *
628 	 * 0	physically contiguous: When set indicates the descriptor
629 	 *			       ring memory is physically contiguous.
630 	 */
631 	uint8_t		ecsq_caps_3;
632 
633 	/*
634 	 * The index of the associated Completion Queue (CQ). The CQ
635 	 * must be created before the SQ.
636 	 */
637 	uint16_t	ecsq_cq_idx;
638 
639 	/* The number of descriptors in this SQ. */
640 	uint16_t	ecsq_num_descs;
641 
642 	/*
643 	 * The base physical address of the SQ. This should not be set
644 	 * for LLQ. Must be page aligned.
645 	 */
646 	enahw_addr_t	ecsq_base;
647 
648 	/*
649 	 * The physical address of the head write-back pointer. Valid
650 	 * only when the completion policy is set to one of the head
651 	 * write-back modes (0x2 or 0x3). Must be cacheline size
652 	 * aligned.
653 	 */
654 	enahw_addr_t	ecsq_head_wb;
655 	uint32_t	ecsq_rsvdw2;
656 	uint32_t	ecsq_rsvdw3;
657 } enahw_cmd_create_sq_t;
658 
659 typedef enum enahw_sq_direction {
660 	ENAHW_SQ_DIRECTION_TX = 1,
661 	ENAHW_SQ_DIRECTION_RX = 2,
662 } enahw_sq_direction_t;
663 
664 typedef enum enahw_placement_policy {
665 	/* Descriptors and headers are in host memory. */
666 	ENAHW_PLACEMENT_POLICY_HOST = 1,
667 
668 	/*
669 	 * Descriptors and headers are in device memory (a.k.a Low
670 	 * Latency Queue).
671 	 */
672 	ENAHW_PLACEMENT_POLICY_DEV = 3,
673 } enahw_placement_policy_t;
674 
675 /*
676  * DESC: Write a CQ entry for each SQ descriptor.
677  *
678  * DESC_ON_DEMAND: Write a CQ entry when requested by the SQ descriptor.
679  *
680  * HEAD_ON_DEMAND: Update head pointer when requested by the SQ
681  *		   descriptor.
682  *
683  * HEAD: Update head pointer for each SQ descriptor.
684  *
685  */
686 typedef enum enahw_completion_policy_type {
687 	ENAHW_COMPLETION_POLICY_DESC		= 0,
688 	ENAHW_COMPLETION_POLICY_DESC_ON_DEMAND	= 1,
689 	ENAHW_COMPLETION_POLICY_HEAD_ON_DEMAND	= 2,
690 	ENAHW_COMPLETION_POLICY_HEAD		= 3,
691 } enahw_completion_policy_type_t;
692 
693 #define	ENAHW_CMD_CREATE_SQ_DIR_SHIFT			5
694 #define	ENAHW_CMD_CREATE_SQ_DIR_MASK			GENMASK(7, 5)
695 #define	ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY_MASK	GENMASK(3, 0)
696 #define	ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_SHIFT	4
697 #define	ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_MASK	GENMASK(6, 4)
698 #define	ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG_MASK		BIT(0)
699 
700 #define	ENAHW_CMD_CREATE_SQ_DIR(cmd, val)				\
701 	(((cmd)->ecsq_dir) |= (((val) << ENAHW_CMD_CREATE_SQ_DIR_SHIFT) & \
702 	    ENAHW_CMD_CREATE_SQ_DIR_MASK))
703 
704 #define	ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY(cmd, val)		\
705 	(((cmd)->ecsq_caps_2) |=				\
706 	    ((val) & ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY_MASK))
707 
708 #define	ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY(cmd, val)			\
709 	(((cmd)->ecsq_caps_2) |=					\
710 	    (((val) << ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_SHIFT) &	\
711 		ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_MASK))
712 
713 #define	ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG(cmd)				\
714 	((cmd)->ecsq_caps_3 |= ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG_MASK)
715 
716 /* common: ena_admin_sq */
717 typedef struct enahw_cmd_destroy_sq {
718 	uint16_t	edsq_idx;
719 	uint8_t		edsq_dir; /* Tx/Rx */
720 	uint8_t		edsq_rsvd;
721 } enahw_cmd_destroy_sq_t;
722 
723 #define	ENAHW_CMD_DESTROY_SQ_DIR_SHIFT	5
724 #define	ENAHW_CMD_DESTROY_SQ_DIR_MASK	GENMASK(7, 5)
725 
726 #define	ENAHW_CMD_DESTROY_SQ_DIR(cmd, val)				\
727 	(((cmd)->edsq_dir) |= (((val) << ENAHW_CMD_DESTROY_SQ_DIR_SHIFT) & \
728 	    ENAHW_CMD_DESTROY_SQ_DIR_MASK))
729 
730 /* common: ena_admin_aq_get_stats_cmd */
731 typedef struct enahw_cmd_get_stats {
732 	struct enahw_ctrl_buff	ecgs_ctrl_buf;
733 	uint8_t			ecgs_type;
734 	uint8_t			ecgs_scope;
735 	uint16_t		ecgs_rsvd;
736 	uint16_t		ecgs_queue_idx;
737 
738 	/*
739 	 * The device ID for which to query stats from. The sentinel
740 	 * value 0xFFFF indicates a query of the current device.
741 	 * According to the common docs, a "privileged device" may
742 	 * query stats for other ENA devices. However the definition
743 	 * of this "privilege device" is not expanded upon.
744 	 */
745 	uint16_t		ecgs_device_id;
746 } enahw_cmd_get_stats_t;
747 
748 /* Query the stats for my device. */
749 #define	ENAHW_CMD_GET_STATS_MY_DEVICE_ID	0xFFFF
750 
751 /*
752  * BASIC: Returns enahw_resp_basic_stats.
753  *
754  * EXTENDED: According to the Linux documentation returns a buffer in
755  * "string format" with additional statistics per queue and per device ID.
756  *
757  * ENI: According to the Linux documentation it returns "extra HW
758  * stats for a specific network interfaces".
759  *
760  * common: ena_admin_get_stats_type
761  */
762 typedef enum enahw_get_stats_type {
763 	ENAHW_GET_STATS_TYPE_BASIC	= 0,
764 	ENAHW_GET_STATS_TYPE_EXTENDED	= 1,
765 	ENAHW_GET_STATS_TYPE_ENI	= 2,
766 } enahw_get_stats_type_t;
767 
768 /* common: ena_admin_get_stats_scope */
769 typedef enum enahw_get_stats_scope {
770 	ENAHW_GET_STATS_SCOPE_QUEUE	= 0,
771 	ENAHW_GET_STATS_SCOPE_ETH	= 1,
772 } enahw_get_stats_scope_t;
773 
774 /* common: ena_admin_aq_entry */
775 typedef struct enahw_cmd_desc {
776 	uint16_t	ecd_cmd_id;
777 	uint8_t		ecd_opcode;
778 	uint8_t		ecd_flags;
779 
780 	union {
781 		uint32_t			ecd_raw[15];
782 		enahw_cmd_get_feat_t		ecd_get_feat;
783 		enahw_cmd_set_feat_t		ecd_set_feat;
784 		enahw_cmd_create_cq_t		ecd_create_cq;
785 		enahw_cmd_destroy_cq_t		ecd_destroy_cq;
786 		enahw_cmd_create_sq_t		ecd_create_sq;
787 		enahw_cmd_destroy_sq_t		ecd_destroy_sq;
788 		enahw_cmd_get_stats_t		ecd_get_stats;
789 	} ecd_cmd;
790 
791 } enahw_cmd_desc_t;
792 
793 /*
794  * top level commands that may be sent to the Admin Queue.
795  *
796  * common: ena_admin_aq_opcode
797  */
798 typedef enum ena_cmd_opcode {
799 	ENAHW_CMD_NONE		= 0,
800 	ENAHW_CMD_CREATE_SQ	= 1,
801 	ENAHW_CMD_DESTROY_SQ	= 2,
802 	ENAHW_CMD_CREATE_CQ	= 3,
803 	ENAHW_CMD_DESTROY_CQ	= 4,
804 	ENAHW_CMD_GET_FEATURE	= 8,
805 	ENAHW_CMD_SET_FEATURE	= 9,
806 	ENAHW_CMD_GET_STATS	= 11,
807 } enahw_cmd_opcode_t;
808 
809 /* common: ENA_ADMIN_AQ_COMMON_DESC */
810 #define	ENAHW_CMD_ID_MASK	GENMASK(11, 0)
811 #define	ENAHW_CMD_PHASE_MASK	BIT(0)
812 
813 #define	ENAHW_CMD_ID(desc, id)					\
814 	(((desc)->ecd_cmd_id) |= ((id) & ENAHW_CMD_ID_MASK))
815 
816 /*
817  * Subcommands for ENA_ADMIN_{GET,SET}_FEATURE.
818  *
819  * common: ena_admin_aq_feature_id
820  */
821 typedef enum enahw_feature_id {
822 	ENAHW_FEAT_DEVICE_ATTRIBUTES		= 1,
823 	ENAHW_FEAT_MAX_QUEUES_NUM		= 2,
824 	ENAHW_FEAT_HW_HINTS			= 3,
825 	ENAHW_FEAT_LLQ				= 4,
826 	ENAHW_FEAT_EXTRA_PROPERTIES_STRINGS	= 5,
827 	ENAHW_FEAT_EXTRA_PROPERTIES_FLAGS	= 6,
828 	ENAHW_FEAT_MAX_QUEUES_EXT		= 7,
829 	ENAHW_FEAT_RSS_HASH_FUNCTION		= 10,
830 	ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG	= 11,
831 	ENAHW_FEAT_RSS_INDIRECTION_TABLE_CONFIG	= 12,
832 	ENAHW_FEAT_MTU				= 14,
833 	ENAHW_FEAT_RSS_HASH_INPUT		= 18,
834 	ENAHW_FEAT_INTERRUPT_MODERATION		= 20,
835 	ENAHW_FEAT_AENQ_CONFIG			= 26,
836 	ENAHW_FEAT_LINK_CONFIG			= 27,
837 	ENAHW_FEAT_HOST_ATTR_CONFIG		= 28,
838 	ENAHW_FEAT_NUM				= 32,
839 } enahw_feature_id_t;
840 
841 /*
842  * The following macros define the maximum version we support for each
843  * feature. These are the feature versions we use to communicate with
844  * the feature command. Linux has these values spread throughout the
845  * code at the various callsites of ena_com_get_feature(). We choose
846  * to centralize our feature versions to make it easier to audit.
847  */
848 #define	ENAHW_FEAT_DEVICE_ATTRIBUTES_VER		0
849 #define	ENAHW_FEAT_MAX_QUEUES_NUM_VER			0
850 #define	ENAHW_FEAT_HW_HINTS_VER				0
851 #define	ENAHW_FEAT_LLQ_VER				0
852 #define	ENAHW_FEAT_EXTRA_PROPERTIES_STRINGS_VER		0
853 #define	ENAHW_FEAT_EXTRA_PROPERTIES_FLAGS_VER		0
854 #define	ENAHW_FEAT_MAX_QUEUES_EXT_VER			1
855 #define	ENAHW_FEAT_RSS_HASH_FUNCTION_VER		0
856 #define	ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG_VER		0
857 #define	ENAHW_FEAT_RSS_INDIRECTION_TABLE_CONFIG_VER	0
858 #define	ENAHW_FEAT_MTU_VER				0
859 #define	ENAHW_FEAT_RSS_HASH_INPUT_VER			0
860 #define	ENAHW_FEAT_INTERRUPT_MODERATION_VER		0
861 #define	ENAHW_FEAT_AENQ_CONFIG_VER			0
862 #define	ENAHW_FEAT_LINK_CONFIG_VER			0
863 #define	ENAHW_FEAT_HOST_ATTR_CONFIG_VER			0
864 
865 /* common: ena_admin_link_types */
866 typedef enum enahw_link_speeds {
867 	ENAHW_LINK_SPEED_1G		= 0x1,
868 	ENAHW_LINK_SPEED_2_HALF_G	= 0x2,
869 	ENAHW_LINK_SPEED_5G		= 0x4,
870 	ENAHW_LINK_SPEED_10G		= 0x8,
871 	ENAHW_LINK_SPEED_25G		= 0x10,
872 	ENAHW_LINK_SPEED_40G		= 0x20,
873 	ENAHW_LINK_SPEED_50G		= 0x40,
874 	ENAHW_LINK_SPEED_100G		= 0x80,
875 	ENAHW_LINK_SPEED_200G		= 0x100,
876 	ENAHW_LINK_SPEED_400G		= 0x200,
877 } enahw_link_speeds_t;
878 
879 /*
880  * Response to ENAHW_FEAT_HW_HINTS.
881  *
882  * Hints from the device to the driver about what values to use for
883  * various communications between the two. A value of 0 indicates
884  * there is no hint and the driver should provide its own default. All
885  * timeout values are in milliseconds.
886  *
887  * common: ena_admin_ena_hw_hints
888  */
889 typedef struct enahw_device_hints {
890 	/*
891 	 * The amount of time the driver should wait for an MMIO read
892 	 * reply before giving up and returning an error.
893 	 */
894 	uint16_t edh_mmio_read_timeout;
895 
896 	/*
897 	 * If the driver has not seen an AENQ keep alive in this
898 	 * timeframe, then consider the device hung and perform a
899 	 * reset.
900 	 */
901 	uint16_t edh_keep_alive_timeout;
902 
903 	/*
904 	 * The timeperiod in which we expect a Tx to report
905 	 * completion, otherwise it is considered "missed". Initiate a
906 	 * device reset when the number of missed completions is
907 	 * greater than the threshold.
908 	 */
909 	uint16_t edh_tx_comp_timeout;
910 	uint16_t edh_missed_tx_reset_threshold;
911 
912 	/*
913 	 * The timeperiod in which we expect an admin command to
914 	 * report completion.
915 	 */
916 	uint16_t edh_admin_comp_timeout;
917 
918 	/*
919 	 * Used by Linux to set the netdevice 'watchdog_timeo' value.
920 	 * This value is used by the networking stack to determine
921 	 * when a pending transmission has stalled. This is similar to
922 	 * the keep alive timeout, except its viewing progress from
923 	 * the perspective of the network stack itself. This differnce
924 	 * is subtle but important: the device could be in a state
925 	 * where it has a functioning keep alive heartbeat, but has a
926 	 * stuck Tx queue impeding forward progress of the networking
927 	 * stack (which in many cases results in a scenario
928 	 * indistinguishable form a complete host hang).
929 	 *
930 	 * The mac layer does not currently provide such
931 	 * functionality, though it could and should be extended to
932 	 * support such a feature.
933 	 */
934 	uint16_t edh_net_wd_timeout;
935 
936 	/*
937 	 * The maximum number of cookies/segments allowed in a DMA
938 	 * scatter-gather list.
939 	 */
940 	uint16_t edh_max_tx_sgl;
941 	uint16_t edh_max_rx_sgl;
942 
943 	uint16_t reserved[8];
944 } enahw_device_hints_t;
945 
946 /*
947  * Response to ENAHW_FEAT_DEVICE_ATTRIBUTES.
948  *
949  * common: ena_admin_device_attr_feature_desc
950  */
951 typedef struct enahw_feat_dev_attr {
952 	uint32_t efda_impl_id;
953 	uint32_t efda_device_version;
954 
955 	/*
956 	 * Bitmap representing supported get/set feature subcommands
957 	 * (enahw_feature_id).
958 	 */
959 	uint32_t efda_supported_features;
960 	uint32_t efda_rsvd1;
961 
962 	/* Number of bits used for physical/vritual address. */
963 	uint32_t efda_phys_addr_width;
964 	uint32_t efda_virt_addr_with;
965 
966 	/* The unicast MAC address in network byte order. */
967 	uint8_t efda_mac_addr[6];
968 	uint8_t efda_rsvd2[2];
969 	uint32_t efda_max_mtu;
970 } enahw_feat_dev_attr_t;
971 
972 /*
973  * Response to ENAHW_FEAT_MAX_QUEUES_NUM.
974  *
975  * common: ena_admin_queue_feature_desc
976  */
977 typedef struct enahw_feat_max_queue {
978 	uint32_t efmq_max_sq_num;
979 	uint32_t efmq_max_sq_depth;
980 	uint32_t efmq_max_cq_num;
981 	uint32_t efmq_max_cq_depth;
982 	uint32_t efmq_max_legacy_llq_num;
983 	uint32_t efmq_max_legacy_llq_depth;
984 	uint32_t efmq_max_header_size;
985 
986 	/*
987 	 * The maximum number of descriptors a single Tx packet may
988 	 * span. This includes the meta descriptor.
989 	 */
990 	uint16_t efmq_max_per_packet_tx_descs;
991 
992 	/*
993 	 * The maximum number of descriptors a single Rx packet may span.
994 	 */
995 	uint16_t efmq_max_per_packet_rx_descs;
996 } enahw_feat_max_queue_t;
997 
998 /*
999  * Response to ENAHW_FEAT_MAX_QUEUES_EXT.
1000  *
1001  * common: ena_admin_queue_ext_feature_desc
1002  */
1003 typedef struct enahw_feat_max_queue_ext {
1004 	uint8_t efmqe_version;
1005 	uint8_t	efmqe_rsvd[3];
1006 
1007 	uint32_t efmqe_max_tx_sq_num;
1008 	uint32_t efmqe_max_tx_cq_num;
1009 	uint32_t efmqe_max_rx_sq_num;
1010 	uint32_t efmqe_max_rx_cq_num;
1011 	uint32_t efmqe_max_tx_sq_depth;
1012 	uint32_t efmqe_max_tx_cq_depth;
1013 	uint32_t efmqe_max_rx_sq_depth;
1014 	uint32_t efmqe_max_rx_cq_depth;
1015 	uint32_t efmqe_max_tx_header_size;
1016 
1017 	/*
1018 	 * The maximum number of descriptors a single Tx packet may
1019 	 * span. This includes the meta descriptor.
1020 	 */
1021 	uint16_t efmqe_max_per_packet_tx_descs;
1022 
1023 	/*
1024 	 * The maximum number of descriptors a single Rx packet may span.
1025 	 */
1026 	uint16_t efmqe_max_per_packet_rx_descs;
1027 } enahw_feat_max_queue_ext_t;
1028 
1029 /*
1030  * Response to ENA_ADMIN_LINK_CONFIG.
1031  *
1032  * common: ena_admin_get_feature_link_desc
1033  */
1034 typedef struct enahw_feat_link_conf {
1035 	/* Link speed in Mbit/s. */
1036 	uint32_t eflc_speed;
1037 
1038 	/* Bit field of enahw_link_speeds_t. */
1039 	uint32_t eflc_supported;
1040 
1041 	/*
1042 	 * 31-2:	reserved
1043 	 * 1:		duplex - Full Duplex
1044 	 * 0:		autoneg
1045 	 */
1046 	uint32_t eflc_flags;
1047 } enahw_feat_link_conf_t;
1048 
1049 #define	ENAHW_FEAT_LINK_CONF_AUTONEG_MASK	BIT(0)
1050 #define	ENAHW_FEAT_LINK_CONF_DUPLEX_SHIFT	1
1051 #define	ENAHW_FEAT_LINK_CONF_DUPLEX_MASK	BIT(1)
1052 
1053 #define	ENAHW_FEAT_LINK_CONF_AUTONEG(f)				\
1054 	((f)->eflc_flags & ENAHW_FEAT_LINK_CONF_AUTONEG_MASK)
1055 
1056 #define	ENAHW_FEAT_LINK_CONF_FULL_DUPLEX(f)				\
1057 	((((f)->eflc_flags & ENAHW_FEAT_LINK_CONF_DUPLEX_MASK) >>	\
1058 	    ENAHW_FEAT_LINK_CONF_DUPLEX_SHIFT) == 1)
1059 
1060 /*
1061  * Response to ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG.
1062  *
1063  * common: ena_admin_feature_offload_desc
1064  */
1065 typedef struct enahw_feat_offload {
1066 	/*
1067 	 * 0 : Tx IPv4 Header Checksum
1068 	 * 1 : Tx L4/IPv4 Partial Checksum
1069 	 *
1070 	 *    The L4 checksum field should be initialized with pseudo
1071 	 *    header checksum.
1072 	 *
1073 	 * 2 : Tx L4/IPv4 Checksum Full
1074 	 * 3 : Tx L4/IPv6 Partial Checksum
1075 	 *
1076 	 *    The L4 checksum field should be initialized with pseudo
1077 	 *    header checksum.
1078 	 *
1079 	 * 4 : Tx L4/IPv6 Checksum Full
1080 	 * 5 : TCP/IPv4 LSO (aka TSO)
1081 	 * 6 : TCP/IPv6 LSO (aka TSO)
1082 	 * 7 : LSO ECN
1083 	 */
1084 	uint32_t efo_tx;
1085 
1086 	/*
1087 	 * Receive side supported stateless offload.
1088 	 *
1089 	 * 0 : Rx IPv4 Header Checksum
1090 	 * 1 : Rx TCP/UDP + IPv4 Full Checksum
1091 	 * 2 : Rx TCP/UDP + IPv6 Full Checksum
1092 	 * 3 : Rx hash calculation
1093 	 */
1094 	uint32_t efo_rx_supported;
1095 
1096 	/* Linux seems to only check rx_supported. */
1097 	uint32_t efo_rx_enabled;
1098 } enahw_feat_offload_t;
1099 
1100 /* Feature Offloads */
1101 #define	ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM_MASK		BIT(0)
1102 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_SHIFT	1
1103 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_MASK	BIT(1)
1104 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_SHIFT	2
1105 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_MASK	BIT(2)
1106 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_SHIFT	3
1107 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_MASK	BIT(3)
1108 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_SHIFT	4
1109 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_MASK	BIT(4)
1110 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV4_SHIFT		5
1111 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV4_MASK		BIT(5)
1112 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV6_SHIFT		6
1113 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV6_MASK		BIT(6)
1114 #define	ENAHW_FEAT_OFFLOAD_TSO_ECN_SHIFT		7
1115 #define	ENAHW_FEAT_OFFLOAD_TSO_ECN_MASK			BIT(7)
1116 #define	ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM_MASK		BIT(0)
1117 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_SHIFT	1
1118 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_MASK		BIT(1)
1119 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_SHIFT	2
1120 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_MASK		BIT(2)
1121 #define	ENAHW_FEAT_OFFLOAD_RX_HASH_SHIFT		3
1122 #define	ENAHW_FEAT_OFFLOAD_RX_HASH_MASK			BIT(3)
1123 
1124 #define	ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM(f)				\
1125 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM_MASK) != 0)
1126 
1127 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART(f)			\
1128 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_MASK) != 0)
1129 
1130 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL(f)			\
1131 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_MASK) != 0)
1132 
1133 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV4(f)				\
1134 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TSO_IPV4_MASK) != 0)
1135 
1136 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART(f)		\
1137 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_MASK) != 0)
1138 
1139 #define	ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL(f)		\
1140 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_MASK) != 0)
1141 
1142 #define	ENAHW_FEAT_OFFLOAD_TSO_IPV6(f)				\
1143 	(((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TSO_IPV6_MASK) != 0)
1144 
1145 #define	ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM(f)				\
1146 	(((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM_MASK) != 0)
1147 
1148 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM(f)				\
1149 	(((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_MASK) != 0)
1150 
1151 #define	ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM(f)				\
1152 	(((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_MASK) != 0)
1153 
1154 typedef union enahw_resp_get_feat {
1155 	uint32_t			ergf_raw[14];
1156 	enahw_feat_dev_attr_t		ergf_dev_attr;
1157 	enahw_feat_max_queue_t		ergf_max_queue;
1158 	enahw_feat_max_queue_ext_t	ergf_max_queue_ext;
1159 	enahw_feat_aenq_t		ergf_aenq;
1160 	enahw_feat_link_conf_t		ergf_link_conf;
1161 	enahw_feat_offload_t		ergf_offload;
1162 } enahw_resp_get_feat_u;
1163 
1164 /*
1165  * common: ena_admin_acq_create_cq_resp_desc
1166  */
1167 typedef struct enahw_resp_create_cq {
1168 	/*
1169 	 * The hardware's index for this queue.
1170 	 */
1171 	uint16_t ercq_idx;
1172 
1173 	/*
1174 	 * Apparently the number of descriptors granted may be
1175 	 * different than that requested.
1176 	 */
1177 	uint16_t ercq_actual_num_descs;
1178 	uint32_t ercq_numa_node_reg_offset;
1179 	uint32_t ercq_head_db_reg_offset; /* doorbell */
1180 	uint32_t ercq_interrupt_mask_reg_offset; /* stop intr */
1181 } enahw_resp_create_cq_t;
1182 
1183 /* common: ena_admin_acq_create_sq_resp_desc */
1184 typedef struct enahw_resp_create_sq {
1185 	uint16_t ersq_idx;
1186 	uint16_t ersq_rsvdw1;
1187 	uint32_t ersq_db_reg_offset;
1188 	uint32_t ersq_llq_descs_reg_offset;
1189 	uint32_t ersq_llq_headers_reg_offset;
1190 } enahw_resp_create_sq_t;
1191 
1192 /* common: ena_admin_basic_stats */
1193 typedef struct enahw_resp_basic_stats {
1194 	uint32_t erbs_tx_bytes_low;
1195 	uint32_t erbs_tx_bytes_high;
1196 	uint32_t erbs_tx_pkts_low;
1197 	uint32_t erbs_tx_pkts_high;
1198 	uint32_t erbs_rx_bytes_low;
1199 	uint32_t erbs_rx_bytes_high;
1200 	uint32_t erbs_rx_pkts_low;
1201 	uint32_t erbs_rx_pkts_high;
1202 	uint32_t erbs_rx_drops_low;
1203 	uint32_t erbs_rx_drops_high;
1204 	uint32_t erbs_tx_drops_low;
1205 	uint32_t erbs_tx_drops_high;
1206 } enahw_resp_basic_stats_t;
1207 
1208 /* common: ena_admin_eni_stats */
1209 typedef struct enahw_resp_eni_stats {
1210 	/*
1211 	 * The number of inbound packets dropped due to aggregate
1212 	 * inbound bandwidth allowance being exceeded.
1213 	 */
1214 	uint64_t eres_bw_in_exceeded;
1215 
1216 	/*
1217 	 * The number of outbound packets dropped due to aggregated outbound
1218 	 * bandwidth allowance being exceeded.
1219 	 */
1220 	uint64_t eres_bw_out_exceeded;
1221 
1222 	/*
1223 	 * The number of packets dropped due to the Packets Per Second
1224 	 * allowance being exceeded.
1225 	 */
1226 	uint64_t eres_pps_exceeded;
1227 
1228 	/*
1229 	 * The number of packets dropped due to connection tracking
1230 	 * allowance being exceeded and leading to failure in
1231 	 * establishment of new connections.
1232 	 */
1233 	uint64_t eres_conns_exceeded;
1234 
1235 	/*
1236 	 * The number of packets dropped due to linklocal packet rate
1237 	 * allowance being exceeded.
1238 	 */
1239 	uint64_t eres_linklocal_exceeded;
1240 } enahw_resp_eni_stats_t;
1241 
1242 /*
1243  * common: ena_admin_acq_entry
1244  */
1245 typedef struct enahw_resp_desc {
1246 	/* The index of the completed command. */
1247 	uint16_t	erd_cmd_id;
1248 
1249 	/* The status of the command (enahw_resp_status_t). */
1250 	uint8_t		erd_status;
1251 
1252 	/*
1253 	 * 7-1	Reserved
1254 	 * 0	Phase
1255 	 */
1256 	uint8_t		erd_flags;
1257 
1258 	/* Extended status. */
1259 	uint16_t	erd_ext_status;
1260 
1261 	/*
1262 	 * The AQ entry (enahw_cmd_desc) index which has been consumed
1263 	 * by the device and can be reused. However, this field is not
1264 	 * used in the other drivers, and it seems to be redundant
1265 	 * with the erd_idx field.
1266 	 */
1267 	uint16_t	erd_sq_head_idx;
1268 
1269 	union {
1270 		uint32_t			raw[14];
1271 		enahw_resp_get_feat_u		erd_get_feat;
1272 		enahw_resp_create_cq_t		erd_create_cq;
1273 		/* destroy_cq: No command-specific response. */
1274 		enahw_resp_create_sq_t		erd_create_sq;
1275 		/* destroy_sq: No command-specific response. */
1276 		enahw_resp_basic_stats_t	erd_basic_stats;
1277 		enahw_resp_eni_stats_t		erd_eni_stats;
1278 	} erd_resp;
1279 } enahw_resp_desc_t;
1280 
1281 /* common: ENA_ADMIN_ACQ_COMMON_DESC */
1282 #define	ENAHW_RESP_CMD_ID_MASK	GENMASK(11, 0)
1283 #define	ENAHW_RESP_PHASE_MASK	0x1
1284 
1285 #define	ENAHW_RESP_CMD_ID(desc)				\
1286 	(((desc)->erd_cmd_id) & ENAHW_RESP_CMD_ID_MASK)
1287 
1288 /*
1289  * The response status of an Admin Queue command.
1290  *
1291  * common: ena_admin_aq_completion_status
1292  */
1293 typedef enum enahw_resp_status {
1294 	ENAHW_RESP_SUCCESS			= 0,
1295 	ENAHW_RESP_RESOURCE_ALLOCATION_FAILURE	= 1,
1296 	ENAHW_RESP_BAD_OPCODE			= 2,
1297 	ENAHW_RESP_UNSUPPORTED_OPCODE		= 3,
1298 	ENAHW_RESP_MALFORMED_REQUEST		= 4,
1299 	/*
1300 	 * At this place in the common code it mentions that there is
1301 	 * "additional status" in the reponse descriptor's
1302 	 * erd_ext_status field. As the common code never actually
1303 	 * uses this field it's hard to know the exact meaning of the
1304 	 * comment. My best guess is the illegal parameter error
1305 	 * stores additional context in the erd_ext_status field. But
1306 	 * how to interpret that additional context is anyone's guess.
1307 	 */
1308 	ENAHW_RESP_ILLEGAL_PARAMETER		= 5,
1309 	ENAHW_RESP_UNKNOWN_ERROR		= 6,
1310 	ENAHW_RESP_RESOURCE_BUSY		= 7,
1311 } enahw_resp_status_t;
1312 
1313 /*
1314  * Not really a device structure, more of a helper to debug register values.
1315  */
1316 typedef struct enahw_reg_nv {
1317 	char		*ern_name;
1318 	uint32_t	ern_offset;
1319 	uint32_t	ern_value;
1320 } enahw_reg_nv_t;
1321 
1322 /*
1323  * I/O macros and strcutures.
1324  * -------------------------
1325  */
1326 
1327 /*
1328  * The device's L3 and L4 protocol numbers. These are specific to the
1329  * ENA device and not to be confused with IANA protocol numbers.
1330  *
1331  * common: ena_eth_io_l3_proto_index
1332  */
1333 typedef enum enahw_io_l3_proto {
1334 	ENAHW_IO_L3_PROTO_UNKNOWN	= 0,
1335 	ENAHW_IO_L3_PROTO_IPV4		= 8,
1336 	ENAHW_IO_L3_PROTO_IPV6		= 11,
1337 	ENAHW_IO_L3_PROTO_FCOE		= 21,
1338 	ENAHW_IO_L3_PROTO_ROCE		= 22,
1339 } enahw_io_l3_proto_t;
1340 
1341 /* common: ena_eth_io_l4_proto_index */
1342 typedef enum enahw_io_l4_proto {
1343 	ENAHW_IO_L4_PROTO_UNKNOWN		= 0,
1344 	ENAHW_IO_L4_PROTO_TCP			= 12,
1345 	ENAHW_IO_L4_PROTO_UDP			= 13,
1346 	ENAHW_IO_L4_PROTO_ROUTEABLE_ROCE	= 23,
1347 } enahw_io_l4_proto_t;
1348 
1349 /* common: ena_eth_io_tx_desc */
1350 typedef struct enahw_tx_data_desc {
1351 	/*
1352 	 * 15-0   Buffer Length (LENGTH)
1353 	 *
1354 	 *	The buffer length in bytes. This should NOT include the
1355 	 *	Ethernet FCS bytes.
1356 	 *
1357 	 * 21-16  Request ID High Bits [15-10] (REQ_ID_HI)
1358 	 * 22	  Reserved Zero
1359 	 * 23	  Metadata Flag always zero (META_DESC)
1360 	 *
1361 	 *	This flag indicates if the descriptor is a metadata
1362 	 *	descriptor or not. In this case we are defining the Tx
1363 	 *	descriptor, so it's always zero.
1364 	 *
1365 	 * 24	  Phase bit (PHASE)
1366 	 * 25	  Reserved Zero
1367 	 * 26	  First Descriptor Bit (FIRST)
1368 	 *
1369 	 *	Indicates this is the first descriptor for the frame.
1370 	 *
1371 	 * 27	  Last Descriptor Bit (LAST)
1372 	 *
1373 	 *	Indicates this is the last descriptor for the frame.
1374 	 *
1375 	 * 28	  Completion Request Bit (COMP_REQ)
1376 	 *
1377 	 *	Indicates if completion should be posted after the
1378 	 *	frame is transmitted. This bit is only valid on the
1379 	 *	first descriptor.
1380 	 *
1381 	 * 31-29  Reserved Zero
1382 	 */
1383 	uint32_t etd_len_ctrl;
1384 
1385 	/*
1386 	 * 3-0	  L3 Protocol Number (L3_PROTO_IDX)
1387 	 *
1388 	 *	The L3 protocol type, one of enahw_io_l3_proto_t. This
1389 	 *	field is required when L3_CSUM_EN or TSO_EN is set.
1390 	 *
1391 	 * 4	  Don't Fragment Bit (DF)
1392 	 *
1393 	 *	The value of IPv4 DF. This value must copy the value
1394 	 *	found in the packet's IPv4 header.
1395 	 *
1396 	 * 6-5	  Reserved Zero
1397 	 * 7	  TSO Bit (TSO_EN)
1398 	 *
1399 	 *	Enable TCP Segment Offload.
1400 	 *
1401 	 * 12-8	  L4 Protocol Number (L4_PROTO_IDX)
1402 	 *
1403 	 *	The L4 protocol type, one of enahw_io_l4_proto_t. This
1404 	 *	field is required when L4_CSUM_EN or TSO_EN are
1405 	 *	set.
1406 	 *
1407 	 * 13	  L3 Checksum Offload (L3_CSUM_EN)
1408 	 *
1409 	 *	Enable IPv4 header checksum offload.
1410 	 *
1411 	 * 14	  L4 Checksum Offload (L4_CSUM_EN)
1412 	 *
1413 	 *	Enable TCP/UDP checksum offload.
1414 	 *
1415 	 * 15	  Ethernet FCS Disable (ETHERNET_FCS_DIS)
1416 	 *
1417 	 *	Disable the device's Ethernet Frame Check sequence.
1418 	 *
1419 	 * 16	  Reserved Zero
1420 	 * 17	  L4 Partial Checksum Present (L4_CSUM_PARTIAL)
1421 	 *
1422 	 *	When set it indicates the host has already provided
1423 	 *	the pseudo-header checksum. Otherwise, it is up to the
1424 	 *	device to calculate it.
1425 	 *
1426 	 *	When set and using TSO the host stack must remember
1427 	 *	not to include the TCP segment length in the supplied
1428 	 *	pseudo-header.
1429 	 *
1430 	 *	The host stack should provide the pseudo-header
1431 	 *	checksum when using IPv6 with Routing Headers.
1432 	 *
1433 	 * 21-18  Reserved Zero
1434 	 * 31-22  Request ID Low [9-0] (REQ_ID_LO)
1435 	 */
1436 	uint32_t etd_meta_ctrl;
1437 
1438 	/* The low 32 bits of the buffer address. */
1439 	uint32_t etd_buff_addr_lo;
1440 
1441 	/*
1442 	 * address high and header size
1443 	 *
1444 	 * 15-0	Buffer Address High [47-32] (ADDR_HI)
1445 	 *
1446 	 *	The upper 15 bits of the buffer address.
1447 	 *
1448 	 * 23-16  Reserved Zero
1449 	 * 31-24  Header Length (HEADER_LENGTH)
1450 	 *
1451 	 *	This field has dubious documentation in the
1452 	 *	common/Linux driver code, even contradicting itself in
1453 	 *	the same sentence. Here's what it says, verbatim:
1454 	 *
1455 	 *	> Header length. For Low Latency Queues, this fields
1456 	 *	> indicates the number of bytes written to the
1457 	 *	> headers' memory. For normal queues, if packet is TCP
1458 	 *	> or UDP, and longer than max_header_size, then this
1459 	 *	> field should be set to the sum of L4 header offset
1460 	 *	> and L4 header size(without options), otherwise, this
1461 	 *	> field should be set to 0. For both modes, this field
1462 	 *	> must not exceed the max_header_size. max_header_size
1463 	 *	> value is reported by the Max Queues Feature
1464 	 *	> descriptor
1465 	 *
1466 	 *	Here's what one _might_ ascertain from the above.
1467 	 *
1468 	 *	1. This field should always be set in the case of
1469 	 *	   LLQs/device placement.
1470 	 *
1471 	 *	2. This field must _never_ exceed the max header size
1472 	 *	   as reported by feature detection. In our code this
1473 	 *	   would be efmq_max_header_size for older ENA devices
1474 	 *	   and efmqe_max_tx_header_size for newer ones. One
1475 	 *	   empirical data point from a t3.small (with newer
1476 	 *	   device) is a max Tx header size of 128 bytes.
1477 	 *
1478 	 *	3. If the packet is TCP or UDP, and the packet (or the
1479 	 *	   headers?) is longer than the max header size, then
1480 	 *	   this field should be set to the total header size
1481 	 *	   with the exception of TCP header options.
1482 	 *	   Otherwise, if the packet is not TCP or UDP, or if
1483 	 *	   the packet (or header length?) _does not_ exceed
1484 	 *	   the max header size, then set this value to 0.
1485 	 *
1486 	 *	One might think, based on (3), that when the header
1487 	 *	size exceeds the max this field needs to be set, but
1488 	 *	that contradicts (2), which dictates that the total
1489 	 *	header size can never exceed the max. Sure enough, the
1490 	 *	Linux code drops all packets with headers that exceed
1491 	 *	the max. So in that case it would mean that "and
1492 	 *	longer than max_header_size" is referring to the total
1493 	 *	packet length. So for most workloads, the TCP/UDP
1494 	 *	packets should have this field set, to indicate their
1495 	 *	header length. This matches with Linux, which seems to
1496 	 *	set header length regardless of IP protocol.
1497 	 *
1498 	 *	However, the FreeBSD code tells a different story. In
1499 	 *	it's non-LLQ Tx path it has the following comment,
1500 	 *	verbatim:
1501 	 *
1502 	 *	> header_len is just a hint for the device. Because
1503 	 *	> FreeBSD is not giving us information about packet
1504 	 *	> header length and it is not guaranteed that all
1505 	 *	> packet headers will be in the 1st mbuf, setting
1506 	 *	> header_len to 0 is making the device ignore this
1507 	 *	> value and resolve header on it's own.
1508 	 *
1509 	 *	According to this we can just set the value to zero
1510 	 *	and let the device figure it out. This maps better to
1511 	 *	illumos, where we also allow the header to potentially
1512 	 *	span multiple mblks (though we do have access to the
1513 	 *	header sizes via mac_ether_offload_info_t).
1514 	 *
1515 	 *	The upshot: for now we take advantage of the device's
1516 	 *	ability to determine the header length on its own, at
1517 	 *	the potential cost of some performance (not measured).
1518 	 */
1519 	uint32_t etd_buff_addr_hi_hdr_sz;
1520 } enahw_tx_data_desc_t;
1521 
1522 #define	ENAHW_TX_DESC_LENGTH_MASK		GENMASK(15, 0)
1523 #define	ENAHW_TX_DESC_REQ_ID_HI_SHIFT		16
1524 #define	ENAHW_TX_DESC_REQ_ID_HI_MASK		GENMASK(21, 16)
1525 #define	ENAHW_TX_DESC_META_DESC_SHIFT		23
1526 #define	ENAHW_TX_DESC_META_DESC_MASK		BIT(23)
1527 #define	ENAHW_TX_DESC_PHASE_SHIFT		24
1528 #define	ENAHW_TX_DESC_PHASE_MASK		BIT(24)
1529 #define	ENAHW_TX_DESC_FIRST_SHIFT		26
1530 #define	ENAHW_TX_DESC_FIRST_MASK		BIT(26)
1531 #define	ENAHW_TX_DESC_LAST_SHIFT		27
1532 #define	ENAHW_TX_DESC_LAST_MASK			BIT(27)
1533 #define	ENAHW_TX_DESC_COMP_REQ_SHIFT		28
1534 #define	ENAHW_TX_DESC_COMP_REQ_MASK		BIT(28)
1535 #define	ENAHW_TX_DESC_L3_PROTO_IDX_MASK		GENMASK(3, 0)
1536 #define	ENAHW_TX_DESC_DF_SHIFT			4
1537 #define	ENAHW_TX_DESC_DF_MASK			BIT(4)
1538 #define	ENAHW_TX_DESC_TSO_EN_SHIFT		7
1539 #define	ENAHW_TX_DESC_TSO_EN_MASK		BIT(7)
1540 #define	ENAHW_TX_DESC_L4_PROTO_IDX_SHIFT	8
1541 #define	ENAHW_TX_DESC_L4_PROTO_IDX_MASK		GENMASK(12, 8)
1542 #define	ENAHW_TX_DESC_L3_CSUM_EN_SHIFT		13
1543 #define	ENAHW_TX_DESC_L3_CSUM_EN_MASK		BIT(13)
1544 #define	ENAHW_TX_DESC_L4_CSUM_EN_SHIFT		14
1545 #define	ENAHW_TX_DESC_L4_CSUM_EN_MASK		BIT(14)
1546 #define	ENAHW_TX_DESC_ETHERNET_FCS_DIS_SHIFT	15
1547 #define	ENAHW_TX_DESC_ETHERNET_FCS_DIS_MASK	BIT(15)
1548 #define	ENAHW_TX_DESC_L4_CSUM_PARTIAL_SHIFT	17
1549 #define	ENAHW_TX_DESC_L4_CSUM_PARTIAL_MASK	BIT(17)
1550 #define	ENAHW_TX_DESC_REQ_ID_LO_SHIFT		22
1551 #define	ENAHW_TX_DESC_REQ_ID_LO_MASK		GENMASK(31, 22)
1552 #define	ENAHW_TX_DESC_ADDR_HI_MASK		GENMASK(15, 0)
1553 #define	ENAHW_TX_DESC_HEADER_LENGTH_SHIFT	24
1554 #define	ENAHW_TX_DESC_HEADER_LENGTH_MASK	GENMASK(31, 24)
1555 
1556 #define	ENAHW_TX_DESC_LENGTH(desc, len)					\
1557 	(((desc)->etd_len_ctrl) |= ((len) & ENAHW_TX_DESC_LENGTH_MASK))
1558 
1559 #define	ENAHW_TX_DESC_FIRST_ON(desc)				\
1560 	(((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_FIRST_MASK)
1561 
1562 #define	ENAHW_TX_DESC_FIRST_OFF(desc)				\
1563 	(((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_FIRST_MASK)
1564 
1565 #define	ENAHW_TX_DESC_REQID_HI(desc, reqid)				\
1566 	(((desc)->etd_len_ctrl) |=					\
1567 	    ((((reqid) >> 10) << ENAHW_TX_DESC_REQ_ID_HI_SHIFT) &	\
1568 		ENAHW_TX_DESC_REQ_ID_HI_MASK))
1569 
1570 #define	ENAHW_TX_DESC_REQID_LO(desc, reqid)				\
1571 	(((desc)->etd_meta_ctrl) |=					\
1572 	    (((reqid) << ENAHW_TX_DESC_REQ_ID_LO_SHIFT) &		\
1573 		ENAHW_TX_DESC_REQ_ID_LO_MASK))
1574 
1575 #define	ENAHW_TX_DESC_PHASE(desc, phase)				\
1576 	(((desc)->etd_len_ctrl) |= (((phase) << ENAHW_TX_DESC_PHASE_SHIFT) & \
1577 	    ENAHW_TX_DESC_PHASE_MASK))
1578 
1579 #define	ENAHW_TX_DESC_LAST_ON(desc)				\
1580 	(((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_LAST_MASK)
1581 
1582 #define	ENAHW_TX_DESC_LAST_OFF(desc)				\
1583 	(((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_LAST_MASK)
1584 
1585 #define	ENAHW_TX_DESC_COMP_REQ_ON(desc)				\
1586 	(((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_COMP_REQ_MASK)
1587 
1588 #define	ENAHW_TX_DESC_COMP_REQ_OFF(desc)				\
1589 	(((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_COMP_REQ_MASK)
1590 
1591 #define	ENAHW_TX_DESC_META_DESC_ON(desc)				\
1592 	(((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_META_DESC_MASK)
1593 
1594 #define	ENAHW_TX_DESC_META_DESC_OFF(desc)				\
1595 	(((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_META_DESC_MASK)
1596 
1597 #define	ENAHW_TX_DESC_ADDR_LO(desc, addr)	\
1598 	(((desc)->etd_buff_addr_lo) = (addr))
1599 
1600 #define	ENAHW_TX_DESC_ADDR_HI(desc, addr)				\
1601 	(((desc)->etd_buff_addr_hi_hdr_sz) |=				\
1602 	    (((addr) >> 32) & ENAHW_TX_DESC_ADDR_HI_MASK))
1603 
1604 #define	ENAHW_TX_DESC_HEADER_LENGTH(desc, len)			\
1605 	(((desc)->etd_buff_addr_hi_hdr_sz) |=			\
1606 	    (((len) << ENAHW_TX_DESC_HEADER_LENGTH_SHIFT) &	\
1607 		ENAHW_TX_DESC_HEADER_LENGTH_MASK))
1608 
1609 #define	ENAHW_TX_DESC_DF_ON(desc)				\
1610 	((desc)->etd_meta_ctrl |= ENAHW_TX_DESC_DF_MASK)
1611 
1612 #define	ENAHW_TX_DESC_TSO_OFF(desc)				\
1613 	(((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_TSO_EN_MASK)
1614 
1615 #define	ENAHW_TX_DESC_L3_CSUM_OFF(desc)				\
1616 	(((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L3_CSUM_EN_MASK)
1617 
1618 #define	ENAHW_TX_DESC_L4_CSUM_OFF(desc)				\
1619 	(((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L4_CSUM_EN_MASK)
1620 
1621 #define	ENAHW_TX_DESC_L4_CSUM_PARTIAL_ON(desc)				\
1622 	(((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L4_CSUM_PARTIAL_MASK)
1623 
1624 /* common: ena_eth_io_tx_meta_desc */
1625 typedef struct enahw_tx_meta_desc {
1626 	/*
1627 	 * 9-0	  Request ID Low [9-0] (REQ_ID_LO)
1628 	 * 13-10  Reserved Zero
1629 	 * 14	  Extended Metadata Valid (EXT_VALID)
1630 	 *
1631 	 *	When set this descriptor contains valid extended
1632 	 *	metadata. The extended metadata includes the L3/L4
1633 	 *	length and offset fields as well as the MSS bits. This
1634 	 *	is needed for TSO.
1635 	 *
1636 	 * 15	  Reserved Zero
1637 	 * 19-16  MSS High Bits (MSS_HI)
1638 	 * 20	  Meta Type (ETH_META_TYPE)
1639 	 *
1640 	 *	If enabled this is an extended metadata descriptor.
1641 	 *	This seems redundant with EXT_VALID.
1642 	 *
1643 	 * 21	  Meta Store (META_STORE)
1644 	 *
1645 	 *	Store the extended metadata in the queue cache.
1646 	 *
1647 	 * 22	  Reserved Zero
1648 	 * 23	  Metadata Flag (META_DESC) -- always one
1649 	 * 24	  Phase (PHASE)
1650 	 * 25	  Reserved Zero
1651 	 * 26	  First Descriptor Bit (FIRST)
1652 	 * 27	  Last Descriptor Bit (LAST)
1653 	 * 28	  Completion Request Bit (COMP_REQ)
1654 	 * 31-29  Reserved Zero
1655 	 */
1656 	uint32_t etmd_len_ctrl;
1657 
1658 	/*
1659 	 * 5-0	  Request ID High Bits [15-10] (REQ_ID_HI)
1660 	 * 31-6	  Reserved Zero
1661 	 */
1662 	uint32_t etmd_word1;
1663 
1664 	/*
1665 	 * 7-0	  L3 Header Length (L3_HDR_LEN)
1666 	 * 15:8	  L3 Header Offset (L3_HDR_OFF)
1667 	 * 21:16  L4 Header Length in Words (L4_HDR_LEN_IN_WORDS)
1668 	 *
1669 	 *    Specifies the L4 header length in words. The device
1670 	 *    assumes the L4 header follows directly after the L3
1671 	 *    header and that the L4 offset is equal to L3_HDR_OFF +
1672 	 *    L3_HDR_LEN.
1673 	 *
1674 	 * 31-22  MSS Low Bits (MSS_LO)
1675 	 */
1676 	uint32_t etmd_word2;
1677 	uint32_t etmd_reserved;
1678 } enahw_tx_meta_desc_t;
1679 
1680 /* common: N/A */
1681 typedef union enahw_tx_desc {
1682 	enahw_tx_data_desc_t etd_data;
1683 	enahw_tx_meta_desc_t etd_meta;
1684 } enahw_tx_desc_t;
1685 
1686 /* common: ena_eth_io_tx_cdesc */
1687 typedef struct enahw_tx_cdesc {
1688 	/*
1689 	 * 15-0	  Request ID Bits
1690 	 * 16	  Reserved Zero
1691 	 */
1692 	uint16_t etc_req_id;
1693 
1694 	/*
1695 	 * Presumably the status of the Tx, though the Linux driver
1696 	 * never checks this field.
1697 	 */
1698 	uint8_t etc_status;
1699 
1700 	/*
1701 	 * 0	  Phase
1702 	 * 7-1	  Reserved Zero
1703 	 */
1704 	uint8_t etc_flags;
1705 
1706 	/*
1707 	 * This isn't documented or used in the Linux driver, but
1708 	 * these probably store the submission queue ID and the
1709 	 * submission queue head index.
1710 	 */
1711 	uint16_t etc_sub_qid;
1712 	uint16_t etc_sq_head_idx;
1713 } enahw_tx_cdesc_t;
1714 
1715 #define	ENAHW_TX_CDESC_PHASE_SHIFT	0
1716 #define	ENAHW_TX_CDESC_PHASE_MASK	BIT(0)
1717 
1718 #define	ENAHW_TX_CDESC_GET_PHASE(cdesc)				\
1719 	((cdesc)->etc_flags & ENAHW_TX_CDESC_PHASE_MASK)
1720 
1721 /* common: ena_eth_io_rx_desc */
1722 typedef struct enahw_rx_desc {
1723 	/*
1724 	 * The length of the buffer provided by the host, in bytes.
1725 	 * Use the value of 0 to indicate 64K.
1726 	 */
1727 	uint16_t erd_length;
1728 	uint8_t erd_reserved1;
1729 
1730 	/*
1731 	 * 0	  Phase (PHASE)
1732 	 * 1	  Reserved Zero
1733 	 * 2	  First (FIRST)
1734 	 *
1735 	 *	Indicates this is the first descriptor for the frame.
1736 	 *
1737 	 * 3	  Last (LAST)
1738 	 *
1739 	 *	Indicates this is the last descriptor for the frame.
1740 	 *
1741 	 * 4	  Completion Request (COMP_REQ)
1742 	 *
1743 	 *	Indicates that a completion request should be generated
1744 	 *	for this descriptor.
1745 	 *
1746 	 * 7-5	  Reserved Zero
1747 	 */
1748 	uint8_t erd_ctrl;
1749 
1750 	/*
1751 	 * 15-0	  Request ID
1752 	 * 16	  Reserved 0
1753 	 */
1754 	uint16_t erd_req_id;
1755 	uint16_t erd_reserved2;
1756 
1757 	/* The physical address of the buffer provided by the host. */
1758 	uint32_t erd_buff_addr_lo;
1759 	uint16_t erd_buff_addr_hi;
1760 	uint16_t erd_reserved3;
1761 } enahw_rx_desc_t;
1762 
1763 #define	ENAHW_RX_DESC_PHASE_MASK	BIT(0)
1764 #define	ENAHW_RX_DESC_FIRST_SHIFT	2
1765 #define	ENAHW_RX_DESC_FIRST_MASK	BIT(2)
1766 #define	ENAHW_RX_DESC_LAST_SHIFT	3
1767 #define	ENAHW_RX_DESC_LAST_MASK		BIT(3)
1768 #define	ENAHW_RX_DESC_COMP_REQ_SHIFT	4
1769 #define	ENAHW_RX_DESC_COMP_REQ_MASK	BIT(4)
1770 
1771 #define	ENAHW_RX_DESC_SET_PHASE(desc, val)				\
1772 	((desc)->erd_ctrl |= ((val) & ENAHW_RX_DESC_PHASE_MASK))
1773 
1774 #define	ENAHW_RX_DESC_SET_FIRST(desc)			\
1775 	((desc)->erd_ctrl |= ENAHW_RX_DESC_FIRST_MASK)
1776 
1777 #define	ENAHW_RX_DESC_SET_LAST(desc)			\
1778 	((desc)->erd_ctrl |= ENAHW_RX_DESC_LAST_MASK)
1779 
1780 #define	ENAHW_RX_DESC_SET_COMP_REQ(desc)			\
1781 	((desc)->erd_ctrl |= ENAHW_RX_DESC_COMP_REQ_MASK)
1782 
1783 /*
1784  * Ethernet parsing information is only valid when last == 1.
1785  *
1786  * common: ena_eth_io_rx_cdesc_base
1787  */
1788 typedef struct enahw_rx_cdesc {
1789 	/*
1790 	 * 4-0	  L3 Protocol Number (L3_PROTO)
1791 	 *
1792 	 *	The L3 protocol type, one of enahw_io_l3_proto_t.
1793 	 *
1794 	 * 6-5	  (SRC_VLAN_CNT)
1795 	 * 7	  Reserved Zero
1796 	 * 12-8	  L4 Protocol Number (L4_PROTO)
1797 	 * 13	  L3 Checksum Error (L3_CSUM_ERR)
1798 	 *
1799 	 *	When set either the L3 checksum failed to match or the
1800 	 *	controller didn't attempt to validate the checksum.
1801 	 *	This bit is valid only when L3_PROTO indicates an IPv4
1802 	 *	packet.
1803 	 *
1804 	 * 14	  L4 Checksum Error (L4_CSUM_ERR)
1805 	 *
1806 	 *	When set either the L4 checksum failed to match or the
1807 	 *	controller didn't attempt to validate the checksum.
1808 	 *	This bit is valid only when L4_PROTO indicates a
1809 	 *	TCP/UDP packet, IPV4_FRAG is not set, and
1810 	 *	L4_CSUM_CHECKED is set.
1811 	 *
1812 	 * 15	  IPv4 Fragmented (IPV4_FRAG)
1813 	 * 16	  L4 Checksum Validated (L4_CSUM_CHECKED)
1814 	 *
1815 	 *	When set it indicates the device attempted to validate
1816 	 *	the L4 checksum.
1817 	 *
1818 	 * 23-17  Reserved Zero
1819 	 * 24	  Phase (PHASE)
1820 	 * 25	  (L3_CSUM2)
1821 	 *
1822 	 *	According to the Linux source this is the "second
1823 	 *	checksum engine result". It's never checked.
1824 	 *
1825 	 * 26	  First Descriptor Bit (FIRST)
1826 	 *
1827 	 *	Indicates the first descriptor for the frame.
1828 	 *
1829 	 * 27	  Last Descriptor Bit (LAST)
1830 	 *
1831 	 *	Indicates the last descriptor for the frame.
1832 	 *
1833 	 * 29-28  Reserved Zero
1834 	 * 30	  Buffer Type (BUFFER)
1835 	 *
1836 	 *	When enabled indicates this is a data descriptor.
1837 	 *	Otherwse, it is a metadata descriptor.
1838 	 *
1839 	 * 31 : reserved31
1840 	 */
1841 	uint32_t erc_status;
1842 	uint16_t erc_length;
1843 	uint16_t erc_req_id;
1844 
1845 	/* 32-bit hash result */
1846 	uint32_t erc_hash;
1847 	uint16_t erc_sub_qid;
1848 
1849 	/*
1850 	 * The device may choose to offset the start of the header
1851 	 * data (which implies this value only applies to the first
1852 	 * descriptor). When and why the device does this is not
1853 	 * documented in the common code. The most likely case would
1854 	 * be for IP header alignment.
1855 	 */
1856 	uint8_t erc_offset;
1857 	uint8_t erc_reserved;
1858 } enahw_rx_cdesc_t;
1859 
1860 #define	ENAHW_RX_CDESC_L3_PROTO_MASK		GENMASK(4, 0)
1861 #define	ENAHW_RX_CDESC_SRC_VLAN_CNT_SHIFT	5
1862 #define	ENAHW_RX_CDESC_SRC_VLAN_CNT_MASK	GENMASK(6, 5)
1863 #define	ENAHW_RX_CDESC_L4_PROTO_SHIFT		8
1864 #define	ENAHW_RX_CDESC_L4_PROTO_MASK		GENMASK(12, 8)
1865 #define	ENAHW_RX_CDESC_L3_CSUM_ERR_SHIFT	13
1866 #define	ENAHW_RX_CDESC_L3_CSUM_ERR_MASK		BIT(13)
1867 #define	ENAHW_RX_CDESC_L4_CSUM_ERR_SHIFT	14
1868 #define	ENAHW_RX_CDESC_L4_CSUM_ERR_MASK		BIT(14)
1869 #define	ENAHW_RX_CDESC_IPV4_FRAG_SHIFT		15
1870 #define	ENAHW_RX_CDESC_IPV4_FRAG_MASK		BIT(15)
1871 #define	ENAHW_RX_CDESC_L4_CSUM_CHECKED_SHIFT	16
1872 #define	ENAHW_RX_CDESC_L4_CSUM_CHECKED_MASK	BIT(16)
1873 #define	ENAHW_RX_CDESC_PHASE_SHIFT		24
1874 #define	ENAHW_RX_CDESC_PHASE_MASK		BIT(24)
1875 #define	ENAHW_RX_CDESC_L3_CSUM2_SHIFT		25
1876 #define	ENAHW_RX_CDESC_L3_CSUM2_MASK		BIT(25)
1877 #define	ENAHW_RX_CDESC_FIRST_SHIFT		26
1878 #define	ENAHW_RX_CDESC_FIRST_MASK		BIT(26)
1879 #define	ENAHW_RX_CDESC_LAST_SHIFT		27
1880 #define	ENAHW_RX_CDESC_LAST_MASK		BIT(27)
1881 #define	ENAHW_RX_CDESC_BUFFER_SHIFT		30
1882 #define	ENAHW_RX_CDESC_BUFFER_MASK		BIT(30)
1883 
1884 #define	ENAHW_RX_CDESC_L3_PROTO(desc)				\
1885 	((desc)->erc_status & ENAHW_RX_CDESC_L3_PROTO_MASK)
1886 
1887 #define	ENAHW_RX_CDESC_L3_CSUM_ERR(desc)				\
1888 	((((desc)->erc_status & ENAHW_RX_CDESC_L3_CSUM_ERR_MASK) >>	\
1889 	    ENAHW_RX_CDESC_L3_CSUM_ERR_SHIFT) != 0)
1890 
1891 #define	ENAHW_RX_CDESC_L4_PROTO(desc)				\
1892 	(((desc)->erc_status & ENAHW_RX_CDESC_L4_PROTO_MASK) >>	\
1893 	    ENAHW_RX_CDESC_L4_PROTO_SHIFT)
1894 
1895 #define	ENAHW_RX_CDESC_L4_CSUM_CHECKED(desc)				\
1896 	((((desc)->erc_status & ENAHW_RX_CDESC_L4_CSUM_CHECKED_MASK) >>	\
1897 	    ENAHW_RX_CDESC_L4_CSUM_CHECKED_SHIFT) != 0)
1898 
1899 #define	ENAHW_RX_CDESC_L4_CSUM_ERR(desc)				\
1900 	((((desc)->erc_status & ENAHW_RX_CDESC_L4_CSUM_ERR_MASK) >>	\
1901 	    ENAHW_RX_CDESC_L4_CSUM_ERR_SHIFT) != 0)
1902 
1903 #define	ENAHW_RX_CDESC_PHASE(desc)			 \
1904 	(((desc)->erc_status & ENAHW_RX_CDESC_PHASE_MASK) >> \
1905 	    ENAHW_RX_CDESC_PHASE_SHIFT)
1906 
1907 #define	ENAHW_RX_CDESC_FIRST(desc)			 \
1908 	((((desc)->erc_status & ENAHW_RX_CDESC_FIRST_MASK) >> \
1909 	    ENAHW_RX_CDESC_FIRST_SHIFT) == 1)
1910 
1911 #define	ENAHW_RX_CDESC_LAST(desc)			 \
1912 	((((desc)->erc_status & ENAHW_RX_CDESC_LAST_MASK) >> \
1913 	    ENAHW_RX_CDESC_LAST_SHIFT) == 1)
1914 
1915 /*
1916  * Controls for the interrupt register mapped to each Rx/Tx CQ.
1917  */
1918 #define	ENAHW_REG_INTR_RX_DELAY_MASK	GENMASK(14, 0)
1919 #define	ENAHW_REG_INTR_TX_DELAY_SHIFT	15
1920 #define	ENAHW_REG_INTR_TX_DELAY_MASK	GENMASK(29, 15)
1921 #define	ENAHW_REG_INTR_UNMASK_SHIFT	30
1922 #define	ENAHW_REG_INTR_UNMASK_MASK	BIT(30)
1923 
1924 #define	ENAHW_REG_INTR_UNMASK(val)		\
1925 	((val) |= ENAHW_REG_INTR_UNMASK_MASK)
1926 
1927 #define	ENAHW_REG_INTR_MASK(val)		\
1928 	((val) &= ~ENAHW_REG_INTR_UNMASK_MASK)
1929 
1930 #endif	/* _ENA_HW_H */
1931