1 /*
2  * Copyright (c) 2007-2015 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  *
30  * Ackowledgement to Fen Systems Ltd.
31  */
32 
33 #ifndef	_SYS_EFX_TYPES_H
34 #define	_SYS_EFX_TYPES_H
35 
36 #include "efsys.h"
37 
38 #ifdef	__cplusplus
39 extern "C" {
40 #endif
41 
42 /*
43  * Bitfield access
44  *
45  * Solarflare NICs make extensive use of bitfields up to 128 bits
46  * wide.  Since there is no native 128-bit datatype on most systems,
47  * and since 64-bit datatypes are inefficient on 32-bit systems and
48  * vice versa, we wrap accesses in a way that uses the most efficient
49  * datatype.
50  *
51  * The NICs are PCI devices and therefore little-endian.  Since most
52  * of the quantities that we deal with are DMAed to/from host memory,
53  * we define	our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
54  * to be little-endian.
55  *
56  * In the less common case of using PIO for individual register
57  * writes, we construct the little-endian datatype in host memory and
58  * then use non-swapping register access primitives, rather than
59  * constructing a native-endian datatype and relying on implicit
60  * byte-swapping.  (We use a similar strategy for register reads.)
61  */
62 
63 /*
64  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
65  *       bit number (LBN) and a width.
66  */
67 
68 #define	EFX_DUMMY_FIELD_LBN 0
69 #define	EFX_DUMMY_FIELD_WIDTH 0
70 
71 #define	EFX_BYTE_0_LBN 0
72 #define	EFX_BYTE_0_WIDTH 8
73 
74 #define	EFX_BYTE_1_LBN 8
75 #define	EFX_BYTE_1_WIDTH 8
76 
77 #define	EFX_BYTE_2_LBN 16
78 #define	EFX_BYTE_2_WIDTH 8
79 
80 #define	EFX_BYTE_3_LBN 24
81 #define	EFX_BYTE_3_WIDTH 8
82 
83 #define	EFX_BYTE_4_LBN 32
84 #define	EFX_BYTE_4_WIDTH 8
85 
86 #define	EFX_BYTE_5_LBN 40
87 #define	EFX_BYTE_5_WIDTH 8
88 
89 #define	EFX_BYTE_6_LBN 48
90 #define	EFX_BYTE_6_WIDTH 8
91 
92 #define	EFX_BYTE_7_LBN 56
93 #define	EFX_BYTE_7_WIDTH 8
94 
95 #define	EFX_WORD_0_LBN 0
96 #define	EFX_WORD_0_WIDTH 16
97 
98 #define	EFX_WORD_1_LBN 16
99 #define	EFX_WORD_1_WIDTH 16
100 
101 #define	EFX_WORD_2_LBN 32
102 #define	EFX_WORD_2_WIDTH 16
103 
104 #define	EFX_WORD_3_LBN 48
105 #define	EFX_WORD_3_WIDTH 16
106 
107 #define	EFX_DWORD_0_LBN 0
108 #define	EFX_DWORD_0_WIDTH 32
109 
110 #define	EFX_DWORD_1_LBN 32
111 #define	EFX_DWORD_1_WIDTH 32
112 
113 #define	EFX_DWORD_2_LBN 64
114 #define	EFX_DWORD_2_WIDTH 32
115 
116 #define	EFX_DWORD_3_LBN 96
117 #define	EFX_DWORD_3_WIDTH 32
118 
119 /* There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
120  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
121  * support field widths larger than 32 bits.
122  */
123 
124 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
125 #define	EFX_VAL(_field, _attribute)					\
126 	_field ## _ ## _attribute
127 
128 /* Lowest bit number of the specified field */
129 #define	EFX_LOW_BIT(_field)						\
130 	EFX_VAL(_field, LBN)
131 
132 /* Width of the specified field */
133 #define	EFX_WIDTH(_field)						\
134 	EFX_VAL(_field, WIDTH)
135 
136 /* Highest bit number of the specified field */
137 #define	EFX_HIGH_BIT(_field)						\
138 	(EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
139 
140 /*
141  * 64-bit mask equal in width to the specified field.
142  *
143  * For example, a field with width 5 would have a mask of 0x000000000000001f.
144  */
145 #define	EFX_MASK64(_field)						\
146 	((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :			\
147 	    (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
148 /*
149  * 32-bit mask equal in width to the specified field.
150  *
151  * For example, a field with width 5 would have a mask of 0x0000001f.
152  */
153 #define	EFX_MASK32(_field)						\
154 	((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :			\
155 	    (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
156 
157 /*
158  * 16-bit mask equal in width to the specified field.
159  *
160  * For example, a field with width 5 would have a mask of 0x001f.
161  */
162 #define	EFX_MASK16(_field)						\
163 	((EFX_WIDTH(_field) == 16) ? 0xffffu :				\
164 	    (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
165 
166 /*
167  * 8-bit mask equal in width to the specified field.
168  *
169  * For example, a field with width 5 would have a mask of 0x1f.
170  */
171 #define	EFX_MASK8(_field)						\
172 	((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
173 
174 #pragma pack(1)
175 
176 /*
177  * A byte (i.e. 8-bit) datatype
178  */
179 typedef union efx_byte_u {
180 	uint8_t eb_u8[1];
181 } efx_byte_t;
182 
183 /*
184  * A word (i.e. 16-bit) datatype
185  *
186  * This datatype is defined to be little-endian.
187  */
188 typedef union efx_word_u {
189 	efx_byte_t ew_byte[2];
190 	uint16_t ew_u16[1];
191 	uint8_t ew_u8[2];
192 } efx_word_t;
193 
194 /*
195  * A doubleword (i.e. 32-bit) datatype
196  *
197  * This datatype is defined to be little-endian.
198  */
199 typedef union efx_dword_u {
200 	efx_byte_t ed_byte[4];
201 	efx_word_t ed_word[2];
202 	uint32_t ed_u32[1];
203 	uint16_t ed_u16[2];
204 	uint8_t ed_u8[4];
205 } efx_dword_t;
206 
207 /*
208  * A quadword (i.e. 64-bit) datatype
209  *
210  * This datatype is defined to be little-endian.
211  */
212 typedef union efx_qword_u {
213 	efx_byte_t eq_byte[8];
214 	efx_word_t eq_word[4];
215 	efx_dword_t eq_dword[2];
216 #if EFSYS_HAS_UINT64
217 	uint64_t eq_u64[1];
218 #endif
219 	uint32_t eq_u32[2];
220 	uint16_t eq_u16[4];
221 	uint8_t eq_u8[8];
222 } efx_qword_t;
223 
224 /*
225  * An octword (i.e. 128-bit) datatype
226  *
227  * This datatype is defined to be little-endian.
228  */
229 typedef union efx_oword_u {
230 	efx_byte_t eo_byte[16];
231 	efx_word_t eo_word[8];
232 	efx_dword_t eo_dword[4];
233 	efx_qword_t eo_qword[2];
234 #if EFSYS_HAS_SSE2_M128
235 	__m128i eo_u128[1];
236 #endif
237 #if EFSYS_HAS_UINT64
238 	uint64_t eo_u64[2];
239 #endif
240 	uint32_t eo_u32[4];
241 	uint16_t eo_u16[8];
242 	uint8_t eo_u8[16];
243 } efx_oword_t;
244 
245 #pragma pack()
246 
247 #define	__SWAP16(_x) 				\
248 	((((_x) & 0xff) << 8) |			\
249 	(((_x) >> 8) & 0xff))
250 
251 #define	__SWAP32(_x)				\
252 	((__SWAP16((_x) & 0xffff) << 16) |	\
253 	__SWAP16(((_x) >> 16) & 0xffff))
254 
255 #define	__SWAP64(_x)				\
256 	((__SWAP32((_x) & 0xffffffff) << 32) |	\
257 	__SWAP32(((_x) >> 32) & 0xffffffff))
258 
259 #define	__NOSWAP16(_x)		(_x)
260 #define	__NOSWAP32(_x)		(_x)
261 #define	__NOSWAP64(_x)		(_x)
262 
263 #if EFSYS_IS_BIG_ENDIAN
264 
265 #define	__CPU_TO_LE_16(_x)	(uint16_t)__SWAP16(_x)
266 #define	__LE_TO_CPU_16(_x)	(uint16_t)__SWAP16(_x)
267 #define	__CPU_TO_BE_16(_x)	(uint16_t)__NOSWAP16(_x)
268 #define	__BE_TO_CPU_16(_x)	(uint16_t)__NOSWAP16(_x)
269 
270 #define	__CPU_TO_LE_32(_x)	(uint32_t)__SWAP32(_x)
271 #define	__LE_TO_CPU_32(_x)	(uint32_t)__SWAP32(_x)
272 #define	__CPU_TO_BE_32(_x)	(uint32_t)__NOSWAP32(_x)
273 #define	__BE_TO_CPU_32(_x)	(uint32_t)__NOSWAP32(_x)
274 
275 #define	__CPU_TO_LE_64(_x)	(uint64_t)__SWAP64(_x)
276 #define	__LE_TO_CPU_64(_x)	(uint64_t)__SWAP64(_x)
277 #define	__CPU_TO_BE_64(_x)	(uint64_t)__NOSWAP64(_x)
278 #define	__BE_TO_CPU_64(_x)	(uint64_t)__NOSWAP64(_x)
279 
280 #elif EFSYS_IS_LITTLE_ENDIAN
281 
282 #define	__CPU_TO_LE_16(_x)	(uint16_t)__NOSWAP16(_x)
283 #define	__LE_TO_CPU_16(_x)	(uint16_t)__NOSWAP16(_x)
284 #define	__CPU_TO_BE_16(_x)	(uint16_t)__SWAP16(_x)
285 #define	__BE_TO_CPU_16(_x)	(uint16_t)__SWAP16(_x)
286 
287 #define	__CPU_TO_LE_32(_x)	(uint32_t)__NOSWAP32(_x)
288 #define	__LE_TO_CPU_32(_x)	(uint32_t)__NOSWAP32(_x)
289 #define	__CPU_TO_BE_32(_x)	(uint32_t)__SWAP32(_x)
290 #define	__BE_TO_CPU_32(_x)	(uint32_t)__SWAP32(_x)
291 
292 #define	__CPU_TO_LE_64(_x)	(uint64_t)__NOSWAP64(_x)
293 #define	__LE_TO_CPU_64(_x)	(uint64_t)__NOSWAP64(_x)
294 #define	__CPU_TO_BE_64(_x)	(uint64_t)__SWAP64(_x)
295 #define	__BE_TO_CPU_64(_x)	(uint64_t)__SWAP64(_x)
296 
297 #else
298 
299 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
300 
301 #endif
302 
303 #define	__NATIVE_8(_x)	(uint8_t)(_x)
304 
305 /* Format string for printing an efx_byte_t */
306 #define	EFX_BYTE_FMT "0x%02x"
307 
308 /* Format string for printing an efx_word_t */
309 #define	EFX_WORD_FMT "0x%04x"
310 
311 /* Format string for printing an efx_dword_t */
312 #define	EFX_DWORD_FMT "0x%08x"
313 
314 /* Format string for printing an efx_qword_t */
315 #define	EFX_QWORD_FMT "0x%08x:%08x"
316 
317 /* Format string for printing an efx_oword_t */
318 #define	EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
319 
320 /* Parameters for printing an efx_byte_t */
321 #define	EFX_BYTE_VAL(_byte)					\
322 	((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
323 
324 /* Parameters for printing an efx_word_t */
325 #define	EFX_WORD_VAL(_word)					\
326 	((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
327 
328 /* Parameters for printing an efx_dword_t */
329 #define	EFX_DWORD_VAL(_dword)					\
330 	((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
331 
332 /* Parameters for printing an efx_qword_t */
333 #define	EFX_QWORD_VAL(_qword)					\
334 	((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),	\
335 	((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
336 
337 /* Parameters for printing an efx_oword_t */
338 #define	EFX_OWORD_VAL(_oword)					\
339 	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),	\
340 	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),	\
341 	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),	\
342 	((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
343 
344 /*
345  * Stop lint complaining about some shifts.
346  */
347 #ifdef	__lint
348 extern int fix_lint;
349 #define	FIX_LINT(_x)	(_x + fix_lint)
350 #else
351 #define	FIX_LINT(_x)	(_x)
352 #endif
353 
354 /*
355  * Extract bit field portion [low,high) from the native-endian element
356  * which contains bits [min,max).
357  *
358  * For example, suppose "element" represents the high 32 bits of a
359  * 64-bit value, and we wish to extract the bits belonging to the bit
360  * field occupying bits 28-45 of this 64-bit value.
361  *
362  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
363  *
364  *   (_element) << 4
365  *
366  * The result will contain the relevant bits filled in in the range
367  * [0,high-low), with garbage in bits [high-low+1,...).
368  */
369 #define	EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)		\
370 	((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?		\
371 		0U :							\
372 		((_low > _min) ?					\
373 			((_element) >> (_low - _min)) :			\
374 			((_element) << (_min - _low))))
375 
376 /*
377  * Extract bit field portion [low,high) from the 64-bit little-endian
378  * element which contains bits [min,max)
379  */
380 #define	EFX_EXTRACT64(_element, _min, _max, _low, _high)		\
381 	EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
382 
383 /*
384  * Extract bit field portion [low,high) from the 32-bit little-endian
385  * element which contains bits [min,max)
386  */
387 #define	EFX_EXTRACT32(_element, _min, _max, _low, _high)		\
388 	EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
389 
390 /*
391  * Extract bit field portion [low,high) from the 16-bit little-endian
392  * element which contains bits [min,max)
393  */
394 #define	EFX_EXTRACT16(_element, _min, _max, _low, _high)		\
395 	EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
396 
397 /*
398  * Extract bit field portion [low,high) from the 8-bit
399  * element which contains bits [min,max)
400  */
401 #define	EFX_EXTRACT8(_element, _min, _max, _low, _high)			\
402 	EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
403 
404 #define	EFX_EXTRACT_OWORD64(_oword, _low, _high)			\
405 	(EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),	\
406 	    _low, _high) |						\
407 	EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),	\
408 	    _low, _high))
409 
410 #define	EFX_EXTRACT_OWORD32(_oword, _low, _high)			\
411 	(EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),	\
412 	    _low, _high) |						\
413 	EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),	\
414 	    _low, _high) |						\
415 	EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),	\
416 	    _low, _high) |						\
417 	EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),	\
418 	    _low, _high))
419 
420 #define	EFX_EXTRACT_QWORD64(_qword, _low, _high)			\
421 	(EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),	\
422 	    _low, _high))
423 
424 #define	EFX_EXTRACT_QWORD32(_qword, _low, _high)			\
425 	(EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),	\
426 	    _low, _high) |						\
427 	EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),	\
428 	    _low, _high))
429 
430 #define	EFX_EXTRACT_DWORD(_dword, _low, _high)				\
431 	(EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),	\
432 	    _low, _high))
433 
434 #define	EFX_EXTRACT_WORD(_word, _low, _high)				\
435 	(EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),	\
436 	    _low, _high))
437 
438 #define	EFX_EXTRACT_BYTE(_byte, _low, _high)				\
439 	(EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),	\
440 	    _low, _high))
441 
442 
443 #define	EFX_OWORD_FIELD64(_oword, _field)				\
444 	((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),	\
445 	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
446 
447 #define	EFX_OWORD_FIELD32(_oword, _field)				\
448 	(EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),		\
449 	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
450 
451 #define	EFX_QWORD_FIELD64(_qword, _field)				\
452 	((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),	\
453 	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
454 
455 #define	EFX_QWORD_FIELD32(_qword, _field)				\
456 	(EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),		\
457 	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
458 
459 #define	EFX_DWORD_FIELD(_dword, _field)					\
460 	(EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),			\
461 	    EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
462 
463 #define	EFX_WORD_FIELD(_word, _field)					\
464 	(EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),			\
465 	    EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
466 
467 #define	EFX_BYTE_FIELD(_byte, _field)					\
468 	(EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),			\
469 	    EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
470 
471 
472 #define	EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)			\
473 	((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&		\
474 	    (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
475 
476 #define	EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)			\
477 	((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&		\
478 	    (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&		\
479 	    (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&		\
480 	    (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
481 
482 #define	EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)			\
483 	((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
484 
485 #define	EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)			\
486 	((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&		\
487 	    (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
488 
489 #define	EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)				\
490 	((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
491 
492 #define	EFX_WORD_IS_EQUAL(_word_a, _word_b)				\
493 	((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
494 
495 #define	EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)				\
496 	((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
497 
498 
499 #define	EFX_OWORD_IS_ZERO64(_oword)					\
500 	(((_oword).eo_u64[0] |						\
501 	    (_oword).eo_u64[1]) == 0)
502 
503 #define	EFX_OWORD_IS_ZERO32(_oword)					\
504 	(((_oword).eo_u32[0] |						\
505 	    (_oword).eo_u32[1] |					\
506 	    (_oword).eo_u32[2] |					\
507 	    (_oword).eo_u32[3]) == 0)
508 
509 #define	EFX_QWORD_IS_ZERO64(_qword)					\
510 	(((_qword).eq_u64[0]) == 0)
511 
512 #define	EFX_QWORD_IS_ZERO32(_qword)					\
513 	(((_qword).eq_u32[0] |						\
514 	    (_qword).eq_u32[1]) == 0)
515 
516 #define	EFX_DWORD_IS_ZERO(_dword)					\
517 	(((_dword).ed_u32[0]) == 0)
518 
519 #define	EFX_WORD_IS_ZERO(_word)						\
520 	(((_word).ew_u16[0]) == 0)
521 
522 #define	EFX_BYTE_IS_ZERO(_byte)						\
523 	(((_byte).eb_u8[0]) == 0)
524 
525 
526 #define	EFX_OWORD_IS_SET64(_oword)					\
527 	(((_oword).eo_u64[0] &						\
528 	    (_oword).eo_u64[1]) == ~((uint64_t)0))
529 
530 #define	EFX_OWORD_IS_SET32(_oword)					\
531 	(((_oword).eo_u32[0] &						\
532 	    (_oword).eo_u32[1] &					\
533 	    (_oword).eo_u32[2] &					\
534 	    (_oword).eo_u32[3]) == ~((uint32_t)0))
535 
536 #define	EFX_QWORD_IS_SET64(_qword)					\
537 	(((_qword).eq_u64[0]) == ~((uint64_t)0))
538 
539 #define	EFX_QWORD_IS_SET32(_qword)					\
540 	(((_qword).eq_u32[0] &						\
541 	    (_qword).eq_u32[1]) == ~((uint32_t)0))
542 
543 #define	EFX_DWORD_IS_SET(_dword)					\
544 	((_dword).ed_u32[0] == ~((uint32_t)0))
545 
546 #define	EFX_WORD_IS_SET(_word)						\
547 	((_word).ew_u16[0] == ~((uint16_t)0))
548 
549 #define	EFX_BYTE_IS_SET(_byte)						\
550 	((_byte).eb_u8[0] == ~((uint8_t)0))
551 
552 /*
553  * Construct bit field portion
554  *
555  * Creates the portion of the bit field [low,high) that lies within
556  * the range [min,max).
557  */
558 
559 #define	EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)		\
560 	(((_low > _max) || (_high < _min)) ?				\
561 		0U :							\
562 		((_low > _min) ?					\
563 			(((uint64_t)(_value)) << (_low - _min)) :	\
564 			(((uint64_t)(_value)) >> (_min - _low))))
565 
566 #define	EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)		\
567 	(((_low > _max) || (_high < _min)) ?				\
568 		0U :							\
569 		((_low > _min) ?					\
570 			(((uint32_t)(_value)) << (_low - _min)) :	\
571 			(((uint32_t)(_value)) >> (_min - _low))))
572 
573 #define	EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)		\
574 	(((_low > _max) || (_high < _min)) ?				\
575 		0U :							\
576 		(uint16_t)((_low > _min) ?				\
577 				((_value) << (_low - _min)) :		\
578 				((_value) >> (_min - _low))))
579 
580 #define	EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)		\
581 	(((_low > _max) || (_high < _min)) ?				\
582 		0U :							\
583 		(uint8_t)((_low > _min) ?				\
584 				((_value) << (_low - _min)) :	\
585 				((_value) >> (_min - _low))))
586 
587 /*
588  * Construct bit field portion
589  *
590  * Creates the portion of the named bit field that lies within the
591  * range [min,max).
592  */
593 #define	EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)		\
594 	EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),		\
595 	    EFX_HIGH_BIT(_field), _value)
596 
597 #define	EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)		\
598 	EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),		\
599 	    EFX_HIGH_BIT(_field), _value)
600 
601 #define	EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)		\
602 	EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),		\
603 	    EFX_HIGH_BIT(_field), _value)
604 
605 #define	EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)		\
606 	EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),		\
607 	    EFX_HIGH_BIT(_field), _value)
608 
609 /*
610  * Construct bit field
611  *
612  * Creates the portion of the named bit fields that lie within the
613  * range [min,max).
614  */
615 #define	EFX_INSERT_FIELDS64(_min, _max,					\
616 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
617 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
618 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
619 	    _field10, _value10)						\
620 	__CPU_TO_LE_64(							\
621 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |	\
622 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |	\
623 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |	\
624 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |	\
625 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |	\
626 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |	\
627 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |	\
628 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |	\
629 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |	\
630 	    EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
631 
632 #define	EFX_INSERT_FIELDS32(_min, _max,					\
633 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
634 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
635 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
636 	    _field10, _value10)						\
637 	__CPU_TO_LE_32(							\
638 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |	\
639 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |	\
640 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |	\
641 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |	\
642 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |	\
643 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |	\
644 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |	\
645 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |	\
646 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |	\
647 	    EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
648 
649 #define	EFX_INSERT_FIELDS16(_min, _max,					\
650 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
651 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
652 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
653 	    _field10, _value10)						\
654 	__CPU_TO_LE_16(							\
655 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |	\
656 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |	\
657 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |	\
658 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |	\
659 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |	\
660 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |	\
661 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |	\
662 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |	\
663 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |	\
664 	    EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
665 
666 #define	EFX_INSERT_FIELDS8(_min, _max,					\
667 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
668 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
669 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
670 	    _field10, _value10)						\
671 	__NATIVE_8(							\
672 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |	\
673 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |	\
674 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |	\
675 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |	\
676 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |	\
677 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |	\
678 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |	\
679 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |	\
680 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |	\
681 	    EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
682 
683 #define	EFX_POPULATE_OWORD64(_oword,					\
684 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
685 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
686 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
687 	    _field10, _value10)						\
688 	do {								\
689 		_NOTE(CONSTANTCONDITION) 				\
690 		(_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,		\
691 		    _field1, _value1, _field2, _value2,			\
692 		    _field3, _value3, _field4, _value4,			\
693 		    _field5, _value5, _field6, _value6,			\
694 		    _field7, _value7, _field8, _value8,			\
695 		    _field9, _value9, _field10, _value10);		\
696 		_NOTE(CONSTANTCONDITION) 				\
697 		(_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,	\
698 		    _field1, _value1, _field2, _value2,			\
699 		    _field3, _value3, _field4, _value4,			\
700 		    _field5, _value5, _field6, _value6,			\
701 		    _field7, _value7, _field8, _value8,			\
702 		    _field9, _value9, _field10, _value10);		\
703 	_NOTE(CONSTANTCONDITION) 					\
704 	} while (B_FALSE)
705 
706 #define	EFX_POPULATE_OWORD32(_oword,					\
707 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
708 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
709 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
710 	    _field10, _value10)						\
711 	do {								\
712 		_NOTE(CONSTANTCONDITION) 				\
713 		(_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
714 		    _field1, _value1, _field2, _value2,			\
715 		    _field3, _value3, _field4, _value4,			\
716 		    _field5, _value5, _field6, _value6,			\
717 		    _field7, _value7, _field8, _value8,			\
718 		    _field9, _value9, _field10, _value10);		\
719 		_NOTE(CONSTANTCONDITION) 				\
720 		(_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,	\
721 		    _field1, _value1, _field2, _value2,			\
722 		    _field3, _value3, _field4, _value4,			\
723 		    _field5, _value5, _field6, _value6,			\
724 		    _field7, _value7, _field8, _value8,			\
725 		    _field9, _value9, _field10, _value10);		\
726 		_NOTE(CONSTANTCONDITION) 				\
727 		(_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,	\
728 		    _field1, _value1, _field2, _value2,			\
729 		    _field3, _value3, _field4, _value4,			\
730 		    _field5, _value5, _field6, _value6,			\
731 		    _field7, _value7, _field8, _value8,			\
732 		    _field9, _value9, _field10, _value10);		\
733 		_NOTE(CONSTANTCONDITION) 				\
734 		(_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,	\
735 		    _field1, _value1, _field2, _value2,			\
736 		    _field3, _value3, _field4, _value4,			\
737 		    _field5, _value5, _field6, _value6,			\
738 		    _field7, _value7, _field8, _value8,			\
739 		    _field9, _value9, _field10, _value10);		\
740 	_NOTE(CONSTANTCONDITION) 					\
741 	} while (B_FALSE)
742 
743 #define	EFX_POPULATE_QWORD64(_qword,					\
744 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
745 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
746 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
747 	    _field10, _value10)						\
748 	do {								\
749 		_NOTE(CONSTANTCONDITION) 				\
750 		(_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,		\
751 		    _field1, _value1, _field2, _value2,			\
752 		    _field3, _value3, _field4, _value4,			\
753 		    _field5, _value5, _field6, _value6,			\
754 		    _field7, _value7, _field8, _value8,			\
755 		    _field9, _value9, _field10, _value10);		\
756 	_NOTE(CONSTANTCONDITION) 					\
757 	} while (B_FALSE)
758 
759 #define	EFX_POPULATE_QWORD32(_qword,					\
760 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
761 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
762 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
763 	    _field10, _value10)						\
764 	do {								\
765 		_NOTE(CONSTANTCONDITION) 				\
766 		(_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
767 		    _field1, _value1, _field2, _value2,			\
768 		    _field3, _value3, _field4, _value4,			\
769 		    _field5, _value5, _field6, _value6,			\
770 		    _field7, _value7, _field8, _value8,			\
771 		    _field9, _value9, _field10, _value10);		\
772 		_NOTE(CONSTANTCONDITION) 				\
773 		(_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,	\
774 		    _field1, _value1, _field2, _value2,			\
775 		    _field3, _value3, _field4, _value4,			\
776 		    _field5, _value5, _field6, _value6,			\
777 		    _field7, _value7, _field8, _value8,			\
778 		    _field9, _value9, _field10, _value10);		\
779 	_NOTE(CONSTANTCONDITION) 					\
780 	} while (B_FALSE)
781 
782 #define	EFX_POPULATE_DWORD(_dword,					\
783 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
784 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
785 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
786 	    _field10, _value10)						\
787 	do {								\
788 		_NOTE(CONSTANTCONDITION) 				\
789 		(_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,		\
790 		    _field1, _value1, _field2, _value2,			\
791 		    _field3, _value3, _field4, _value4,			\
792 		    _field5, _value5, _field6, _value6,			\
793 		    _field7, _value7, _field8, _value8,			\
794 		    _field9, _value9, _field10, _value10);		\
795 	_NOTE(CONSTANTCONDITION)					\
796 	} while (B_FALSE)
797 
798 #define	EFX_POPULATE_WORD(_word,					\
799 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
800 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
801 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
802 	    _field10, _value10)						\
803 	do {								\
804 		_NOTE(CONSTANTCONDITION) 				\
805 		(_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,		\
806 		    _field1, _value1, _field2, _value2,			\
807 		    _field3, _value3, _field4, _value4,			\
808 		    _field5, _value5, _field6, _value6,			\
809 		    _field7, _value7, _field8, _value8,			\
810 		    _field9, _value9, _field10, _value10);		\
811 	_NOTE(CONSTANTCONDITION)					\
812 	} while (B_FALSE)
813 
814 #define	EFX_POPULATE_BYTE(_byte,					\
815 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
816 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
817 	    _field7, _value7, _field8, _value8,	_field9, _value9,	\
818 	    _field10, _value10)						\
819 	do {								\
820 		_NOTE(CONSTANTCONDITION) 				\
821 		(_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,		\
822 		    _field1, _value1, _field2, _value2,			\
823 		    _field3, _value3, _field4, _value4,			\
824 		    _field5, _value5, _field6, _value6,			\
825 		    _field7, _value7, _field8, _value8,			\
826 		    _field9, _value9, _field10, _value10);		\
827 	_NOTE(CONSTANTCONDITION)					\
828 	} while (B_FALSE)
829 
830 /* Populate an octword field with various numbers of arguments */
831 #define	EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
832 
833 #define	EFX_POPULATE_OWORD_9(_oword,					\
834 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
835 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
836 	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
837 	EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,		\
838 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
839 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
840 	    _field7, _value7, _field8, _value8,	_field9, _value9)
841 
842 #define	EFX_POPULATE_OWORD_8(_oword,					\
843 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
844 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
845 	    _field7, _value7, _field8, _value8)				\
846 	EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,		\
847 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
848 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
849 	    _field7, _value7, _field8, _value8)
850 
851 #define	EFX_POPULATE_OWORD_7(_oword,					\
852 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
853 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
854 	    _field7, _value7)						\
855 	EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,		\
856 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
857 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
858 	    _field7, _value7)
859 
860 #define	EFX_POPULATE_OWORD_6(_oword,					\
861 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
862 	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
863 	EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,		\
864 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
865 	    _field4, _value4, _field5, _value5,	_field6, _value6)
866 
867 #define	EFX_POPULATE_OWORD_5(_oword,					\
868 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
869 	    _field4, _value4, _field5, _value5)				\
870 	EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,		\
871 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
872 	    _field4, _value4, _field5, _value5)
873 
874 #define	EFX_POPULATE_OWORD_4(_oword,					\
875 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
876 	    _field4, _value4)						\
877 	EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,		\
878 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
879 	    _field4, _value4)
880 
881 #define	EFX_POPULATE_OWORD_3(_oword,					\
882 	    _field1, _value1, _field2, _value2, _field3, _value3)	\
883 	EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,		\
884 	    _field1, _value1, _field2, _value2, _field3, _value3)
885 
886 #define	EFX_POPULATE_OWORD_2(_oword,					\
887 	    _field1, _value1, _field2, _value2)				\
888 	EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,		\
889 	    _field1, _value1, _field2, _value2)
890 
891 #define	EFX_POPULATE_OWORD_1(_oword,					\
892 	    _field1, _value1)						\
893 	EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,		\
894 	    _field1, _value1)
895 
896 #define	EFX_ZERO_OWORD(_oword)						\
897 	EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
898 
899 #define	EFX_SET_OWORD(_oword)						\
900 	EFX_POPULATE_OWORD_4(_oword,					\
901 	    EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,		\
902 	    EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
903 
904 /* Populate a quadword field with various numbers of arguments */
905 #define	EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
906 
907 #define	EFX_POPULATE_QWORD_9(_qword,					\
908 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
909 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
910 	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
911 	EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,		\
912 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
913 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
914 	    _field7, _value7, _field8, _value8,	_field9, _value9)
915 
916 #define	EFX_POPULATE_QWORD_8(_qword,					\
917 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
918 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
919 	    _field7, _value7, _field8, _value8)				\
920 	EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,		\
921 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
922 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
923 	    _field7, _value7, _field8, _value8)
924 
925 #define	EFX_POPULATE_QWORD_7(_qword,					\
926 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
927 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
928 	    _field7, _value7)						\
929 	EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,		\
930 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
931 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
932 	    _field7, _value7)
933 
934 #define	EFX_POPULATE_QWORD_6(_qword,					\
935 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
936 	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
937 	EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,		\
938 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
939 	    _field4, _value4, _field5, _value5,	_field6, _value6)
940 
941 #define	EFX_POPULATE_QWORD_5(_qword,					\
942 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
943 	    _field4, _value4, _field5, _value5)				\
944 	EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,		\
945 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
946 	    _field4, _value4, _field5, _value5)
947 
948 #define	EFX_POPULATE_QWORD_4(_qword,					\
949 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
950 	    _field4, _value4)						\
951 	EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,		\
952 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
953 	    _field4, _value4)
954 
955 #define	EFX_POPULATE_QWORD_3(_qword,					\
956 	    _field1, _value1, _field2, _value2, _field3, _value3)	\
957 	EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,		\
958 	    _field1, _value1, _field2, _value2, _field3, _value3)
959 
960 #define	EFX_POPULATE_QWORD_2(_qword,					\
961 	    _field1, _value1, _field2, _value2)				\
962 	EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,		\
963 	    _field1, _value1, _field2, _value2)
964 
965 #define	EFX_POPULATE_QWORD_1(_qword,					\
966 	    _field1, _value1)						\
967 	EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,		\
968 	    _field1, _value1)
969 
970 #define	EFX_ZERO_QWORD(_qword)						\
971 	EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
972 
973 #define	EFX_SET_QWORD(_qword)						\
974 	EFX_POPULATE_QWORD_2(_qword,					\
975 	    EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
976 
977 /* Populate a dword field with various numbers of arguments */
978 #define	EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
979 
980 #define	EFX_POPULATE_DWORD_9(_dword,					\
981 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
982 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
983 	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
984 	EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,		\
985 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
986 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
987 	    _field7, _value7, _field8, _value8,	_field9, _value9)
988 
989 #define	EFX_POPULATE_DWORD_8(_dword,					\
990 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
991 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
992 	    _field7, _value7, _field8, _value8)				\
993 	EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,		\
994 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
995 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
996 	    _field7, _value7, _field8, _value8)
997 
998 #define	EFX_POPULATE_DWORD_7(_dword,					\
999 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1000 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1001 	    _field7, _value7)						\
1002 	EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,		\
1003 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1004 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1005 	    _field7, _value7)
1006 
1007 #define	EFX_POPULATE_DWORD_6(_dword,					\
1008 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1009 	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
1010 	EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,		\
1011 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1012 	    _field4, _value4, _field5, _value5,	_field6, _value6)
1013 
1014 #define	EFX_POPULATE_DWORD_5(_dword,					\
1015 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1016 	    _field4, _value4, _field5, _value5)				\
1017 	EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,		\
1018 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1019 	    _field4, _value4, _field5, _value5)
1020 
1021 #define	EFX_POPULATE_DWORD_4(_dword,					\
1022 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1023 	    _field4, _value4)						\
1024 	EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,		\
1025 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1026 	    _field4, _value4)
1027 
1028 #define	EFX_POPULATE_DWORD_3(_dword,					\
1029 	    _field1, _value1, _field2, _value2, _field3, _value3)	\
1030 	EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,		\
1031 	    _field1, _value1, _field2, _value2, _field3, _value3)
1032 
1033 #define	EFX_POPULATE_DWORD_2(_dword,					\
1034 	    _field1, _value1, _field2, _value2)				\
1035 	EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,		\
1036 	    _field1, _value1, _field2, _value2)
1037 
1038 #define	EFX_POPULATE_DWORD_1(_dword,					\
1039 	    _field1, _value1)						\
1040 	EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,		\
1041 	    _field1, _value1)
1042 
1043 #define	EFX_ZERO_DWORD(_dword)						\
1044 	EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1045 
1046 #define	EFX_SET_DWORD(_dword)						\
1047 	EFX_POPULATE_DWORD_1(_dword,					\
1048 	    EFX_DWORD_0, 0xffffffff)
1049 
1050 /* Populate a word field with various numbers of arguments */
1051 #define	EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1052 
1053 #define	EFX_POPULATE_WORD_9(_word,					\
1054 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1055 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1056 	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
1057 	EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,			\
1058 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1059 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1060 	    _field7, _value7, _field8, _value8,	_field9, _value9)
1061 
1062 #define	EFX_POPULATE_WORD_8(_word,					\
1063 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1064 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1065 	    _field7, _value7, _field8, _value8)				\
1066 	EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,			\
1067 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1068 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1069 	    _field7, _value7, _field8, _value8)
1070 
1071 #define	EFX_POPULATE_WORD_7(_word,					\
1072 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1073 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1074 	    _field7, _value7)						\
1075 	EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,			\
1076 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1077 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1078 	    _field7, _value7)
1079 
1080 #define	EFX_POPULATE_WORD_6(_word,					\
1081 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1082 	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
1083 	EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,			\
1084 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1085 	    _field4, _value4, _field5, _value5,	_field6, _value6)
1086 
1087 #define	EFX_POPULATE_WORD_5(_word,					\
1088 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1089 	    _field4, _value4, _field5, _value5)				\
1090 	EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,			\
1091 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1092 	    _field4, _value4, _field5, _value5)
1093 
1094 #define	EFX_POPULATE_WORD_4(_word,					\
1095 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1096 	    _field4, _value4)						\
1097 	EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,			\
1098 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1099 	    _field4, _value4)
1100 
1101 #define	EFX_POPULATE_WORD_3(_word,					\
1102 	    _field1, _value1, _field2, _value2, _field3, _value3)	\
1103 	EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,			\
1104 	    _field1, _value1, _field2, _value2, _field3, _value3)
1105 
1106 #define	EFX_POPULATE_WORD_2(_word,					\
1107 	    _field1, _value1, _field2, _value2)				\
1108 	EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,			\
1109 	    _field1, _value1, _field2, _value2)
1110 
1111 #define	EFX_POPULATE_WORD_1(_word,					\
1112 	    _field1, _value1)						\
1113 	EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,			\
1114 	    _field1, _value1)
1115 
1116 #define	EFX_ZERO_WORD(_word)						\
1117 	EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1118 
1119 #define	EFX_SET_WORD(_word)						\
1120 	EFX_POPULATE_WORD_1(_word,					\
1121 	    EFX_WORD_0, 0xffff)
1122 
1123 /* Populate a byte field with various numbers of arguments */
1124 #define	EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1125 
1126 #define	EFX_POPULATE_BYTE_9(_byte,					\
1127 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1128 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1129 	    _field7, _value7, _field8, _value8,	_field9, _value9)	\
1130 	EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,			\
1131 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1132 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1133 	    _field7, _value7, _field8, _value8,	_field9, _value9)
1134 
1135 #define	EFX_POPULATE_BYTE_8(_byte,					\
1136 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1137 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1138 	    _field7, _value7, _field8, _value8)				\
1139 	EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,			\
1140 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1141 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1142 	    _field7, _value7, _field8, _value8)
1143 
1144 #define	EFX_POPULATE_BYTE_7(_byte,					\
1145 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1146 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1147 	    _field7, _value7)						\
1148 	EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,			\
1149 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1150 	    _field4, _value4, _field5, _value5,	_field6, _value6,	\
1151 	    _field7, _value7)
1152 
1153 #define	EFX_POPULATE_BYTE_6(_byte,					\
1154 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1155 	    _field4, _value4, _field5, _value5,	_field6, _value6)	\
1156 	EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,			\
1157 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1158 	    _field4, _value4, _field5, _value5,	_field6, _value6)
1159 
1160 #define	EFX_POPULATE_BYTE_5(_byte,					\
1161 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1162 	    _field4, _value4, _field5, _value5)				\
1163 	EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,			\
1164 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1165 	    _field4, _value4, _field5, _value5)
1166 
1167 #define	EFX_POPULATE_BYTE_4(_byte,					\
1168 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1169 	    _field4, _value4)						\
1170 	EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,			\
1171 	    _field1, _value1, _field2, _value2, _field3, _value3,	\
1172 	    _field4, _value4)
1173 
1174 #define	EFX_POPULATE_BYTE_3(_byte,					\
1175 	    _field1, _value1, _field2, _value2, _field3, _value3)	\
1176 	EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,			\
1177 	    _field1, _value1, _field2, _value2, _field3, _value3)
1178 
1179 #define	EFX_POPULATE_BYTE_2(_byte,					\
1180 	    _field1, _value1, _field2, _value2)				\
1181 	EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,			\
1182 	    _field1, _value1, _field2, _value2)
1183 
1184 #define	EFX_POPULATE_BYTE_1(_byte,					\
1185 	    _field1, _value1)						\
1186 	EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,			\
1187 	    _field1, _value1)
1188 
1189 #define	EFX_ZERO_BYTE(_byte)						\
1190 	EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1191 
1192 #define	EFX_SET_BYTE(_byte)						\
1193 	EFX_POPULATE_BYTE_1(_byte,					\
1194 	    EFX_BYTE_0, 0xff)
1195 
1196 /*
1197  * Modify a named field within an already-populated structure.  Used
1198  * for read-modify-write operations.
1199  */
1200 
1201 #define	EFX_INSERT_FIELD64(_min, _max, _field, _value)			\
1202 	__CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1203 
1204 #define	EFX_INSERT_FIELD32(_min, _max, _field, _value)			\
1205 	__CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1206 
1207 #define	EFX_INSERT_FIELD16(_min, _max, _field, _value)			\
1208 	__CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1209 
1210 #define	EFX_INSERT_FIELD8(_min, _max, _field, _value)			\
1211 	__NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1212 
1213 #define	EFX_INPLACE_MASK64(_min, _max, _field)			       	\
1214 	EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1215 
1216 #define	EFX_INPLACE_MASK32(_min, _max, _field)			       	\
1217 	EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1218 
1219 #define	EFX_INPLACE_MASK16(_min, _max, _field)			       	\
1220 	EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1221 
1222 #define	EFX_INPLACE_MASK8(_min, _max, _field)			       	\
1223 	EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1224 
1225 #define	EFX_SET_OWORD_FIELD64(_oword, _field, _value)			\
1226 	do {								\
1227 		_NOTE(CONSTANTCONDITION) 				\
1228 		(_oword).eo_u64[0] = (((_oword).eo_u64[0] &		\
1229 		    ~EFX_INPLACE_MASK64(0, 63, _field)) |		\
1230 		    EFX_INSERT_FIELD64(0, 63, _field, _value));		\
1231 		_NOTE(CONSTANTCONDITION) 				\
1232 		(_oword).eo_u64[1] = (((_oword).eo_u64[1] &		\
1233 		    ~EFX_INPLACE_MASK64(64, 127, _field)) |		\
1234 		    EFX_INSERT_FIELD64(64, 127, _field, _value));	\
1235 	_NOTE(CONSTANTCONDITION) 					\
1236 	} while (B_FALSE)
1237 
1238 #define	EFX_SET_OWORD_FIELD32(_oword, _field, _value)			\
1239 	do {								\
1240 		_NOTE(CONSTANTCONDITION) 				\
1241 		(_oword).eo_u32[0] = (((_oword).eo_u32[0] &		\
1242 		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
1243 		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
1244 		_NOTE(CONSTANTCONDITION) 				\
1245 		(_oword).eo_u32[1] = (((_oword).eo_u32[1] &		\
1246 		    ~EFX_INPLACE_MASK32(32, 63, _field)) |		\
1247 		    EFX_INSERT_FIELD32(32, 63, _field, _value));	\
1248 		_NOTE(CONSTANTCONDITION) 				\
1249 		(_oword).eo_u32[2] = (((_oword).eo_u32[2] &		\
1250 		    ~EFX_INPLACE_MASK32(64, 95, _field)) |		\
1251 		    EFX_INSERT_FIELD32(64, 95, _field, _value));	\
1252 		_NOTE(CONSTANTCONDITION) 				\
1253 		(_oword).eo_u32[3] = (((_oword).eo_u32[3] &		\
1254 		    ~EFX_INPLACE_MASK32(96, 127, _field)) |		\
1255 		    EFX_INSERT_FIELD32(96, 127, _field, _value));	\
1256 	_NOTE(CONSTANTCONDITION) 					\
1257 	} while (B_FALSE)
1258 
1259 #define	EFX_SET_QWORD_FIELD64(_qword, _field, _value)			\
1260 	do {								\
1261 		_NOTE(CONSTANTCONDITION) 				\
1262 		(_qword).eq_u64[0] = (((_qword).eq_u64[0] &		\
1263 		    ~EFX_INPLACE_MASK64(0, 63, _field)) |		\
1264 		    EFX_INSERT_FIELD64(0, 63, _field, _value));		\
1265 	_NOTE(CONSTANTCONDITION) 					\
1266 	} while (B_FALSE)
1267 
1268 #define	EFX_SET_QWORD_FIELD32(_qword, _field, _value)			\
1269 	do {								\
1270 		_NOTE(CONSTANTCONDITION) 				\
1271 		(_qword).eq_u32[0] = (((_qword).eq_u32[0] &		\
1272 		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
1273 		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
1274 		_NOTE(CONSTANTCONDITION) 				\
1275 		(_qword).eq_u32[1] = (((_qword).eq_u32[1] &		\
1276 		    ~EFX_INPLACE_MASK32(32, 63, _field)) |		\
1277 		    EFX_INSERT_FIELD32(32, 63, _field, _value));	\
1278 	_NOTE(CONSTANTCONDITION) 					\
1279 	} while (B_FALSE)
1280 
1281 #define	EFX_SET_DWORD_FIELD(_dword, _field, _value)			\
1282 	do {								\
1283 		_NOTE(CONSTANTCONDITION) 				\
1284 		(_dword).ed_u32[0] = (((_dword).ed_u32[0] &		\
1285 		    ~EFX_INPLACE_MASK32(0, 31, _field)) |		\
1286 		    EFX_INSERT_FIELD32(0, 31, _field, _value));		\
1287 	_NOTE(CONSTANTCONDITION) 					\
1288 	} while (B_FALSE)
1289 
1290 #define	EFX_SET_WORD_FIELD(_word, _field, _value)			\
1291 	do {								\
1292 		_NOTE(CONSTANTCONDITION) 				\
1293 		(_word).ew_u16[0] = (((_word).ew_u16[0] &		\
1294 		    ~EFX_INPLACE_MASK16(0, 15, _field)) |		\
1295 		    EFX_INSERT_FIELD16(0, 15, _field, _value));		\
1296 	_NOTE(CONSTANTCONDITION) 					\
1297 	} while (B_FALSE)
1298 
1299 #define	EFX_SET_BYTE_FIELD(_byte, _field, _value)			\
1300 	do {								\
1301 		_NOTE(CONSTANTCONDITION) 				\
1302 		(_byte).eb_u8[0] = (((_byte).eb_u8[0] &			\
1303 		    ~EFX_INPLACE_MASK8(0, 7, _field)) |			\
1304 		    EFX_INSERT_FIELD8(0, 7, _field, _value));		\
1305 	_NOTE(CONSTANTCONDITION) 					\
1306 	} while (B_FALSE)
1307 
1308 /*
1309  * Set or clear a numbered bit within an octword.
1310  */
1311 
1312 #define	EFX_SHIFT64_0(_bit)						\
1313 	(((_bit) < 64) ? ((uint64_t)1 << (_bit)) : 0U)
1314 
1315 #define	EFX_SHIFT64(_bit, _base)					\
1316 	(((_bit) >= (_base) && (_bit) < (_base) + 64) ?			\
1317 		((uint64_t)1 << ((_bit) - (_base))) :			\
1318 		0U)
1319 
1320 #define	EFX_SHIFT32_0(_bit)						\
1321 	((_bit) < (32) ? ((uint32_t)1 << (_bit)) : 0U)
1322 
1323 #define	EFX_SHIFT32(_bit, _base)					\
1324 	(((_bit) >= (_base) && (_bit) < (_base) + 32) ?			\
1325 		((uint32_t)1 << ((_bit) - (_base))) :			\
1326 		0U)
1327 
1328 #define	EFX_SHIFT16(_bit, _base)					\
1329 	(((_bit) >= (_base) && (_bit) < (_base) + 16) ?			\
1330 		(uint16_t)(1 << ((_bit) - (_base))) :			\
1331 		0U)
1332 
1333 #define	EFX_SHIFT8(_bit, _base)						\
1334 	(((_bit) >= (_base) && (_bit) < (_base) + 8) ?			\
1335 		(uint8_t)(1 << ((_bit) - (_base))) :			\
1336 		0U)
1337 
1338 #define	EFX_SET_OWORD_BIT64(_oword, _bit)				\
1339 	do {								\
1340 		_NOTE(CONSTANTCONDITION) 				\
1341 		(_oword).eo_u64[0] |=					\
1342 		    __CPU_TO_LE_64(EFX_SHIFT64_0(_bit));		\
1343 		(_oword).eo_u64[1] |=					\
1344 		    __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));	\
1345 	_NOTE(CONSTANTCONDITION) 					\
1346 	} while (B_FALSE)
1347 
1348 #define	EFX_SET_OWORD_BIT32(_oword, _bit)				\
1349 	do {								\
1350 		_NOTE(CONSTANTCONDITION) 				\
1351 		(_oword).eo_u32[0] |=					\
1352 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit));		\
1353 		(_oword).eo_u32[1] |=					\
1354 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));	\
1355 		(_oword).eo_u32[2] |=					\
1356 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));	\
1357 		(_oword).eo_u32[3] |=					\
1358 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));	\
1359 	_NOTE(CONSTANTCONDITION) 					\
1360 	} while (B_FALSE)
1361 
1362 #define	EFX_CLEAR_OWORD_BIT64(_oword, _bit)				\
1363 	do {								\
1364 		_NOTE(CONSTANTCONDITION) 				\
1365 		(_oword).eo_u64[0] &=					\
1366 		    __CPU_TO_LE_64(~EFX_SHIFT64_0(_bit));		\
1367 		(_oword).eo_u64[1] &=					\
1368 		    __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));	\
1369 	_NOTE(CONSTANTCONDITION) 					\
1370 	} while (B_FALSE)
1371 
1372 #define	EFX_CLEAR_OWORD_BIT32(_oword, _bit)				\
1373 	do {								\
1374 		_NOTE(CONSTANTCONDITION) 				\
1375 		(_oword).eo_u32[0] &=					\
1376 		    __CPU_TO_LE_32(~EFX_SHIFT32_0(_bit));		\
1377 		(_oword).eo_u32[1] &=					\
1378 		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));	\
1379 		(_oword).eo_u32[2] &=					\
1380 		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));	\
1381 		(_oword).eo_u32[3] &=					\
1382 		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));	\
1383 	_NOTE(CONSTANTCONDITION) 					\
1384 	} while (B_FALSE)
1385 
1386 #define	EFX_TEST_OWORD_BIT64(_oword, _bit)				\
1387 	(((_oword).eo_u64[0] &						\
1388 		    __CPU_TO_LE_64(EFX_SHIFT64_0(_bit))) ||		\
1389 	((_oword).eo_u64[1] &						\
1390 		    __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1391 
1392 #define	EFX_TEST_OWORD_BIT32(_oword, _bit)				\
1393 	(((_oword).eo_u32[0] &						\
1394 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit))) ||		\
1395 	((_oword).eo_u32[1] &						\
1396 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) ||	\
1397 	((_oword).eo_u32[2] &						\
1398 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) ||	\
1399 	((_oword).eo_u32[3] &						\
1400 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1401 
1402 
1403 #define	EFX_SET_QWORD_BIT64(_qword, _bit)				\
1404 	do {								\
1405 		_NOTE(CONSTANTCONDITION) 				\
1406 		(_qword).eq_u64[0] |=					\
1407 		    __CPU_TO_LE_64(EFX_SHIFT64_0(_bit));		\
1408 	_NOTE(CONSTANTCONDITION) 					\
1409 	} while (B_FALSE)
1410 
1411 #define	EFX_SET_QWORD_BIT32(_qword, _bit)				\
1412 	do {								\
1413 		_NOTE(CONSTANTCONDITION) 				\
1414 		(_qword).eq_u32[0] |=					\
1415 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit));		\
1416 		(_qword).eq_u32[1] |=					\
1417 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));	\
1418 	_NOTE(CONSTANTCONDITION) 					\
1419 	} while (B_FALSE)
1420 
1421 #define	EFX_CLEAR_QWORD_BIT64(_qword, _bit)				\
1422 	do {								\
1423 		_NOTE(CONSTANTCONDITION) 				\
1424 		(_qword).eq_u64[0] &=					\
1425 		    __CPU_TO_LE_64(~EFX_SHIFT64_0(_bit));		\
1426 	_NOTE(CONSTANTCONDITION) 					\
1427 	} while (B_FALSE)
1428 
1429 #define	EFX_CLEAR_QWORD_BIT32(_qword, _bit)				\
1430 	do {								\
1431 		_NOTE(CONSTANTCONDITION) 				\
1432 		(_qword).eq_u32[0] &=					\
1433 		    __CPU_TO_LE_32(~EFX_SHIFT32_0(_bit));		\
1434 		(_qword).eq_u32[1] &=					\
1435 		    __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));	\
1436 	_NOTE(CONSTANTCONDITION) 					\
1437 	} while (B_FALSE)
1438 
1439 #define	EFX_TEST_QWORD_BIT64(_qword, _bit)				\
1440 	(((_qword).eq_u64[0] &						\
1441 		    __CPU_TO_LE_64(EFX_SHIFT64_0(_bit))) != 0)
1442 
1443 #define	EFX_TEST_QWORD_BIT32(_qword, _bit)				\
1444 	(((_qword).eq_u32[0] &						\
1445 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit))) ||		\
1446 	((_qword).eq_u32[1] &						\
1447 		    __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1448 
1449 
1450 #define	EFX_SET_DWORD_BIT(_dword, _bit)					\
1451 	do {								\
1452 		(_dword).ed_u32[0] |=					\
1453 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit));		\
1454 	_NOTE(CONSTANTCONDITION) 					\
1455 	} while (B_FALSE)
1456 
1457 #define	EFX_CLEAR_DWORD_BIT(_dword, _bit)				\
1458 	do {								\
1459 		(_dword).ed_u32[0] &=					\
1460 		    __CPU_TO_LE_32(~EFX_SHIFT32_0(_bit));		\
1461 	_NOTE(CONSTANTCONDITION) 					\
1462 	} while (B_FALSE)
1463 
1464 #define	EFX_TEST_DWORD_BIT(_dword, _bit)				\
1465 	(((_dword).ed_u32[0] &						\
1466 		    __CPU_TO_LE_32(EFX_SHIFT32_0(_bit))) != 0)
1467 
1468 
1469 #define	EFX_SET_WORD_BIT(_word, _bit)					\
1470 	do {								\
1471 		(_word).ew_u16[0] |=					\
1472 		    __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));	\
1473 	_NOTE(CONSTANTCONDITION) 					\
1474 	} while (B_FALSE)
1475 
1476 #define	EFX_CLEAR_WORD_BIT(_word, _bit)					\
1477 	do {								\
1478 		(_word).ew_u32[0] &=					\
1479 		    __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));	\
1480 	_NOTE(CONSTANTCONDITION) 					\
1481 	} while (B_FALSE)
1482 
1483 #define	EFX_TEST_WORD_BIT(_word, _bit)					\
1484 	(((_word).ew_u16[0] &						\
1485 		    __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1486 
1487 
1488 #define	EFX_SET_BYTE_BIT(_byte, _bit)					\
1489 	do {								\
1490 		(_byte).eb_u8[0] |=					\
1491 		    __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));		\
1492 	_NOTE(CONSTANTCONDITION) 					\
1493 	} while (B_FALSE)
1494 
1495 #define	EFX_CLEAR_BYTE_BIT(_byte, _bit)					\
1496 	do {								\
1497 		(_byte).eb_u8[0] &=					\
1498 		    __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));		\
1499 	_NOTE(CONSTANTCONDITION) 					\
1500 	} while (B_FALSE)
1501 
1502 #define	EFX_TEST_BYTE_BIT(_byte, _bit)					\
1503 	(((_byte).eb_u8[0] &						\
1504 		    __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1505 
1506 
1507 #define	EFX_OR_OWORD64(_oword1, _oword2)				\
1508 	do {								\
1509 		(_oword1).eo_u64[0] |= (_oword2).eo_u64[0];		\
1510 		(_oword1).eo_u64[1] |= (_oword2).eo_u64[1];		\
1511 	_NOTE(CONSTANTCONDITION) 					\
1512 	} while (B_FALSE)
1513 
1514 #define	EFX_OR_OWORD32(_oword1, _oword2)				\
1515 	do {								\
1516 		(_oword1).eo_u32[0] |= (_oword2).eo_u32[0];		\
1517 		(_oword1).eo_u32[1] |= (_oword2).eo_u32[1];		\
1518 		(_oword1).eo_u32[2] |= (_oword2).eo_u32[2];		\
1519 		(_oword1).eo_u32[3] |= (_oword2).eo_u32[3];		\
1520 	_NOTE(CONSTANTCONDITION) 					\
1521 	} while (B_FALSE)
1522 
1523 #define	EFX_AND_OWORD64(_oword1, _oword2)				\
1524 	do {								\
1525 		(_oword1).eo_u64[0] &= (_oword2).eo_u64[0];		\
1526 		(_oword1).eo_u64[1] &= (_oword2).eo_u64[1];		\
1527 	_NOTE(CONSTANTCONDITION) 					\
1528 	} while (B_FALSE)
1529 
1530 #define	EFX_AND_OWORD32(_oword1, _oword2)				\
1531 	do {								\
1532 		(_oword1).eo_u32[0] &= (_oword2).eo_u32[0];		\
1533 		(_oword1).eo_u32[1] &= (_oword2).eo_u32[1];		\
1534 		(_oword1).eo_u32[2] &= (_oword2).eo_u32[2];		\
1535 		(_oword1).eo_u32[3] &= (_oword2).eo_u32[3];		\
1536 	_NOTE(CONSTANTCONDITION) 					\
1537 	} while (B_FALSE)
1538 
1539 #define	EFX_OR_QWORD64(_qword1, _qword2)				\
1540 	do {								\
1541 		(_qword1).eq_u64[0] |= (_qword2).eq_u64[0];		\
1542 	_NOTE(CONSTANTCONDITION) 					\
1543 	} while (B_FALSE)
1544 
1545 #define	EFX_OR_QWORD32(_qword1, _qword2)				\
1546 	do {								\
1547 		(_qword1).eq_u32[0] |= (_qword2).eq_u32[0];		\
1548 		(_qword1).eq_u32[1] |= (_qword2).eq_u32[1];		\
1549 	_NOTE(CONSTANTCONDITION) 					\
1550 	} while (B_FALSE)
1551 
1552 #define	EFX_AND_QWORD64(_qword1, _qword2)				\
1553 	do {								\
1554 		(_qword1).eq_u64[0] &= (_qword2).eq_u64[0];		\
1555 	_NOTE(CONSTANTCONDITION) 					\
1556 	} while (B_FALSE)
1557 
1558 #define	EFX_AND_QWORD32(_qword1, _qword2)				\
1559 	do {								\
1560 		(_qword1).eq_u32[0] &= (_qword2).eq_u32[0];		\
1561 		(_qword1).eq_u32[1] &= (_qword2).eq_u32[1];		\
1562 	_NOTE(CONSTANTCONDITION) 					\
1563 	} while (B_FALSE)
1564 
1565 #define	EFX_OR_DWORD(_dword1, _dword2)					\
1566 	do {								\
1567 		(_dword1).ed_u32[0] |= (_dword2).ed_u32[0];		\
1568 	_NOTE(CONSTANTCONDITION) 					\
1569 	} while (B_FALSE)
1570 
1571 #define	EFX_AND_DWORD(_dword1, _dword2)					\
1572 	do {								\
1573 		(_dword1).ed_u32[0] &= (_dword2).ed_u32[0];		\
1574 	_NOTE(CONSTANTCONDITION) 					\
1575 	} while (B_FALSE)
1576 
1577 #define	EFX_OR_WORD(_word1, _word2)					\
1578 	do {								\
1579 		(_word1).ew_u16[0] |= (_word2).ew_u16[0];		\
1580 	_NOTE(CONSTANTCONDITION) 					\
1581 	} while (B_FALSE)
1582 
1583 #define	EFX_AND_WORD(_word1, _word2)					\
1584 	do {								\
1585 		(_word1).ew_u16[0] &= (_word2).ew_u16[0];		\
1586 	_NOTE(CONSTANTCONDITION) 					\
1587 	} while (B_FALSE)
1588 
1589 #define	EFX_OR_BYTE(_byte1, _byte2)					\
1590 	do {								\
1591 		(_byte1).eb_u8[0] |= (_byte2).eb_u8[0];			\
1592 	_NOTE(CONSTANTCONDITION) 					\
1593 	} while (B_FALSE)
1594 
1595 #define	EFX_AND_BYTE(_byte1, _byte2)					\
1596 	do {								\
1597 		(_byte1).eb_u8[0] &= (_byte2).eb_u8[0];			\
1598 	_NOTE(CONSTANTCONDITION) 					\
1599 	} while (B_FALSE)
1600 
1601 #if EFSYS_USE_UINT64
1602 #define	EFX_OWORD_FIELD		EFX_OWORD_FIELD64
1603 #define	EFX_QWORD_FIELD		EFX_QWORD_FIELD64
1604 #define	EFX_OWORD_IS_EQUAL	EFX_OWORD_IS_EQUAL64
1605 #define	EFX_QWORD_IS_EQUAL	EFX_QWORD_IS_EQUAL64
1606 #define	EFX_OWORD_IS_ZERO	EFX_OWORD_IS_ZERO64
1607 #define	EFX_QWORD_IS_ZERO	EFX_QWORD_IS_ZERO64
1608 #define	EFX_OWORD_IS_SET	EFX_OWORD_IS_SET64
1609 #define	EFX_QWORD_IS_SET	EFX_QWORD_IS_SET64
1610 #define	EFX_POPULATE_OWORD	EFX_POPULATE_OWORD64
1611 #define	EFX_POPULATE_QWORD	EFX_POPULATE_QWORD64
1612 #define	EFX_SET_OWORD_FIELD	EFX_SET_OWORD_FIELD64
1613 #define	EFX_SET_QWORD_FIELD	EFX_SET_QWORD_FIELD64
1614 #define	EFX_SET_OWORD_BIT	EFX_SET_OWORD_BIT64
1615 #define	EFX_CLEAR_OWORD_BIT	EFX_CLEAR_OWORD_BIT64
1616 #define	EFX_TEST_OWORD_BIT	EFX_TEST_OWORD_BIT64
1617 #define	EFX_SET_QWORD_BIT	EFX_SET_QWORD_BIT64
1618 #define	EFX_CLEAR_QWORD_BIT	EFX_CLEAR_QWORD_BIT64
1619 #define	EFX_TEST_QWORD_BIT	EFX_TEST_QWORD_BIT64
1620 #define	EFX_OR_OWORD		EFX_OR_OWORD64
1621 #define	EFX_AND_OWORD		EFX_AND_OWORD64
1622 #define	EFX_OR_QWORD		EFX_OR_QWORD64
1623 #define	EFX_AND_QWORD		EFX_AND_QWORD64
1624 #else
1625 #define	EFX_OWORD_FIELD		EFX_OWORD_FIELD32
1626 #define	EFX_QWORD_FIELD		EFX_QWORD_FIELD32
1627 #define	EFX_OWORD_IS_EQUAL	EFX_OWORD_IS_EQUAL32
1628 #define	EFX_QWORD_IS_EQUAL	EFX_QWORD_IS_EQUAL32
1629 #define	EFX_OWORD_IS_ZERO	EFX_OWORD_IS_ZERO32
1630 #define	EFX_QWORD_IS_ZERO	EFX_QWORD_IS_ZERO32
1631 #define	EFX_OWORD_IS_SET	EFX_OWORD_IS_SET32
1632 #define	EFX_QWORD_IS_SET	EFX_QWORD_IS_SET32
1633 #define	EFX_POPULATE_OWORD	EFX_POPULATE_OWORD32
1634 #define	EFX_POPULATE_QWORD	EFX_POPULATE_QWORD32
1635 #define	EFX_SET_OWORD_FIELD	EFX_SET_OWORD_FIELD32
1636 #define	EFX_SET_QWORD_FIELD	EFX_SET_QWORD_FIELD32
1637 #define	EFX_SET_OWORD_BIT	EFX_SET_OWORD_BIT32
1638 #define	EFX_CLEAR_OWORD_BIT	EFX_CLEAR_OWORD_BIT32
1639 #define	EFX_TEST_OWORD_BIT	EFX_TEST_OWORD_BIT32
1640 #define	EFX_SET_QWORD_BIT	EFX_SET_QWORD_BIT32
1641 #define	EFX_CLEAR_QWORD_BIT	EFX_CLEAR_QWORD_BIT32
1642 #define	EFX_TEST_QWORD_BIT	EFX_TEST_QWORD_BIT32
1643 #define	EFX_OR_OWORD		EFX_OR_OWORD32
1644 #define	EFX_AND_OWORD		EFX_AND_OWORD32
1645 #define	EFX_OR_QWORD		EFX_OR_QWORD32
1646 #define	EFX_AND_QWORD		EFX_AND_QWORD32
1647 #endif
1648 
1649 #ifdef	__cplusplus
1650 }
1651 #endif
1652 
1653 #endif	/* _SYS_EFX_TYPES_H */
1654