1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_TAVOR_HW_H
28 #define	_SYS_IB_ADAPTERS_TAVOR_HW_H
29 
30 /*
31  * tavor_hw.h
32  *    Contains all the structure definitions and #defines for all Tavor
33  *    hardware resources and registers (as defined by the Tavor register
34  *    specification).  Wherever possible, the names in the Tavor spec
35  *    have been preserved in the structure and field names below.
36  */
37 
38 #include <sys/types.h>
39 #include <sys/conf.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 
43 #ifdef __cplusplus
44 extern "C" {
45 #endif
46 
47 
48 /*
49  * Offsets into the CMD BAR (BAR 0) for many of the more interesting hardware
50  * registers.  These registers include the HCR (more below), the Event Cause
51  * Register (ECR) and its related clear register, the Interrupt Clear register
52  * (CLR_INT), and the software reset register (SW_RESET).
53  */
54 #define	TAVOR_CMD_HCR_OFFSET		0x80680
55 #define	TAVOR_CMD_ECR_OFFSET		0x80700
56 #define	TAVOR_CMD_CLR_ECR_OFFSET	0x80708
57 #define	TAVOR_CMD_CLR_INT_OFFSET	0xF00D8
58 #define	TAVOR_CMD_SW_RESET_OFFSET	0xF0010
59 
60 /*
61  * Ownership flags used to define hardware or software ownership for
62  * various Tavor resources
63  */
64 #define	TAVOR_HW_OWNER			0x1
65 #define	TAVOR_SW_OWNER			0x0
66 
67 /*
68  * Determines whether or not virtual-to-physical address translation is
69  * required.  Several of the Tavor hardware structures can be optionally
70  * accessed by Tavor without going through the TPT address translation
71  * tables.
72  */
73 #define	TAVOR_VA2PA_XLAT_ENABLED	0x1
74 #define	TAVOR_VA2PA_XLAT_DISABLED	0x0
75 
76 /*
77  * HCA Command Register (HCR)
78  *    The HCR command interface provides privileged access to the HCA in
79  *    order to query, configure and modify HCA execution.  It is the
80  *    primary mechanism through which mailboxes may be posted to Tavor
81  *    firmware.  To use this interface software fills the HCR with pointers
82  *    to input and output mailboxes.  Some commands support immediate
83  *    parameters, however, and for these commands the HCR will contain the
84  *    input or output parameters. Command execution completion can be
85  *    detected either by the software polling the HCR or by waiting for a
86  *    command completion event.
87  */
88 struct tavor_hw_hcr_s {
89 	uint32_t	in_param0;
90 	uint32_t	in_param1;
91 	uint32_t	input_modifier;
92 	uint32_t	out_param0;
93 	uint32_t	out_param1;
94 	uint32_t	token;
95 	uint32_t	cmd;
96 };
97 #define	TAVOR_HCR_TOKEN_MASK		0xFFFF0000
98 #define	TAVOR_HCR_TOKEN_SHIFT		16
99 
100 #define	TAVOR_HCR_CMD_STATUS_MASK	0xFF000000
101 #define	TAVOR_HCR_CMD_GO_MASK		0x00800000
102 #define	TAVOR_HCR_CMD_E_MASK		0x00400000
103 #define	TAVOR_HCR_CMD_OPMOD_MASK	0x0000F000
104 #define	TAVOR_HCR_CMD_OPCODE_MASK	0x00000FFF
105 #define	TAVOR_HCR_CMD_STATUS_SHFT	24
106 #define	TAVOR_HCR_CMD_GO_SHFT		23
107 #define	TAVOR_HCR_CMD_E_SHFT		22
108 #define	TAVOR_HCR_CMD_OPMOD_SHFT	12
109 
110 
111 /*
112  * Tavor "QUERY_DEV_LIM" command
113  *    The QUERY_DEV_LIM command returns the device limits and capabilities
114  *    supported by the Tavor device.  This command should be run before
115  *    running the INIT_HCA command (below) in order to determine the maximum
116  *    capabilities of the device and which optional features are supported.
117  */
118 #ifdef	_LITTLE_ENDIAN
119 struct tavor_hw_querydevlim_s {
120 	uint32_t	rsrv0[4];
121 	uint32_t	log_max_ee	:5;
122 	uint32_t			:3;
123 	uint32_t	log_rsvd_ee	:4;
124 	uint32_t			:4;
125 	uint32_t	log_max_srq	:5;
126 	uint32_t			:7;
127 	uint32_t	log_rsvd_srq	:4;
128 	uint32_t	log_max_qp	:5;
129 	uint32_t			:3;
130 	uint32_t	log_rsvd_qp	:4;
131 	uint32_t			:4;
132 	uint32_t	log_max_qp_sz	:8;
133 	uint32_t	log_max_srq_sz	:8;
134 	uint32_t	log_max_eq	:3;
135 	uint32_t			:5;
136 	uint32_t	num_rsvd_eq	:4;
137 	uint32_t			:4;
138 	uint32_t	log_max_mpt	:6;
139 	uint32_t			:10;
140 	uint32_t	log_max_cq	:5;
141 	uint32_t			:3;
142 	uint32_t	log_rsvd_cq	:4;
143 	uint32_t			:4;
144 	uint32_t	log_max_cq_sz	:8;
145 	uint32_t			:8;
146 	uint32_t	log_max_av	:6;
147 	uint32_t			:26;
148 	uint32_t	log_max_mttseg	:6;
149 	uint32_t			:2;
150 	uint32_t	log_rsvd_mpt	:4;
151 	uint32_t			:4;
152 	uint32_t	log_max_mrw_sz	:8;
153 	uint32_t			:4;
154 	uint32_t	log_rsvd_mttseg	:4;
155 	uint32_t	log_max_ra_glob	:6;
156 	uint32_t			:26;
157 	uint32_t	log_max_ras_qp	:6;
158 	uint32_t			:10;
159 	uint32_t	log_max_raq_qp	:6;
160 	uint32_t			:10;
161 	uint32_t	num_ports	:4;
162 	uint32_t	max_vl		:4;
163 	uint32_t	max_port_width	:4;
164 	uint32_t	max_mtu		:4;
165 	uint32_t	ca_ack_delay	:5;
166 	uint32_t			:11;
167 	uint32_t			:32;
168 	uint32_t	log_max_pkey	:4;
169 	uint32_t			:12;
170 	uint32_t	stat_rate_sup	:16;
171 	uint32_t	log_max_gid	:4;
172 	uint32_t			:28;
173 	uint32_t	rc		:1;
174 	uint32_t	uc		:1;
175 	uint32_t	ud		:1;
176 	uint32_t	rd		:1;
177 	uint32_t	raw_ipv6	:1;
178 	uint32_t	raw_ether	:1;
179 	uint32_t	srq		:1;
180 	uint32_t			:1;
181 	uint32_t	pkey_v		:1;
182 	uint32_t	qkey_v		:1;
183 	uint32_t			:6;
184 	uint32_t	mem_win		:1;
185 	uint32_t	apm		:1;
186 	uint32_t	atomic		:1;
187 	uint32_t	raw_multi	:1;
188 	uint32_t	avp		:1;
189 	uint32_t	ud_multi	:1;
190 	uint32_t			:2;
191 	uint32_t	pg_on_demand	:1;
192 	uint32_t	router		:1;
193 	uint32_t			:6;
194 	uint32_t			:32;
195 	uint32_t			:32;
196 	uint32_t	log_pg_sz	:8;
197 	uint32_t			:8;
198 	uint32_t	log_max_uar_sz	:6;
199 	uint32_t			:6;
200 	uint32_t	num_rsvd_uar	:4;
201 	uint32_t			:32;
202 	uint32_t	max_desc_sz	:16;
203 	uint32_t	max_sg		:8;
204 	uint32_t			:8;
205 	uint32_t	rsrv1[2];
206 	uint32_t	log_max_rdd	:6;
207 	uint32_t			:6;
208 	uint32_t	num_rsvd_rdd	:4;
209 	uint32_t	log_max_pd	:6;
210 	uint32_t			:6;
211 	uint32_t	num_rsvd_pd	:4;
212 	uint32_t	log_max_mcg	:8;
213 	uint32_t	num_rsvd_mcg	:4;
214 	uint32_t			:4;
215 	uint32_t	log_max_qp_mcg	:8;
216 	uint32_t			:8;
217 	uint32_t	rsrv2[6];
218 	uint32_t	eqpc_entry_sz	:16;
219 	uint32_t	eeec_entry_sz	:16;
220 	uint32_t	qpc_entry_sz	:16;
221 	uint32_t	eec_entry_sz	:16;
222 	uint32_t	uarscr_entry_sz	:16;
223 	uint32_t	srq_entry_sz	:16;
224 	uint32_t	cqc_entry_sz	:16;
225 	uint32_t	eqc_entry_sz	:16;
226 	uint32_t	rsrv3[28];
227 };
228 #else
229 struct tavor_hw_querydevlim_s {
230 	uint32_t	rsrv0[4];
231 	uint32_t	log_max_srq_sz	:8;
232 	uint32_t	log_max_qp_sz	:8;
233 	uint32_t			:4;
234 	uint32_t	log_rsvd_qp	:4;
235 	uint32_t			:3;
236 	uint32_t	log_max_qp	:5;
237 	uint32_t	log_rsvd_srq	:4;
238 	uint32_t			:7;
239 	uint32_t	log_max_srq	:5;
240 	uint32_t			:4;
241 	uint32_t	log_rsvd_ee	:4;
242 	uint32_t			:3;
243 	uint32_t	log_max_ee	:5;
244 	uint32_t			:8;
245 	uint32_t	log_max_cq_sz	:8;
246 	uint32_t			:4;
247 	uint32_t	log_rsvd_cq	:4;
248 	uint32_t			:3;
249 	uint32_t	log_max_cq	:5;
250 	uint32_t			:10;
251 	uint32_t	log_max_mpt	:6;
252 	uint32_t			:4;
253 	uint32_t	num_rsvd_eq	:4;
254 	uint32_t			:5;
255 	uint32_t	log_max_eq	:3;
256 	uint32_t	log_rsvd_mttseg	:4;
257 	uint32_t			:4;
258 	uint32_t	log_max_mrw_sz	:8;
259 	uint32_t			:4;
260 	uint32_t	log_rsvd_mpt	:4;
261 	uint32_t			:2;
262 	uint32_t	log_max_mttseg	:6;
263 	uint32_t			:26;
264 	uint32_t	log_max_av	:6;
265 	uint32_t			:10;
266 	uint32_t	log_max_raq_qp	:6;
267 	uint32_t			:10;
268 	uint32_t	log_max_ras_qp	:6;
269 	uint32_t			:26;
270 	uint32_t	log_max_ra_glob	:6;
271 	uint32_t			:32;
272 	uint32_t			:11;
273 	uint32_t	ca_ack_delay	:5;
274 	uint32_t	max_mtu		:4;
275 	uint32_t	max_port_width	:4;
276 	uint32_t	max_vl		:4;
277 	uint32_t	num_ports	:4;
278 	uint32_t			:28;
279 	uint32_t	log_max_gid	:4;
280 	uint32_t	stat_rate_sup	:16;
281 	uint32_t			:12;
282 	uint32_t	log_max_pkey	:4;
283 	uint32_t			:32;
284 	uint32_t			:6;
285 	uint32_t	router		:1;
286 	uint32_t	pg_on_demand	:1;
287 	uint32_t			:2;
288 	uint32_t	ud_multi	:1;
289 	uint32_t	avp		:1;
290 	uint32_t	raw_multi	:1;
291 	uint32_t	atomic		:1;
292 	uint32_t	apm		:1;
293 	uint32_t	mem_win		:1;
294 	uint32_t			:6;
295 	uint32_t	qkey_v		:1;
296 	uint32_t	pkey_v		:1;
297 	uint32_t			:1;
298 	uint32_t	srq		:1;
299 	uint32_t	raw_ether	:1;
300 	uint32_t	raw_ipv6	:1;
301 	uint32_t	rd		:1;
302 	uint32_t	ud		:1;
303 	uint32_t	uc		:1;
304 	uint32_t	rc		:1;
305 	uint32_t	num_rsvd_uar	:4;
306 	uint32_t			:6;
307 	uint32_t	log_max_uar_sz	:6;
308 	uint32_t			:8;
309 	uint32_t	log_pg_sz	:8;
310 	uint32_t			:32;
311 	uint32_t			:8;
312 	uint32_t	max_sg		:8;
313 	uint32_t	max_desc_sz	:16;
314 	uint32_t			:32;
315 	uint32_t	rsrv1[2];
316 	uint32_t			:8;
317 	uint32_t	log_max_qp_mcg	:8;
318 	uint32_t			:4;
319 	uint32_t	num_rsvd_mcg	:4;
320 	uint32_t	log_max_mcg	:8;
321 	uint32_t	num_rsvd_pd	:4;
322 	uint32_t			:6;
323 	uint32_t	log_max_pd	:6;
324 	uint32_t	num_rsvd_rdd	:4;
325 	uint32_t			:6;
326 	uint32_t	log_max_rdd	:6;
327 	uint32_t	rsrv2[6];
328 	uint32_t	eec_entry_sz	:16;
329 	uint32_t	qpc_entry_sz	:16;
330 	uint32_t	eeec_entry_sz	:16;
331 	uint32_t	eqpc_entry_sz	:16;
332 	uint32_t	eqc_entry_sz	:16;
333 	uint32_t	cqc_entry_sz	:16;
334 	uint32_t	srq_entry_sz	:16;
335 	uint32_t	uarscr_entry_sz	:16;
336 	uint32_t	rsrv3[28];
337 };
338 #endif
339 
340 
341 /*
342  * Tavor "QUERY_FW" command
343  *    The QUERY_FW command retrieves the firmware revision and the Command
344  *    Interface revision.  The command also returns the HCA attached local
345  *    memory area (DDR) which is used by the firmware.  Below we also
346  *    include some defines which are used to enforce a minimum firmware
347  *    version check (see tavor_fw_version_check() for more details).
348  */
349 #ifdef	_LITTLE_ENDIAN
350 struct tavor_hw_queryfw_s {
351 	uint32_t	fw_rev_minor	:16;
352 	uint32_t	fw_rev_subminor	:16;
353 	uint32_t	fw_rev_major	:16;
354 	uint32_t			:16;
355 	uint32_t	log_max_cmd	:8;
356 	uint32_t			:23;
357 	uint32_t	dbg_trace	:1;
358 	uint32_t	cmd_intf_rev	:16;
359 	uint32_t			:16;
360 	uint32_t	rsrv0[4];
361 	uint64_t	fw_baseaddr;
362 	uint64_t	fw_endaddr;
363 	uint64_t	error_buf_addr;
364 	uint32_t			:32;
365 	uint32_t	error_buf_sz;
366 	uint32_t	rsrv1[48];
367 };
368 #else
369 struct tavor_hw_queryfw_s {
370 	uint32_t			:16;
371 	uint32_t	fw_rev_major	:16;
372 	uint32_t	fw_rev_subminor	:16;
373 	uint32_t	fw_rev_minor	:16;
374 	uint32_t			:16;
375 	uint32_t	cmd_intf_rev	:16;
376 	uint32_t	dbg_trace	:1;
377 	uint32_t			:23;
378 	uint32_t	log_max_cmd	:8;
379 	uint32_t	rsrv0[4];
380 	uint64_t	fw_baseaddr;
381 	uint64_t	fw_endaddr;
382 	uint64_t	error_buf_addr;
383 	uint32_t	error_buf_sz;
384 	uint32_t	rsrv1[49];
385 };
386 #endif
387 #define	TAVOR_FW_VER_MAJOR		0x0003
388 #define	TAVOR_FW_VER_MINOR		0x0001
389 #define	TAVOR_FW_VER_SUBMINOR		0x0000
390 #define	TAVOR_COMPAT_FW_VER_MAJOR	0x0004
391 #define	TAVOR_COMPAT_FW_VER_MINOR	0x0005
392 #define	TAVOR_COMPAT_FW_VER_SUBMINOR	0x0003
393 
394 
395 /*
396  * Tavor "QUERY_DDR" command
397  *    The QUERY_DDR command retrieves information regarding the HCA attached
398  *    local memory area (DDR). This information includes:  the DIMM PCI BAR,
399  *    the total address space provided by the HCA attached local memory, and
400  *    some DIMM-specific information.  Note:  Some of the HCA attached local
401  *    memory is reserved for use by firmware.  This extent of this reserved
402  *    area can be obtained through the QUERY_FW command (above).
403  *
404  *    Below we first define the tavor_hw_queryddr_dimm_t or "Logical DIMM
405  *    Information" structure.  Four of these are present in the QUERY_DDR
406  *    command.
407  */
408 #ifdef	_LITTLE_ENDIAN
409 typedef struct tavor_hw_queryddr_dimm_s {
410 	uint32_t	spd		:1;
411 	uint32_t	sladr		:3;
412 	uint32_t	sock_num	:2;
413 	uint32_t	syn		:4;
414 	uint32_t			:22;
415 	uint32_t	dimmsz		:16;
416 	uint32_t			:8;
417 	uint32_t	dimmstatus	:1;
418 	uint32_t	dimm_hidden	:1;
419 	uint32_t	write_only	:1;
420 	uint32_t			:5;
421 	uint32_t	vendor_id_l;
422 	uint32_t	vendor_id_h;
423 	uint32_t	dimm_baseaddr_l;
424 	uint32_t	dimm_baseaddr_h;
425 	uint32_t	rsrv0[2];
426 } tavor_hw_queryddr_dimm_t;
427 #else
428 typedef struct tavor_hw_queryddr_dimm_s {
429 	uint32_t			:5;
430 	uint32_t	write_only	:1;
431 	uint32_t	dimm_hidden	:1;
432 	uint32_t	dimmstatus	:1;
433 	uint32_t			:8;
434 	uint32_t	dimmsz		:16;
435 	uint32_t			:22;
436 	uint32_t	syn		:4;
437 	uint32_t	sock_num	:2;
438 	uint32_t	sladr		:3;
439 	uint32_t	spd		:1;
440 	uint32_t	vendor_id_h;
441 	uint32_t	vendor_id_l;
442 	uint32_t	dimm_baseaddr_h;
443 	uint32_t	dimm_baseaddr_l;
444 	uint32_t	rsrv0[2];
445 } tavor_hw_queryddr_dimm_t;
446 #endif
447 #define	TAVOR_DIMMSTATUS_ENABLED	0x0
448 #define	TAVOR_DIMMSTATUS_DISABLED	0x1
449 
450 #define	TAVOR_DIMM_ERR_NONE		0x0
451 #define	TAVOR_DIMM_ERR_SPD		0x1
452 #define	TAVOR_DIMM_ERR_BOUNDS		0x2
453 #define	TAVOR_DIMM_ERR_CONFLICT		0x3
454 #define	TAVOR_DIMM_ERR_SIZETRIM		0x5
455 
456 #define	TAVOR_DIMM_SPD_FROM_DIMM	0x0
457 #define	TAVOR_DIMM_SPD_FROM_NVMEM	0x1
458 
459 #ifdef	_LITTLE_ENDIAN
460 struct tavor_hw_queryddr_s {
461 	uint64_t	ddr_baseaddr;
462 	uint64_t	ddr_endaddr;
463 	uint32_t			:32;
464 	uint32_t	data_integrity	:2;
465 	uint32_t	auto_precharge	:2;
466 	uint32_t	ddr_hidden	:1;
467 	uint32_t			:27;
468 	uint32_t	rsrv0[10];
469 	tavor_hw_queryddr_dimm_t	dimm[4];
470 	uint32_t	rsrv1[16];
471 };
472 #else
473 struct tavor_hw_queryddr_s {
474 	uint64_t	ddr_baseaddr;
475 	uint64_t	ddr_endaddr;
476 	uint32_t			:27;
477 	uint32_t	ddr_hidden	:1;
478 	uint32_t	auto_precharge	:2;
479 	uint32_t	data_integrity	:2;
480 	uint32_t			:32;
481 	uint32_t	rsrv0[10];
482 	tavor_hw_queryddr_dimm_t	dimm[4];
483 	uint32_t	rsrv1[16];
484 };
485 #endif
486 #define	TAVOR_AUTO_PRECHRG_NONE		0x0
487 #define	TAVOR_AUTO_PRECHRG_PER_TRANS	0x1
488 #define	TAVOR_AUTO_PRECHRG_PER_64B	0x2
489 
490 #define	TAVOR_DATA_INT_NONE		0x0
491 #define	TAVOR_DATA_INT_PARITY		0x1
492 #define	TAVOR_DATA_INT_ECC_DETECT_ONLY	0x2
493 #define	TAVOR_DATA_INT_ECC_CORRECT	0x3
494 
495 
496 /*
497  * Tavor "QUERY_ADAPTER" command
498  *    The QUERY_ADAPTER command retrieves adapter specific parameters. The
499  *    command also retrieves the PCI(X) interrupt pin routing for each of
500  *    the INTx# pins supported by the device.  This information is used by
501  *    the driver during interrupt processing in order to clear the appropriate
502  *    interrupt bit.
503  */
504 #ifdef	_LITTLE_ENDIAN
505 struct tavor_hw_queryadapter_s {
506 	uint32_t	device_id;
507 	uint32_t	vendor_id;
508 	uint32_t			:32;
509 	uint32_t	rev_id;
510 	uint32_t			:32;
511 	uint32_t			:24;
512 	uint32_t	inta_pin	:8;
513 	uint32_t	rsrv0[58];
514 };
515 #else
516 struct tavor_hw_queryadapter_s {
517 	uint32_t	vendor_id;
518 	uint32_t	device_id;
519 	uint32_t	rev_id;
520 	uint32_t			:32;
521 	uint32_t	inta_pin	:8;
522 	uint32_t			:24;
523 	uint32_t			:32;
524 	uint32_t	rsrv0[58];
525 };
526 #endif
527 #define	TAVOR_REV_A0	0xA0
528 #define	TAVOR_REV_A1	0xA1
529 
530 
531 /*
532  * Tavor "INIT_HCA" and "QUERY_HCA" commands
533  *    The INIT_HCA command configures all HCA resources in HCA attached local
534  *    memory and some system relevant information.  The same mailbox output
535  *    format is used by the QUERY_HCA command.  All parameters, which are
536  *    specifically the output of the QUERY_HCA command are marked as
537  *    "QUERY_HCA only".  These parameters are not configurable through the
538  *    INIT_HCA command, but can be retrieved as read-only through the
539  *    QUERY_HCA command.
540  *
541  *    Below we first define several structures which help make up the whole
542  *    of the INIT_HCA/QUERY_HCA command.  These are:
543  *    tavor_hw_qp_ee_cq_eq_rdb_t for "QPC/EEC/CQC/EQC/RDB Parameters",
544  *    tavor_udav_mem_param_t for "Memory Access Parameters for UDAV Table",
545  *    tavor_multicast_param_t for "Multicast Support Parameters",
546  *    tavor_tpt_param_t for "Translation and Protection Table Parameters",
547  *    and tavor_uar_param_t for Tavor "UAR Parameters".
548  */
549 #ifdef	_LITTLE_ENDIAN
550 typedef struct tavor_hw_qp_ee_cq_eq_rdb_s {
551 	uint32_t	rsrv0[4];
552 	uint32_t	log_num_qp	:5;
553 	uint32_t			:2;
554 	uint32_t	qpc_baseaddr_l	:25;
555 	uint32_t	qpc_baseaddr_h;
556 	uint32_t	rsrv1[2];
557 	uint32_t	log_num_ee	:5;
558 	uint32_t			:2;
559 	uint32_t	eec_baseaddr_l	:25;
560 	uint32_t	eec_baseaddr_h;
561 	uint32_t	log_num_srq	:5;
562 	uint32_t	srqc_baseaddr_l	:27;
563 	uint32_t	srqc_baseaddr_h;
564 	uint32_t	log_num_cq	:5;
565 	uint32_t			:1;
566 	uint32_t	cqc_baseaddr_l	:26;
567 	uint32_t	cqc_baseaddr_h;
568 	uint32_t	rsrv2[2];
569 	uint64_t	eqpc_baseaddr;
570 	uint32_t	rsrv3[2];
571 	uint64_t	eeec_baseaddr;
572 	uint32_t	rsrv4[2];
573 	uint32_t	log_num_eq	:4;
574 	uint32_t			:2;
575 	uint32_t	eqc_baseaddr_l	:26;
576 	uint32_t	eqc_baseaddr_h;
577 	uint32_t	rsrv5[2];
578 	uint32_t	rdb_baseaddr_l;
579 	uint32_t	rdb_baseaddr_h;
580 	uint32_t	rsrv6[2];
581 } tavor_hw_qp_ee_cq_eq_rdb_t;
582 #else
583 typedef struct tavor_hw_qp_ee_cq_eq_rdb_s {
584 	uint32_t	rsrv0[4];
585 	uint32_t	qpc_baseaddr_h;
586 	uint32_t	qpc_baseaddr_l	:25;
587 	uint32_t			:2;
588 	uint32_t	log_num_qp	:5;
589 	uint32_t	rsrv1[2];
590 	uint32_t	eec_baseaddr_h;
591 	uint32_t	eec_baseaddr_l	:25;
592 	uint32_t			:2;
593 	uint32_t	log_num_ee	:5;
594 	uint32_t	srqc_baseaddr_h;
595 	uint32_t	srqc_baseaddr_l	:27;
596 	uint32_t	log_num_srq	:5;
597 	uint32_t	cqc_baseaddr_h;
598 	uint32_t	cqc_baseaddr_l	:26;
599 	uint32_t			:1;
600 	uint32_t	log_num_cq	:5;
601 	uint32_t	rsrv2[2];
602 	uint64_t	eqpc_baseaddr;
603 	uint32_t	rsrv3[2];
604 	uint64_t	eeec_baseaddr;
605 	uint32_t	rsrv4[2];
606 	uint32_t	eqc_baseaddr_h;
607 	uint32_t	eqc_baseaddr_l	:26;
608 	uint32_t			:2;
609 	uint32_t	log_num_eq	:4;
610 	uint32_t	rsrv5[2];
611 	uint32_t	rdb_baseaddr_h;
612 	uint32_t	rdb_baseaddr_l;
613 	uint32_t	rsrv6[2];
614 } tavor_hw_qp_ee_cq_eq_rdb_t;
615 #endif
616 
617 #ifdef	_LITTLE_ENDIAN
618 typedef struct tavor_udav_mem_param_s {
619 	uint32_t	udav_pd		:24;
620 	uint32_t			:5;
621 	uint32_t	udav_xlat_en	:1;
622 	uint32_t			:2;
623 	uint32_t	udav_lkey;
624 } tavor_udav_mem_param_t;
625 #else
626 typedef struct tavor_udav_mem_param_s {
627 	uint32_t	udav_lkey;
628 	uint32_t			:2;
629 	uint32_t	udav_xlat_en	:1;
630 	uint32_t			:5;
631 	uint32_t	udav_pd		:24;
632 } tavor_udav_mem_param_t;
633 #endif
634 
635 #ifdef	_LITTLE_ENDIAN
636 typedef struct tavor_multicast_param_s {
637 	uint64_t	mc_baseaddr;
638 	uint32_t	rsrv0[2];
639 	uint32_t	mc_tbl_hash_sz	:17;
640 	uint32_t			:15;
641 	uint32_t	log_mc_tbl_ent	:16;
642 	uint32_t			:16;
643 	uint32_t			:32;
644 	uint32_t	log_mc_tbl_sz	:5;
645 	uint32_t			:19;
646 	uint32_t	mc_hash_fn	:3;
647 	uint32_t			:5;
648 } tavor_multicast_param_t;
649 #else
650 typedef struct tavor_multicast_param_s {
651 	uint64_t	mc_baseaddr;
652 	uint32_t	rsrv0[2];
653 	uint32_t			:16;
654 	uint32_t	log_mc_tbl_ent	:16;
655 	uint32_t			:15;
656 	uint32_t	mc_tbl_hash_sz	:17;
657 	uint32_t			:5;
658 	uint32_t	mc_hash_fn	:3;
659 	uint32_t			:19;
660 	uint32_t	log_mc_tbl_sz	:5;
661 	uint32_t			:32;
662 } tavor_multicast_param_t;
663 #endif
664 #define	TAVOR_MCG_DEFAULT_HASH_FN	0x0
665 
666 #ifdef	_LITTLE_ENDIAN
667 typedef struct tavor_tpt_param_s {
668 	uint64_t	mpt_baseaddr;
669 	uint32_t	mtt_version	:8;
670 	uint32_t			:24;
671 	uint32_t	log_mpt_sz	:6;
672 	uint32_t			:2;
673 	uint32_t	pgfault_rnr_to	:5;
674 	uint32_t			:3;
675 	uint32_t	mttseg_sz	:3;
676 	uint32_t			:13;
677 	uint64_t	mtt_baseaddr;
678 	uint32_t	rsrv0[2];
679 } tavor_tpt_param_t;
680 #else
681 typedef struct tavor_tpt_param_s {
682 	uint64_t	mpt_baseaddr;
683 	uint32_t			:13;
684 	uint32_t	mttseg_sz	:3;
685 	uint32_t			:3;
686 	uint32_t	pgfault_rnr_to	:5;
687 	uint32_t			:2;
688 	uint32_t	log_mpt_sz	:6;
689 	uint32_t			:24;
690 	uint32_t	mtt_version	:8;
691 	uint64_t	mtt_baseaddr;
692 	uint32_t	rsrv0[2];
693 } tavor_tpt_param_t;
694 #endif
695 
696 #ifdef	_LITTLE_ENDIAN
697 typedef struct tavor_uar_param_s {
698 	uint32_t			:20;
699 	uint32_t	uar_baseaddr_l	:12;	/* QUERY_HCA only */
700 	uint32_t	uar_baseaddr_h;		/* QUERY_HCA only */
701 	uint32_t			:32;
702 	uint32_t	uar_pg_sz	:8;
703 	uint32_t			:24;
704 	uint64_t	uarscr_baseaddr;
705 	uint32_t	rsrv0[2];
706 } tavor_uar_param_t;
707 #else
708 typedef struct tavor_uar_param_s {
709 	uint32_t	uar_baseaddr_h;		/* QUERY_HCA only */
710 	uint32_t	uar_baseaddr_l	:12;	/* QUERY_HCA only */
711 	uint32_t			:20;
712 	uint32_t			:24;
713 	uint32_t	uar_pg_sz	:8;
714 	uint32_t			:32;
715 	uint64_t	uarscr_baseaddr;
716 	uint32_t	rsrv0[2];
717 } tavor_uar_param_t;
718 #endif
719 
720 #ifdef	_LITTLE_ENDIAN
721 struct tavor_hw_initqueryhca_s {
722 	uint32_t	rsrv0[2];
723 	uint32_t			:24;
724 	uint32_t	hca_core_clock	:8;	/* QUERY_HCA only */
725 	uint32_t			:32;
726 	uint32_t	udav_port_chk	:1;
727 	uint32_t	big_endian	:1;
728 	uint32_t	udav_chk	:1;
729 	uint32_t			:5;
730 	uint32_t	responder_exu	:4;
731 	uint32_t			:4;
732 	uint32_t	wqe_quota	:15;
733 	uint32_t	wqe_quota_en	:1;
734 	uint32_t			:8;
735 	uint32_t	router_qp	:16;
736 	uint32_t			:7;
737 	uint32_t	router_en	:1;
738 	uint32_t	rsrv1[2];
739 	tavor_hw_qp_ee_cq_eq_rdb_t	context;
740 	uint32_t	rsrv2[4];
741 	tavor_udav_mem_param_t		udav;
742 	uint32_t	rsrv3[2];
743 	tavor_multicast_param_t		multi;
744 	uint32_t	rsrv4[4];
745 	tavor_tpt_param_t		tpt;
746 	uint32_t	rsrv5[4];
747 	tavor_uar_param_t		uar;
748 	uint32_t	rsrv6[48];
749 };
750 #else
751 struct tavor_hw_initqueryhca_s {
752 	uint32_t	rsrv0[2];
753 	uint32_t			:32;
754 	uint32_t	hca_core_clock	:8;	/* QUERY_HCA only */
755 	uint32_t			:24;
756 	uint32_t	router_en	:1;
757 	uint32_t			:7;
758 	uint32_t	router_qp	:16;
759 	uint32_t			:8;
760 	uint32_t	wqe_quota_en	:1;
761 	uint32_t	wqe_quota	:15;
762 	uint32_t			:4;
763 	uint32_t	responder_exu	:4;
764 	uint32_t			:5;
765 	uint32_t	udav_chk	:1;
766 	uint32_t	big_endian	:1;
767 	uint32_t	udav_port_chk	:1;
768 	uint32_t	rsrv1[2];
769 	tavor_hw_qp_ee_cq_eq_rdb_t	context;
770 	uint32_t	rsrv2[4];
771 	tavor_udav_mem_param_t		udav;
772 	uint32_t	rsrv3[2];
773 	tavor_multicast_param_t		multi;
774 	uint32_t	rsrv4[4];
775 	tavor_tpt_param_t		tpt;
776 	uint32_t	rsrv5[4];
777 	tavor_uar_param_t		uar;
778 	uint32_t	rsrv6[48];
779 };
780 #endif
781 #define	TAVOR_UDAV_PROTECT_DISABLED	0x0
782 #define	TAVOR_UDAV_PROTECT_ENABLED	0x1
783 #define	TAVOR_UDAV_PORTCHK_DISABLED	0x0
784 #define	TAVOR_UDAV_PORTCHK_ENABLED	0x1
785 
786 
787 /*
788  * Tavor "INIT_IB" command
789  *    The INIT_IB command enables the physical layer of a given IB port.
790  *    It provides control over the IB port attributes.  The capabilities
791  *    requested here should not exceed the device limits, as retrieved by
792  *    the QUERY_DEV_LIM command (above).  To query information about the IB
793  *    port or node, the driver may submit GetPortInfo or GetNodeInfo MADs
794  *    through the Tavor MAD_IFC command.
795  */
796 #ifdef	_LITTLE_ENDIAN
797 struct tavor_hw_initib_s {
798 	uint32_t	max_gid		:16;
799 	uint32_t			:16;
800 	uint32_t			:4;
801 	uint32_t	vl_cap		:4;
802 	uint32_t	port_width_cap	:4;
803 	uint32_t	mtu_cap		:4;
804 	uint32_t	set_port_guid0	:1;
805 	uint32_t	set_node_guid	:1;
806 	uint32_t	set_sysimg_guid	:1;
807 	uint32_t			:13;
808 	uint32_t			:32;
809 	uint32_t	max_pkey	:16;
810 	uint32_t			:16;
811 	uint64_t	guid0;
812 	uint64_t	node_guid;
813 	uint64_t	sysimg_guid;
814 	uint32_t	rsrv0[54];
815 };
816 #else
817 struct tavor_hw_initib_s {
818 	uint32_t			:13;
819 	uint32_t	set_sysimg_guid	:1;
820 	uint32_t	set_node_guid	:1;
821 	uint32_t	set_port_guid0	:1;
822 	uint32_t	mtu_cap		:4;
823 	uint32_t	port_width_cap	:4;
824 	uint32_t	vl_cap		:4;
825 	uint32_t			:4;
826 	uint32_t			:16;
827 	uint32_t	max_gid		:16;
828 	uint32_t			:16;
829 	uint32_t	max_pkey	:16;
830 	uint32_t			:32;
831 	uint64_t	guid0;
832 	uint64_t	node_guid;
833 	uint64_t	sysimg_guid;
834 	uint32_t	rsrv0[54];
835 };
836 #endif
837 
838 /*
839  * Tavor Memory Protection Table (MPT) entries
840  *    The Memory Protection Table (MPT) contains the information associated
841  *    with all the regions and windows. The MPT table resides in a physically-
842  *    contiguous area in HCA attached local memory, and the memory key (R_Key
843  *    or L_Key) is used to calculate the physical address for accessing the
844  *    entries in the table.
845  *
846  *    The following structure is used in the SW2HW_MPT, QUERY_MPT, and
847  *    HW2SW_MPT commands.
848  *    The SW2HW_MPT command transfers ownership of an MPT entry from software
849  *    to hardware. The command takes the MPT entry from the input mailbox and
850  *    stores it in the MPT in the hardware. The command will fail if the
851  *    requested MPT entry is already owned by the hardware or if the MPT index
852  *    given in the command is inconsistent with the MPT entry memory key.
853  *    The QUERY_MPT command retrieves a snapshot of an MPT entry. The command
854  *    takes the current state of an MPT entry from the hardware and stores it
855  *    in the output mailbox.  The command will fail if the requested MPT entry
856  *    is already owned by software.
857  *    Finally, the HW2SW_MPT command transfers ownership of an MPT entry from
858  *    the hardware to the software. The command takes the MPT entry from the
859  *    hardware, invalidates it, and stores it in the output mailbox. The
860  *    command will fail if the requested entry is already owned by software.
861  *    The command will also fail if the MPT entry in question is a Memory
862  *    Region which has Memory Windows currently bound to it.
863  */
864 #ifdef	_LITTLE_ENDIAN
865 struct tavor_hw_mpt_s {
866 	uint32_t	page_sz		:5;
867 	uint32_t			:27;
868 	uint32_t	ver		:4;
869 	uint32_t			:4;
870 	uint32_t	reg_win		:1;
871 	uint32_t	phys_addr	:1;
872 	uint32_t	lr		:1;
873 	uint32_t	lw		:1;
874 	uint32_t	rr		:1;
875 	uint32_t	rw		:1;
876 	uint32_t	atomic		:1;
877 	uint32_t	en_bind		:1;
878 	uint32_t			:1;
879 	uint32_t	m_io		:1;
880 	uint32_t			:10;
881 	uint32_t	status		:4;
882 	uint32_t	pd		:24;
883 	uint32_t			:8;
884 	uint32_t	mem_key;
885 	uint64_t	start_addr;
886 	uint64_t	reg_win_len;
887 	uint32_t	win_cnt;
888 	uint32_t	lkey;
889 	uint32_t	mttseg_addr_h;
890 	uint32_t	win_cnt_limit;
891 	uint32_t			:32;
892 	uint32_t			:6;
893 	uint32_t	mttseg_addr_l	:26;
894 	uint32_t	rsrv0[2];
895 };
896 #else
897 struct tavor_hw_mpt_s {
898 	uint32_t	status		:4;
899 	uint32_t			:10;
900 	uint32_t	m_io		:1;
901 	uint32_t			:1;
902 	uint32_t	en_bind		:1;
903 	uint32_t	atomic		:1;
904 	uint32_t	rw		:1;
905 	uint32_t	rr		:1;
906 	uint32_t	lw		:1;
907 	uint32_t	lr		:1;
908 	uint32_t	phys_addr	:1;
909 	uint32_t	reg_win		:1;
910 	uint32_t			:4;
911 	uint32_t	ver		:4;
912 	uint32_t			:27;
913 	uint32_t	page_sz		:5;
914 	uint32_t	mem_key;
915 	uint32_t			:8;
916 	uint32_t	pd		:24;
917 	uint64_t	start_addr;
918 	uint64_t	reg_win_len;
919 	uint32_t	lkey;
920 	uint32_t	win_cnt;
921 	uint32_t	win_cnt_limit;
922 	uint32_t	mttseg_addr_h;
923 	uint32_t	mttseg_addr_l	:26;
924 	uint32_t			:6;
925 	uint32_t			:32;
926 	uint32_t	rsrv0[2];
927 };
928 #endif
929 #define	TAVOR_MEM_CYCLE_GENERATE	0x1
930 #define	TAVOR_IO_CYCLE_GENERATE		0x0
931 
932 #define	TAVOR_MPT_IS_WINDOW		0x0
933 #define	TAVOR_MPT_IS_REGION		0x1
934 
935 #define	TAVOR_MPT_DEFAULT_VERSION	0x0
936 
937 #define	TAVOR_UNLIMITED_WIN_BIND	0x0
938 
939 #define	TAVOR_PHYSADDR_ENABLED		0x1
940 #define	TAVOR_PHYSADDR_DISABLED		0x0
941 
942 
943 /*
944  * Tavor Memory Translation Table (MTT) entries
945  *    After accessing the MPT table (above) and validating the access rights
946  *    to the region/window, Tavor address translation moves to the next step
947  *    where it translates the virtual address to a physical address.  This
948  *    translation is performed using the Memory Translation Table entries
949  *    (MTT).  Note: The MTT in hardware is organized into segments and each
950  *    segment contains multiple address translation pages (MTT entries).
951  *    Each memory region (MPT above) points to the first segment in the MTT
952  *    that corresponds to that region.
953  */
954 #ifdef	_LITTLE_ENDIAN
955 struct tavor_hw_mtt_s {
956 	uint32_t	present		:1;
957 	uint32_t			:11;
958 	uint32_t	ptag_l		:20;
959 	uint32_t	ptag_h;
960 };
961 #else
962 struct tavor_hw_mtt_s {
963 	uint32_t	ptag_h;
964 	uint32_t	ptag_l		:20;
965 	uint32_t			:11;
966 	uint32_t	present		:1;
967 };
968 #endif
969 #define	TAVOR_MTT_ENTRY_NOTPRESET	0x0
970 #define	TAVOR_MTT_ENTRY_PRESET		0x1
971 
972 
973 /*
974  * Tavor Event Queue Context Table (EQC) entries
975  *    Tavor supports 64 Event Queues, and the status of Event Queues is stored
976  *    in the Event Queue Context (EQC) table.  The EQC table is a physically-
977  *    contiguous memory structure in the HCA attached local memory.  Each EQC
978  *    table entry contains Event Queue status and information required by
979  *    the hardware in order to access the event queue.
980  *
981  *    The following structure is used in the SW2HW_EQ, QUERY_EQ, and HW2SW_EQ
982  *    commands.
983  *    The SW2HW_EQ command transfers ownership of an EQ context from software
984  *    to hardware. The command takes the EQC entry from the input mailbox and
985  *    stores it in the EQC in the hardware. The command will fail if the
986  *    requested EQC entry is already owned by the hardware.
987  *    The QUERY_EQ command retrieves a snapshot of an EQC entry. The command
988  *    stores the snapshot in the output mailbox.  The EQC state and its values
989  *    are not affected by the QUERY_EQ command.
990  *    Finally, the HW2SW_EQ command transfers ownership of an EQC entry from
991  *    the hardware to the software. The command takes the EQC entry from the
992  *    hardware and stores it in the output mailbox. The EQC entry will be
993  *    invalidated as a result of the command.  It is the responsibility of the
994  *    software to unmap all the events, which might have been previously
995  *    mapped to the EQ, prior to issuing the HW2SW_EQ command.
996  */
997 #ifdef	_LITTLE_ENDIAN
998 struct tavor_hw_eqc_s {
999 	uint32_t	start_addr_h;
1000 	uint32_t			:8;
1001 	uint32_t	state		:2;
1002 	uint32_t			:7;
1003 	uint32_t	overrun_ignore	:1;
1004 	uint32_t	xlat		:1;
1005 	uint32_t			:5;
1006 	uint32_t	owner		:4;
1007 	uint32_t	status		:4;
1008 	uint32_t	usr_page	:24;
1009 	uint32_t	log_eq_sz	:5;
1010 	uint32_t			:3;
1011 	uint32_t	start_addr_l;
1012 	uint32_t	intr		:8;
1013 	uint32_t			:24;
1014 	uint32_t	pd		:24;
1015 	uint32_t			:8;
1016 	uint32_t	lkey;
1017 	uint32_t	lost_cnt;
1018 	uint32_t	rsrv0[2];
1019 	uint32_t	prod_indx;
1020 	uint32_t	cons_indx;
1021 	uint32_t	rsrv1[4];
1022 };
1023 #else
1024 struct tavor_hw_eqc_s {
1025 	uint32_t	status		:4;
1026 	uint32_t	owner		:4;
1027 	uint32_t			:5;
1028 	uint32_t	xlat		:1;
1029 	uint32_t	overrun_ignore	:1;
1030 	uint32_t			:7;
1031 	uint32_t	state		:2;
1032 	uint32_t			:8;
1033 	uint32_t	start_addr_h;
1034 	uint32_t	start_addr_l;
1035 	uint32_t			:3;
1036 	uint32_t	log_eq_sz	:5;
1037 	uint32_t	usr_page	:24;
1038 	uint32_t			:8;
1039 	uint32_t	pd		:24;
1040 	uint32_t			:24;
1041 	uint32_t	intr		:8;
1042 	uint32_t	lost_cnt;
1043 	uint32_t	lkey;
1044 	uint32_t	rsrv0[2];
1045 	uint32_t	cons_indx;
1046 	uint32_t	prod_indx;
1047 	uint32_t	rsrv1[4];
1048 };
1049 #endif
1050 #define	TAVOR_EQ_STATUS_OK		0x0
1051 #define	TAVOR_EQ_STATUS_OVERFLOW	0x9
1052 #define	TAVOR_EQ_STATUS_WRITE_FAILURE	0xA
1053 
1054 #define	TAVOR_EQ_ARMED			0x1
1055 #define	TAVOR_EQ_FIRED			0x2
1056 #define	TAVOR_EQ_ALWAYS_ARMED		0x3
1057 
1058 
1059 /*
1060  * Tavor Event Queue Entries (EQE)
1061  *    Each EQE contains enough information for the software to identify the
1062  *    source of the event.  The following structures are used to define each
1063  *    of the various kinds of events that the Tavor hardware will generate.
1064  *    Note: The tavor_hw_eqe_t below is the generic "Event Queue Entry".  All
1065  *    other EQEs differ only in the contents of their "event_data" field.
1066  *
1067  *    Below we first define several structures which define the contents of
1068  *    the "event_data" fields:
1069  *    tavor_hw_eqe_cq_t for "Completion Queue Events"
1070  *    tavor_hw_eqe_cqerr_t for "Completion Queue Error Events"
1071  *    tavor_hw_eqe_portstate_t for "Port State Change Events"
1072  *    tavor_hw_eqe_cmdcmpl_t for "Command Interface Completion Events"
1073  *    tavor_hw_eqe_qp_evt_t for "Queue Pair Events" such as Path Migration
1074  *        Succeeded, Path Migration Failed, Communication Established, Send
1075  *        Queue Drained, Local WQ Catastrophic Error, Invalid Request Local
1076  *        WQ Error, and Local Access Violation WQ Error.
1077  *    tavor_hw_eqe_operr_t for "Operational and Catastrophic Error Events"
1078  *        such as EQ Overflow, Misbehaved UAR page, Internal Parity Error,
1079  *        Uplink bus error, and DDR data error.
1080  *    tavor_hw_eqe_pgflt_t for "Not-present Page Fault on WQE or Data
1081  *        Buffer Access".  (Note: Currently, this event is unsupported).
1082  *
1083  *    Note also: The following structures are not #define'd with both
1084  *    little-endian and big-endian definitions.  This is because their
1085  *    individual fields are not directly accessed except through the macros
1086  *    defined below.
1087  */
1088 typedef struct tavor_hw_eqe_cq_s {
1089 	uint32_t			:8;
1090 	uint32_t	cqn		:24;
1091 	uint32_t	rsrv0[5];
1092 } tavor_hw_eqe_cq_t;
1093 
1094 typedef struct tavor_hw_eqe_cqerr_s {
1095 	uint32_t			:8;
1096 	uint32_t	cqn		:24;
1097 	uint32_t			:32;
1098 	uint32_t			:24;
1099 	uint32_t	syndrome	:8;
1100 	uint32_t	rsrv0[3];
1101 } tavor_hw_eqe_cqerr_t;
1102 #define	TAVOR_CQERR_OVERFLOW		0x1
1103 #define	TAVOR_CQERR_ACCESS_VIOLATION	0x2
1104 
1105 typedef struct tavor_hw_eqe_portstate_s {
1106 	uint32_t	rsrv0[2];
1107 	uint32_t			:2;
1108 	uint32_t	port		:2;
1109 	uint32_t			:28;
1110 	uint32_t	rsrv1[3];
1111 } tavor_hw_eqe_portstate_t;
1112 #define	TAVOR_PORT_LINK_ACTIVE		0x4
1113 #define	TAVOR_PORT_LINK_DOWN		0x1
1114 
1115 typedef struct tavor_hw_eqe_cmdcmpl_s {
1116 	uint32_t			:16;
1117 	uint32_t	token		:16;
1118 	uint32_t			:32;
1119 	uint32_t			:24;
1120 	uint32_t	status		:8;
1121 	uint32_t	out_param0;
1122 	uint32_t	out_param1;
1123 	uint32_t			:32;
1124 } tavor_hw_eqe_cmdcmpl_t;
1125 
1126 typedef struct tavor_hw_eqe_qp_evt_s {
1127 	uint32_t			:8;
1128 	uint32_t	qpn		:24;
1129 	uint32_t			:32;
1130 	uint32_t			:3;
1131 	uint32_t	qp_ee		:1;
1132 	uint32_t			:28;
1133 	uint32_t	rsrv0[3];
1134 } tavor_hw_eqe_qpevt_t;
1135 
1136 typedef struct tavor_hw_eqe_operr_s {
1137 	uint32_t	rsrv0[2];
1138 	uint32_t			:24;
1139 	uint32_t	error_type	:8;
1140 	uint32_t	data;
1141 	uint32_t	rsrv1[2];
1142 } tavor_hw_eqe_operr_t;
1143 #define	TAVOR_ERREVT_EQ_OVERFLOW	0x1
1144 #define	TAVOR_ERREVT_BAD_UARPG		0x2
1145 #define	TAVOR_ERREVT_UPLINK_BUSERR	0x3
1146 #define	TAVOR_ERREVT_DDR_DATAERR	0x4
1147 #define	TAVOR_ERREVT_INTERNAL_PARITY	0x5
1148 
1149 typedef struct tavor_hw_eqe_pgflt_s {
1150 	uint32_t	rsrv0[2];
1151 	uint32_t			:24;
1152 	uint32_t	fault_type	:4;
1153 	uint32_t	wqv		:1;
1154 	uint32_t	wqe_data	:1;
1155 	uint32_t	rem_loc		:1;
1156 	uint32_t	snd_rcv		:1;
1157 	uint32_t	vaddr_h;
1158 	uint32_t	vaddr_l;
1159 	uint32_t	mem_key;
1160 } tavor_hw_eqe_pgflt_t;
1161 #define	TAVOR_PGFLT_PG_NOTPRESENT	0x8
1162 #define	TAVOR_PGFLT_PG_WRACC_VIOL	0xA
1163 #define	TAVOR_PGFLT_UNSUP_NOTPRESENT	0xE
1164 #define	TAVOR_PGFLT_UNSUP_WRACC_VIOL	0xF
1165 #define	TAVOR_PGFLT_WQE_CAUSED		0x1
1166 #define	TAVOR_PGFLT_DATA_CAUSED		0x0
1167 #define	TAVOR_PGFLT_REMOTE_CAUSED	0x1
1168 #define	TAVOR_PGFLT_LOCAL_CAUSED	0x0
1169 #define	TAVOR_PGFLT_SEND_CAUSED		0x1
1170 #define	TAVOR_PGFLT_RECV_CAUSED		0x0
1171 #define	TAVOR_PGFLT_DESC_CONSUMED	0x1
1172 #define	TAVOR_PGFLT_DESC_NOTCONSUMED	0x0
1173 
1174 typedef struct tavor_hw_eqe_ecc_s {
1175 	uint32_t	rsrcv0[4];
1176 	uint32_t	overflow	:1;
1177 	uint32_t			:15;
1178 	uint32_t			:2;
1179 	uint32_t	err_ba		:2;
1180 	uint32_t	err_da		:2;
1181 	uint32_t	err_src_id	:3;
1182 	uint32_t	err_rmw		:1;
1183 	uint32_t			:2;
1184 	uint32_t	cause_msb	:1;
1185 	uint32_t			:2;
1186 	uint32_t	cause_lsb	:1;
1187 
1188 	uint32_t	err_ca		:16;
1189 	uint32_t	err_ra		:16;
1190 } tavor_hw_eqe_ecc_t;
1191 
1192 struct tavor_hw_eqe_s {
1193 	uint32_t			:8;
1194 	uint32_t	event_type	:8;
1195 	uint32_t			:8;
1196 	uint32_t	event_subtype	:8;
1197 	union {
1198 		tavor_hw_eqe_cq_t		eqe_cq;
1199 		tavor_hw_eqe_cqerr_t		eqe_cqerr;
1200 		tavor_hw_eqe_portstate_t	eqe_portstate;
1201 		tavor_hw_eqe_cmdcmpl_t		eqe_cmdcmpl;
1202 		tavor_hw_eqe_qpevt_t		eqe_qpevt;
1203 		tavor_hw_eqe_operr_t		eqe_operr;
1204 		tavor_hw_eqe_pgflt_t		eqe_pgflt;
1205 		tavor_hw_eqe_ecc_t		eqe_ecc;
1206 	} event_data;
1207 	uint32_t			:24;
1208 	uint32_t	owner		:1;
1209 	uint32_t			:7;
1210 };
1211 #define	eqe_cq				event_data.eqe_cq
1212 #define	eqe_cqerr			event_data.eqe_cqerr
1213 #define	eqe_portstate			event_data.eqe_portstate
1214 #define	eqe_cmdcmpl			event_data.eqe_cmdcmpl
1215 #define	eqe_qpevt			event_data.eqe_qpevt
1216 #define	eqe_operr			event_data.eqe_operr
1217 #define	eqe_pgflt			event_data.eqe_pgflt
1218 
1219 /*
1220  * The following macros are used for extracting (and in some cases filling in)
1221  * information from EQEs
1222  */
1223 #define	TAVOR_EQE_EVTTYPE_MASK		0x00FF0000
1224 #define	TAVOR_EQE_EVTTYPE_SHIFT		16
1225 #define	TAVOR_EQE_EVTSUBTYPE_MASK	0x000000FF
1226 #define	TAVOR_EQE_EVTSUBTYPE_SHIFT	0
1227 #define	TAVOR_EQE_CQNUM_MASK		0x00FFFFFF
1228 #define	TAVOR_EQE_CQNUM_SHIFT		0
1229 #define	TAVOR_EQE_QPNUM_MASK		0x00FFFFFF
1230 #define	TAVOR_EQE_QPNUM_SHIFT		0
1231 #define	TAVOR_EQE_PORTNUM_MASK		0x30000000
1232 #define	TAVOR_EQE_PORTNUM_SHIFT		28
1233 #define	TAVOR_EQE_CMDTOKEN_MASK		0x0000FFFF
1234 #define	TAVOR_EQE_CMDTOKEN_SHIFT	0
1235 #define	TAVOR_EQE_CMDSTATUS_MASK	0x000000FF
1236 #define	TAVOR_EQE_CMDSTATUS_SHIFT	0
1237 #define	TAVOR_EQE_OPERRTYPE_MASK 	0x000000FF
1238 #define	TAVOR_EQE_OPERRTYPE_SHIFT	0
1239 #define	TAVOR_EQE_OWNER_MASK		0x00000080
1240 #define	TAVOR_EQE_OWNER_SHIFT		7
1241 
1242 #define	TAVOR_EQE_EVTTYPE_GET(eq, eqe)					\
1243 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1244 	    &((uint32_t *)(eqe))[0]) & TAVOR_EQE_EVTTYPE_MASK) >>	\
1245 	    TAVOR_EQE_EVTTYPE_SHIFT)
1246 #define	TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe)				\
1247 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1248 	    &((uint32_t *)(eqe))[0]) & TAVOR_EQE_EVTSUBTYPE_MASK) >>	\
1249 	    TAVOR_EQE_EVTSUBTYPE_SHIFT)
1250 #define	TAVOR_EQE_CQNUM_GET(eq, eqe)					\
1251 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1252 	    &((uint32_t *)(eqe))[1]) & TAVOR_EQE_CQNUM_MASK) >>		\
1253 	    TAVOR_EQE_CQNUM_SHIFT)
1254 #define	TAVOR_EQE_QPNUM_GET(eq, eqe)					\
1255 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1256 	&((uint32_t *)(eqe))[1]) & TAVOR_EQE_QPNUM_MASK) >>		\
1257 	TAVOR_EQE_QPNUM_SHIFT)
1258 #define	TAVOR_EQE_PORTNUM_GET(eq, eqe)					\
1259 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1260 	    &((uint32_t *)(eqe))[3]) & TAVOR_EQE_PORTNUM_MASK) >>	\
1261 	    TAVOR_EQE_PORTNUM_SHIFT)
1262 #define	TAVOR_EQE_CMDTOKEN_GET(eq, eqe)					\
1263 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1264 	    &((uint32_t *)(eqe))[1]) & TAVOR_EQE_CMDTOKEN_MASK) >>	\
1265 	    TAVOR_EQE_CMDTOKEN_SHIFT)
1266 #define	TAVOR_EQE_CMDSTATUS_GET(eq, eqe)				\
1267 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1268 	    &((uint32_t *)(eqe))[3]) & TAVOR_EQE_CMDSTATUS_MASK) >>	\
1269 	    TAVOR_EQE_CMDSTATUS_SHIFT)
1270 #define	TAVOR_EQE_CMDOUTP0_GET(eq, eqe)					\
1271 	(ddi_get32((eq)->eq_eqinfo.qa_acchdl, &((uint32_t *)(eqe))[4]))
1272 #define	TAVOR_EQE_CMDOUTP1_GET(eq, eqe)					\
1273 	(ddi_get32((eq)->eq_eqinfo.qa_acchdl, &((uint32_t *)(eqe))[5]))
1274 #define	TAVOR_EQE_OPERRTYPE_GET(eq, eqe)				\
1275 	((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1276 	    &((uint32_t *)(eqe))[3]) & TAVOR_EQE_OPERRTYPE_MASK) >>	\
1277 	    TAVOR_EQE_OPERRTYPE_SHIFT)
1278 #define	TAVOR_EQE_OPERRDATA_GET(eq, eqe)				\
1279 	(ddi_get32((eq)->eq_eqinfo.qa_acchdl, &((uint32_t *)(eqe))[4]))
1280 #define	TAVOR_EQE_OWNER_IS_SW(eq, eqe)					\
1281 	(((ddi_get32((eq)->eq_eqinfo.qa_acchdl,				\
1282 	    &((uint32_t *)(eqe))[7]) & TAVOR_EQE_OWNER_MASK) >>		\
1283 	    TAVOR_EQE_OWNER_SHIFT) == TAVOR_SW_OWNER)
1284 #define	TAVOR_EQE_OWNER_SET_HW(eq, eqe)					\
1285 	(ddi_put32((eq)->eq_eqinfo.qa_acchdl, &((uint32_t *)(eqe))[7],	\
1286 	    ((TAVOR_HW_OWNER <<	TAVOR_EQE_OWNER_SHIFT) &		\
1287 	    TAVOR_EQE_OWNER_MASK)))
1288 
1289 
1290 /*
1291  * Tavor Completion Queue Context Table (CQC) entries
1292  *    The CQC table is a physically-contiguous memory area residing in HCA
1293  *    attached local memory.  Each CQC table entry contains information
1294  *    required by the hardware to access the completion queue to post
1295  *    completions (CQE).
1296  *
1297  *    The following structure is used in the SW2HW_CQ, QUERY_CQ, RESIZE_CQ,
1298  *    and HW2SW_CQ commands.
1299  *    The SW2HW_CQ command transfers ownership of an CQ context from software
1300  *    to hardware. The command takes the CQC entry from the input mailbox and
1301  *    stores it in the CQC in the hardware. The command will fail if the
1302  *    requested CQC entry is already owned by the hardware.
1303  *    The QUERY_CQ command retrieves a snapshot of a CQC entry. The command
1304  *    stores the snapshot in the output mailbox.  The CQC state and its values
1305  *    are not affected by the QUERY_CQ command.
1306  *    Finally, the HW2SW_CQ command transfers ownership of a CQC entry from
1307  *    the hardware to the software. The command takes the CQC entry from the
1308  *    hardware and stores it in the output mailbox. The CQC entry will be
1309  *    invalidated as a result of the command.
1310  */
1311 #ifdef	_LITTLE_ENDIAN
1312 struct tavor_hw_cqc_s {
1313 	uint32_t	start_addr_h;
1314 	uint32_t			:8;
1315 	uint32_t	state		:4;
1316 	uint32_t			:5;
1317 	uint32_t	overrun_ignore	:1;
1318 	uint32_t	xlat		:1;
1319 	uint32_t			:9;
1320 	uint32_t	status		:4;
1321 	uint32_t	usr_page	:24;
1322 	uint32_t	log_cq_sz	:5;
1323 	uint32_t			:3;
1324 	uint32_t	start_addr_l;
1325 	uint32_t	c_eqn		:8;
1326 	uint32_t			:24;
1327 	uint32_t	e_eqn		:8;
1328 	uint32_t			:24;
1329 	uint32_t	lkey;
1330 	uint32_t	pd		:24;
1331 	uint32_t			:8;
1332 	uint32_t	solicit_prod_indx;
1333 	uint32_t	last_notified_indx;
1334 	uint32_t	prod_indx;
1335 	uint32_t	cons_indx;
1336 	uint32_t			:32;
1337 	uint32_t	cqn		:24;
1338 	uint32_t			:8;
1339 	uint32_t	rsrv0[2];
1340 };
1341 #else
1342 struct tavor_hw_cqc_s {
1343 	uint32_t	status		:4;
1344 	uint32_t			:9;
1345 	uint32_t	xlat		:1;
1346 	uint32_t	overrun_ignore	:1;
1347 	uint32_t			:5;
1348 	uint32_t	state		:4;
1349 	uint32_t			:8;
1350 	uint32_t	start_addr_h;
1351 	uint32_t	start_addr_l;
1352 	uint32_t			:3;
1353 	uint32_t	log_cq_sz	:5;
1354 	uint32_t	usr_page	:24;
1355 	uint32_t			:24;
1356 	uint32_t	e_eqn		:8;
1357 	uint32_t			:24;
1358 	uint32_t	c_eqn		:8;
1359 	uint32_t			:8;
1360 	uint32_t	pd		:24;
1361 	uint32_t	lkey;
1362 	uint32_t	last_notified_indx;
1363 	uint32_t	solicit_prod_indx;
1364 	uint32_t	cons_indx;
1365 	uint32_t	prod_indx;
1366 	uint32_t			:8;
1367 	uint32_t	cqn		:24;
1368 	uint32_t			:32;
1369 	uint32_t	rsrv0[2];
1370 };
1371 #endif
1372 #define	TAVOR_CQ_STATUS_OK		0x0
1373 #define	TAVOR_CQ_STATUS_OVERFLOW	0x9
1374 #define	TAVOR_CQ_STATUS_WRITE_FAILURE	0xA
1375 
1376 #define	TAVOR_CQ_DISARMED		0x0
1377 #define	TAVOR_CQ_ARMED			0x1
1378 #define	TAVOR_CQ_ARMED_SOLICITED	0x4
1379 #define	TAVOR_CQ_FIRED			0xA
1380 
1381 /*
1382  * Tavor Completion Queue Entries (CQE)
1383  *    Each CQE contains enough information for the software to associate the
1384  *    completion with the Work Queue Element (WQE) to which it corresponds.
1385  *
1386  *    Note: The following structure is not #define'd with both little-endian
1387  *    and big-endian definitions.  This is because each CQE's individual
1388  *    fields are not directly accessed except through the macros defined below.
1389  */
1390 struct tavor_hw_cqe_s {
1391 	uint32_t	ver		:4;
1392 	uint32_t			:4;
1393 	uint32_t	my_qpn		:24;
1394 	uint32_t			:8;
1395 	uint32_t	my_ee		:24;
1396 	uint32_t			:8;
1397 	uint32_t	rqpn		:24;
1398 	uint32_t	sl		:4;
1399 	uint32_t			:4;
1400 	uint32_t	grh		:1;
1401 	uint32_t	ml_path		:7;
1402 	uint32_t	rlid		:16;
1403 	uint32_t	imm_eth_pkey_cred;
1404 	uint32_t	byte_cnt;
1405 	uint32_t	wqe_addr	:26;
1406 	uint32_t	wqe_sz		:6;
1407 	uint32_t	opcode		:8;
1408 	uint32_t	send_or_recv	:1;
1409 	uint32_t			:15;
1410 	uint32_t	owner		:1;
1411 	uint32_t			:7;
1412 };
1413 #define	TAVOR_COMPLETION_RECV		0x0
1414 #define	TAVOR_COMPLETION_SEND		0x1
1415 
1416 #define	TAVOR_CQE_DEFAULT_VERSION	0x0
1417 
1418 /*
1419  * The following macros are used for extracting (and in some cases filling in)
1420  * information from CQEs
1421  */
1422 #define	TAVOR_CQE_QPNUM_MASK		0x00FFFFFF
1423 #define	TAVOR_CQE_QPNUM_SHIFT		0
1424 #define	TAVOR_CQE_DQPN_MASK		0x00FFFFFF
1425 #define	TAVOR_CQE_DQPN_SHIFT		0
1426 #define	TAVOR_CQE_SL_MASK		0xF0000000
1427 #define	TAVOR_CQE_SL_SHIFT		28
1428 #define	TAVOR_CQE_GRH_MASK		0x00800000
1429 #define	TAVOR_CQE_GRH_SHIFT		23
1430 #define	TAVOR_CQE_PATHBITS_MASK		0x007F0000
1431 #define	TAVOR_CQE_PATHBITS_SHIFT	16
1432 #define	TAVOR_CQE_DLID_MASK		0x0000FFFF
1433 #define	TAVOR_CQE_DLID_SHIFT		0
1434 #define	TAVOR_CQE_OPCODE_MASK		0xFF000000
1435 #define	TAVOR_CQE_OPCODE_SHIFT		24
1436 #define	TAVOR_CQE_SENDRECV_MASK		0x00800000
1437 #define	TAVOR_CQE_SENDRECV_SHIFT	23
1438 #define	TAVOR_CQE_OWNER_MASK		0x00000080
1439 #define	TAVOR_CQE_OWNER_SHIFT		7
1440 
1441 #define	TAVOR_CQE_QPNUM_GET(cq, cqe)					\
1442 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1443 	    &((uint32_t *)(cqe))[0]) & TAVOR_CQE_QPNUM_MASK) >>		\
1444 	    TAVOR_CQE_QPNUM_SHIFT)
1445 #define	TAVOR_CQE_DQPN_GET(cq, cqe)					\
1446 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1447 	    &((uint32_t *)(cqe))[2]) & TAVOR_CQE_DQPN_MASK) >>		\
1448 	    TAVOR_CQE_DQPN_SHIFT)
1449 #define	TAVOR_CQE_SL_GET(cq, cqe)					\
1450 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1451 	    &((uint32_t *)(cqe))[3]) & TAVOR_CQE_SL_MASK) >>		\
1452 	    TAVOR_CQE_SL_SHIFT)
1453 #define	TAVOR_CQE_GRH_GET(cq, cqe)					\
1454 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1455 	    &((uint32_t *)(cqe))[3]) & TAVOR_CQE_GRH_MASK) >>		\
1456 	    TAVOR_CQE_GRH_SHIFT)
1457 #define	TAVOR_CQE_PATHBITS_GET(cq, cqe)					\
1458 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1459 	    &((uint32_t *)(cqe))[3]) & TAVOR_CQE_PATHBITS_MASK) >>	\
1460 	    TAVOR_CQE_PATHBITS_SHIFT)
1461 #define	TAVOR_CQE_DLID_GET(cq, cqe)					\
1462 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1463 	    &((uint32_t *)(cqe))[3]) & TAVOR_CQE_DLID_MASK) >>		\
1464 	    TAVOR_CQE_DLID_SHIFT)
1465 #define	TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe)			\
1466 	(ddi_get32((cq)->cq_cqinfo.qa_acchdl, &((uint32_t *)(cqe))[4]))
1467 #define	TAVOR_CQE_IMM_ETH_PKEY_CRED_SET(cq, cqe, arg)			\
1468 	(ddi_put32((cq)->cq_cqinfo.qa_acchdl, &((uint32_t *)(cqe))[4],	\
1469 	    (arg)))
1470 #define	TAVOR_CQE_BYTECNT_GET(cq, cqe)					\
1471 	(ddi_get32((cq)->cq_cqinfo.qa_acchdl, &((uint32_t *)(cqe))[5]))
1472 #define	TAVOR_CQE_WQEADDRSZ_GET(cq, cqe)				\
1473 	(ddi_get32((cq)->cq_cqinfo.qa_acchdl, &((uint32_t *)(cqe))[6]))
1474 #define	TAVOR_CQE_WQEADDRSZ_SET(cq, cqe, arg)				\
1475 	(ddi_put32((cq)->cq_cqinfo.qa_acchdl, &((uint32_t *)(cqe))[6],	\
1476 	    (arg)))
1477 #define	TAVOR_CQE_OPCODE_GET(cq, cqe)					\
1478 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1479 	    &((uint32_t *)(cqe))[7]) & TAVOR_CQE_OPCODE_MASK) >>	\
1480 	    TAVOR_CQE_OPCODE_SHIFT)
1481 #define	TAVOR_CQE_SENDRECV_GET(cq, cqe)					\
1482 	((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1483 	    &((uint32_t *)(cqe))[7]) & TAVOR_CQE_SENDRECV_MASK) >>	\
1484 	    TAVOR_CQE_SENDRECV_SHIFT)
1485 #define	TAVOR_CQE_OWNER_IS_SW(cq, cqe)					\
1486 	(((ddi_get32((cq)->cq_cqinfo.qa_acchdl,				\
1487 	    &((uint32_t *)(cqe))[7]) & TAVOR_CQE_OWNER_MASK) >>		\
1488 	    TAVOR_CQE_OWNER_SHIFT) == TAVOR_SW_OWNER)
1489 #ifdef	_LITTLE_ENDIAN
1490 #define	TAVOR_CQE_OWNER_SET_HW(cq, cqe)					\
1491 	{								\
1492 	    if ((cq)->cq_is_umap) {					\
1493 		((uint32_t *)(cqe))[7] = 0x80000000;			\
1494 	    } else {							\
1495 		ddi_put32((cq)->cq_cqinfo.qa_acchdl,			\
1496 		    &((uint32_t *)(cqe))[7], 0x00000080);		\
1497 	    }								\
1498 	}
1499 #else
1500 #define	TAVOR_CQE_OWNER_SET_HW(cq, cqe)					\
1501 	{								\
1502 	    if ((cq)->cq_is_umap) {					\
1503 		((uint32_t *)(cqe))[7] = 0x00000080;			\
1504 	    } else {							\
1505 		ddi_put32((cq)->cq_cqinfo.qa_acchdl,			\
1506 		    &((uint32_t *)(cqe))[7], 0x00000080);		\
1507 	    }								\
1508 	}
1509 #endif
1510 
1511 /*
1512  * Tavor Shared Receive Queue (SRQ) Context Entry Format
1513  */
1514 #ifdef _LITTLE_ENDIAN
1515 struct tavor_hw_srqc_s {
1516 	uint32_t	ds			:5;
1517 	uint32_t	next_wqe_addr_l		:27;
1518 	uint32_t	wqe_addr_h;
1519 
1520 	uint32_t	lkey;
1521 	uint32_t	pd			:24;
1522 	uint32_t				:4;
1523 	uint32_t	state			:4;
1524 
1525 	uint32_t	wqe_cnt			:16;
1526 	uint32_t				:16;
1527 	uint32_t	uar			:24;
1528 	uint32_t				:8;
1529 };
1530 #else
1531 struct tavor_hw_srqc_s {
1532 	uint32_t	wqe_addr_h;
1533 	uint32_t	next_wqe_addr_l		:27;
1534 	uint32_t	ds			:5;
1535 
1536 	uint32_t	state			:4;
1537 	uint32_t				:4;
1538 	uint32_t	pd			:24;
1539 	uint32_t	lkey;
1540 
1541 	uint32_t				:8;
1542 	uint32_t	uar			:24;
1543 	uint32_t				:16;
1544 	uint32_t	wqe_cnt			:16;
1545 };
1546 #endif
1547 
1548 /*
1549  * Tavor MOD_STAT_CFG input mailbox structure
1550  */
1551 #ifdef _LITTLE_ENDIAN
1552 struct tavor_hw_mod_stat_cfg_s {
1553 	uint32_t				:32;
1554 	uint32_t	log_max_srq		:5;
1555 	uint32_t				:1;
1556 	uint32_t	srq			:1;
1557 	uint32_t	srq_m			:1;
1558 	uint32_t				:24;
1559 	uint32_t	reserved[62];
1560 };
1561 #else
1562 struct tavor_hw_mod_stat_cfg_s {
1563 	uint32_t				:24;
1564 	uint32_t	srq_m			:1;
1565 	uint32_t	srq			:1;
1566 	uint32_t				:1;
1567 	uint32_t	log_max_srq		:5;
1568 	uint32_t				:32;
1569 	uint32_t	reserved[62];
1570 };
1571 #endif
1572 
1573 /*
1574  * Tavor UD Address Vector (UDAV)
1575  *    Tavor UDAV are used in conjunction with Unreliable Datagram (UD) send
1576  *    WQEs. Each UD send message specifies an address vector that denotes its
1577  *    link and (optional) network layer destination address.  The IBA verbs
1578  *    interface enables the separation of the address administration from the
1579  *    send WQE posting. The verbs consumer must use special verbs to create
1580  *    and modify address handles (which represent hardware address vectors).
1581  *    When posting send WQEs to UD QP, the verbs consumer must supply a
1582  *    valid address handle/UDAV.
1583  */
1584 #ifdef	_LITTLE_ENDIAN
1585 struct tavor_hw_udav_s {
1586 	uint32_t	rlid		:16;
1587 	uint32_t	ml_path		:7;
1588 	uint32_t	grh		:1;
1589 	uint32_t			:8;
1590 	uint32_t	pd		:24;
1591 	uint32_t	portnum		:2;
1592 	uint32_t			:6;
1593 	uint32_t	flow_label	:20;
1594 	uint32_t	tclass		:8;
1595 	uint32_t	sl		:4;
1596 	uint32_t	hop_limit	:8;
1597 	uint32_t	max_stat_rate	:3;
1598 	uint32_t			:1;
1599 	uint32_t	msg_sz		:2;
1600 	uint32_t			:2;
1601 	uint32_t	mgid_index	:6;
1602 	uint32_t			:10;
1603 	uint64_t	rgid_h;
1604 	uint64_t	rgid_l;
1605 };
1606 #else
1607 struct tavor_hw_udav_s {
1608 	uint32_t			:6;
1609 	uint32_t	portnum		:2;
1610 	uint32_t	pd		:24;
1611 	uint32_t			:8;
1612 	uint32_t	grh		:1;
1613 	uint32_t	ml_path		:7;
1614 	uint32_t	rlid		:16;
1615 	uint32_t			:10;
1616 	uint32_t	mgid_index	:6;
1617 	uint32_t			:2;
1618 	uint32_t	msg_sz		:2;
1619 	uint32_t			:1;
1620 	uint32_t	max_stat_rate	:3;
1621 	uint32_t	hop_limit	:8;
1622 	uint32_t	sl		:4;
1623 	uint32_t	tclass		:8;
1624 	uint32_t	flow_label	:20;
1625 	uint64_t	rgid_h;
1626 	uint64_t	rgid_l;
1627 };
1628 #endif
1629 #define	TAVOR_UDAV_MODIFY_MASK0		0xFCFFFFFFFF000000ULL
1630 #define	TAVOR_UDAV_MODIFY_MASK1		0xFFC0F80000000000ULL
1631 
1632 
1633 /*
1634  * Tavor Queue Pair Context Table (QPC) entries
1635  *    The QPC table is a physically-contiguous memory area residing in HCA
1636  *    attached local memory.  Each QPC entry is accessed for reads and writes
1637  *    by the HCA while executing work requests on the associated QP.
1638  *
1639  *    The following structure is used in the RST2INIT_QP, INIT2INIT_QP,
1640  *    INIT2RTR_QP, RTR2RTS_QP, RTS2RTS_QP, SQERR2RTS_QP, TOERR_QP, RTS2SQD_QP,
1641  *    SQD2RTS_QP, TORST_QP, and QUERY_QP commands.
1642  *    With the exception of the QUERY_QP command, each of these commands reads
1643  *    from some portion of the QPC in the input mailbox and modified the QPC
1644  *    stored in the hardware.  The QUERY_QP command retrieves a snapshot of a
1645  *    QPC entry. The command stores the snapshot in the output mailbox.  The
1646  *    QPC state and its values are not affected by the QUERY_QP command.
1647  *
1648  *    Below we first define the tavor_hw_addr_path_t or "Tavor Address Path"
1649  *    structure.  This structure is used to provide address path information
1650  *    (both primary and secondary) for each QP context.  Note:  Since this
1651  *    structure is _very_ similar to the tavor_hw_udav_t structure above,
1652  *    we are able to leverage the similarity with filling in and reading from
1653  *    the two types of structures.  See tavor_get_addr_path() and
1654  *    tavor_set_addr_path() in tavor_misc.c for more details.
1655  */
1656 #ifdef	_LITTLE_ENDIAN
1657 struct tavor_hw_addr_path_s {
1658 	uint32_t	rlid		:16;
1659 	uint32_t	ml_path		:7;
1660 	uint32_t	grh		:1;
1661 	uint32_t			:5;
1662 	uint32_t	rnr_retry	:3;
1663 	uint32_t	pkey_indx	:7;
1664 	uint32_t			:17;
1665 	uint32_t	portnum		:2;
1666 	uint32_t			:6;
1667 	uint32_t	flow_label	:20;
1668 	uint32_t	tclass		:8;
1669 	uint32_t	sl		:4;
1670 	uint32_t	hop_limit	:8;
1671 	uint32_t	max_stat_rate	:3;
1672 	uint32_t			:5;
1673 	uint32_t	mgid_index	:6;
1674 	uint32_t			:5;
1675 	uint32_t	ack_timeout	:5;
1676 	uint64_t	rgid_h;
1677 	uint64_t	rgid_l;
1678 };
1679 #else
1680 struct tavor_hw_addr_path_s {
1681 	uint32_t			:6;
1682 	uint32_t	portnum		:2;
1683 	uint32_t			:17;
1684 	uint32_t	pkey_indx	:7;
1685 	uint32_t	rnr_retry	:3;
1686 	uint32_t			:5;
1687 	uint32_t	grh		:1;
1688 	uint32_t	ml_path		:7;
1689 	uint32_t	rlid		:16;
1690 	uint32_t	ack_timeout	:5;
1691 	uint32_t			:5;
1692 	uint32_t	mgid_index	:6;
1693 	uint32_t			:5;
1694 	uint32_t	max_stat_rate	:3;
1695 	uint32_t	hop_limit	:8;
1696 	uint32_t	sl		:4;
1697 	uint32_t	tclass		:8;
1698 	uint32_t	flow_label	:20;
1699 	uint64_t	rgid_h;
1700 	uint64_t	rgid_l;
1701 };
1702 #endif
1703 
1704 #ifdef	_LITTLE_ENDIAN
1705 struct tavor_hw_qpc_s {
1706 	uint32_t	sched_q		:4;
1707 	uint32_t			:28;
1708 	uint32_t			:8;
1709 	uint32_t	de		:1;
1710 	uint32_t			:2;
1711 	uint32_t	pm_state	:2;
1712 	uint32_t			:3;
1713 	uint32_t	serv_type	:3;
1714 	uint32_t			:9;
1715 	uint32_t	state		:4;
1716 	uint32_t	usr_page	:24;
1717 	uint32_t			:8;
1718 	uint32_t			:24;
1719 	uint32_t	msg_max		:5;
1720 	uint32_t	mtu		:3;
1721 	uint32_t	rem_qpn		:24;
1722 	uint32_t			:8;
1723 	uint32_t	loc_qpn		:24;
1724 	uint32_t			:8;
1725 	uint32_t			:32;
1726 	uint32_t			:32;
1727 	tavor_hw_addr_path_t	pri_addr_path;
1728 	tavor_hw_addr_path_t	alt_addr_path;
1729 	uint32_t	pd		:24;
1730 	uint32_t			:8;
1731 	uint32_t	rdd		:24;
1732 	uint32_t			:8;
1733 	uint32_t	wqe_lkey;
1734 	uint32_t	wqe_baseaddr;
1735 	uint32_t			:32;
1736 	uint32_t			:3;
1737 	uint32_t	ssc		:1;
1738 	uint32_t	sic		:1;
1739 	uint32_t	cur_retry_cnt	:3;
1740 	uint32_t	cur_rnr_retry	:3;
1741 	uint32_t			:2;
1742 	uint32_t	sae		:1;
1743 	uint32_t	swe		:1;
1744 	uint32_t	sre		:1;
1745 	uint32_t	retry_cnt	:3;
1746 	uint32_t			:2;
1747 	uint32_t	sra_max		:3;
1748 	uint32_t	flight_lim	:4;
1749 	uint32_t	ack_req_freq	:4;
1750 	uint32_t	cqn_snd		:24;
1751 	uint32_t			:8;
1752 	uint32_t	next_snd_psn	:24;
1753 	uint32_t			:8;
1754 	uint64_t	next_snd_wqe;
1755 	uint32_t	ssn		:24;
1756 	uint32_t			:8;
1757 	uint32_t	last_acked_psn	:24;
1758 	uint32_t			:8;
1759 	uint32_t	next_rcv_psn	:24;
1760 	uint32_t	min_rnr_nak	:5;
1761 	uint32_t			:3;
1762 	uint32_t			:3;
1763 	uint32_t	rsc		:1;
1764 	uint32_t	ric		:1;
1765 	uint32_t			:8;
1766 	uint32_t	rae		:1;
1767 	uint32_t	rwe		:1;
1768 	uint32_t	rre		:1;
1769 	uint32_t			:5;
1770 	uint32_t	rra_max		:3;
1771 	uint32_t			:8;
1772 	uint32_t	cqn_rcv		:24;
1773 	uint32_t			:8;
1774 	uint32_t			:5;
1775 	uint32_t	ra_buff_indx	:27;
1776 	uint64_t	next_rcv_wqe;
1777 	uint32_t	srq_number	:24;
1778 	uint32_t	srq_en		:1;
1779 	uint32_t			:7;
1780 	uint32_t	qkey;
1781 	uint32_t			:32;
1782 	uint32_t	rmsn		:24;
1783 	uint32_t			:8;
1784 	uint32_t	rsrv0[18];
1785 };
1786 #else
1787 struct tavor_hw_qpc_s {
1788 	uint32_t	state		:4;
1789 	uint32_t			:9;
1790 	uint32_t	serv_type	:3;
1791 	uint32_t			:3;
1792 	uint32_t	pm_state	:2;
1793 	uint32_t			:2;
1794 	uint32_t	de		:1;
1795 	uint32_t			:8;
1796 	uint32_t			:28;
1797 	uint32_t	sched_q		:4;
1798 	uint32_t	mtu		:3;
1799 	uint32_t	msg_max		:5;
1800 	uint32_t			:24;
1801 	uint32_t			:8;
1802 	uint32_t	usr_page	:24;
1803 	uint32_t			:8;
1804 	uint32_t	loc_qpn		:24;
1805 	uint32_t			:8;
1806 	uint32_t	rem_qpn		:24;
1807 	uint32_t			:32;
1808 	uint32_t			:32;
1809 	tavor_hw_addr_path_t	pri_addr_path;
1810 	tavor_hw_addr_path_t	alt_addr_path;
1811 	uint32_t			:8;
1812 	uint32_t	rdd		:24;
1813 	uint32_t			:8;
1814 	uint32_t	pd		:24;
1815 	uint32_t	wqe_baseaddr;
1816 	uint32_t	wqe_lkey;
1817 	uint32_t	ack_req_freq	:4;
1818 	uint32_t	flight_lim	:4;
1819 	uint32_t	sra_max		:3;
1820 	uint32_t			:2;
1821 	uint32_t	retry_cnt	:3;
1822 	uint32_t	sre		:1;
1823 	uint32_t	swe		:1;
1824 	uint32_t	sae		:1;
1825 	uint32_t			:2;
1826 	uint32_t	cur_rnr_retry	:3;
1827 	uint32_t	cur_retry_cnt	:3;
1828 	uint32_t	sic		:1;
1829 	uint32_t	ssc		:1;
1830 	uint32_t			:3;
1831 	uint32_t			:32;
1832 	uint32_t			:8;
1833 	uint32_t	next_snd_psn	:24;
1834 	uint32_t			:8;
1835 	uint32_t	cqn_snd		:24;
1836 	uint64_t	next_snd_wqe;
1837 	uint32_t			:8;
1838 	uint32_t	last_acked_psn	:24;
1839 	uint32_t			:8;
1840 	uint32_t	ssn		:24;
1841 	uint32_t			:8;
1842 	uint32_t	rra_max		:3;
1843 	uint32_t			:5;
1844 	uint32_t	rre		:1;
1845 	uint32_t	rwe		:1;
1846 	uint32_t	rae		:1;
1847 	uint32_t			:8;
1848 	uint32_t	ric		:1;
1849 	uint32_t	rsc		:1;
1850 	uint32_t			:3;
1851 	uint32_t			:3;
1852 	uint32_t	min_rnr_nak	:5;
1853 	uint32_t	next_rcv_psn	:24;
1854 	uint32_t	ra_buff_indx	:27;
1855 	uint32_t			:5;
1856 	uint32_t			:8;
1857 	uint32_t	cqn_rcv		:24;
1858 	uint64_t	next_rcv_wqe;
1859 	uint32_t	qkey;
1860 	uint32_t			:7;
1861 	uint32_t	srq_en		:1;
1862 	uint32_t	srq_number	:24;
1863 	uint32_t			:8;
1864 	uint32_t	rmsn		:24;
1865 	uint32_t			:32;
1866 	uint32_t	rsrv0[18];
1867 };
1868 #endif
1869 #define	TAVOR_QP_RESET			0x0
1870 #define	TAVOR_QP_INIT			0x1
1871 #define	TAVOR_QP_RTR			0x2
1872 #define	TAVOR_QP_RTS			0x3
1873 #define	TAVOR_QP_SQERR			0x4
1874 #define	TAVOR_QP_SQD			0x5
1875 #define	TAVOR_QP_ERR			0x6
1876 #define	TAVOR_QP_SQDRAINING		0x7
1877 
1878 #define	TAVOR_QP_RC			0x0
1879 #define	TAVOR_QP_UC			0x1
1880 #define	TAVOR_QP_UD			0x3
1881 #define	TAVOR_QP_MLX			0x7
1882 
1883 #define	TAVOR_QP_PMSTATE_MIGRATED	0x3
1884 #define	TAVOR_QP_PMSTATE_ARMED		0x0
1885 #define	TAVOR_QP_PMSTATE_REARM		0x1
1886 
1887 #define	TAVOR_QP_DESC_EVT_DISABLED	0x0
1888 #define	TAVOR_QP_DESC_EVT_ENABLED	0x1
1889 
1890 #define	TAVOR_QP_FLIGHT_LIM_UNLIMITED	0xF
1891 
1892 #define	TAVOR_QP_SQ_ALL_SIGNALED	0x1
1893 #define	TAVOR_QP_SQ_WR_SIGNALED		0x0
1894 #define	TAVOR_QP_RQ_ALL_SIGNALED	0x1
1895 #define	TAVOR_QP_RQ_WR_SIGNALED		0x0
1896 
1897 #define	TAVOR_QP_SRQ_ENABLED	0x1
1898 #define	TAVOR_QP_SRQ_DISABLED	0x0
1899 
1900 
1901 /*
1902  * Tavor Multicast Group Member (MCG)
1903  *    Tavor MCG are organized in a physically-contiguous memory table (the
1904  *    Multicast Group Table) in the HCA attached local memory.  This table is
1905  *    actually comprised of two consecutive tables: the Multicast Group Hash
1906  *    Table (MGHT) and the Additional Multicast Group Members Table (AMGM).
1907  *    Each such entry contains an MGID and a list of QPs that are attached to
1908  *    the multicast group.  Each such entry may also include an index to an
1909  *    Additional Multicast Group Member Table (AMGM) entry.  The AMGMs are
1910  *    used to form a linked list of MCG entries that all map to the same hash
1911  *    value.  The MCG entry size is configured through the INIT_HCA command.
1912  *    Note:  An MCG actually consists of a single tavor_hw_mcg_t and some
1913  *    number of tavor_hw_mcg_qp_list_t (such that the combined structure is a
1914  *    power-of-2).
1915  *
1916  *    The following structures are used in the READ_MGM and WRITE_MGM commands.
1917  *    The READ_MGM command reads an MCG entry from the multicast table and
1918  *    returns it in the output mailbox.  Note: This operation does not affect
1919  *    the MCG entry state or values.
1920  *    The WRITE_MGM command retrieves an MCG entry from the input mailbox and
1921  *    stores it in the multicast group table at the index specified in the
1922  *    command.  Once the command has finished execution, the multicast group
1923  *    table is updated.  The old entry contents are lost.
1924  */
1925 #ifdef	_LITTLE_ENDIAN
1926 struct tavor_hw_mcg_s {
1927 	uint32_t			:32;
1928 	uint32_t			:6;
1929 	uint32_t	next_gid_indx	:26;
1930 	uint32_t			:32;
1931 	uint32_t			:32;
1932 	uint64_t	mgid_h;
1933 	uint64_t	mgid_l;
1934 };
1935 #else
1936 struct tavor_hw_mcg_s {
1937 	uint32_t	next_gid_indx	:26;
1938 	uint32_t			:6;
1939 	uint32_t			:32;
1940 	uint32_t			:32;
1941 	uint32_t			:32;
1942 	uint64_t	mgid_h;
1943 	uint64_t	mgid_l;
1944 };
1945 #endif
1946 
1947 /* Multicast Group Member - QP List entries */
1948 #ifdef	_LITTLE_ENDIAN
1949 struct tavor_hw_mcg_qp_list_s {
1950 	uint32_t	qpn		:24;
1951 	uint32_t			:7;
1952 	uint32_t	q		:1;
1953 };
1954 #else
1955 struct tavor_hw_mcg_qp_list_s {
1956 	uint32_t	q		:1;
1957 	uint32_t			:7;
1958 	uint32_t	qpn		:24;
1959 };
1960 #endif
1961 #define	TAVOR_MCG_QPN_INVALID		0x0
1962 #define	TAVOR_MCG_QPN_VALID		0x1
1963 
1964 /*
1965  * Structure for getting the peformance counters from the HCA
1966  */
1967 
1968 #ifdef _LITTLE_ENDIAN
1969 struct tavor_hw_sm_perfcntr_s {
1970 	uint32_t	linkdown	:8;
1971 	uint32_t	linkerrrec	:8;
1972 	uint32_t	symerr		:16;
1973 
1974 	uint32_t	cntrsel		:16;
1975 	uint32_t	portsel		:8;
1976 	uint32_t			:8;
1977 
1978 	uint32_t	portxmdiscard	:16;
1979 	uint32_t	portrcvswrelay	:16;
1980 
1981 	uint32_t	portrcvrem	:16;
1982 	uint32_t	portrcv		:16;
1983 
1984 	uint32_t	vl15drop	:16;
1985 	uint32_t			:16;
1986 
1987 	uint32_t	xsbuffovrun	:4;
1988 	uint32_t	locallinkint	:4;
1989 	uint32_t			:8;
1990 	uint32_t	portrcconstr	:8;
1991 	uint32_t	portxmconstr	:8;
1992 
1993 	uint32_t	portrcdata;
1994 
1995 	uint32_t	portxmdata;
1996 
1997 	uint32_t	portrcpkts;
1998 
1999 	uint32_t	portxmpkts;
2000 
2001 	uint32_t	reserved;
2002 
2003 	uint32_t	portxmwait;
2004 };
2005 #else	/* BIG ENDIAN */
2006 struct tavor_hw_sm_perfcntr_s {
2007 	uint32_t			:8;
2008 	uint32_t	portsel		:8;
2009 	uint32_t	cntrsel		:16;
2010 
2011 	uint32_t	symerr		:16;
2012 	uint32_t	linkerrrec	:8;
2013 	uint32_t	linkdown	:8;
2014 
2015 	uint32_t	portrcv		:16;
2016 	uint32_t	portrcvrem	:16;
2017 
2018 	uint32_t	portrcvswrelay	:16;
2019 	uint32_t	portxmdiscard	:16;
2020 
2021 	uint32_t	portxmconstr	:8;
2022 	uint32_t	portrcconstr	:8;
2023 	uint32_t			:8;
2024 	uint32_t	locallinkint	:4;
2025 	uint32_t	xsbuffovrun	:4;
2026 
2027 	uint32_t			:16;
2028 	uint32_t	vl15drop	:16;
2029 
2030 	uint32_t	portxmdata;
2031 
2032 	uint32_t	portrcdata;
2033 
2034 	uint32_t	portxmpkts;
2035 
2036 	uint32_t	portrcpkts;
2037 
2038 	uint32_t	portxmwait;
2039 
2040 	uint32_t	reserved;
2041 };
2042 #endif
2043 
2044 
2045 /*
2046  * Tavor User Access Region (UAR)
2047  *    Tavor doorbells are each rung by writing to the doorbell registers that
2048  *    form a User Access Region (UAR).  A doorbell is a write-only hardware
2049  *    register which enables passing information from software to hardware
2050  *    with minimum software latency. A write operation from the host software
2051  *    to these doorbell registers passes information about the HCA resources
2052  *    and initiates processing of the doorbell data.  There are 6 types of
2053  *    doorbells in Tavor.
2054  *
2055  *    "Send Doorbell" for synchronizing the attachment of a WQE (or a chain
2056  *        of WQEs) to the send queue.
2057  *    "RD Send Doorbell" (Same as above, except for RD QPs) is not supported.
2058  *    "Receive Doorbell" for synchronizing the attachment of a WQE (or a chain
2059  *        of WQEs) to the receive queue.
2060  *    "CQ Doorbell" for updating the CQ consumer index and requesting
2061  *        completion notifications.
2062  *    "EQ Doorbell" for updating the EQ consumer index, arming interrupt
2063  *        triggering, and disarming CQ notification requests.
2064  *    "InfiniBlast" (which would have enabled access to the "InfiniBlast
2065  *        buffer") is not supported.
2066  *
2067  *    Note: The tavor_hw_uar_t below is the container for all of the various
2068  *    doorbell types.  Below we first define several structures which make up
2069  *    the contents of those doorbell types.
2070  *
2071  *    Note also: The following structures are not #define'd with both little-
2072  *    endian and big-endian definitions.  This is because each doorbell type
2073  *    is not directly accessed except through a single ddi_put64() operation
2074  *    (see tavor_qp_send_doorbell, tavor_qp_recv_doorbell, tavor_cq_doorbell,
2075  *    or tavor_eq_doorbell)
2076  */
2077 typedef struct tavor_hw_uar_send_s {
2078 	uint32_t	nda		:26;
2079 	uint32_t	fence		:1;
2080 	uint32_t	nopcode		:5;
2081 	uint32_t	qpn		:24;
2082 	uint32_t			:2;
2083 	uint32_t	nds		:6;
2084 } tavor_hw_uar_send_t;
2085 #define	TAVOR_QPSNDDB_NDA_MASK		0xFFFFFFC0
2086 #define	TAVOR_QPSNDDB_NDA_SHIFT		0x20
2087 #define	TAVOR_QPSNDDB_F_SHIFT		0x25
2088 #define	TAVOR_QPSNDDB_NOPCODE_SHIFT	0x20
2089 #define	TAVOR_QPSNDDB_QPN_SHIFT		0x8
2090 
2091 typedef struct tavor_hw_uar_recv_s {
2092 	uint32_t	nda		:26;
2093 	uint32_t	nds		:6;
2094 	uint32_t	qpn		:24;
2095 	uint32_t	credits		:8;
2096 } tavor_hw_uar_recv_t;
2097 #define	TAVOR_QPRCVDB_NDA_MASK		0xFFFFFFC0
2098 #define	TAVOR_QPRCVDB_NDA_SHIFT		0x20
2099 #define	TAVOR_QPRCVDB_NDS_SHIFT		0x20
2100 #define	TAVOR_QPRCVDB_QPN_SHIFT		0x8
2101 /* Max descriptors per Tavor doorbell */
2102 #define	TAVOR_QP_MAXDESC_PER_DB		256
2103 
2104 typedef struct tavor_hw_uar_cq_s {
2105 	uint32_t	cmd		:8;
2106 	uint32_t	cqn		:24;
2107 	uint32_t	param;
2108 } tavor_hw_uar_cq_t;
2109 #define	TAVOR_CQDB_CMD_SHIFT		0x38
2110 #define	TAVOR_CQDB_CQN_SHIFT		0x20
2111 
2112 #define	TAVOR_CQDB_INCR_CONSINDX	0x01
2113 #define	TAVOR_CQDB_NOTIFY_CQ		0x02
2114 #define	TAVOR_CQDB_NOTIFY_CQ_SOLICIT	0x03
2115 #define	TAVOR_CQDB_SET_CONSINDX		0x04
2116 /* Default value for use in NOTIFY_CQ doorbell */
2117 #define	TAVOR_CQDB_DEFAULT_PARAM	0xFFFFFFFF
2118 
2119 typedef struct tavor_hw_uar_eq_s {
2120 	uint32_t	cmd		:8;
2121 	uint32_t			:18;
2122 	uint32_t	eqn		:6;
2123 	uint32_t	param;
2124 } tavor_hw_uar_eq_t;
2125 #define	TAVOR_EQDB_CMD_SHIFT		0x38
2126 #define	TAVOR_EQDB_EQN_SHIFT		0x20
2127 
2128 #define	TAVOR_EQDB_INCR_CONSINDX	0x01
2129 #define	TAVOR_EQDB_REARM_EQ		0x02
2130 #define	TAVOR_EQDB_DISARM_CQ		0x03
2131 #define	TAVOR_EQDB_SET_CONSINDX		0x04
2132 #define	TAVOR_EQDB_SET_ALWAYSARMED	0x05
2133 
2134 struct tavor_hw_uar_s {
2135 	uint32_t		rsrv0[4];	/* "RD Send" unsupported */
2136 	tavor_hw_uar_send_t	send;
2137 	tavor_hw_uar_recv_t	recv;
2138 	tavor_hw_uar_cq_t	cq;
2139 	tavor_hw_uar_eq_t	eq;
2140 	uint32_t		rsrv1[244];
2141 	uint32_t		iblast[256];	/* "InfiniBlast" unsupported */
2142 };
2143 
2144 
2145 /*
2146  * Tavor Send Work Queue Element (WQE)
2147  *    A Tavor Send WQE is built of the following segments, each of which is a
2148  *    multiple of 16 bytes.  Note: Each individual WQE may contain only a
2149  *    subset of these segments described below (according to the operation type
2150  *    and transport type of the QP).
2151  *
2152  *    The first 16 bytes of ever WQE are formed from the "Next/Ctrl" segment.
2153  *    This segment contains the address of the next WQE to be executed and the
2154  *    information required in order to allocate the resources to execute the
2155  *    next WQE.  The "Ctrl" part of this segment contains the control
2156  *    information required to execute the WQE, including the opcode and other
2157  *    control information.
2158  *    The "Datagram" segment contains address information required in order to
2159  *    form a UD message.
2160  *    The "Bind" segment contains the parameters required for a Bind Memory
2161  *    Window operation.
2162  *    The "Remote Address" segment is present only in RDMA or Atomic WQEs and
2163  *    specifies remote virtual addresses and RKey, respectively.  Length of
2164  *    the remote access is calculated from the scatter/gather list (for
2165  *    RDMA-write/RDMA-read) or set to eight (for Atomic).
2166  *    The "Atomic" segment is present only in Atomic WQEs and specifies
2167  *    Swap/Add and Compare data.
2168  *
2169  *    Note: The following structures are not #define'd with both little-endian
2170  *    and big-endian definitions.  This is because their individual fields are
2171  *    not directly accessed except through macros defined below.
2172  */
2173 struct tavor_hw_snd_wqe_nextctrl_s {
2174 	uint32_t	next_wqe_addr	:26;
2175 	uint32_t			:1;
2176 	uint32_t	nopcode		:5;
2177 	uint32_t	next_eec	:24;
2178 	uint32_t	dbd		:1;
2179 	uint32_t	fence		:1;
2180 	uint32_t	nds		:6;
2181 
2182 	uint32_t			:28;
2183 	uint32_t	c		:1;
2184 	uint32_t	e		:1;
2185 	uint32_t	s		:1;
2186 	uint32_t	i		:1;
2187 	uint32_t	immediate	:32;
2188 };
2189 #define	TAVOR_WQE_NDA_MASK		0x00000000FFFFFFC0ull
2190 #define	TAVOR_WQE_NDS_MASK		0x3F
2191 #define	TAVOR_WQE_DBD_MASK		0x80
2192 
2193 #define	TAVOR_WQE_SEND_FENCE_MASK	0x40
2194 #define	TAVOR_WQE_SEND_NOPCODE_RDMAW	0x8
2195 #define	TAVOR_WQE_SEND_NOPCODE_RDMAWI	0x9
2196 #define	TAVOR_WQE_SEND_NOPCODE_SEND	0xA
2197 #define	TAVOR_WQE_SEND_NOPCODE_SENDI	0xB
2198 #define	TAVOR_WQE_SEND_NOPCODE_RDMAR	0x10
2199 #define	TAVOR_WQE_SEND_NOPCODE_ATMCS	0x11
2200 #define	TAVOR_WQE_SEND_NOPCODE_ATMFA	0x12
2201 #define	TAVOR_WQE_SEND_NOPCODE_BIND	0x18
2202 
2203 #define	TAVOR_WQE_SEND_SIGNALED_MASK	0x0000000800000000ull
2204 #define	TAVOR_WQE_SEND_SOLICIT_MASK	0x0000000200000000ull
2205 #define	TAVOR_WQE_SEND_IMMEDIATE_MASK	0x0000000100000000ull
2206 
2207 struct tavor_hw_snd_wqe_ud_s {
2208 	uint32_t			:32;
2209 	uint32_t	lkey		:32;
2210 	uint32_t	av_addr_h	:32;
2211 	uint32_t	av_addr_l	:27;
2212 	uint32_t			:5;
2213 	uint32_t	rsrv0[4];
2214 	uint32_t			:8;
2215 	uint32_t	dest_qp		:24;
2216 	uint32_t	qkey		:32;
2217 	uint32_t			:32;
2218 	uint32_t			:32;
2219 };
2220 #define	TAVOR_WQE_SENDHDR_UD_AV_MASK	0xFFFFFFFFFFFFFFE0ull
2221 #define	TAVOR_WQE_SENDHDR_UD_DQPN_MASK	0xFFFFFF
2222 
2223 struct tavor_hw_snd_wqe_bind_s {
2224 	uint32_t	ae		:1;
2225 	uint32_t	rw		:1;
2226 	uint32_t	rr		:1;
2227 	uint32_t			:29;
2228 	uint32_t			:32;
2229 	uint32_t	new_rkey;
2230 	uint32_t	reg_lkey;
2231 	uint64_t	addr;
2232 	uint64_t	len;
2233 };
2234 #define	TAVOR_WQE_SENDHDR_BIND_ATOM	0x8000000000000000ull
2235 #define	TAVOR_WQE_SENDHDR_BIND_WR	0x4000000000000000ull
2236 #define	TAVOR_WQE_SENDHDR_BIND_RD	0x2000000000000000ull
2237 
2238 struct tavor_hw_snd_wqe_remaddr_s {
2239 	uint64_t	vaddr;
2240 	uint32_t	rkey;
2241 	uint32_t			:32;
2242 };
2243 
2244 struct tavor_hw_snd_wqe_atomic_s {
2245 	uint64_t	swap_add;
2246 	uint64_t	compare;
2247 };
2248 
2249 
2250 /*
2251  * Tavor "MLX transport" Work Queue Element (WQE)
2252  *    The format of the MLX WQE is similar to that of the Send WQE (above)
2253  *    with the following exceptions.  MLX WQEs are used for sending MADs on
2254  *    special QPs 0 and 1.  Everything following the "Next/Ctrl" header
2255  *    (defined below) consists of scatter-gather list entries.  The contents
2256  *    of these SGLs (also defined below) will be put on the wire exactly as
2257  *    they appear in the buffers.  In addition, the VCRC and the ICRC of each
2258  *    sent packet can be modified by changing values in the following header
2259  *    or in the payload of the packet itself.
2260  */
2261 struct tavor_hw_mlx_wqe_nextctrl_s {
2262 	uint32_t	next_wqe_addr	:26;
2263 	uint32_t			:1;
2264 	uint32_t	nopcode		:5;
2265 	uint32_t			:24;
2266 	uint32_t	dbd		:1;
2267 	uint32_t			:1;
2268 	uint32_t	nds		:6;
2269 
2270 	uint32_t			:14;
2271 	uint32_t	vl15		:1;
2272 	uint32_t	slr		:1;
2273 	uint32_t			:1;
2274 	uint32_t	max_srate	:3;
2275 	uint32_t	sl		:4;
2276 	uint32_t			:4;
2277 	uint32_t	c		:1;
2278 	uint32_t	e		:1;
2279 	uint32_t			:2;
2280 	uint32_t	rlid		:16;
2281 	uint32_t	vcrc		:16;
2282 };
2283 #define	TAVOR_WQE_MLXHDR_VL15_MASK	0x0002000000000000ull
2284 #define	TAVOR_WQE_MLXHDR_SLR_MASK	0x0001000000000000ull
2285 #define	TAVOR_WQE_MLXHDR_SRATE_SHIFT	44
2286 #define	TAVOR_WQE_MLXHDR_SL_SHIFT	40
2287 #define	TAVOR_WQE_MLXHDR_SIGNALED_MASK	0x0000000800000000ull
2288 #define	TAVOR_WQE_MLXHDR_RLID_SHIFT	16
2289 
2290 /*
2291  * Tavor Receive Work Queue Element (WQE)
2292  *    Like the Send WQE, the Receive WQE is built of 16-byte segments. The
2293  *    segment is the "Next/Ctrl" segment (defined below).  It is followed by
2294  *    some number of scatter list entries for the incoming message.
2295  *
2296  *    The format of the scatter-gather list entries is also shown below.  For
2297  *    Receive WQEs the "inline_data" field must be cleared (i.e. data segments
2298  *    cannot contain inline data).
2299  */
2300 struct tavor_hw_rcv_wqe_nextctrl_s {
2301 	uint32_t	next_wqe_addr	:26;
2302 	uint32_t			:5;
2303 	uint32_t	one		:1;
2304 	uint32_t			:24;
2305 	uint32_t	dbd		:1;
2306 	uint32_t			:1;
2307 	uint32_t	nds		:6;
2308 
2309 	uint32_t			:28;
2310 	uint32_t	c		:1;
2311 	uint32_t	e		:1;
2312 	uint32_t			:2;
2313 	uint32_t			:32;
2314 };
2315 
2316 /*
2317  * This bit must be set in the next/ctrl field of all Receive WQEs
2318  * as a workaround to a Tavor hardware erratum related to having
2319  * the first 32-bits in the WQE set to zero.
2320  */
2321 #define	TAVOR_RCV_WQE_NDA0_WA_MASK	0x0000000100000000ull
2322 
2323 struct tavor_hw_wqe_sgl_s {
2324 	uint32_t	inline_data	:1;
2325 	uint32_t	byte_cnt	:31;
2326 	uint32_t	lkey;
2327 	uint64_t	addr;
2328 };
2329 #define	TAVOR_WQE_SGL_BYTE_CNT_MASK	0x7FFFFFFF
2330 #define	TAVOR_WQE_SGL_INLINE_MASK	0x80000000
2331 
2332 /*
2333  * The following defines are used when building descriptors for special QP
2334  * work requests (i.e. MLX transport WQEs).  Note: Because Tavor MLX transport
2335  * requires the driver to build actual IB packet headers, we use these defines
2336  * for the most common fields in those headers.
2337  */
2338 #define	TAVOR_MLX_VL15_LVER		0xF0000000
2339 #define	TAVOR_MLX_VL0_LVER		0x00000000
2340 #define	TAVOR_MLX_IPVER_TC_FLOW		0x60000000
2341 #define	TAVOR_MLX_TC_SHIFT		20
2342 #define	TAVOR_MLX_DEF_PKEY		0xFFFF
2343 #define	TAVOR_MLX_GSI_QKEY		0x80010000
2344 #define	TAVOR_MLX_UDSEND_OPCODE		0x64000000
2345 #define	TAVOR_MLX_DQPN_MASK		0xFFFFFF
2346 
2347 /*
2348  * The following macros are used for building each of the individual
2349  * segments that can make up a Tavor WQE.  Note: We try not to use the
2350  * structures (with their associated bitfields) here, instead opting to
2351  * build and put 64-bit or 32-bit chunks to the WQEs as appropriate,
2352  * primarily because using the bitfields appears to force more read-modify-
2353  * write operations.
2354  *
2355  *    TAVOR_WQE_BUILD_UD		- Builds Unreliable Datagram Segment
2356  *
2357  *    TAVOR_WQE_BUILD_REMADDR		- Builds Remote Address Segment using
2358  *					    RDMA info from the work request
2359  *    TAVOR_WQE_BUILD_RC_ATOMIC_REMADDR	- Builds Remote Address Segment
2360  *					    for RC Atomic work requests
2361  *    TAVOR_WQE_BUILD_ATOMIC		- Builds Atomic Segment using atomic
2362  *					    info from the work request
2363  *    TAVOR_WQE_BUILD_BIND		- Builds the Bind Memory Window
2364  *					    Segment using bind info from the
2365  *					    work request
2366  *    TAVOR_WQE_BUILD_DATA_SEG		- Builds the individual Data Segments
2367  *					    for Send, Receive, and MLX WQEs
2368  *    TAVOR_WQE_BUILD_INLINE		- Builds an "inline" Data Segment
2369  *					    (primarily for MLX transport)
2370  *    TAVOR_WQE_BUILD_INLINE_ICRC	- Also builds an "inline" Data Segment
2371  *					    (but used primarily in the ICRC
2372  *					    portion of MLX transport WQEs)
2373  *    TAVOR_WQE_LINKNEXT		- Links the current WQE to the
2374  *					    previous one
2375  *    TAVOR_WQE_LINKFIRST		- Links the first WQE on the current
2376  *					    chain to the previous WQE
2377  *    TAVOR_WQE_BUILD_MLX_LRH		- Builds the inline LRH header for
2378  *					    MLX transport MADs
2379  *    TAVOR_WQE_BUILD_MLX_GRH		- Builds the inline GRH header for
2380  *					    MLX transport MADs
2381  *    TAVOR_WQE_BUILD_MLX_BTH		- Builds the inline BTH header for
2382  *					    MLX transport MADs
2383  *    TAVOR_WQE_BUILD_MLX_DETH		- Builds the inline DETH header for
2384  *					    MLX transport MADs
2385  */
2386 #define	TAVOR_WQE_BUILD_UD(qp, ud, ah, wr)				\
2387 {									\
2388 	uint64_t		*tmp;					\
2389 									\
2390 	tmp	= (uint64_t *)(ud);					\
2391 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0],			\
2392 	    (uint64_t)(ah)->ah_mrhdl->mr_lkey);				\
2393 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2394 	    (ah)->ah_mrhdl->mr_bindinfo.bi_addr &			\
2395 	    TAVOR_WQE_SENDHDR_UD_AV_MASK);				\
2396 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[2], 0x0);		\
2397 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[3], 0x0);		\
2398 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[4],			\
2399 	    (((uint64_t)((wr)->wr.ud.udwr_dest->ud_dst_qpn &		\
2400 	    TAVOR_WQE_SENDHDR_UD_DQPN_MASK) << 32) |			\
2401 	    (wr)->wr.ud.udwr_dest->ud_qkey));				\
2402 }
2403 
2404 #define	TAVOR_WQE_BUILD_REMADDR(qp, ra, wr_rdma)			\
2405 {									\
2406 	uint64_t		*tmp;					\
2407 									\
2408 	tmp	= (uint64_t *)(ra);					\
2409 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0],			\
2410 	    (wr_rdma)->rdma_raddr);					\
2411 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2412 	    (uint64_t)(wr_rdma)->rdma_rkey << 32);			\
2413 }
2414 
2415 #define	TAVOR_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr)			\
2416 {									\
2417 	uint64_t		*tmp;					\
2418 									\
2419 	tmp	= (uint64_t *)(rc);					\
2420 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0],			\
2421 	    (wr)->wr.rc.rcwr.atomic->atom_raddr);			\
2422 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2423 	    (uint64_t)(wr)->wr.rc.rcwr.atomic->atom_rkey << 32);	\
2424 }
2425 
2426 #define	TAVOR_WQE_BUILD_ATOMIC(qp, at, wr_atom)				\
2427 {									\
2428 	uint64_t		*tmp;					\
2429 									\
2430 	tmp	= (uint64_t *)(at);					\
2431 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0],			\
2432 	    (wr_atom)->atom_arg2);					\
2433 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2434 	    (wr_atom)->atom_arg1);					\
2435 }
2436 
2437 #define	TAVOR_WQE_BUILD_BIND(qp, bn, wr_bind)				\
2438 {									\
2439 	uint64_t		*tmp;					\
2440 	uint64_t		bn0_tmp;				\
2441 	ibt_bind_flags_t	bind_flags;				\
2442 									\
2443 	tmp	   = (uint64_t *)(bn);					\
2444 	bind_flags = (wr_bind)->bind_flags;				\
2445 	bn0_tmp	   = (bind_flags & IBT_WR_BIND_ATOMIC) ?		\
2446 	    TAVOR_WQE_SENDHDR_BIND_ATOM : 0;				\
2447 	bn0_tmp	  |= (bind_flags & IBT_WR_BIND_WRITE) ?			\
2448 	    TAVOR_WQE_SENDHDR_BIND_WR : 0;				\
2449 	bn0_tmp	  |= (bind_flags & IBT_WR_BIND_READ) ?			\
2450 	    TAVOR_WQE_SENDHDR_BIND_RD : 0;				\
2451 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0], bn0_tmp);		\
2452 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2453 	    (((uint64_t)(wr_bind)->bind_rkey_out << 32) |		\
2454 	    (wr_bind)->bind_lkey));					\
2455 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[2],			\
2456 	    (wr_bind)->bind_va);					\
2457 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[3],			\
2458 	    (wr_bind)->bind_len);					\
2459 }
2460 
2461 #define	TAVOR_WQE_BUILD_DATA_SEG(qp, ds, sgl)				\
2462 {									\
2463 	uint64_t		*tmp;					\
2464 									\
2465 	tmp	= (uint64_t *)(ds);					\
2466 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[0],			\
2467 	    (((uint64_t)((sgl)->ds_len &				\
2468 	    TAVOR_WQE_SGL_BYTE_CNT_MASK) << 32) | (sgl)->ds_key));	\
2469 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &tmp[1], (sgl)->ds_va);	\
2470 }
2471 
2472 #define	TAVOR_WQE_BUILD_DATA_SEG_SRQ(srq, ds, sgl)			\
2473 {									\
2474 	uint64_t		*tmp;					\
2475 									\
2476 	tmp	= (uint64_t *)(ds);					\
2477 	ddi_put64((srq)->srq_wqinfo.qa_acchdl, &tmp[0],			\
2478 	    (((uint64_t)((sgl)->ds_len &				\
2479 	    TAVOR_WQE_SGL_BYTE_CNT_MASK) << 32) | (sgl)->ds_key));	\
2480 	ddi_put64((srq)->srq_wqinfo.qa_acchdl, &tmp[1], (sgl)->ds_va);	\
2481 }
2482 
2483 #define	TAVOR_WQE_BUILD_INLINE(qp, ds, sz)				\
2484 {									\
2485 	uint32_t		*tmp;					\
2486 	uint32_t		inline_tmp;				\
2487 									\
2488 	tmp	   = (uint32_t *)(ds);					\
2489 	inline_tmp = TAVOR_WQE_SGL_INLINE_MASK | sz;			\
2490 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], inline_tmp);	\
2491 }
2492 
2493 #define	TAVOR_WQE_BUILD_INLINE_ICRC(qp, ds, sz, icrc)			\
2494 {									\
2495 	uint32_t		*tmp;					\
2496 	uint32_t		inline_tmp;				\
2497 									\
2498 	tmp = (uint32_t *)(ds);						\
2499 	inline_tmp = TAVOR_WQE_SGL_INLINE_MASK | sz;			\
2500 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], inline_tmp);	\
2501 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1], icrc);		\
2502 }
2503 
2504 #define	TAVOR_WQE_LINKNEXT(qp, prev, ctrl, next)			\
2505 {									\
2506 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &((uint64_t *)(prev))[1],	\
2507 	    (ctrl));							\
2508 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &((uint64_t *)(prev))[0],	\
2509 	    (next));							\
2510 }
2511 
2512 #define	TAVOR_WQE_LINKNEXT_SRQ(srq, prev, ctrl, next)			\
2513 {									\
2514 	ddi_put64((srq)->srq_wqinfo.qa_acchdl, &((uint64_t *)(prev))[1],\
2515 	    (ctrl));							\
2516 	ddi_put64((srq)->srq_wqinfo.qa_acchdl, &((uint64_t *)(prev))[0],\
2517 	    (next));							\
2518 }
2519 
2520 #define	TAVOR_WQE_LINKFIRST(qp, prev, next)				\
2521 {									\
2522 	ddi_put64((qp)->qp_wqinfo.qa_acchdl, &((uint64_t *)(prev))[0],	\
2523 	    (next));							\
2524 }
2525 
2526 #define	TAVOR_WQE_BUILD_MLX_LRH(lrh, qp, udav, pktlen)			\
2527 {									\
2528 	uint32_t		*tmp;					\
2529 	uint32_t		lrh_tmp;				\
2530 									\
2531 	tmp	 = (uint32_t *)(lrh);					\
2532 									\
2533 	if ((qp)->qp_is_special == TAVOR_QP_SMI) {			\
2534 		lrh_tmp = TAVOR_MLX_VL15_LVER;				\
2535 	} else {							\
2536 		lrh_tmp = TAVOR_MLX_VL0_LVER | ((udav).sl << 20);	\
2537 	}								\
2538 	if ((udav).grh) {						\
2539 		lrh_tmp |= (IB_LRH_NEXT_HDR_GRH << 16);			\
2540 	} else {							\
2541 		lrh_tmp |= (IB_LRH_NEXT_HDR_BTH << 16);			\
2542 	}								\
2543 	lrh_tmp |= (udav).rlid;						\
2544 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], lrh_tmp);		\
2545 									\
2546 	lrh_tmp	 = (pktlen) << 16;					\
2547 	if ((udav).rlid == IB_LID_PERMISSIVE) {				\
2548 		lrh_tmp |= IB_LID_PERMISSIVE;				\
2549 	} else {							\
2550 		lrh_tmp |= (udav).ml_path;				\
2551 	}								\
2552 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1], lrh_tmp);		\
2553 }
2554 
2555 /*
2556  * Note: The GRH payload length, calculated below, is the overall packet
2557  * length (in bytes) minus LRH header and GRH headers.
2558  *
2559  * Also note: Filling in the GIDs in the way we do below is helpful because
2560  * it avoids potential alignment restrictions and/or conflicts.
2561  */
2562 #define	TAVOR_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen)		\
2563 {									\
2564 	uint32_t		*tmp;					\
2565 	uint32_t		grh_tmp;				\
2566 	ib_gid_t		sgid;					\
2567 									\
2568 	tmp	 = (uint32_t *)(grh);					\
2569 									\
2570 	grh_tmp	 = TAVOR_MLX_IPVER_TC_FLOW;				\
2571 	grh_tmp |= (udav).tclass << TAVOR_MLX_TC_SHIFT;			\
2572 	grh_tmp |= (udav).flow_label;					\
2573 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], grh_tmp);		\
2574 									\
2575 	grh_tmp	 = (((pktlen) << 2) - (sizeof (ib_lrh_hdr_t) +		\
2576 	    sizeof (ib_grh_t))) << 16;					\
2577 	grh_tmp |= (IB_GRH_NEXT_HDR_BTH << 8);				\
2578 	grh_tmp |= (udav).hop_limit;					\
2579 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1], grh_tmp);		\
2580 									\
2581 	TAVOR_SGID_FROM_INDX_GET((state), (qp)->qp_portnum,		\
2582 	    (udav).mgid_index, &sgid);					\
2583 	bcopy(&sgid, &tmp[2], sizeof (ib_gid_t));			\
2584 	bcopy(&(udav).rgid_h, &tmp[6], sizeof (ib_gid_t));		\
2585 }
2586 
2587 #define	TAVOR_WQE_BUILD_MLX_BTH(state, bth, qp, wr)			\
2588 {									\
2589 	uint32_t		*tmp;					\
2590 	uint32_t		bth_tmp;				\
2591 									\
2592 	tmp	 = (uint32_t *)(bth);					\
2593 									\
2594 	bth_tmp	 = TAVOR_MLX_UDSEND_OPCODE;				\
2595 	if ((wr)->wr_flags & IBT_WR_SEND_SOLICIT) {			\
2596 		bth_tmp |= (IB_BTH_SOLICITED_EVENT_MASK << 16);		\
2597 	}								\
2598 	if (qp->qp_is_special == TAVOR_QP_SMI) {			\
2599 		bth_tmp |= TAVOR_MLX_DEF_PKEY;				\
2600 	} else {							\
2601 		bth_tmp |= TAVOR_PKEY_FROM_INDX_GET((state),		\
2602 		    (qp)->qp_portnum, (qp)->qp_pkeyindx);		\
2603 	}								\
2604 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], bth_tmp);		\
2605 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1],			\
2606 	    (wr)->wr.ud.udwr_dest->ud_dst_qpn &				\
2607 	    TAVOR_MLX_DQPN_MASK);					\
2608 	ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[2], 0x0);		\
2609 }
2610 
2611 #define	TAVOR_WQE_BUILD_MLX_DETH(deth, qp)				\
2612 {									\
2613 	uint32_t		*tmp;					\
2614 									\
2615 	tmp	 = (uint32_t *)(deth);					\
2616 									\
2617 	if ((qp)->qp_is_special == TAVOR_QP_SMI) {			\
2618 		ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0], 0x0);	\
2619 		ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1], 0x0);	\
2620 	} else {							\
2621 		ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[0],		\
2622 		    TAVOR_MLX_GSI_QKEY);				\
2623 		ddi_put32((qp)->qp_wqinfo.qa_acchdl, &tmp[1], 0x1);	\
2624 	}								\
2625 }
2626 
2627 
2628 /*
2629  * Undocumented:
2630  *    The following registers (and the macros to access them) are not defined
2631  *    in the Tavor PRM.  But we have high confidence that these offsets are
2632  *    unlikely to change in the lifetime of the Tavor hardware.
2633  */
2634 #define	TAVOR_HW_PORTINFO_LMC_OFFSET		0x10020
2635 #define	TAVOR_HW_PORTINFO_BASELID_OFFSET	0x10010
2636 #define	TAVOR_HW_PORTINFO_MASTERSMLID_OFFSET	0x10010
2637 #define	TAVOR_HW_PORTINFO_LINKWIDTH_OFFSET	0x1001C
2638 #define	TAVOR_HW_PORT_SIZE			0x800
2639 
2640 #define	TAVOR_HW_PMEG_PORTXMITDATA_OFFSET	0x10120
2641 #define	TAVOR_HW_PMEG_PORTRECVDATA_OFFSET	0x10124
2642 #define	TAVOR_HW_PMEG_PORTXMITPKTS_OFFSET	0x10128
2643 #define	TAVOR_HW_PMEG_PORTRECVPKTS_OFFSET	0x1012C
2644 #define	TAVOR_HW_PMEG_PORTRECVERR_OFFSET	0x10130
2645 #define	TAVOR_HW_PMEG_PORTXMITDISCARD_OFFSET	0x10134
2646 #define	TAVOR_HW_PMEG_VL15DROPPED_OFFSET	0x10138
2647 #define	TAVOR_HW_PMEG_PORTXMITWAIT_OFFSET	0x1013C
2648 #define	TAVOR_HW_PMEG_PORTRECVREMPHYSERR_OFFSET	0x10144
2649 #define	TAVOR_HW_PMEG_PORTXMITCONSTERR_OFFSET	0x10148
2650 #define	TAVOR_HW_PMEG_PORTRECVCONSTERR_OFFSET	0x1014C
2651 #define	TAVOR_HW_PMEG_SYMBOLERRCNT_OFFSET	0x10150
2652 #define	TAVOR_HW_PMEG_LINKERRRECOVERCNT_OFFSET	0x10154
2653 #define	TAVOR_HW_PMEG_LINKDOWNEDCNT_OFFSET	0x10154
2654 #define	TAVOR_HW_PMEG_EXCESSBUFOVERRUN_OFFSET	0x10164
2655 #define	TAVOR_HW_PMEG_LOCALLINKINTERR_OFFSET	0x10164
2656 
2657 #define	TAVOR_HW_GUIDTABLE_OFFSET		0x4C800
2658 #define	TAVOR_HW_GUIDTABLE_PORT_SIZE		0x200
2659 #define	TAVOR_HW_GUIDTABLE_GID_SIZE		0x10
2660 #define	TAVOR_HW_GUIDTABLE_GIDPREFIX_SIZE	0x8
2661 
2662 #define	TAVOR_HW_PKEYTABLE_OFFSET		0x4D800
2663 #define	TAVOR_HW_PKEYTABLE_PORT_SIZE		0x100
2664 #define	TAVOR_HW_PKEYTABLE_PKEY_SIZE		0x4
2665 
2666 #define	TAVOR_PORT_LMC_GET(state, port)				\
2667 	((ddi_get32((state)->ts_reg_cmdhdl,			\
2668 	(uint32_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2669 	TAVOR_HW_PORTINFO_LMC_OFFSET +				\
2670 	(TAVOR_HW_PORT_SIZE * (port)))) >> 8) & 0x7);
2671 
2672 #define	TAVOR_PORT_BASELID_GET(state, port)			\
2673 	(ddi_get32((state)->ts_reg_cmdhdl,			\
2674 	(uint32_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2675 	TAVOR_HW_PORTINFO_BASELID_OFFSET +			\
2676 	(TAVOR_HW_PORT_SIZE * (port)))) >> 16);
2677 
2678 #define	TAVOR_PORT_MASTERSMLID_GET(state, port)			\
2679 	(ddi_get32((state)->ts_reg_cmdhdl,			\
2680 	(uint32_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2681 	TAVOR_HW_PORTINFO_MASTERSMLID_OFFSET +			\
2682 	(TAVOR_HW_PORT_SIZE * (port)))) & 0xFFFF);
2683 
2684 #define	TAVOR_PORT_LINKWIDTH_ACTIVE_GET(state, port)		\
2685 	(ddi_get32((state)->ts_reg_cmdhdl,			\
2686 	(uint32_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2687 	TAVOR_HW_PORTINFO_LINKWIDTH_OFFSET +			\
2688 	(TAVOR_HW_PORT_SIZE * (port)))) & 0xF);
2689 
2690 #define	TAVOR_SGID_FROM_INDX_GET(state, port, sgid_ix, sgid)	\
2691 	(sgid)->gid_prefix = ddi_get64((state)->ts_reg_cmdhdl,	\
2692 	(uint64_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2693 	TAVOR_HW_GUIDTABLE_OFFSET +				\
2694 	((port) * TAVOR_HW_GUIDTABLE_PORT_SIZE) +		\
2695 	((sgid_ix) * TAVOR_HW_GUIDTABLE_GID_SIZE)));		\
2696 	(sgid)->gid_guid = ddi_get64((state)->ts_reg_cmdhdl,	\
2697 	(uint64_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2698 	TAVOR_HW_GUIDTABLE_OFFSET +				\
2699 	((port) * TAVOR_HW_GUIDTABLE_PORT_SIZE) +		\
2700 	((sgid_ix) * TAVOR_HW_GUIDTABLE_GID_SIZE) +		\
2701 	TAVOR_HW_GUIDTABLE_GIDPREFIX_SIZE));
2702 
2703 #define	TAVOR_PKEY_FROM_INDX_GET(state, port, pkey_ix)		\
2704 	(ddi_get32((state)->ts_reg_cmdhdl,			\
2705 	(uint32_t *)((uintptr_t)(state)->ts_reg_cmd_baseaddr +	\
2706 	TAVOR_HW_PKEYTABLE_OFFSET +				\
2707 	((port) * TAVOR_HW_PKEYTABLE_PORT_SIZE) +		\
2708 	((pkey_ix) * TAVOR_HW_PKEYTABLE_PKEY_SIZE))) & 0xFFFF)
2709 
2710 
2711 /*
2712  * Flash interface:
2713  *    Below we have PCI config space space offsets for flash interface
2714  *    access, offsets within Tavor CR space for accessing flash-specific
2715  *    information or settings, masks used for flash settings, and
2716  *    timeout values for flash operations.
2717  */
2718 #define	TAVOR_HW_FLASH_CFG_HWREV		8
2719 #define	TAVOR_HW_FLASH_CFG_ADDR			88
2720 #define	TAVOR_HW_FLASH_CFG_DATA			92
2721 
2722 #define	TAVOR_HW_FLASH_RESET_AMD		0xF0
2723 #define	TAVOR_HW_FLASH_RESET_INTEL		0xFF
2724 #define	TAVOR_HW_FLASH_CPUMODE			0xF0150
2725 #define	TAVOR_HW_FLASH_ADDR			0xF01A4
2726 #define	TAVOR_HW_FLASH_DATA			0xF01A8
2727 #define	TAVOR_HW_FLASH_GPIO_SEMA		0xF03FC
2728 #define	TAVOR_HW_FLASH_GPIO_DIR			0xF008C
2729 #define	TAVOR_HW_FLASH_GPIO_POL			0xF0094
2730 #define	TAVOR_HW_FLASH_GPIO_MOD			0xF009C
2731 #define	TAVOR_HW_FLASH_GPIO_DAT			0xF0084
2732 #define	TAVOR_HW_FLASH_GPIO_DATACLEAR		0xF00D4
2733 #define	TAVOR_HW_FLASH_GPIO_DATASET		0xF00DC
2734 
2735 #define	TAVOR_HW_FLASH_CPU_MASK			0xC0000000
2736 #define	TAVOR_HW_FLASH_CPU_SHIFT		30
2737 #define	TAVOR_HW_FLASH_ADDR_MASK		0x0007FFFC
2738 #define	TAVOR_HW_FLASH_CMD_MASK			0xE0000000
2739 #define	TAVOR_HW_FLASH_BANK_MASK		0xFFF80000
2740 
2741 #define	TAVOR_HW_FLASH_TIMEOUT_WRITE		300
2742 #define	TAVOR_HW_FLASH_TIMEOUT_ERASE		1000000
2743 #define	TAVOR_HW_FLASH_TIMEOUT_GPIO_SEMA	1000
2744 #define	TAVOR_HW_FLASH_TIMEOUT_CONFIG		50
2745 
2746 /* Intel Command Set */
2747 #define	TAVOR_HW_FLASH_ICS_ERASE		0x20
2748 #define	TAVOR_HW_FLASH_ICS_ERROR		0x3E
2749 #define	TAVOR_HW_FLASH_ICS_WRITE		0x40
2750 #define	TAVOR_HW_FLASH_ICS_STATUS		0x70
2751 #define	TAVOR_HW_FLASH_ICS_READY		0x80
2752 #define	TAVOR_HW_FLASH_ICS_CONFIRM		0xD0
2753 #define	TAVOR_HW_FLASH_ICS_READ			0xFF
2754 
2755 #ifdef __cplusplus
2756 }
2757 #endif
2758 
2759 #endif	/* _SYS_IB_ADAPTERS_TAVOR_HW_H */
2760