1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #ifndef	_SYS_IB_ADAPTERS_TAVOR_H
28 #define	_SYS_IB_ADAPTERS_TAVOR_H
29 
30 
31 /*
32  * tavor.h
33  *    Contains the #defines and typedefs necessary for the Tavor softstate
34  *    structure and for proper attach() and detach() processing.  Also
35  *    includes all the other Tavor header files (and so is the only header
36  *    file that is directly included by the Tavor source files).
37  *    Lastly, this file includes everything necessary for implementing the
38  *    devmap interface and for maintaining the "mapped resource database".
39  */
40 
41 #include <sys/types.h>
42 #include <sys/conf.h>
43 #include <sys/ddi.h>
44 #include <sys/sunddi.h>
45 #include <sys/taskq.h>
46 
47 #include <sys/ib/ibtl/ibci.h>
48 #include <sys/ib/adapters/mlnx_umap.h>
49 
50 /*
51  * First include all the Tavor typedefs, then include all the other Tavor
52  * specific headers (many of which depend on the typedefs having already
53  * been defined.
54  */
55 #include <sys/ib/adapters/tavor/tavor_typedef.h>
56 
57 #include <sys/ib/adapters/tavor/tavor_agents.h>
58 #include <sys/ib/adapters/tavor/tavor_cfg.h>
59 #include <sys/ib/adapters/tavor/tavor_cmd.h>
60 #include <sys/ib/adapters/tavor/tavor_cq.h>
61 #include <sys/ib/adapters/tavor/tavor_event.h>
62 #include <sys/ib/adapters/tavor/tavor_hw.h>
63 #include <sys/ib/adapters/tavor/tavor_ioctl.h>
64 #include <sys/ib/adapters/tavor/tavor_misc.h>
65 #include <sys/ib/adapters/tavor/tavor_mr.h>
66 #include <sys/ib/adapters/tavor/tavor_qp.h>
67 #include <sys/ib/adapters/tavor/tavor_srq.h>
68 #include <sys/ib/adapters/tavor/tavor_rsrc.h>
69 #include <sys/ib/adapters/tavor/tavor_wr.h>
70 
71 #ifdef __cplusplus
72 extern "C" {
73 #endif
74 
75 #define	TAVOR_VPD_HDR_DWSIZE		0x10 /* 16 Dwords */
76 #define	TAVOR_VPD_HDR_BSIZE		0x40 /* 64 Bytes */
77 
78 /*
79  * Number of initial states to setup. Used in call to ddi_soft_state_init()
80  */
81 #define	TAVOR_INITIAL_STATES		3
82 
83 /*
84  * Macro and defines used to calculate device instance number from minor
85  * number (and vice versa).
86  */
87 #define	TAVOR_MINORNUM_SHIFT		3
88 #define	TAVOR_DEV_INSTANCE(dev)	(getminor((dev)) &			\
89 	((1 << TAVOR_MINORNUM_SHIFT) - 1))
90 
91 /*
92  * Locations for the various Tavor hardware PCI BARs (CMD, UAR, DDR)
93  */
94 #define	TAVOR_CMD_BAR			1
95 #define	TAVOR_UAR_BAR			2
96 #define	TAVOR_DDR_BAR			3
97 
98 /*
99  * Some defines for the software reset.  These define the value that should
100  * be written to begin the reset (TAVOR_SW_RESET_START), the delay before
101  * beginning to poll for completion (TAVOR_SW_RESET_DELAY), the in-between
102  * polling delay (TAVOR_SW_RESET_POLL_DELAY), and the value that indicates
103  * that the reset has not completed (TAVOR_SW_RESET_NOTDONE).
104  */
105 #define	TAVOR_SW_RESET_START		0x00000001
106 #define	TAVOR_SW_RESET_DELAY		100000		/* 100 ms */
107 #define	TAVOR_SW_RESET_POLL_DELAY	100		/* 100 us */
108 #define	TAVOR_SW_RESET_NOTDONE		0xFFFFFFFF
109 
110 /*
111  * These defines are used in the Tavor software reset operation.  They define
112  * the total number PCI registers to read/restore during the reset.  And they
113  * also specify two config registers which should not be read or restored.
114  */
115 #define	TAVOR_SW_RESET_NUMREGS		0x40
116 #define	TAVOR_SW_RESET_REG22_RSVD	0x16
117 #define	TAVOR_SW_RESET_REG23_RSVD	0x17
118 
119 /*
120  * Macro used to output Tavor warning messages.  Note: Tavor warning messages
121  * are only generated when an unexpected condition has been detected.  This
122  * can be the result of a software bug or some other problem, but it is more
123  * often an indication that the Tavor firmware (and/or hardware) has done
124  * something unexpected.  This warning message means that the driver state
125  * in unpredictable and that shutdown/restart is suggested.
126  */
127 #define	TAVOR_WARNING(state, string)					\
128 	cmn_err(CE_WARN, "tavor%d: "string, (state)->ts_instance)
129 
130 /*
131  * Macro used to set attach failure messages.  Also, the attach message buf
132  * size is set here.
133  */
134 #define	TAVOR_ATTACH_MSGSIZE	80
135 #define	TAVOR_ATTACH_MSG(attach_buf, attach_msg)			\
136 	(void) snprintf((attach_buf), TAVOR_ATTACH_MSGSIZE, (attach_msg));
137 #define	TAVOR_ATTACH_MSG_INIT(attach_buf)				\
138 	(attach_buf)[0] = '\0';
139 
140 /*
141  * Macros used for controlling whether or not event callbacks will be forwarded
142  * to the IBTF.  This is necessary because there are certain race conditions
143  * that can occur (e.g. calling IBTF with an asynch event before the IBTF
144  * registration has successfully completed or handling an event after we've
145  * detached from the IBTF.)
146  *
147  * TAVOR_ENABLE_IBTF_CALLB() initializes the "ts_ibtfpriv" field in the Tavor
148  *    softstate.  When "ts_ibtfpriv" is non-NULL, it is OK to forward asynch
149  *    and CQ events to the IBTF.
150  *
151  * TAVOR_DO_IBTF_ASYNC_CALLB() and TAVOR_DO_IBTF_CQ_CALLB() both set and clear
152  *    the "ts_in_evcallb" flag, as necessary, to indicate that an IBTF
153  *    callback is currently in progress.  This is necessary so that we can
154  *    block on this condition in tavor_detach().
155  *
156  * TAVOR_QUIESCE_IBTF_CALLB() is used in tavor_detach() to set the
157  *    "ts_ibtfpriv" to NULL (thereby disabling any further IBTF callbacks)
158  *    and to poll on the "ts_in_evcallb" flag.  When this flag is zero, all
159  *    IBTF callbacks have quiesced and it is safe to continue with detach
160  *    (i.e. continue detaching from IBTF).
161  */
162 #define	TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv)			\
163 	(state)->ts_ibtfpriv = (tmp_ibtfpriv);
164 
165 #define	TAVOR_DO_IBTF_ASYNC_CALLB(state, type, event)			\
166 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS((state)->ts_in_evcallb))	\
167 	(state)->ts_in_evcallb = 1;					\
168 	ibc_async_handler((state)->ts_ibtfpriv, (type), (event));	\
169 	(state)->ts_in_evcallb = 0;
170 
171 #define	TAVOR_DO_IBTF_CQ_CALLB(state, cq)				\
172 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS((state)->ts_in_evcallb))	\
173 	(state)->ts_in_evcallb = 1;					\
174 	ibc_cq_handler((state)->ts_ibtfpriv, (cq)->cq_hdlrarg);		\
175 	(state)->ts_in_evcallb = 0;
176 
177 #define	TAVOR_QUIESCE_IBTF_CALLB(state)					\
178 {									\
179 	uint_t		count = 0;					\
180 									\
181 	state->ts_ibtfpriv = NULL;					\
182 	while (((state)->ts_in_evcallb != 0) &&				\
183 	    (count++ < TAVOR_QUIESCE_IBTF_CALLB_POLL_MAX)) {		\
184 		drv_usecwait(TAVOR_QUIESCE_IBTF_CALLB_POLL_DELAY);	\
185 	}								\
186 }
187 
188 /*
189  * Defines used by the TAVOR_QUIESCE_IBTF_CALLB() macro to determine the
190  * duration and number of times (at maximum) to poll while waiting for IBTF
191  * callbacks to quiesce.
192  */
193 #define	TAVOR_QUIESCE_IBTF_CALLB_POLL_DELAY	1
194 #define	TAVOR_QUIESCE_IBTF_CALLB_POLL_MAX	1000000
195 
196 /*
197  * Define used to determine the device mode to which Tavor driver has been
198  * attached.  TAVOR_IS_MAINTENANCE_MODE() returns true when the device has
199  * come up in the "maintenance mode".  In this mode, no InfiniBand interfaces
200  * are enabled, but the device's firmware can be updated/flashed (and
201  * test/debug interfaces should be useable).
202  * TAVOR_IS_HCA_MODE() returns true when the device has come up in the normal
203  * HCA mode.  In this mode, all necessary InfiniBand interfaces are enabled
204  * (and, if necessary, Tavor firmware can be updated/flashed).
205  */
206 #define	TAVOR_IS_MAINTENANCE_MODE(dip)					\
207 	(((ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
208 	"device-id", -1) == 0x5a45) ||					\
209 	(ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
210 	"device-id", -1) == 0x6279)) &&					\
211 	(ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
212 	"vendor-id", -1) == 0x15b3))
213 #define	TAVOR_IS_COMPAT_MODE(dip)					\
214 	((ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
215 	"device-id", -1) == 0x6278) &&					\
216 	(ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
217 	"vendor-id", -1) == 0x15b3))
218 #define	TAVOR_IS_HCA_MODE(dip)						\
219 	((ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
220 	"device-id", -1) == 0x5a44) &&					\
221 	(ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
222 	"vendor-id", -1) == 0x15b3))
223 
224 #define	TAVOR_MAINTENANCE_MODE		1
225 #define	TAVOR_COMPAT_MODE		2
226 #define	TAVOR_HCA_MODE			3
227 
228 /*
229  * Used to determine if the device is operational, or not in maintenance mode.
230  * This means either the driver has attached successfully against an arbel
231  * device in tavor compatibility mode, or against a tavor device in full HCA
232  * mode.
233  */
234 #define	TAVOR_IS_OPERATIONAL(mode)					\
235 	(mode == TAVOR_COMPAT_MODE || mode == TAVOR_HCA_MODE)
236 
237 /*
238  * Used to determine if parent bridge is a PCI bridge; used in software reset
239  */
240 #define	TAVOR_PARENT_IS_BRIDGE(dip)					\
241 	((ddi_prop_get_int(DDI_DEV_T_ANY, (dip), DDI_PROP_DONTPASS,	\
242 	"device-id", -1) == 0x5a46))
243 
244 /*
245  * The following define is used (in tavor_umap_db_set_onclose_cb()) to
246  * indicate that a cleanup callback is needed to undo initialization done
247  * by the firmware flash burn code.
248  */
249 #define	TAVOR_ONCLOSE_FLASH_INPROGRESS		(1 << 0)
250 
251 /*
252  * The following enumerated type and structures are used during driver
253  * initialization.  Note: The TAVOR_DRV_CLEANUP_ALL type is used as a marker
254  * for end of the cleanup steps.  No cleanup steps should be added after
255  * TAVOR_DRV_CLEANUP_ALL.  Any addition steps should be added before it.
256  */
257 typedef enum {
258 	TAVOR_DRV_CLEANUP_LEVEL0,
259 	TAVOR_DRV_CLEANUP_LEVEL1,
260 	TAVOR_DRV_CLEANUP_LEVEL2,
261 	TAVOR_DRV_CLEANUP_LEVEL3,
262 	TAVOR_DRV_CLEANUP_LEVEL4,
263 	TAVOR_DRV_CLEANUP_LEVEL5,
264 	TAVOR_DRV_CLEANUP_LEVEL6,
265 	TAVOR_DRV_CLEANUP_LEVEL7,
266 	TAVOR_DRV_CLEANUP_LEVEL8,
267 	TAVOR_DRV_CLEANUP_LEVEL9,
268 	TAVOR_DRV_CLEANUP_LEVEL10,
269 	TAVOR_DRV_CLEANUP_LEVEL11,
270 	TAVOR_DRV_CLEANUP_LEVEL12,
271 	TAVOR_DRV_CLEANUP_LEVEL13,
272 	TAVOR_DRV_CLEANUP_LEVEL14,
273 	/* No more driver cleanup steps below this point! */
274 	TAVOR_DRV_CLEANUP_ALL
275 } tavor_drv_cleanup_level_t;
276 
277 /*
278  *  tavor_mem_alloc_hdl_t structure store DMA handles for the new
279  * ibc_alloc_io_mem calls
280  */
281 typedef struct tavor_mem_alloc_hdl_s {
282 	ddi_dma_handle_t tavor_dma_hdl;
283 	ddi_acc_handle_t tavor_acc_hdl;
284 } *tavor_mem_alloc_hdl_t;
285 
286 
287 /*
288  * The tavor_cmd_reg_t structure is used to hold the address of the each of
289  * the most frequently accessed hardware registers.  Specifically, it holds
290  * the HCA Command Registers (HCR, used to pass command and mailbox
291  * information back and forth to Tavor firmware) and the lock used to guarantee
292  * mutually exclusive access to the registers.  It also holds the Event Cause
293  * Register (ECR) and its related clear register.  These are used to indicate
294  * during interrupt processing which EQs have fired and require servicing.
295  * Related to this, is the "clr_int" register which is used to clear the
296  * interrupt once all EQs have been services.
297  * Finally, there is the software reset register which is used to reinitialize
298  * the Tavor device and to put it into a known state at driver startup time.
299  * Below we also have the offsets (into the CMD register space) for each of
300  * the various registers.
301  */
302 typedef struct tavor_cmd_reg_s {
303 	tavor_hw_hcr_t	*hcr;
304 	kmutex_t	hcr_lock;
305 	uint64_t	*ecr;
306 	uint64_t	*clr_ecr;
307 	uint64_t	*clr_int;
308 	uint32_t	*sw_reset;
309 } tavor_cmd_reg_t;
310 _NOTE(MUTEX_PROTECTS_DATA(tavor_cmd_reg_t::hcr_lock,
311     tavor_cmd_reg_t::hcr))
312 
313 
314 /*
315  * The tavor_state_t structure is the Tavor software state structure.  It
316  * contains all the pointers and placeholder for everything that the Tavor
317  * driver needs to properly operate.  One of these structures exists for
318  * every instance of the Tavor driver.
319  */
320 struct tavor_state_s {
321 	dev_info_t		*ts_dip;
322 	int			ts_instance;
323 
324 	/* Tavor interrupt/MSI information */
325 	int			ts_intr_types_avail;
326 	uint_t			ts_intr_type_chosen;
327 	int			ts_intrmsi_count;
328 	int			ts_intrmsi_avail;
329 	int			ts_intrmsi_allocd;
330 	ddi_intr_handle_t	ts_intrmsi_hdl;
331 	uint_t			ts_intrmsi_pri;
332 	int			ts_intrmsi_cap;
333 
334 	/* Tavor device operational mode */
335 	int			ts_operational_mode;
336 
337 	/* Attach buffer saved per state to store detailed attach errors */
338 	char			ts_attach_buf[TAVOR_ATTACH_MSGSIZE];
339 
340 	/*
341 	 * Tavor NodeGUID, SystemImageGUID, NodeDescription, HCA name,
342 	 * and HCA part number.
343 	 */
344 	uint64_t		ts_nodeguid;
345 	uint64_t		ts_sysimgguid;
346 	char			ts_nodedesc[64];
347 	char			ts_hca_name[64];
348 	char			ts_hca_pn[64];
349 	int			ts_hca_pn_len;
350 
351 	/* Info passed to IBTF during registration */
352 	ibc_hca_info_t		ts_ibtfinfo;
353 	ibc_clnt_hdl_t		ts_ibtfpriv;
354 
355 	/*
356 	 * Tavor register mapping.  Holds the device access attributes,
357 	 * kernel mapped addresses, and DDI access handles for each of
358 	 * Tavor's three types of address register (CMD, UAR, and DDR).
359 	 */
360 	ddi_device_acc_attr_t	ts_reg_accattr;
361 	caddr_t			ts_reg_cmd_baseaddr;	/* Tavor CMD BAR */
362 	ddi_acc_handle_t	ts_reg_cmdhdl;
363 	caddr_t			ts_reg_uar_baseaddr;	/* Tavor UAR BAR */
364 	ddi_acc_handle_t	ts_reg_uarhdl;
365 	caddr_t			ts_reg_ddr_baseaddr;	/* Tavor DDR BAR */
366 	ddi_acc_handle_t	ts_reg_ddrhdl;
367 
368 	/*
369 	 * Tavor PCI config space registers.  These two arrays are used to
370 	 * save and restore the PCI config registers before and after a
371 	 * software reset.  Note: We must save away both our own registers
372 	 * and our parent's (the "virtual" PCI bridge in the device) because
373 	 * the software reset will reset both sets.
374 	 */
375 	uint32_t		ts_cfg_data[TAVOR_SW_RESET_NUMREGS];
376 	uint32_t		ts_cfg_pdata[TAVOR_SW_RESET_NUMREGS];
377 
378 	/*
379 	 * Tavor UAR page resources.  Holds the resource pointers for
380 	 * UAR page #0 (reserved) and for UAR page #1 (used for kernel
381 	 * driver doorbells).  In addition, we save a pointer to the
382 	 * UAR page #1 doorbells which will be used throughout the driver
383 	 * whenever it is necessary to ring one of them.  And, in case we
384 	 * are unable to do 64-bit writes to the page (because of system
385 	 * architecture), we include a lock (to ensure atomic 64-bit access).
386 	 */
387 	tavor_rsrc_t		*ts_uarpg0_rsrc_rsrvd;
388 	tavor_rsrc_t		*ts_uarpg1_rsrc;
389 	tavor_hw_uar_t		*ts_uar;
390 	kmutex_t		ts_uar_lock;
391 
392 	/*
393 	 * Used during a call to open() if we are in maintenance mode, this
394 	 * field serves as a semi-unique rolling count index value, used only
395 	 * in the setup of umap_db entries.  This is primarily needed to
396 	 * firmware device access ioctl operations can still be guaranteed to
397 	 * close in the event of an unplanned process exit, even in maintenance
398 	 * mode.
399 	 */
400 	uint_t			ts_open_tr_indx;
401 
402 	/*
403 	 * Tavor command registers.  This structure contains the addresses
404 	 * for each of the most frequently accessed CMD registers.  Since
405 	 * almost all accesses to the Tavor hardware are through the Tavor
406 	 * command interface (i.e. the HCR), we save away the pointer to
407 	 * the HCR, as well as pointers to the ECR and INT registers (as
408 	 * well as their corresponding "clear" registers) for interrupt
409 	 * processing.  And we also save away a pointer to the software
410 	 * reset register (see above).
411 	 */
412 	tavor_cmd_reg_t		ts_cmd_regs;
413 
414 	/*
415 	 * Tavor resource pointers.  The following are pointers to the vmem
416 	 * arena (created to manage the DDR memory), the kmem cache (from
417 	 * which the Tavor resource handles are allocated), and the array
418 	 * of "resource pools" (which store all the pertinent information
419 	 * necessary to manage each of the various types of resources that
420 	 * are used by the Tavor driver.  See tavor_rsrc.h for more detail.
421 	 */
422 	vmem_t			*ts_ddrvmem;
423 	kmem_cache_t		*ts_rsrc_cache;
424 	tavor_rsrc_pool_info_t	*ts_rsrc_hdl;
425 
426 	/*
427 	 * Tavor mailbox lists.  These hold the information necessary to
428 	 * manage the pools of pre-allocated Tavor mailboxes (both "In" and
429 	 * "Out" type).  See tavor_cmd.h for more detail.
430 	 */
431 	tavor_mboxlist_t	ts_in_mblist;
432 	tavor_mboxlist_t	ts_out_mblist;
433 
434 	/*
435 	 * Tavor interrupt mailbox lists.  We allocate both an "In" mailbox
436 	 * and an "Out" type mailbox for the interrupt context.  This is in
437 	 * order to guarantee that a mailbox entry will always be available in
438 	 * the interrupt context, and we can NOSLEEP without having to worry
439 	 * about possible failure allocating the mbox.  We create this as an
440 	 * mboxlist so that we have the potential for having multiple mboxes
441 	 * available based on the number of interrupts we can receive at once.
442 	 */
443 	tavor_mboxlist_t		ts_in_intr_mblist;
444 	tavor_mboxlist_t		ts_out_intr_mblist;
445 
446 	/*
447 	 * Tavor outstanding command list.  Used to hold all the information
448 	 * necessary to manage the Tavor "outstanding command list".  See
449 	 * tavor_cmd.h for more detail.
450 	 */
451 	tavor_cmdlist_t		ts_cmd_list;
452 
453 	/*
454 	 * This structure contains the Tavor driver's "configuration profile".
455 	 * This is the collected set of configuration information, such as
456 	 * number of QPs, CQs, mailboxes and other resources, sizes of
457 	 * individual resources, other system level configuration information,
458 	 * etc.  See tavor_cfg.h for more detail.
459 	 */
460 	tavor_cfg_profile_t	*ts_cfg_profile;
461 
462 	/*
463 	 * This flag contains the profile setting, selecting which profile the
464 	 * driver would use.  This is needed in the case where we have to
465 	 * fallback to a smaller profile based on some DDR conditions.  If we
466 	 * don't fallback, then it is set to the size of DDR in the system.
467 	 */
468 	uint32_t		ts_cfg_profile_setting;
469 
470 	/*
471 	 * The following are a collection of resource handles used by the
472 	 * Tavor driver (internally).  First is the protection domain (PD)
473 	 * handle that is used when mapping all kernel memory (work queues,
474 	 * completion queues, etc).  Next is an array of EQ handles.  This
475 	 * array is indexed by EQ number and allows the Tavor driver to quickly
476 	 * convert an EQ number into the software structure associated with the
477 	 * given EQ.  Likewise, we have three arrays for CQ, QP and SRQ
478 	 * handles.  These arrays are also indexed by CQ, QP or SRQ number and
479 	 * allow the driver to quickly find the corresponding CQ, QP or SRQ
480 	 * software structure.  Note: while the EQ table is of fixed size
481 	 * (because there are a maximum of 64 EQs), each of the CQ, QP and SRQ
482 	 * handle lists must be allocated at driver startup.
483 	 */
484 	tavor_pdhdl_t		ts_pdhdl_internal;
485 	tavor_eqhdl_t		ts_eqhdl[TAVOR_NUM_EQ];
486 	tavor_cqhdl_t		*ts_cqhdl;
487 	tavor_qphdl_t		*ts_qphdl;
488 	tavor_srqhdl_t		*ts_srqhdl;
489 
490 	/*
491 	 * The AVL tree is used to store information regarding QP number
492 	 * allocations.  The lock protects access to the AVL tree.
493 	 */
494 	avl_tree_t		ts_qpn_avl;
495 	kmutex_t		ts_qpn_avl_lock;
496 
497 	/*
498 	 * This field is used to indicate whether or not the Tavor driver is
499 	 * currently in an IBTF event callback elsewhere in the system.  Note:
500 	 * It is "volatile" because we intend to poll on this value - in
501 	 * tavor_detach() - until we are assured that no further IBTF callbacks
502 	 * are currently being processed.
503 	 */
504 	volatile uint32_t	ts_in_evcallb;
505 
506 	/*
507 	 * The following structures are used to store the results of several
508 	 * device query commands passed to the Tavor hardware at startup.
509 	 * Specifically, we have hung onto the results of QUERY_DDR (which
510 	 * gives information about how much DDR memory is present and where
511 	 * it is located), QUERY_FW (which gives information about firmware
512 	 * version numbers and the location and extent of firmware's footprint
513 	 * in DDR, QUERY_DEVLIM (which gives the device limitations/resource
514 	 * maximums), QUERY_ADAPTER (which gives additional miscellaneous
515 	 * information), and INIT/QUERY_HCA (which serves the purpose of
516 	 * recording what configuration information was passed to the firmware
517 	 * when the HCA was initialized).
518 	 */
519 	struct tavor_hw_queryddr_s	ts_ddr;
520 	struct tavor_hw_queryfw_s	ts_fw;
521 	struct tavor_hw_querydevlim_s	ts_devlim;
522 	struct tavor_hw_queryadapter_s	ts_adapter;
523 	struct tavor_hw_initqueryhca_s	ts_hcaparams;
524 
525 	/*
526 	 * The following are used for managing special QP resources.
527 	 * Specifically, we have a lock, a set of flags (in "ts_spec_qpflags")
528 	 * used to track the special QP resources, and two Tavor resource
529 	 * handle pointers.  Each resource handle actually corresponds to two
530 	 * consecutive QP contexts (one per port) for each special QP type.
531 	 */
532 	kmutex_t		ts_spec_qplock;
533 	uint_t			ts_spec_qpflags;
534 	tavor_rsrc_t		*ts_spec_qp0;
535 	tavor_rsrc_t		*ts_spec_qp1;
536 
537 	/*
538 	 * Related in some ways to the special QP handling above are these
539 	 * resources which are used specifically for implementing the Tavor
540 	 * agents (SMA, PMA, and BMA).  Although, each of these agents does
541 	 * little more that intercept the appropriate incoming MAD and forward
542 	 * it along to the firmware (see tavor_agents.c for more details), we
543 	 * do still use a task queue to queue them up.  We can also configure
544 	 * the driver to force firmware handling for certain classes of MAD,
545 	 * and, therefore, we require the agent list and number of agents
546 	 * in order to know what needs to be torn down at detach() time.
547 	 */
548 	tavor_agent_list_t	*ts_agents;
549 	ddi_taskq_t		*ts_taskq_agents;
550 	uint_t			ts_num_agents;
551 
552 	/*
553 	 * Multicast group lists.  These are used to track the "shadow" MCG
554 	 * lists that speed up the processing of attach and detach multicast
555 	 * group operations.  See tavor_misc.h for more details.  Note: we
556 	 * need the pointer to the "temporary" MCG entry here primarily
557 	 * because the size of a given MCG entry is configurable.  Therefore,
558 	 * it is impossible to put this variable on the stack.  And rather
559 	 * than allocate and deallocate the entry multiple times, we choose
560 	 * instead to preallocate it once and reuse it over and over again.
561 	 */
562 	kmutex_t		ts_mcglock;
563 	tavor_mcghdl_t		ts_mcghdl;
564 	tavor_hw_mcg_t		*ts_mcgtmp;
565 
566 	/*
567 	 * Used for tracking Tavor kstat information
568 	 */
569 	tavor_ks_info_t		*ts_ks_info;
570 
571 	/*
572 	 * Used for Tavor info ioctl used by VTS
573 	 */
574 	kmutex_t		ts_info_lock;
575 
576 	/*
577 	 * Used for Tavor FW flash burning.  They are used exclusively
578 	 * within the ioctl calls for use when accessing the tavor
579 	 * flash device.
580 	 */
581 	kmutex_t		ts_fw_flashlock;
582 	int			ts_fw_flashstarted;
583 	dev_t			ts_fw_flashdev;
584 	uint32_t		ts_fw_log_sector_sz;
585 	uint32_t		ts_fw_device_sz;
586 	uint32_t		ts_fw_flashbank;
587 	uint32_t		*ts_fw_sector;
588 	uint32_t		ts_fw_gpio[4];
589 	ddi_acc_handle_t	ts_pci_cfghdl;		/* PCI cfg handle */
590 	int			ts_fw_cmdset;
591 
592 	/* Tavor fastreboot support */
593 	boolean_t		ts_quiescing;		/* in fastreboot */
594 };
595 _NOTE(MUTEX_PROTECTS_DATA(tavor_state_s::ts_fw_flashlock,
596     tavor_state_s::ts_fw_flashstarted
597     tavor_state_s::ts_fw_flashdev
598     tavor_state_s::ts_fw_log_sector_sz
599     tavor_state_s::ts_fw_device_sz))
600 _NOTE(MUTEX_PROTECTS_DATA(tavor_state_s::ts_spec_qplock,
601     tavor_state_s::ts_spec_qpflags
602     tavor_state_s::ts_spec_qp0
603     tavor_state_s::ts_spec_qp1))
604 _NOTE(MUTEX_PROTECTS_DATA(tavor_state_s::ts_mcglock,
605     tavor_state_s::ts_mcghdl
606     tavor_state_s::ts_mcgtmp))
607 _NOTE(DATA_READABLE_WITHOUT_LOCK(tavor_state_s::ts_in_evcallb
608     tavor_state_s::ts_fw_log_sector_sz
609     tavor_state_s::ts_fw_device_sz
610     tavor_state_s::ts_fw_sector
611     tavor_state_s::ts_spec_qpflags
612     tavor_state_s::ts_spec_qp0
613     tavor_state_s::ts_spec_qp1))
614 _NOTE(MUTEX_PROTECTS_DATA(tavor_state_s::ts_qpn_avl_lock,
615     tavor_state_s::ts_qpn_avl))
616 
617 /*
618  * TAVOR_IN_FASTREBOOT() shows if Hermon driver is at fastreboot.
619  * This macro should be used to check if the mutex lock can be used
620  * since the lock cannot be used if the driver is in the quiesce mode.
621  */
622 #define	TAVOR_IN_FASTREBOOT(state)	(state->ts_quiescing == B_TRUE)
623 
624 /*
625  * Bit positions in the "ts_spec_qpflags" field above.  The flags are (from
626  * least significant to most): (QP0,Port1), (QP0,Port2), (QP1,Port1), and
627  * (QP1,Port2).  The masks are there to help with some specific allocation
628  * and freeing operations
629  */
630 #define	TAVOR_SPECIAL_QP0_RSRC		0
631 #define	TAVOR_SPECIAL_QP0_RSRC_MASK	0x3
632 #define	TAVOR_SPECIAL_QP1_RSRC		2
633 #define	TAVOR_SPECIAL_QP1_RSRC_MASK	0xC
634 
635 
636 /*
637  * These flags specifies additional behaviors on database access.
638  * TAVOR_UMAP_DB_REMOVE, for example, specifies that (if found) the database
639  * entry should be removed from the database.  TAVOR_UMAP_DB_IGNORE_INSTANCE
640  * specifies that a particular database query should ignore value in the
641  * "tdb_instance" field as a criterion for the search.
642  */
643 #define	TAVOR_UMAP_DB_REMOVE		(1 << 0)
644 #define	TAVOR_UMAP_DB_IGNORE_INSTANCE	(1 << 1)
645 
646 
647 /*
648  * The tavor_umap_db_t structure contains what is referred to throughout the
649  * driver code as the "userland resources database".  This structure contains
650  * all the necessary information to track resources that have been prepared
651  * for direct-from-userland access.  There is an AVL tree ("tdl_umapdb_avl")
652  * which consists of the "tavor_umap_db_entry_t" (below) and a lock to ensure
653  * atomic access when adding or removing entries from the database.
654  */
655 typedef struct tavor_umap_db_s {
656 	kmutex_t		tdl_umapdb_lock;
657 	avl_tree_t		tdl_umapdb_avl;
658 } tavor_umap_db_t;
659 
660 /*
661  * The tavor_umap_db_priv_t structure currently contains information necessary
662  * to provide the "on close" callback to the firmware flash interfaces.  It
663  * is intended that this structure could be extended to enable other "on
664  * close" callbacks as well.
665  */
666 typedef struct tavor_umap_db_priv_s {
667 	void		(*tdp_cb)(void *);
668 	void		*tdp_arg;
669 } tavor_umap_db_priv_t;
670 
671 /*
672  * The tavor_umap_db_common_t structure contains fields which are common
673  * between the database entries ("tavor_umap_db_entry_t") and the structure
674  * used to contain the search criteria ("tavor_umap_db_query_t").  This
675  * structure contains a key, a resource type (described above), an instance
676  * (corresponding to the driver instance which inserted the database entry),
677  * and a "value" field.  Typically, "tdb_value" is a pointer to a Tavor
678  * resource object.  Although for memory regions, the value field corresponds
679  * to the ddi_umem_cookie_t for the pinned userland memory.
680  * The structure also includes a placeholder for private data ("tdb_priv").
681  * Currently this data is being used for holding "on close" callback
682  * information to allow certain kinds of cleanup even if a userland process
683  * prematurely exits.
684  */
685 typedef struct tavor_umap_db_common_s {
686 	uint64_t		tdb_key;
687 	uint64_t		tdb_value;
688 	uint_t			tdb_type;
689 	uint_t			tdb_instance;
690 	void			*tdb_priv;
691 } tavor_umap_db_common_t;
692 
693 /*
694  * The tavor_umap_db_entry_t structure is the entry in "userland resources
695  * database".  As required by the AVL framework, each entry contains an
696  * "avl_node_t".  Then, as required to implement the database, each entry
697  * contains a "tavor_umap_db_common_t" structure used to contain all of the
698  * relevant entries.
699  */
700 typedef struct tavor_umap_db_entry_s {
701 	avl_node_t		tdbe_avlnode;
702 	tavor_umap_db_common_t	tdbe_common;
703 } tavor_umap_db_entry_t;
704 
705 /*
706  * The tavor_umap_db_query_t structure is used in queries to the "userland
707  * resources database".  In addition to the "tavor_umap_db_common_t" structure
708  * used to contain the various search criteria, this structure also contains
709  * a flags field "tqdb_flags" which can be used to specify additional behaviors
710  * (as described above).  Specifically, the flags field can be used to specify
711  * that an entry should be removed from the database, if found, and to
712  * specify whether the database lookup should consider "tdb_instance" in the
713  * search.
714  */
715 typedef struct tavor_umap_db_query_s {
716 	uint_t			tqdb_flags;
717 	tavor_umap_db_common_t	tqdb_common;
718 } tavor_umap_db_query_t;
719 _NOTE(MUTEX_PROTECTS_DATA(tavor_umap_db_s::tdl_umapdb_lock,
720     tavor_umap_db_entry_s::tdbe_avlnode
721     tavor_umap_db_entry_s::tdbe_common.tdb_key
722     tavor_umap_db_entry_s::tdbe_common.tdb_value
723     tavor_umap_db_entry_s::tdbe_common.tdb_type
724     tavor_umap_db_entry_s::tdbe_common.tdb_instance))
725 
726 /*
727  * The tavor_devmap_track_t structure contains all the necessary information
728  * to track resources that have been mapped through devmap.  There is a
729  * back-pointer to the Tavor softstate, the logical offset corresponding with
730  * the mapped resource, the size of the mapped resource (zero indicates an
731  * "invalid mapping"), and a reference count and lock used to determine when
732  * to free the structure (specifically, this is necessary to handle partial
733  * unmappings).
734  */
735 typedef struct tavor_devmap_track_s {
736 	tavor_state_t	*tdt_state;
737 	uint64_t	tdt_offset;
738 	uint_t		tdt_size;
739 	int		tdt_refcnt;
740 	kmutex_t	tdt_lock;
741 } tavor_devmap_track_t;
742 
743 
744 /* Defined in tavor_umap.c */
745 int tavor_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
746     size_t *maplen, uint_t model);
747 ibt_status_t tavor_umap_ci_data_in(tavor_state_t *state,
748     ibt_ci_data_flags_t flags, ibt_object_type_t object, void *hdl,
749     void *data_p, size_t data_sz);
750 ibt_status_t tavor_umap_ci_data_out(tavor_state_t *state,
751     ibt_ci_data_flags_t flags, ibt_object_type_t object, void *hdl,
752     void *data_p, size_t data_sz);
753 void tavor_umap_db_init(void);
754 void tavor_umap_db_fini(void);
755 tavor_umap_db_entry_t *tavor_umap_db_alloc(uint_t instance, uint64_t key,
756     uint_t type, uint64_t value);
757 void tavor_umap_db_free(tavor_umap_db_entry_t *umapdb);
758 void tavor_umap_db_add(tavor_umap_db_entry_t *umapdb);
759 void tavor_umap_db_add_nolock(tavor_umap_db_entry_t *umapdb);
760 int tavor_umap_db_find(uint_t instance, uint64_t key, uint_t type,
761     uint64_t *value, uint_t flags, tavor_umap_db_entry_t **umapdb);
762 int tavor_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
763     uint64_t *value, uint_t flags, tavor_umap_db_entry_t **umapdb);
764 void tavor_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie);
765 int tavor_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
766     void (*callback)(void *), void *arg);
767 int tavor_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag);
768 void tavor_umap_db_handle_onclose_cb(tavor_umap_db_priv_t *priv);
769 
770 #ifdef __cplusplus
771 }
772 #endif
773 
774 #endif	/* _SYS_IB_ADAPTERS_TAVOR_H */
775