1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #ifndef	_SYS_IB_ADAPTERS_HERMON_CQ_H
27 #define	_SYS_IB_ADAPTERS_HERMON_CQ_H
28 
29 /*
30  * hermon_cq.h
31  *    Contains all of the prototypes, #defines, and structures necessary
32  *    for the Completion Queue Processing routines.
33  *    Specifically it contains the various completion types, flags,
34  *    structures used for managing Hermon completion queues, and prototypes
35  *    for many of the functions consumed by other parts of the Hermon driver
36  *    (including those routines directly exposed through the IBTF CI
37  *    interface).
38  *
39  *    Most of the values defined below establish default values which,
40  *    where indicated, can be controlled via their related patchable values,
41  *    if 'hermon_alt_config_enable' is set.
42  */
43 
44 #include <sys/types.h>
45 #include <sys/conf.h>
46 #include <sys/ddi.h>
47 #include <sys/sunddi.h>
48 
49 #include <sys/ib/adapters/hermon/hermon_misc.h>
50 
51 #ifdef __cplusplus
52 extern "C" {
53 #endif
54 
55 /*
56  * The following defines the default number of Completion Queues. This
57  * is controllable via the "hermon_log_num_cq" configuration variable.
58  * We also have a define for the minimum size of a CQ.  CQs allocated
59  * with size "less than a page" will always get back a page.
60  */
61 #define	HERMON_NUM_CQ_SHIFT		0x12
62 
63 #define	HERMON_CQ_MIN_SIZE	((PAGESIZE / 32) - 1)
64 
65 /*
66  * These are the defines for the Hermon CQ completion statuses.
67  */
68 #define	HERMON_CQE_SUCCESS		0x0
69 #define	HERMON_CQE_LOC_LEN_ERR		0x1
70 #define	HERMON_CQE_LOC_OP_ERR		0x2
71 #define	HERMON_CQE_LOC_PROT_ERR		0x4
72 #define	HERMON_CQE_WR_FLUSHED_ERR	0x5
73 #define	HERMON_CQE_MW_BIND_ERR		0x6
74 #define	HERMON_CQE_BAD_RESPONSE_ERR	0x10
75 #define	HERMON_CQE_LOCAL_ACCESS_ERR	0x11
76 #define	HERMON_CQE_REM_INV_REQ_ERR	0x12
77 #define	HERMON_CQE_REM_ACC_ERR		0x13
78 #define	HERMON_CQE_REM_OP_ERR		0x14
79 #define	HERMON_CQE_TRANS_TO_ERR		0x15
80 #define	HERMON_CQE_RNRNAK_TO_ERR	0x16
81 #define	HERMON_CQE_EEC_REM_ABORTED_ERR	0x22
82 
83 /*
84  * These are the defines for the Hermon CQ entry types. They indicate what type
85  * of work request is completing (for successful completions).  Note: The
86  * "SND" or "RCV" in each define is used to indicate whether the completion
87  * work request was from the Send work queue or the Receive work queue on
88  * the associated QP.
89  */
90 #define	HERMON_CQE_SND_NOP		0x0
91 #define	HERMON_CQE_SND_SEND_INV		0x1
92 #define	HERMON_CQE_SND_RDMAWR		0x8
93 #define	HERMON_CQE_SND_RDMAWR_IMM	0x9
94 #define	HERMON_CQE_SND_SEND		0xA
95 #define	HERMON_CQE_SND_SEND_IMM		0xB
96 #define	HERMON_CQE_SND_LSO		0xE
97 #define	HERMON_CQE_SND_RDMARD		0x10
98 #define	HERMON_CQE_SND_ATOMIC_CS	0x11
99 #define	HERMON_CQE_SND_ATOMIC_FA	0x12
100 #define	HERMON_CQE_SND_ATOMIC_CS_EX	0x14
101 #define	HERMON_CQE_SND_ATOMIC_FC_EX	0x15
102 #define	HERMON_CQE_SND_FRWR		0x19
103 #define	HERMON_CQE_SND_LCL_INV		0x1B
104 #define	HERMON_CQE_SND_CONFIG		0x1F
105 #define	HERMON_CQE_SND_BIND_MW		0x18
106 
107 #define	HERMON_CQE_RCV_RDMAWR_IMM	0x00
108 #define	HERMON_CQE_RCV_SEND		0x01
109 #define	HERMON_CQE_RCV_SEND_IMM		0x02
110 #define	HERMON_CQE_RCV_SEND_INV		0x03
111 #define	HERMON_CQE_RCV_ERROR_CODE	0x1E
112 #define	HERMON_CQE_RCV_RESIZE_CODE	0x16
113 
114 
115 /* Define for maximum CQ number mask (CQ number is 24 bits) */
116 #define	HERMON_CQ_MAXNUMBER_MSK		0xFFFFFF
117 
118 /*
119  * CQ Sched Management
120  *
121  *	Each hermon_cq_sched struct defines a range of cq handler_id's
122  *	assigned to the cq_sched instance.  Also, the "next_alloc"
123  *	member is used to allocate handler_id's in a round robin fashion.
124  *
125  *	Valid cq handler_id's are in the range of 1 to hs_intrmsi_allocd.
126  *	They are indexes into the hs_intrmsi_hdl array.
127  */
128 #define	HERMON_CQH_MAX	32
129 typedef struct hermon_cq_sched_s {
130 	char	cqs_name[HERMON_CQH_MAX];
131 	uint_t	cqs_start_hid;
132 	uint_t	cqs_len;
133 	uint_t	cqs_next_alloc;
134 	uint_t	cqs_desired;
135 	uint_t	cqs_minimum;
136 	uint_t	cqs_refcnt;	/* could be alloc'ed more than once */
137 } hermon_cq_sched_t;
138 
139 /*
140  * new EQ mgmt - per domain (when it gets there).
141  * The first hs_rsvd_eqs are reserved by the firmware.
142  * The next hs_intrmsi_allocd are for CQ Completions.
143  * Each of these "completion" EQs has a unique interrupt vector.
144  * The EQs following that are:
145  *
146  *	1 for CQ Errors
147  *	1 for Asyncs and Command Completions, and finally
148  *	1 for All Other events.
149  *
150  * share the last of the interrupt vectors.
151  */
152 #define	HERMON_CQSCHED_NEXT_HID(cq_schedp)				\
153 	((atomic_inc_uint_nv(&(cq_schedp)->cqs_next_alloc) %		\
154 	    (cq_schedp)->cqs_len) + (cq_schedp)->cqs_start_hid)
155 
156 #define	HERMON_HID_TO_EQNUM(state, hid)					\
157 	((state)->hs_rsvd_eqs + (hid) - 1)
158 
159 #define	HERMON_HID_VALID(state, hid)					\
160 	((uint_t)((hid) - 1) < (state)->hs_intrmsi_allocd)
161 
162 #define	HERMON_EQNUM_TO_HID(state, eqnum)				\
163 	((eqnum) - (state)->hs_rsvd_eqs + 1)
164 
165 #define	HERMON_CQ_ERREQNUM_GET(state)					\
166 	(state)->hs_cq_erreqnum
167 
168 /*
169  * The following defines are used for Hermon CQ error handling.  Note: For
170  * CQEs which correspond to error events, the Hermon device requires some
171  * special handling by software.  These defines are used to identify and
172  * extract the necessary information from each error CQE, including status
173  * code (above), doorbell count, and whether a error completion is for a
174  * send or receive work request.
175  */
176 #define	HERMON_CQE_ERR_STATUS_SHIFT	0
177 #define	HERMON_CQE_ERR_STATUS_MASK	0xFF
178 #define	HERMON_CQE_ERR_DBDCNT_MASK	0xFFFF
179 #define	HERMON_CQE_SEND_ERR_OPCODE	0x1E
180 #define	HERMON_CQE_RECV_ERR_OPCODE	0x1E
181 
182 /* Defines for tracking whether a CQ is being used with special QP or not */
183 #define	HERMON_CQ_IS_NORMAL		0
184 #define	HERMON_CQ_IS_SPECIAL		1
185 
186 /*
187  * The hermon_sw_cq_s structure is also referred to using the "hermon_cqhdl_t"
188  * typedef (see hermon_typedef.h).  It encodes all the information necessary
189  * to track the various resources needed to allocate, initialize, poll, resize,
190  * and (later) free a completion queue (CQ).
191  *
192  * Specifically, it has a consumer index and a lock to ensure single threaded
193  * access to it.  It has pointers to the various resources allocated for the
194  * completion queue, i.e. a CQC resource and the memory for the completion
195  * queue itself. It also has a reference count and the number(s) of the EQs
196  * to which it is associated (for success and for errors).
197  *
198  * Additionally, it has a pointer to the associated MR handle (for the mapped
199  * queue memory) and a void pointer that holds the argument that should be
200  * passed back to the IBTF when events are generated on the CQ.
201  *
202  * We also have the always necessary backpointer to the resource for the
203  * CQ handle structure itself.  But we also have pointers to the "Work Request
204  * ID" processing lists (both the lock and the regular list, as well as the
205  * head and tail for the "reapable" list).  See hermon_wrid.c for more details.
206  */
207 
208 #define	HERMON_CQ_DEF_UAR_DOORBELL	0x11	/* cmd_sn = 1, req solicited */
209 #define	HERMON_CD_DEF_UAR_DB_SHIFT	0x38	/* decimal 56 */
210 
211 struct hermon_sw_cq_s {
212 	kmutex_t		cq_lock;
213 	struct hermon_sw_cq_s 	*cq_resize_hdl; /* points to tranistory hdl */
214 	uint32_t		cq_consindx;
215 	uint32_t		cq_cqnum;
216 	hermon_hw_cqe_t		*cq_buf;
217 	hermon_mrhdl_t		cq_mrhdl;
218 	uint32_t		cq_bufsz;
219 	uint32_t		cq_log_cqsz;
220 	uint_t			cq_refcnt;
221 	uint32_t		cq_eqnum;
222 	uint32_t		cq_erreqnum;
223 	uint_t			cq_is_special;
224 	uint_t			cq_is_umap;
225 	uint32_t		cq_uarpg;
226 	devmap_cookie_t		cq_umap_dhp;
227 	hermon_rsrc_t		*cq_cqcrsrcp;
228 	hermon_rsrc_t		*cq_rsrcp;
229 	uint_t			cq_intmod_count;
230 	uint_t			cq_intmod_usec;
231 
232 	/* DoorBell Record Information */
233 	ddi_acc_handle_t	cq_arm_ci_dbr_acchdl;
234 	hermon_dbr_t		*cq_arm_ci_vdbr;
235 	uint64_t		cq_arm_ci_pdbr;
236 	uint64_t		cq_dbr_mapoffset;	/* user mode access */
237 
238 	void			*cq_hdlrarg;
239 
240 	/* For Work Request ID processing */
241 	avl_tree_t		cq_wrid_wqhdr_avl_tree;
242 
243 	struct hermon_qalloc_info_s cq_cqinfo;
244 };
245 _NOTE(READ_ONLY_DATA(hermon_sw_cq_s::cq_cqnum
246     hermon_sw_cq_s::cq_erreqnum
247     hermon_sw_cq_s::cq_cqcrsrcp
248     hermon_sw_cq_s::cq_rsrcp
249     hermon_sw_cq_s::cq_hdlrarg
250     hermon_sw_cq_s::cq_is_umap
251     hermon_sw_cq_s::cq_uarpg))
252 _NOTE(DATA_READABLE_WITHOUT_LOCK(hermon_sw_cq_s::cq_bufsz
253     hermon_sw_cq_s::cq_consindx
254     hermon_sw_cq_s::cq_cqinfo))
255 _NOTE(MUTEX_PROTECTS_DATA(hermon_sw_cq_s::cq_lock,
256     hermon_sw_cq_s::cq_buf
257     hermon_sw_cq_s::cq_eqnum
258     hermon_sw_cq_s::cq_mrhdl
259     hermon_sw_cq_s::cq_refcnt
260     hermon_sw_cq_s::cq_is_special
261     hermon_sw_cq_s::cq_umap_dhp))
262 _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
263     hermon_sw_cq_s::cq_intmod_count
264     hermon_sw_cq_s::cq_intmod_usec
265     hermon_sw_cq_s::cq_resize_hdl))
266 
267 int hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
268     ibt_cq_attr_t *attr_p, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
269     uint_t sleepflag);
270 int hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl,
271     uint_t sleepflag);
272 int hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cqhdl,
273     uint_t req_size, uint_t *actual_size, uint_t sleepflag);
274 int hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cqhdl,
275     uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag);
276 int hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cqhdl,
277     ibt_cq_notify_flags_t flags);
278 int hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cqhdl, ibt_wc_t *wc_p,
279     uint_t num_wc, uint_t *num_polled);
280 int hermon_cq_sched_alloc(hermon_state_t *state, ibt_cq_sched_attr_t *attr,
281     hermon_cq_sched_t **cq_sched_pp);
282 int hermon_cq_sched_free(hermon_state_t *state, hermon_cq_sched_t *cq_schedp);
283 int hermon_cq_handler(hermon_state_t *state, hermon_eqhdl_t eq,
284     hermon_hw_eqe_t *eqe);
285 int hermon_cq_err_handler(hermon_state_t *state, hermon_eqhdl_t eq,
286     hermon_hw_eqe_t *eqe);
287 int hermon_cq_refcnt_inc(hermon_cqhdl_t cq, uint_t is_special);
288 void hermon_cq_refcnt_dec(hermon_cqhdl_t cq);
289 hermon_cqhdl_t hermon_cqhdl_from_cqnum(hermon_state_t *state, uint_t cqnum);
290 void hermon_cq_entries_flush(hermon_state_t *state, hermon_qphdl_t qp);
291 void hermon_cq_resize_helper(hermon_state_t *state, hermon_cqhdl_t cq);
292 int hermon_cq_sched_init(hermon_state_t *state);
293 void hermon_cq_sched_fini(hermon_state_t *state);
294 
295 #ifdef __cplusplus
296 }
297 #endif
298 
299 #endif	/* _SYS_IB_ADAPTERS_HERMON_CQ_H */
300