xref: /illumos-gate/usr/src/uts/common/sys/ib/ibtl/ibti.h (revision 6a634c9d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #ifndef	_SYS_IB_IBTL_IBTI_H
26 #define	_SYS_IB_IBTL_IBTI_H
27 
28 /*
29  * ibti.h
30  *
31  * This file contains the IBTI prototypes and associated data structures.
32  * It is the only header file that should be included by IBTI clients.
33  */
34 #include <sys/ib/ibtl/ibti_common.h>
35 
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39 
40 /*
41  * Allocate channel flags.
42  */
43 typedef enum ibt_chan_alloc_flags_e {
44 	IBT_ACHAN_NO_FLAGS		= 0,
45 	IBT_ACHAN_CLONE			= (1 << 0),
46 	IBT_ACHAN_USER_MAP		= (1 << 1),
47 	IBT_ACHAN_DEFER_ALLOC		= (1 << 2),
48 	IBT_ACHAN_USES_SRQ		= (1 << 3),
49 	IBT_ACHAN_USES_RSS		= (1 << 4),
50 
51 	/* UD variants for FC support */
52 	IBT_ACHAN_USES_RFCI		= (1 << 5),	/* from RFCI pool */
53 	IBT_ACHAN_USES_FCMD		= (1 << 6),
54 	IBT_ACHAN_USES_FEXCH		= (1 << 7)	/* from FEXCH pool */
55 } ibt_chan_alloc_flags_t;
56 
57 
58 /*
59  * Allocate RC channel ibt_alloc_rc_channel() argument.
60  */
61 typedef struct ibt_rc_chan_alloc_args_s {
62 	ibt_attr_flags_t	rc_flags;	/* Signal type etc */
63 	ibt_cep_flags_t		rc_control;
64 	uint8_t			rc_hca_port_num;
65 	ibt_chan_sizes_t	rc_sizes;
66 	ibt_cq_hdl_t		rc_scq;	/* Send CQ */
67 	ibt_cq_hdl_t		rc_rcq;	/* Recv CQ */
68 	ibt_pd_hdl_t		rc_pd;	/* PD */
69 	ibt_channel_hdl_t	rc_clone_chan;	/* Optional Clone handle */
70 	ibt_srq_hdl_t		rc_srq;		/* Optional Shared Rcv Queue */
71 } ibt_rc_chan_alloc_args_t;
72 
73 /*
74  * RC channel query attributes structure.
75  */
76 typedef struct ibt_rc_chan_query_attr_s {
77 	ib_guid_t		rc_hca_guid;	/* Local HCA GUID */
78 	ibt_cq_hdl_t		rc_scq;		/* SendCQ handle */
79 	ibt_cq_hdl_t		rc_rcq;		/* RecvCQ handle */
80 	ibt_pd_hdl_t		rc_pd;		/* PD Handle. */
81 	ibt_cep_state_t		rc_state;	/* Channel state */
82 	ib_mtu_t		rc_path_mtu;
83 	uint8_t			rc_path_retry_cnt:3;
84 	ibt_rnr_retry_cnt_t	rc_path_rnr_retry_cnt;
85 	ibt_rnr_nak_time_t	rc_min_rnr_nak;	/* min RNR-NAK timer */
86 	ibt_cep_path_t		rc_prim_path;
87 	ibt_cep_path_t		rc_alt_path;
88 	ibt_chan_sizes_t	rc_chan_sizes;	/* Queue/SGL sizes */
89 	uint8_t			rc_rdma_ra_out;	/* max RDMA-Reads/Atomics out */
90 	uint8_t			rc_rdma_ra_in;	/* max RDMA-Reads/Atomics in */
91 	ibt_attr_flags_t	rc_flags;	/* SQ Signaling Type etc */
92 	ibt_cep_flags_t		rc_control;	/* Control Flags */
93 	ibt_cep_cmstate_t	rc_mig_state;
94 	ib_qpn_t		rc_qpn;		/* Local QPN */
95 	ib_qpn_t		rc_dst_qpn;	/* Destination QPN */
96 	ibt_srq_hdl_t		rc_srq;		/* Optional Shared Rcv Queue */
97 } ibt_rc_chan_query_attr_t;
98 
99 /*
100  * RC Channel Modify Attributes definition.
101  *
102  * It is only possible to modify a channel that has previously been
103  * opened. The channel must either be in operational state (IBT_STATE_RTS)
104  * or paused (IBT_STATE_SQD). If channel is in paused state, then a modify
105  * operation will unpause the channel.
106  *
107  * See the ibt_modify_rc_channel() for details of the required corresponding
108  * modify flags.
109  * Not specified attributes should be set to "NULL" or "0".
110  */
111 typedef struct ibt_rc_chan_modify_attr_s {
112 	ibt_cep_flags_t		rc_control;	/* Channel Control Flags */
113 	uint_t			rc_sq_sz;	/* Set SQ Max outstanding WRs */
114 	uint_t			rc_rq_sz;	/* Set RQ Max outstanding WRs */
115 
116 	ibt_adds_vect_t		rc_prim_adds_vect; /* Primary Path Address */
117 	ibt_adds_vect_t		rc_alt_adds_vect; /* Alternate Path Address */
118 	uint8_t			rc_path_retry_cnt:3;
119 	ibt_rnr_retry_cnt_t	rc_path_rnr_retry_cnt;
120 	ibt_rnr_nak_time_t	rc_min_rnr_nak;	/* min RNR-NAK timer */
121 	uint8_t			rc_prim_port_num; /* Port of Primary Path */
122 	uint8_t			rc_alt_port_num; /* Port of Alternate Path */
123 	uint8_t			rc_rdma_ra_out;	/* Initiator Depth, Number of */
124 						/* RDMA RD's & Atomics */
125 						/* outstanding. */
126 	uint8_t			rc_rdma_ra_in; /* Responder resources for */
127 						/* handling incoming RDMA rds */
128 						/* and Atomics. */
129 } ibt_rc_chan_modify_attr_t;
130 
131 /*
132  * UD remote destination query attributes
133  */
134 typedef struct ibt_ud_dest_query_attr_s {
135 	ibt_hca_hdl_t		ud_hca_hdl;	/* Local HCA Handle */
136 	ib_qpn_t		ud_dst_qpn;	/* Destination QPN */
137 	ib_qkey_t		ud_qkey;	/* Q_Key */
138 	ibt_adds_vect_t 	ud_addr_vect;	/* Address Information */
139 	ibt_pd_hdl_t		ud_pd;
140 } ibt_ud_dest_query_attr_t;
141 
142 /*
143  * Allocate UD channel ibt_alloc_ud_channel() arguments; see below at
144  * ibt_alloc_ud_channel() for a description of what's required and optional.
145  */
146 typedef struct ibt_ud_chan_alloc_args_s {
147 	ibt_attr_flags_t	ud_flags;	/* Sig type etc */
148 	uint8_t			ud_hca_port_num;
149 	uint16_t		ud_pkey_ix;	/* P_Key Index */
150 	ibt_chan_sizes_t	ud_sizes;
151 	ib_qkey_t		ud_qkey;	/* Q_Key */
152 	ibt_cq_hdl_t		ud_scq;		/* Send CQ */
153 	ibt_cq_hdl_t		ud_rcq;		/* Recv CQ */
154 	ibt_pd_hdl_t		ud_pd;		/* PD */
155 	ibt_channel_hdl_t	ud_clone_chan;	/* Optional clone handle */
156 	ibt_srq_hdl_t		ud_srq;		/* Optional Shared Rcv Queue */
157 	ibt_rss_attr_t		ud_rss;
158 	ibt_fc_attr_t		ud_fc;
159 } ibt_ud_chan_alloc_args_t;
160 
161 /*
162  * UD channel query attributes.
163  */
164 typedef struct ibt_ud_chan_query_attr_s {
165 	ib_qpn_t		ud_qpn;			/* QPN */
166 	ib_guid_t		ud_hca_guid;		/* Local HCA GUID */
167 	ibt_cq_hdl_t		ud_scq;			/* SendCQ handle. */
168 	ibt_cq_hdl_t		ud_rcq;			/* RecvCQ handle. */
169 	ibt_pd_hdl_t		ud_pd;			/* PD Handle. */
170 	uint8_t			ud_hca_port_num;	/* Local HCA port */
171 	ibt_cep_state_t		ud_state;		/* Channel state */
172 	uint16_t		ud_pkey_ix;		/* P_Key Index */
173 	ib_qkey_t		ud_qkey;		/* Q_Key */
174 	ibt_chan_sizes_t	ud_chan_sizes;		/* Queue/SGL sizes */
175 	ibt_attr_flags_t	ud_flags;		/* Signaling Type etc */
176 	ibt_srq_hdl_t		ud_srq;		/* Optional Shared Rcv Queue */
177 	ibt_rss_attr_t		ud_rss;
178 	ibt_fc_attr_t		ud_fc;
179 	ibt_fexch_query_attr_t	ud_query_fc;	/* query only set */
180 } ibt_ud_chan_query_attr_t;
181 
182 /*
183  * UD Channel Modify Attributes definition.
184  *
185  * It is only possible to modify a channel that is either in the operational
186  * state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If channel is in paused
187  * state, then a modify operation will unpause the channel.
188  *
189  * See the ibt_modify_ud_channel() for details of the required corresponding
190  * modify flags.
191  */
192 typedef struct ibt_ud_chan_modify_attr_s {
193 	uint_t			ud_sq_sz;	/* Set SQ Max outstanding WRs */
194 	uint_t			ud_rq_sz;	/* Set RQ Max outstanding WRs */
195 	ib_qkey_t		ud_qkey;	/* Set Q_Key */
196 	ibt_rss_attr_t		ud_rss;		/* Set RSS stuff */
197 	ibt_fc_attr_t		ud_fc;
198 } ibt_ud_chan_modify_attr_t;
199 
200 
201 /*
202  * FUNCTION PROTOTYPES
203  */
204 
205 /*
206  * CONNECTION ESTABLISHMENT/TEAR DOWN FUNCTIONS.
207  */
208 
209 /*
210  * ibt_alloc_rc_channel
211  * 	Allocates a RC communication channels that satisfy the specified
212  *	channel attributes.
213  */
214 ibt_status_t ibt_alloc_rc_channel(ibt_hca_hdl_t hca_hdl,
215     ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args,
216     ibt_channel_hdl_t *rc_chan_p, ibt_chan_sizes_t *sizes);
217 
218 /*
219  * ibt_flush_channel
220  *	Flush the specified channel. Outstanding work requests are flushed
221  *	so that the client can do the associated clean up. After that, the
222  *	client will usually deregister the previously registered memory,
223  *	then free the channel by calling ibt_free_channel().  RC channels
224  *	that have been successfully opened will fail this call, as they
225  *	need to instead be handled by ibt_close_rc_channel().
226  */
227 ibt_status_t ibt_flush_channel(ibt_channel_hdl_t chan);
228 
229 /*
230  * ibt_free_channel
231  *	Releases the resources associated with the specified channel
232  */
233 ibt_status_t ibt_free_channel(ibt_channel_hdl_t chan);
234 
235 /*
236  * ibt_query_rc_channel
237  *	Query an RC channel's attributes. Should only be called on an
238  *	opened RC channel. If called on a channel before it is opened,
239  *	some channel attributes may change when the channel is opened.
240  */
241 ibt_status_t ibt_query_rc_channel(ibt_channel_hdl_t rc_chan,
242     ibt_rc_chan_query_attr_t *chan_attrs);
243 
244 /*
245  * ibt_modify_rc_channel()
246  * 	Modifies a previous opened operational or paused RC channel's
247  *	attributes.
248  *
249  * NOTE:
250  *	It is only possible to modify a channel that has previously been opened.
251  *	The channel must either be in operational state (IBT_STATE_RTS) or
252  *	paused (IBT_STATE_SQD). If channel is in paused state, then a modify
253  *	operation will will unpause the channel.
254  */
255 ibt_status_t ibt_modify_rc_channel(ibt_channel_hdl_t rc_chan,
256     ibt_cep_modify_flags_t flags, ibt_rc_chan_modify_attr_t *attrs,
257     ibt_queue_sizes_t *actual_sz);
258 
259 /*
260  * ibt_alloc_ud_channel
261  *	Allocate UD channels that satisfy the specified channel attributes.
262  */
263 ibt_status_t ibt_alloc_ud_channel(ibt_hca_hdl_t hca_hdl,
264     ibt_chan_alloc_flags_t flags, ibt_ud_chan_alloc_args_t *args,
265     ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes);
266 
267 /*
268  * ibt_alloc_ud_channel_range
269  *	Allocate a range of UD channels that have consecutive QPNs for RSS.
270  */
271 ibt_status_t ibt_alloc_ud_channel_range(ibt_hca_hdl_t hca_hdl,
272     uint_t log2, ibt_chan_alloc_flags_t flags,
273     ibt_ud_chan_alloc_args_t *args, ibt_cq_hdl_t *send_cq_p,
274     ibt_cq_hdl_t *recv_cq_p, ib_qpn_t *base_qpn_p,
275     ibt_channel_hdl_t *ud_chan_p, ibt_chan_sizes_t *sizes);
276 
277 /*
278  * ibt_query_ud_channel
279  *	Query a UD channel's attributes.
280  */
281 ibt_status_t ibt_query_ud_channel(ibt_channel_hdl_t ud_chan,
282     ibt_ud_chan_query_attr_t *ud_chan_attrs);
283 
284 /*
285  * ibt_modify_ud_channel()
286  * 	Modifies an UD channel's attributes, as specified by a
287  *	ibt_cep_modify_flags_t parameter to those specified in the
288  *	ibt_ud_chan_modify_attr_t structure.
289  *
290  * NOTE:
291  *	It is only possible to modify a channel that is either in the
292  *	operational state (IBT_STATE_RTS) or paused (IBT_STATE_SQD). If
293  *	channel is in paused state, then a modify operation will unpause the
294  *	channel.
295  */
296 ibt_status_t ibt_modify_ud_channel(ibt_channel_hdl_t ud_chan,
297     ibt_cep_modify_flags_t flags, ibt_ud_chan_modify_attr_t *attrs,
298     ibt_queue_sizes_t *actual_sz);
299 
300 /*
301  * ibt_recover_ud_channel()
302  *	Recover an UD Channel which has transitioned to SQ Error state. The
303  *	ibt_recover_ud_channel() transitions the channel from SQ Error state
304  *	to Ready-To-Send channel state.
305  *
306  *	If a work request posted to a UD channel's send queue completes with
307  *	an error (see ibt_wc_status_t), the channel gets transitioned to SQ
308  *	Error state. In order to reuse this channel, ibt_recover_ud_channel()
309  *	can be used to recover the channel to a usable (Ready-to-Send) state.
310  */
311 ibt_status_t ibt_recover_ud_channel(ibt_channel_hdl_t ud_chan);
312 
313 
314 /*
315  * ibt_alloc_ud_dest
316  *	Allocate a UD destination handle.  This allocates local resources
317  *	that will need subsequent modification/initialization before use
318  *	(in send work requests).  Functions that can be used to do this are
319  *	ibt_modify_ud_dest (data supplied by caller), ibt_modify_reply_ud_dest
320  *	(data supplied from a successfully completed receive work request),
321  *	and ibt_request_ud_dest (data retrieved using SIDR protocol).
322  */
323 ibt_status_t ibt_alloc_ud_dest(ibt_hca_hdl_t hca_hdl,
324     ibt_ud_dest_flags_t flags, ibt_pd_hdl_t pd, ibt_ud_dest_hdl_t *ud_dest_p);
325 
326 /*
327  * ibt_modify_ud_dest
328  *	Modify a previously allocated UD destination handle from the
329  *	argument data.  After the ud_dest has already been made usable,
330  *	the adds_vect argument to this function is optional (NULL).
331  */
332 ibt_status_t ibt_modify_ud_dest(ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey,
333     ib_qpn_t dest_qpn, ibt_adds_vect_t *adds_vect);
334 
335 /*
336  * ibt_modify_reply_ud_dest
337  *	Modify a previously allocated UD destination handle, so that it
338  *	can be used to reply to the sender of the datagram contained in the
339  *	specified work request completion.
340  */
341 ibt_status_t ibt_modify_reply_ud_dest(ibt_channel_hdl_t ud_chan,
342     ibt_ud_dest_hdl_t ud_dest, ib_qkey_t qkey, ibt_wc_t *wc,
343     ib_vaddr_t recv_buf);
344 
345 /*
346  * ibt_request_ud_dest
347  *	Modify a previously allocated UD destination handle based on the
348  *	data retrieved by making an SIDR request.
349  */
350 ibt_status_t ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest,
351     ibt_execution_mode_t mode, ibt_ud_dest_attr_t *dest_attrs,
352     ibt_ud_returns_t *ud_ret_args);
353 
354 /*
355  * ibt_free_ud_dest
356  *	Releases the resources associated with the specified UD destination
357  *	handle.
358  */
359 ibt_status_t ibt_free_ud_dest(ibt_ud_dest_hdl_t ud_dest);
360 
361 /*
362  * ibt_query_ud_dest
363  *	Query a UD destination's attributes.
364  */
365 ibt_status_t ibt_query_ud_dest(ibt_ud_dest_hdl_t ud_dest,
366     ibt_ud_dest_query_attr_t *dest_attrs);
367 
368 
369 /*
370  * ibt_is_privileged_ud_dest
371  *	Determine if a UD destination Handle is a privileged handle.
372  */
373 boolean_t ibt_is_privileged_ud_dest(ibt_ud_dest_hdl_t ud_dest);
374 
375 
376 /*
377  * ibt_update_channel_qkey
378  *
379  * ibt_update_channel_qkey() set's the Qkey in the specified channel context
380  * to the Qkey in the specified destination handle. This function can be used
381  * to enable sends to a privileged destination. All posted Send Work Requests
382  * that contain a privileged destination handle now use the Qkey in the
383  * channel context.
384  *
385  * ibt_update_channel_qkey() can also be used to enable the caller to receive
386  * from the specified remote destination on the specified channel.
387  *
388  */
389 ibt_status_t ibt_update_channel_qkey(ibt_channel_hdl_t ud_chan,
390     ibt_ud_dest_hdl_t ud_dest);
391 
392 
393 /*
394  * ibt_set_chan_private()
395  * ibt_get_chan_private()
396  * 	Set/get a pointer to client private data.
397  *	Applicable for both RC and UD channels.
398  */
399 void ibt_set_chan_private(ibt_channel_hdl_t chan, void *clnt_private);
400 
401 void *ibt_get_chan_private(ibt_channel_hdl_t chan);
402 
403 /*
404  * ibt_channel_to_hca_guid()
405  *
406  *	A helper function to retrieve HCA GUID for the specified Channel.
407  */
408 ib_guid_t ibt_channel_to_hca_guid(ibt_channel_hdl_t chan);
409 
410 #ifdef __cplusplus
411 }
412 #endif
413 
414 #endif /* _SYS_IB_IBTL_IBTI_H */
415