1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright (c) 2005 SilverStorm Technologies, Inc. All rights reserved.
27  *
28  * This software is available to you under a choice of one of two
29  * licenses.  You may choose to be licensed under the terms of the GNU
30  * General Public License (GPL) Version 2, available from the file
31  * COPYING in the main directory of this source tree, or the
32  * OpenIB.org BSD license below:
33  *
34  *     Redistribution and use in source and binary forms, with or
35  *     without modification, are permitted provided that the following
36  *     conditions are met:
37  *
38  *	- Redistributions of source code must retain the above
39  *	  copyright notice, this list of conditions and the following
40  *	  disclaimer.
41  *
42  *	- Redistributions in binary form must reproduce the above
43  *	  copyright notice, this list of conditions and the following
44  *	  disclaimer in the documentation and/or other materials
45  *	  provided with the distribution.
46  *
47  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
48  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
49  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
50  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
51  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
52  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
53  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
54  * SOFTWARE.
55  *
56  */
57 /*
58  * Sun elects to include this software in Sun product
59  * under the OpenIB BSD license.
60  *
61  *
62  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
63  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
64  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
65  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
66  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
67  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
68  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
69  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
70  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
71  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
72  * POSSIBILITY OF SUCH DAMAGE.
73  */
74 
75 #ifndef _RDSIB_EP_H
76 #define	_RDSIB_EP_H
77 
78 #ifdef __cplusplus
79 extern "C" {
80 #endif
81 
82 #include <netinet/in.h>
83 
84 /*
85  * Control channel or Data channel
86  */
87 typedef enum rds_ep_type_s {
88 	RDS_EP_TYPE_CTRL		= 1,
89 	RDS_EP_TYPE_DATA		= 2
90 } rds_ep_type_t;
91 
92 /*
93  * Channel States
94  *
95  * RDS_EP_STATE_UNCONNECTED - Initial state when rds_ep_t is created
96  * RDS_EP_STATE_ACTIVE_PENDING - Active side connection in progress
97  * RDS_EP_STATE_PASSIVE_PENDING - Passice side connection in progress
98  * RDS_EP_STATE_CONNECTED - Channel is connected
99  * RDS_EP_STATE_DESTROY_TIMEWAIT - Channel is closed
100  */
101 typedef enum rds_ep_state_s {
102 	RDS_EP_STATE_UNCONNECTED		= 0,
103 	RDS_EP_STATE_ACTIVE_PENDING		= 1,
104 	RDS_EP_STATE_PASSIVE_PENDING		= 2,
105 	RDS_EP_STATE_CONNECTED			= 3,
106 	RDS_EP_STATE_CLOSING			= 4,
107 	RDS_EP_STATE_CLOSED			= 5,
108 	RDS_EP_STATE_ERROR			= 6
109 } rds_ep_state_t;
110 
111 /*
112  * Session State Machine Diagram
113  *
114  *                     -----------------
115  *                    |       (6)       |
116  *                    |                 |
117  *                    v                 |
118  *             --> (Created)-------->(Failed)
119  *            |     |         (5)       ^
120  *            |     |(1)                |
121  *            |     |                   |(9)
122  *            |     v                   |
123  *            |    (Init)<--------------|
124  *            |     | |       (8)       |
125  *            |     | |                 |
126  *            |  (2)|  --------------   |
127  *        (11)|     |         (7)    |  |
128  *            |     v                v  |
129  *            |    (Connected)------>(Error)
130  *            |     |         (10)
131  *            |     |(3)
132  *            |     |
133  *            |     v
134  *            |    (Closed)
135  *            |     |
136  *            |     |(4)
137  *            |     |
138  *            |     v
139  *             --- (Fini) ------->(Destroy)
140  *                         (12)
141  *
142  *	(1) rds_session_init()
143  *	(2) rds_session_open()
144  *	(3) rds_session_close()
145  *	(4) rds_session_fini()
146  *	(4) rds_passive_session_fini()
147  *	(5) Failure in rds_session_init()
148  *	(6) rds_sendmsg(3SOCKET)/Incoming CM REQ
149  *	(7) Failure in rds_session_open()
150  *	(8) rds_session_close(), rds_get_ibaddr() and rds_session_reinit()
151  *	(9) rds_session_close() and rds_session_fini()
152  *	(9) rds_cleanup_passive_session() and rds_passive_session_fini()
153  *	(10) Connection Error/Incoming REQ
154  *	(11) rds_sendmsg(3SOCKET)/Incoming REQ
155  *
156  *
157  * Created   - Session is allocated and inserted into the sessionlist but
158  *             not all members are initialized.
159  * Init      - All members are initialized, send buffer pool is allocated.
160  * Connected - Data and ctrl RC channels are opened.
161  * Closed    - Data and ctrl RC channels are closed.
162  * Fini      - Send buffer pool and buffers in the receive pool are freed.
163  * Destroy   - Session is removed from the session list and is ready to be
164  *             freed.
165  * Failed    - Session initialization has failed (send buffer pool allocation).
166  * Error     - (1) Failed to open the RC channels.
167  *             (2) An error occurred on the RC channels while sending.
168  *             (3) Received a new CM REQ message on the existing connection.
169  */
170 typedef enum rds_session_state_s {
171 	RDS_SESSION_STATE_CREATED		= 0,
172 	RDS_SESSION_STATE_FAILED		= 1,
173 	RDS_SESSION_STATE_INIT			= 2,
174 	RDS_SESSION_STATE_CONNECTED		= 3,
175 	RDS_SESSION_STATE_HCA_CLOSING		= 4,
176 	RDS_SESSION_STATE_ERROR			= 5,
177 	RDS_SESSION_STATE_ACTIVE_CLOSING	= 6,
178 	RDS_SESSION_STATE_PASSIVE_CLOSING	= 7,
179 	RDS_SESSION_STATE_CLOSED		= 8,
180 	RDS_SESSION_STATE_FINI			= 9,
181 	RDS_SESSION_STATE_DESTROY		= 10
182 } rds_session_state_t;
183 
184 #define	RDS_SESSION_TRANSITION(sp, state)			\
185 		rw_enter(&sp->session_lock, RW_WRITER);		\
186 		sp->session_state = state;			\
187 		rw_exit(&sp->session_lock)
188 
189 /* Active or Passive */
190 #define	RDS_SESSION_ACTIVE	1
191 #define	RDS_SESSION_PASSIVE	2
192 
193 /*
194  * RDS QP Information
195  *
196  * lock  - Synchronize access
197  * depth - Max number of WRs that can be posted.
198  * level - Number of outstanding WRs in the QP
199  * lwm   - Water mark at which to post more receive WRs.
200  * taskqpending - Indicates if a taskq thread is dispatched to post receive
201  *		WRs in the RQ
202  */
203 typedef struct rds_qp_s {
204 	kmutex_t		qp_lock;
205 	uint32_t		qp_depth;
206 	uint32_t		qp_level;
207 	uint32_t		qp_lwm;
208 	boolean_t		qp_taskqpending;
209 } rds_qp_t;
210 
211 /*
212  * RDS EndPoint(One end of RC connection)
213  *
214  * sp        - Parent Session
215  * type      - Control or Data Channel
216  * remip     - Same as session_remip
217  * myip      - Same as session_myip
218  * snd_lkey  - LKey for the send buffer pool
219  * hca_guid  - HCA guid
220  * snd_mrhdl - Memory handle for the send buffer pool
221  * lock      - Protects the members
222  * state     - See rds_ep_state_t
223  * chanhdl   - RC channel handle
224  * sendcq    - Send CQ handle
225  * recvcq    - Recv CQ handle
226  * sndpool   - Send buffer Pool
227  * rcvpool   - Recv buffer Pool
228  * segfbp    - First packet of a segmented message.
229  * seglbp    - Last packet of a segmented message.
230  * lbufid    - Last successful buffer that was received by the remote.
231  *             Valid only during session failover/reconnect.
232  * rbufid    - Last buffer (remote buffer) that was received successfully
233  *             from the remote node.
234  * ds        - SGL used for send acknowledgement.
235  * ackwr     - WR to send acknowledgement.
236  * ackhdl    - Memory handle for 'ack_addr'.
237  * ack_rkey  - RKey for 'ack_addr'.
238  * ack_addr  - Memory region to receive RDMA acknowledgement from remote.
239  */
240 typedef struct rds_ep_s {
241 	struct rds_session_s	*ep_sp;
242 	rds_ep_type_t		ep_type;
243 	ipaddr_t		ep_remip;
244 	ipaddr_t		ep_myip;
245 	ibt_lkey_t		ep_snd_lkey;
246 	ib_guid_t		ep_hca_guid;
247 	ibt_mr_hdl_t		ep_snd_mrhdl;
248 	kmutex_t		ep_lock;
249 	rds_ep_state_t		ep_state;
250 	ibt_channel_hdl_t	ep_chanhdl;
251 	ibt_cq_hdl_t		ep_sendcq;
252 	ibt_cq_hdl_t		ep_recvcq;
253 	rds_bufpool_t		ep_sndpool;
254 	rds_bufpool_t		ep_rcvpool;
255 	rds_qp_t		ep_recvqp;
256 	uint_t			ep_rdmacnt;
257 	rds_buf_t		*ep_segfbp;
258 	rds_buf_t		*ep_seglbp;
259 	uintptr_t		ep_lbufid;
260 	uintptr_t		ep_rbufid;
261 	ibt_wr_ds_t		ep_ackds;
262 	ibt_send_wr_t		ep_ackwr;
263 	ibt_mr_hdl_t		ep_ackhdl;
264 	ibt_rkey_t		ep_ack_rkey;
265 	uintptr_t		ep_ack_addr;
266 } rds_ep_t;
267 
268 /*
269  * One end of an RDS session
270  *
271  * nextp   - Pointer to the next session in the session list.
272  *           This is protected by rds_state_t:rds_sessionlock.
273  * remip   - IP address of the node having the remote end of the session.
274  * myip    - IP address of this end of the session.
275  * lgid    - IB local (source) gid, hosting "myip".
276  * rgid    - IB remote (destination) gid, hosting "remip".
277  * lock    - Provides read/write access to members of the session.
278  * type    - Identifies which end of session (active or passive).
279  * state   - State of session (rds_session_state_t).
280  * dataep  - Data endpoint
281  * ctrlep  - Control endpoint
282  * failover- Flag to indicate that an error occured and the session is
283  *           re-connecting.
284  * portmap_lock - To serialize access to portmap.
285  * portmap - Bitmap of sockets.
286  *           The maximum number of sockets seem to be 65536, the portmap has
287  *           1 bit for each remote socket. A set bit indicates that the
288  *           corresponding remote socket is stalled and vice versa.
289  */
290 typedef struct rds_session_s {
291 	struct rds_session_s	*session_nextp;
292 	ipaddr_t		session_remip;
293 	ipaddr_t		session_myip;
294 	ib_guid_t		session_hca_guid;
295 	ib_gid_t		session_lgid;
296 	ib_gid_t		session_rgid;
297 	krwlock_t		session_lock;
298 	uint8_t			session_type;
299 	uint8_t			session_state;
300 	struct rds_ep_s		session_dataep;
301 	struct rds_ep_s		session_ctrlep;
302 	uint_t			session_failover;
303 	krwlock_t		session_local_portmap_lock;
304 	krwlock_t		session_remote_portmap_lock;
305 	uint8_t			session_local_portmap[RDS_PORT_MAP_SIZE];
306 	uint8_t			session_remote_portmap[RDS_PORT_MAP_SIZE];
307 	ibt_path_info_t		session_pinfo;
308 } rds_session_t;
309 
310 /* defined in rds_ep.c */
311 int rds_ep_init(rds_ep_t *ep, ib_guid_t hca_guid);
312 rds_session_t *rds_session_create(rds_state_t *statep, ipaddr_t destip,
313     ipaddr_t srcip, ibt_cm_req_rcv_t *reqp, uint8_t type);
314 int rds_session_init(rds_session_t *sp);
315 int rds_session_reinit(rds_session_t *sp, ib_gid_t lgid);
316 void rds_session_open(rds_session_t *sp);
317 void rds_session_close(rds_session_t *sp, ibt_execution_mode_t mode,
318     uint_t wait);
319 rds_session_t *rds_session_lkup(rds_state_t *statep, ipaddr_t destip,
320     ib_guid_t node_guid);
321 void rds_recycle_session(rds_session_t *sp);
322 void rds_session_active(rds_session_t *sp);
323 void rds_close_sessions(void *arg);
324 void rds_received_msg(rds_ep_t *ep, rds_buf_t *bp);
325 void rds_handle_control_message(rds_session_t *sp, rds_ctrl_pkt_t *cp);
326 void rds_handle_send_error(rds_ep_t *ep);
327 void rds_session_fini(rds_session_t *sp);
328 void rds_passive_session_fini(rds_session_t *sp);
329 void rds_cleanup_passive_session(void *arg);
330 
331 /* defined in rds_ib.c */
332 ibt_channel_hdl_t rds_ep_alloc_rc_channel(rds_ep_t *ep, uint8_t hca_port);
333 void rds_ep_free_rc_channel(rds_ep_t *ep);
334 void rds_post_recv_buf(void *arg);
335 void rds_poll_send_completions(ibt_cq_hdl_t cq, struct rds_ep_s *ep,
336     boolean_t lock);
337 
338 /* defined in rds_cm.c */
339 int rds_open_rc_channel(rds_ep_t *ep, ibt_path_info_t *pinfo,
340     ibt_execution_mode_t mode, ibt_channel_hdl_t *chanhdl);
341 int rds_close_rc_channel(ibt_channel_hdl_t chanhdl, ibt_execution_mode_t mode);
342 
343 int rds_deliver_new_msg(mblk_t *mp, ipaddr_t local_addr, ipaddr_t rem_addr,
344     in_port_t local_port, in_port_t rem_port, zoneid_t zoneid);
345 
346 /* defined in rds_sc.c */
347 int rds_sc_path_lookup(ipaddr_t *localip, ipaddr_t *remip);
348 
349 #ifdef __cplusplus
350 }
351 #endif
352 
353 #endif	/* _RDSIB_EP_H */
354