1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef _ISER_IB_H
27 #define	_ISER_IB_H
28 
29 #ifdef	__cplusplus
30 extern "C" {
31 #endif
32 
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <netinet/in.h>
36 #include <sys/ib/ibtl/ibti.h>
37 #include <sys/iscsi_protocol.h>
38 
39 /*
40  * iser_ib.h
41  *	Definitions and macros related to iSER InfiniBand transport,
42  * 	and the use of the Solaris IBTI (InfiniBand Transport Interface).
43  */
44 
45 struct iser_state_s;
46 
47 extern struct iser_state_s	*iser_state;
48 extern ddi_taskq_t	*iser_taskq;
49 
50 /*
51  * iser_hca_s holds all the information about the Infinband HCAs in use.
52  */
53 typedef struct iser_hca_s {
54 	list_node_t		hca_node;
55 	boolean_t		hca_failed;
56 	ibt_clnt_hdl_t		hca_clnt_hdl;
57 	ibt_hca_hdl_t		hca_hdl;
58 	ibt_hca_attr_t		hca_attr;
59 	ibt_pd_hdl_t		hca_pdhdl;
60 	ib_guid_t		hca_guid;
61 	uint_t			hca_num_ports;
62 	ibt_hca_portinfo_t	*hca_port_info;
63 	uint_t			hca_port_info_sz;
64 
65 	/* Per PD (per HCA) message and data buffer caches */
66 	struct iser_vmem_mr_pool_s *hca_msg_pool; /* Use iser_msg_cache */
67 	kmem_cache_t		*iser_msg_cache;
68 	struct iser_vmem_mr_pool_s *hca_buf_pool; /* Use iser_buf_cache */
69 	kmem_cache_t		*iser_buf_cache;
70 } iser_hca_t;
71 
72 /* RQ low water mark percentage */
73 #define	ISER_IB_RQ_LWM_PCT	80
74 
75 /* Maximum number of WRs to post on the RQ at a time */
76 #define	ISER_IB_RQ_POST_MAX	64
77 
78 /* Maximum number of SCQ WCs to poll at a time */
79 #define	ISER_IB_SCQ_POLL_MAX	8
80 
81 /*
82  * iser_qp_t encodes data related to a Queue Pair (QP) in use by
83  * iSER. Each QP consists of two Work Queues (WQs), one Send Queue
84  * (SQ) and on Receive Queue (RQ). Most of the data in the QP
85  * handle relates to monitoring the posted depth of the RQ.
86  *
87  * Note that we are explicitly using slightly less than a power-of-2
88  * number for our queue sizes.  The HCA driver will round up for us,
89  * and this affords us some headroom.
90  */
91 #ifdef _LP64
92 #define	ISER_IB_RECVQ_SIZE	400
93 #else
94 /* Memory is very limited on 32-bit kernels */
95 #define	ISER_IB_RECVQ_SIZE	100
96 #endif
97 #define	ISER_IB_SENDQ_SIZE	2000
98 #define	ISER_IB_SGLIST_SIZE	1
99 
100 #define	ISER_IB_DEFAULT_IRD	2
101 #define	ISER_IB_DEFAULT_ORD	4
102 
103 typedef struct iser_qp_s {
104 	kmutex_t	qp_lock;
105 	uint_t		sq_size;
106 	uint_t		rq_size;
107 	uint32_t	rq_depth;
108 	uint32_t	rq_level;
109 	uint32_t	rq_min_post_level;
110 	uint32_t	rq_lwm;
111 	boolean_t	rq_taskqpending;
112 } iser_qp_t;
113 
114 /*
115  * iSER RC channel information
116  */
117 typedef struct iser_chan_s {
118 	kmutex_t		ic_chan_lock;
119 
120 	/* IBT channel handle */
121 	ibt_channel_hdl_t	ic_chanhdl;
122 
123 	/* local and remote IP addresses and port numbers */
124 	ibt_ip_addr_t		ic_localip;
125 	ibt_ip_addr_t		ic_remoteip;
126 	in_port_t		ic_lport;
127 	in_port_t		ic_rport;
128 
129 	/*
130 	 * The local HCA GUID, the service ID, Destination GID, Source GID
131 	 * the primary hca port on which the channel is connected is
132 	 * stored in ic_ibt_path
133 	 */
134 	ibt_path_info_t		ic_ibt_path;
135 
136 	/*
137 	 * Information related to the HCA handle and the queues.
138 	 */
139 	iser_hca_t		*ic_hca;
140 	ibt_cq_hdl_t		ic_sendcq;
141 	ibt_cq_hdl_t		ic_recvcq;
142 	uint_t			ic_sendcq_sz;
143 	uint_t			ic_recvcq_sz;
144 	iser_qp_t		ic_qp;
145 
146 	/* Used to track the number of WRs posted on the SQ */
147 	kmutex_t		ic_sq_post_lock;
148 	uint_t			ic_sq_post_count;
149 	uint_t			ic_sq_max_post_count;
150 
151 	/*
152 	 * To help identify the channel end point and some connection
153 	 * specifics, maintain a pointer to the connection on which
154 	 * this channel originated
155 	 */
156 	struct iser_conn_s	*ic_conn;
157 } iser_chan_t;
158 
159 int iser_ib_init(void);
160 
161 int iser_ib_fini(void);
162 
163 int iser_ib_register_service(idm_svc_t *idm_svc);
164 
165 int iser_ib_bind_service(idm_svc_t *idm_svc);
166 
167 void iser_ib_unbind_service(idm_svc_t *idm_svc);
168 
169 void iser_ib_deregister_service(idm_svc_t *idm_svc);
170 
171 void iser_ib_conv_sockaddr2ibtaddr(idm_sockaddr_t *saddr,
172     ibt_ip_addr_t *ibt_addr);
173 
174 void iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage *ss,
175     ibt_ip_addr_t *ibt_addr, in_port_t port);
176 
177 int iser_ib_get_paths(
178     ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip, ibt_path_info_t *path,
179     ibt_path_ip_src_t *path_src_ip);
180 
181 iser_chan_t *iser_ib_alloc_channel_pathlookup(
182     ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip);
183 
184 iser_chan_t *iser_ib_alloc_channel_nopathlookup(
185     ib_guid_t hca_guid, uint8_t hca_port);
186 
187 iser_chan_t *iser_ib_alloc_rc_channel(iser_hca_t *hca, uint8_t hca_port);
188 
189 int iser_ib_open_rc_channel(iser_chan_t *chan);
190 
191 void iser_ib_close_rc_channel(iser_chan_t *chan);
192 
193 void iser_ib_free_rc_channel(iser_chan_t *chan);
194 
195 int iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl);
196 
197 void iser_ib_post_recv(ibt_channel_hdl_t chanhdl);
198 
199 void iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);
200 
201 void iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);
202 
203 void iser_ib_async_handler(void *clntp, ibt_hca_hdl_t hdl,
204     ibt_async_code_t code, ibt_async_event_t *event);
205 
206 #ifdef	__cplusplus
207 }
208 #endif
209 
210 #endif /* _ISER_IB_H */
211