1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H
27 #define	_SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H
28 
29 /*
30  * saa_impl.h
31  */
32 
33 #ifdef __cplusplus
34 extern "C" {
35 #endif
36 
37 #include <sys/ib/mgt/ibmf/ibmf_saa.h>
38 #include <sys/ib/mgt/ibmf/ibmf_impl.h>
39 
40 #define	SAA_MAX_CLIENTS_PER_PORT	100
41 #define	SAA_MAD_BASE_VERSION		1
42 #define	SAA_MAD_CLASS_VERSION		2
43 #define	IBMF_SAA_RETRANS_RETRIES 	0
44 #define	IBMF_SAA_MAX_SUBNET_TIMEOUT 	20
45 #define	IBMF_SAA_MAX_RESP_TIME		20
46 #define	IBMF_SAA_MAX_BUSY_RETRY_COUNT	10
47 #define	IBMF_SAA_MAX_WAIT_TIME_IN_SECS	60
48 #define	IBMF_SAA_TRANS_WAIT_TIME_IN_SECS 240
49 #define	IBMF_SAA_BUSY_RETRY_SLEEP_SECS	1	/* seconds between retry */
50 
51 /*
52  * saa_port_s:
53  * Linked list of ports that saa is using. Each port is registered to ibmf.
54  * Multiple saa clients can associate with an saa port
55  */
56 typedef enum saa_port_state_s {
57 	IBMF_SAA_PORT_STATE_REGISTERING,
58 	IBMF_SAA_PORT_STATE_READY,
59 	IBMF_SAA_PORT_STATE_INVALID, 	/* client MUST close */
60 	IBMF_SAA_PORT_STATE_PURGING	/* being purged */
61 } saa_port_state_t;
62 
63 typedef struct saa_port_s {
64 
65 	struct saa_port_s	*next;
66 
67 	kmutex_t		saa_pt_mutex;
68 
69 	/* registration synchronization: only one client registers to ibmf */
70 	kcondvar_t		saa_pt_ibmf_reg_cv;
71 
72 	/* state and client reference counts */
73 	saa_port_state_t	saa_pt_state;
74 	int			saa_pt_reference_count;
75 
76 	/* port identification and ibmf registration info */
77 	ib_guid_t		saa_pt_port_guid;
78 	ibmf_register_info_t	saa_pt_ibmf_reginfo;
79 
80 	ibmf_handle_t		saa_pt_ibmf_handle;
81 	ibmf_impl_caps_t 	saa_pt_ibmf_impl_features;
82 	ibmf_qp_handle_t	saa_pt_qp_handle;
83 	ib_qpn_t		saa_pt_qpn;
84 
85 	/* transaction parameters */
86 	int			saa_pt_timeout; 	/* from portinfo */
87 	uint16_t		saa_pt_sa_cap_mask;  	/* from classportinfo */
88 
89 	ibmf_addr_info_t	saa_pt_ibmf_addr_info;
90 	ibmf_global_addr_info_t	saa_pt_ibmf_global_addr;
91 	uint32_t		saa_pt_ibmf_msg_flags;
92 	boolean_t		saa_pt_redirect_active;	/* SA has redirected */
93 
94 	ibmf_retrans_t		saa_pt_ibmf_retrans;
95 	uint64_t 		saa_pt_current_tid;
96 	int			saa_pt_num_outstanding_trans;
97 
98 	/* kstats */
99 	kmutex_t		saa_pt_kstat_mutex;
100 	struct kstat		*saa_pt_kstatp;
101 
102 	/* sync. for receiving informinfo req packets */
103 	kmutex_t		saa_pt_event_sub_mutex;
104 	uint8_t			saa_pt_event_sub_arrive_mask;
105 	uint8_t			saa_pt_event_sub_success_mask;
106 	uint8_t			saa_pt_event_sub_last_success_mask;
107 	struct saa_client_data_s	*saa_pt_event_sub_client_list;
108 
109 	/* node guid and port num, saved for easy ibt_queries */
110 	ib_guid_t		saa_pt_node_guid;
111 	uint8_t			saa_pt_port_num;
112 
113 	/* latest hrtime that packet from SA was received */
114 	hrtime_t		saa_pt_sa_uptime;
115 } saa_port_t;
116 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_mutex,
117     saa_port_t::saa_pt_reference_count
118     saa_port_t::saa_pt_ibmf_reg_cv
119     saa_port_t::saa_pt_ibmf_retrans
120     saa_port_t::saa_pt_current_tid
121     saa_port_t::saa_pt_num_outstanding_trans
122     saa_port_t::saa_pt_timeout
123     saa_port_t::saa_pt_ibmf_addr_info
124     saa_port_t::saa_pt_ibmf_global_addr
125     saa_port_t::saa_pt_ibmf_msg_flags
126     saa_port_t::saa_pt_redirect_active))
127 _NOTE(MUTEX_PROTECTS_DATA(saa_port_t::saa_pt_kstat_mutex,
128     saa_port_t::saa_pt_kstatp))
129 
130 
131 #define	IBMF_SAA_PORT_EVENT_SUB_ALL_ARRIVE		\
132 	(IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_CA |	\
133 	IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SWITCH |	\
134 	IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_ROUTER |	\
135 	IBMF_SAA_EVENT_STATUS_MASK_PRODUCER_SM)
136 
137 typedef struct ibmf_saa_kstat_s {
138 	kstat_named_t	clients_registered;	/* # saa registrants */
139 	kstat_named_t	clients_reg_failed;	/* # failed registrants */
140 	kstat_named_t	outstanding_requests;	/* # outstanding requests */
141 	kstat_named_t	total_requests;		/* # requests ever made */
142 	kstat_named_t	failed_requests;	/* # failed requests */
143 	kstat_named_t	requests_timedout;	/* # requests that timedout */
144 } ibmf_saa_kstat_t;
145 
146 #define	IBMF_SAA_ADD32_KSTATS(subnetp, xx, val)				\
147 	if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) {	\
148 		ibmf_saa_kstat_t	*kp;				\
149 		kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\
150 		kp->xx.value.ui32 += val;				\
151 	}
152 
153 #define	IBMF_SAA_SUB32_KSTATS(subnetp, xx, val)				\
154 	if ((subnetp != NULL) && (subnetp->saa_pt_kstatp != NULL)) {	\
155 		ibmf_saa_kstat_t	*kp;				\
156 		kp = (ibmf_saa_kstat_t *)subnetp->saa_pt_kstatp->ks_data;\
157 		kp->xx.value.ui32 -= val;				\
158 	}
159 
160 typedef enum _saa_client_state_e {
161 	SAA_CLIENT_STATE_ACTIVE,
162 	SAA_CLIENT_STATE_WAITING,
163 	SAA_CLIENT_STATE_CLOSED
164 } saa_client_state_t;
165 
166 typedef struct saa_client_data_s {
167 	void				*next;
168 
169 	/* set for valid handles */
170 	void				*saa_client_sig;
171 	saa_port_t			*saa_client_port;
172 	kmutex_t			saa_client_mutex;
173 	int				saa_client_num_pending_trans;
174 	kcondvar_t			saa_client_state_cv;
175 	saa_client_state_t		saa_client_state;
176 	ib_smkey_t			saa_client_sm_key;
177 
178 	int				saa_client_event_cb_num_active;
179 	kcondvar_t			saa_client_event_cb_cv;
180 
181 	ibmf_saa_subnet_event_cb_t	saa_client_event_cb;
182 	void				*saa_client_event_cb_arg;
183 } saa_client_data_t;
184 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_port))
185 _NOTE(READ_ONLY_DATA(saa_client_data_t::saa_client_sig))
186 
187 
188 typedef struct saa_state_s {
189 
190 	saa_port_t	*saa_port_list;
191 	kmutex_t	saa_port_list_mutex;
192 	taskq_t		*saa_event_taskq;
193 } saa_state_t;
194 _NOTE(MUTEX_PROTECTS_DATA(saa_state_t::saa_port_list_mutex,
195     saa_port_t::next
196     saa_state_t::saa_port_list))
197 _NOTE(READ_ONLY_DATA(saa_state_t::saa_event_taskq))
198 
199 /*
200  * special callback used specifically for handling informinfo responses;
201  * extra parameter is producer_type
202  */
203 typedef void (*ibmf_saa_sub_cb_t) (void *, size_t, char *, int, uint32_t);
204 
205 /*
206  * saa_impl_trans_info_t:
207  * Convenience structure wich allows ibmf_access_sa to group all the fields
208  * into one structure as a parameter to the send request function.
209  * This structure is allocated by ibmf_access_sa() and freed by ibmf_access_sa()
210  * in the sync case and by the ibmf_msg_transport callback in the async case
211  */
212 typedef struct saa_impl_trans_info_t {
213 
214 	saa_client_data_t	*si_trans_client_data;
215 	saa_port_t		*si_trans_port;
216 
217 	/* used to tell send_request about request mad */
218 	size_t			si_trans_template_length; /* for unknown attr */
219 	uint16_t		si_trans_attr_id;
220 	uint64_t		si_trans_component_mask;
221 	void			*si_trans_template;
222 	uint8_t			si_trans_method;
223 
224 	/* used for async call to tell send_request which callback to use */
225 	ibmf_saa_cb_t		si_trans_callback;
226 	void			*si_trans_callback_arg;
227 
228 	/*
229 	 * used to tell ibmf_access_sa about response if the request was sync.
230 	 * If the request was async, the ibmf_msg_transport callback function
231 	 * will fill these values directly into the si_trans_callback
232 	 */
233 	int			si_trans_status;
234 	void			*si_trans_result;
235 	size_t			si_trans_length;
236 
237 	/* fields needed for specific case of handling InformInfo requests */
238 
239 	/*
240 	 * producer_type indicates the notice producer type that the
241 	 * subscription was for.  There is no way to tell which type the
242 	 * response packet is for.
243 	 */
244 	uint32_t		si_trans_sub_producer_type;
245 
246 	/*
247 	 * separate callback typedef to provide the producer type to the
248 	 * subscription response handler (ibmf_saa_impl_get_informinfo_cb)
249 	 */
250 	ibmf_saa_sub_cb_t	si_trans_sub_callback;
251 
252 	/*
253 	 * some unsubscribe requests are sequenced, others are unsequenced
254 	 * (depending on context that generated unsubscribe request)
255 	 */
256 	boolean_t		si_trans_unseq_unsubscribe;
257 
258 	/* trans flags saved in case sm lid changes and msg must be resent */
259 	uint8_t			si_trans_transport_flags;
260 
261 	/* counter for retrying requests which return MAD_BUSY status */
262 	uint8_t			si_trans_retry_busy_count;
263 
264 	/* hrtime that we initiated this transaction */
265 	hrtime_t		si_trans_send_time;
266 } saa_impl_trans_info_t;
267 _NOTE(SCHEME_PROTECTS_DATA("private callback arg", saa_impl_trans_info_t))
268 
269 typedef struct ibmf_saa_event_taskq_args_s {
270 	saa_client_data_t		*et_client;
271 	ibmf_saa_subnet_event_t		et_subnet_event;
272 	ibmf_saa_event_details_t	*et_event_details;
273 	ib_mad_notice_t			*et_notice;
274 	ibmf_saa_subnet_event_cb_t	et_callback;
275 	void				*et_callback_arg;
276 } ibmf_saa_event_taskq_args_t;
277 _NOTE(READ_ONLY_DATA(ibmf_saa_event_taskq_args_t::et_subnet_event
278     ibmf_saa_event_taskq_args_t::et_event_details
279     ibmf_saa_event_taskq_args_t::et_client
280     ibmf_saa_event_taskq_args_t::et_notice
281     ibmf_saa_event_taskq_args_t::et_callback
282     ibmf_saa_event_taskq_args_t::et_callback_arg))
283 
284 /*
285  * Public Functions
286  */
287 int ibmf_saa_impl_init();
288 int ibmf_saa_impl_fini();
289 boolean_t ibmf_saa_is_valid(saa_port_t *saa_portp, int add_ref);
290 void ibmf_saa_impl_purge();
291 int ibmf_saa_impl_add_client(saa_port_t *saa_portp);
292 int ibmf_saa_impl_create_port(ib_guid_t	pt_guid, saa_port_t **saa_portpp);
293 int ibmf_saa_impl_init_kstats(saa_port_t *saa_portp);
294 void ibmf_saa_impl_register_failed(saa_port_t *saa_portp);
295 int ibmf_saa_impl_register_port(saa_port_t *saa_portp);
296 void ibmf_saa_impl_get_classportinfo(saa_port_t *saa_portp);
297 int ibmf_saa_impl_send_request(saa_impl_trans_info_t *trans_info);
298 
299 void ibmf_saa_async_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
300     void *args);
301 
302 void
303 ibmf_saa_add_event_subscriber(saa_client_data_t *client,
304     ibmf_saa_subnet_event_args_t *event_args);
305 
306 void ibmf_saa_subscribe_events(saa_port_t *saa_portp, boolean_t subscribe,
307     boolean_t seq_unsubscribe);
308 
309 void ibmf_saa_subscribe_sm_events(saa_port_t *saa_portp);
310 
311 void
312 ibmf_saa_notify_event_clients(saa_port_t *saa_portp,
313     ibmf_saa_event_details_t *event_details,
314     ibmf_saa_subnet_event_t subnet_event,
315     saa_client_data_t *registering_client);
316 
317 void
318 ibmf_saa_report_cb(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
319     void *args);
320 #ifdef __cplusplus
321 }
322 #endif
323 
324 #endif /* _SYS_IB_MGT_IBMF_IBMF_SAA_IMPL_H */
325