1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * sol_uverbs_hca.c
27  *
28  * Provides the Solaris OFED User Verbs thin common hca interface for
29  * sharing of IBT client handle, device list, and asynchronous event
30  * delivery.
31  */
32 #include <sys/vfs.h>
33 #ifdef VFS_OPS
34 #include <sys/vfs_opreg.h>
35 #include <sys/vnode.h>
36 #endif
37 #include <sys/errno.h>
38 #include <sys/cred.h>
39 #include <sys/uio.h>
40 #include <sys/semaphore.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 
44 #include <sys/ib/ibtl/ibvti.h>
45 #include <sys/ib/clients/of/ofa_solaris.h>
46 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_hca.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
49 
50 extern char	*sol_uverbs_dbg_str;
51 
52 /*
53  * Globals for managing the list of HCA's and the registered clients.
54  */
55 kmutex_t	sol_uverbs_hca_lock;
56 llist_head_t	sol_uverbs_hca_list;
57 llist_head_t	sol_uverbs_client_list;
58 static uint32_t	sol_uverbs_common_hca_initialized = 0;
59 
60 typedef struct sol_uverbs_hca_client_data {
61 	llist_head_t		list;
62 	sol_uverbs_ib_client_t	*client;
63 	void			*data;
64 } sol_uverbs_hca_client_data_t;
65 
66 static
67 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
68 	sol_uverbs_ib_client_t *client);
69 
70 /*
71  * Function:
72  *	sol_uverbs_ib_register_client
73  * Input:
74  *	client  	- Pointer to the client structure
75  * Output:
76  *	None
77  * Returns:
78  *	Zero on success, else error code.
79  * Description:
80  *	The Solaris User Verbs kernel agent provides a single
81  *	common view of the IBTF devices.  This function allows
82  *	Solaris OFA kernel implementations to share this view
83  *	by registerng a client callback for notification of HCA
84  *	addtion and removal.  Note that when this function is
85  *	called, the client will	get an "add" callback for all
86  *	existing devices.
87  */
88 int
sol_uverbs_ib_register_client(sol_uverbs_ib_client_t * client)89 sol_uverbs_ib_register_client(sol_uverbs_ib_client_t *client)
90 {
91 	llist_head_t		*entry;
92 	sol_uverbs_hca_t	*hca;
93 
94 	ASSERT(client != NULL);
95 	mutex_enter(&sol_uverbs_hca_lock);
96 	llist_head_init(&client->list, client);
97 	llist_add_tail(&client->list, &sol_uverbs_client_list);
98 	list_for_each(entry, &sol_uverbs_hca_list) {
99 		hca = (sol_uverbs_hca_t *)entry->ptr;
100 
101 		if (client->add &&
102 		    !sol_uverbs_hca_add_client_context(hca, client)) {
103 			client->add(hca);
104 		}
105 	}
106 	mutex_exit(&sol_uverbs_hca_lock);
107 
108 	return (0);
109 }
110 
111 /*
112  * Function:
113  *	sol_uverbs_ib_unregister_client
114  * Input:
115  *	client  - Pointer to the client structure
116  * Output:
117  *	None
118  * Returns:
119  *	None
120  * Description:
121  *	Removes a client registration previously created with
122  *	the sol_uverbs_ib_register_client() call.
123  */
124 void
sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t * client)125 sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t *client)
126 {
127 	llist_head_t			*entry, *centry, *tmp;
128 	sol_uverbs_hca_t		*hca;
129 	sol_uverbs_hca_client_data_t	*context;
130 
131 	ASSERT(client != NULL);
132 	mutex_enter(&sol_uverbs_hca_lock);
133 
134 	list_for_each(entry, &sol_uverbs_hca_list) {
135 		hca = (sol_uverbs_hca_t *)entry->ptr;
136 
137 		ASSERT(hca != NULL);
138 
139 		if (client->remove) {
140 			client->remove(hca);
141 		}
142 		mutex_enter(&hca->client_data_lock);
143 		centry = hca->client_data_list.nxt;
144 		tmp = centry->nxt;
145 
146 		while (centry != &hca->client_data_list) {
147 			ASSERT(centry);
148 			context = (sol_uverbs_hca_client_data_t *)centry->ptr;
149 			ASSERT(context != NULL);
150 
151 			if (context->client == client) {
152 				llist_del(centry);
153 				kmem_free(context, sizeof (*context));
154 			}
155 			centry = tmp;
156 			tmp = centry->nxt;
157 		}
158 		mutex_exit(&hca->client_data_lock);
159 	}
160 	llist_del(&client->list);
161 	mutex_exit(&sol_uverbs_hca_lock);
162 }
163 
164 /*
165  * Function:
166  *	sol_uverbs_ib_get_client_data
167  * Input:
168  *	hca    	- Pointer to HCA struct passed in the client
169  *                add function callback.
170  *     client   - A pointer to the client structure.
171  * Output:
172  *	None
173  * Returns:
174  *	The client data, or NULL on error.
175  * Description:
176  *	Returns the client data associated with the given
177  *      HCA. The data is set/specified via the
178  *	sol_uverbs_ib_set_client_data() function.
179  */
180 void *
sol_uverbs_ib_get_client_data(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client)181 sol_uverbs_ib_get_client_data(sol_uverbs_hca_t *hca,
182 					sol_uverbs_ib_client_t *client)
183 {
184 	llist_head_t			*entry;
185 	sol_uverbs_hca_client_data_t	*context;
186 	void				*data = NULL;
187 
188 	ASSERT(hca != NULL);
189 	ASSERT(client != NULL);
190 
191 	mutex_enter(&hca->client_data_lock);
192 
193 	list_for_each(entry, &hca->client_data_list) {
194 		context = (sol_uverbs_hca_client_data_t *)entry->ptr;
195 
196 		ASSERT(context != NULL);
197 
198 		if (context->client == client) {
199 			data = context->data;
200 			break;
201 		}
202 	}
203 	mutex_exit(&hca->client_data_lock);
204 	return (data);
205 }
206 
207 /*
208  * Function:
209  *	sol_uverbs_ib_set_client_data
210  * Input:
211  *	hca    	- Pointer to HCA struct passed in the client
212  *                add function.
213  *     client   - A pointer to the client structure.
214  *     data     - The client data to associate with the HCA.
215  * Output:
216  *	None
217  * Returns:
218  *	None
219  * Description:
220  *	Sets the client data associated with the given
221  *      HCA.
222  */
223 void
sol_uverbs_ib_set_client_data(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client,void * data)224 sol_uverbs_ib_set_client_data(sol_uverbs_hca_t *hca,
225 	sol_uverbs_ib_client_t *client, void *data)
226 {
227 	llist_head_t			*entry;
228 	sol_uverbs_hca_client_data_t	*context;
229 
230 	ASSERT(hca != NULL);
231 	ASSERT(client != NULL);
232 
233 	mutex_enter(&hca->client_data_lock);
234 
235 	list_for_each(entry, &hca->client_data_list) {
236 		context = (sol_uverbs_hca_client_data_t *)entry->ptr;
237 
238 		ASSERT(context != NULL);
239 
240 		if (context->client == client) {
241 			context->data = data;
242 			goto out;
243 		}
244 	}
245 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
246 	    "HCA SET CLIENT DATA: No client found for %s\n",
247 	    client->name != NULL ? client->name : "NULL Client Name");
248 
249 out:
250 	mutex_exit(&hca->client_data_lock);
251 }
252 
253 /*
254  * Function:
255  *	sol_uverbs_ib_register_event_handler
256  * Input:
257  *	handler  - Pointer to handler structure
258  * Output:
259  *	None
260  * Returns:
261  *	Zero
262  * Description:
263  *	Register to receive ansynchronous notifications
264  *	for the HCA defined in the handler struct.  The notifications
265  *	are delivered via the callback function defined in the handler
266  *	struct.
267  */
268 int
sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t * handler)269 sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t *handler)
270 {
271 	ASSERT(handler != NULL);
272 	ASSERT(handler->hca != NULL);
273 
274 	mutex_enter(&handler->hca->event_handler_lock);
275 	llist_head_init(&handler->list, handler);
276 	llist_add_tail(&handler->list, &handler->hca->event_handler_list);
277 	mutex_exit(&handler->hca->event_handler_lock);
278 	return (0);
279 }
280 
281 /*
282  * Function:
283  *	sol_uverbs_ib_unregister_event_handler
284  * Input:
285  *	handler  - Pointer to handler structure
286  * Output:
287  *	None
288  * Returns:
289  *	Zero
290  * Description:
291  *	Unregister a ansynchronous notification handler previously
292  *	registered via the osl_uverbs_ib_register_event_handler() call.
293  */
294 int
sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t * handler)295 sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t *handler)
296 {
297 	ASSERT(handler != NULL);
298 	ASSERT(handler->hca != NULL);
299 
300 	mutex_enter(&handler->hca->event_handler_lock);
301 	llist_del(&handler->list);
302 	mutex_exit(&handler->hca->event_handler_lock);
303 	return (0);
304 }
305 
306 /*
307  * Function:
308  *	sol_uverbs_common_hca_init
309  * Input:
310  *	None
311  * Output:
312  *	None
313  * Returns:
314  *	Zero
315  * Description:
316  *	Perform initialization required by the common hca client API.
317  */
318 int
sol_uverbs_common_hca_init()319 sol_uverbs_common_hca_init()
320 {
321 	llist_head_init(&sol_uverbs_hca_list, NULL);
322 	llist_head_init(&sol_uverbs_client_list, NULL);
323 	mutex_init(&sol_uverbs_hca_lock, NULL, MUTEX_DRIVER, NULL);
324 	sol_uverbs_common_hca_initialized = 1;
325 	return (0);
326 }
327 
328 /*
329  * Function:
330  *	sol_uverbs_common_hca_fini
331  * Input:
332  *	None
333  * Output:
334  *	None
335  * Returns:
336  *	None
337  * Description:
338  *	Perform cleanup required by the common hca client API.
339  */
340 void
sol_uverbs_common_hca_fini()341 sol_uverbs_common_hca_fini()
342 {
343 	ASSERT(llist_empty(&sol_uverbs_client_list));
344 	sol_uverbs_common_hca_initialized = 0;
345 	mutex_destroy(&sol_uverbs_hca_lock);
346 }
347 
348 /*
349  * Helpers for internal use only
350  */
351 /*
352  * Function:
353  *	sol_uverbs_hca_add_client_context
354  * Input:
355  *	hca	- Pointer to the hca struct to add a client context.
356  *	client  - Pointer to the client.
357  * Output:
358  *	None
359  * Returns:
360  *	0 on success, else the error.
361  * Description:
362  *	Create a context for the specified client and attach it to
363  *	the specified hca.
364  */
365 static
sol_uverbs_hca_add_client_context(sol_uverbs_hca_t * hca,sol_uverbs_ib_client_t * client)366 int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
367     sol_uverbs_ib_client_t *client)
368 {
369 	sol_uverbs_hca_client_data_t   *context;
370 
371 	context = kmem_zalloc(sizeof (*context), KM_NOSLEEP);
372 
373 	if (!context) {
374 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
375 		    "HCA: Couldn't allocate client context for %s",
376 		    client->name ? client->name : "Name is NULL");
377 		return (ENOMEM);
378 	}
379 
380 	context->client = client;
381 	context->data   = NULL;
382 	llist_head_init(&context->list, context);
383 
384 	mutex_enter(&hca->client_data_lock);
385 	llist_add(&context->list, &hca->client_data_list);
386 	mutex_exit(&hca->client_data_lock);
387 	return (0);
388 }
389 
390 /*
391  * Function:
392  *	sol_uverbs_ibt_hdl_to_hca
393  * Input:
394  *	hca_hdl - IBT handle to an HCA.
395  * Output:
396  *	None
397  * Returns:
398  *	A pointer to the sol_uverbs HCA structure associated with the handle,
399  *	or NULL if no associated HCA is found.
400  * Description:
401  *	Given an IBT hca handle, return the user verbs HCA structure associated
402  *	with that handle.
403  */
404 sol_uverbs_hca_t *
sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl)405 sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl)
406 {
407 	llist_head_t		*entry;
408 	sol_uverbs_hca_t	*hca;
409 	sol_uverbs_hca_t	*ret = NULL;
410 
411 	mutex_enter(&sol_uverbs_hca_lock);
412 	list_for_each(entry, &sol_uverbs_hca_list) {
413 		hca = (sol_uverbs_hca_t *)entry->ptr;
414 
415 		if (hca->hdl == hca_hdl) {
416 			ret = hca;
417 			break;
418 		}
419 	}
420 	mutex_exit(&sol_uverbs_hca_lock);
421 
422 	return (ret);
423 }
424