1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2017 Joyent, Inc.
25  */
26 
27 /*
28  * sol_uverbs_event.c
29  *
30  * OFED User Verbs Kernel Async Event funtions
31  *
32  */
33 #include <sys/file.h>
34 #include <sys/fcntl.h>
35 #include <sys/vfs.h>
36 #include <sys/errno.h>
37 #include <sys/cred.h>
38 #include <sys/uio.h>
39 #include <sys/semaphore.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 
43 #include <sys/ib/ibtl/ibvti.h>
44 #include <sys/ib/clients/of/ofa_solaris.h>
45 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
46 #include <sys/ib/clients/of/ofed_kernel.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
49 
50 extern char	*sol_uverbs_dbg_str;
51 
52 static void
53 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t,
54     llist_head_t *, uint32_t *);
55 
56 /*
57  * Function:
58  *      sol_uverbs_event_file_close
59  * Input:
60  *      ufile	- Pointer to the event ufile
61  *
62  * Output:
63  *      None
64  * Returns:
65  *      Zero on success, else error code.
66  * Description:
67  *	Called when all kernel references to the event file have been
68  *	removed and the kernel (asynchronous) or user library (completion)
69  *	have closed the file.
70  */
71 void
sol_uverbs_event_file_close(uverbs_ufile_uobj_t * ufile)72 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile)
73 {
74 	if (!ufile) {
75 		SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
76 		    "UFILE CLOSE: Ufile NULL\n");
77 		return;
78 	}
79 
80 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
81 	    "UFILE CLOSE: Is async? %s",
82 	    ufile->is_async ? "yes" : "no");
83 
84 	/*
85 	 * Remove the user file from the user object table and
86 	 * releases appropriate references.  The object resources
87 	 * are freed when it is no longer referenced.
88 	 *
89 	 * If sol_ofs_uobj_remove() returns NULL then the obj was already
90 	 * removed.
91 	 */
92 	rw_enter(&(ufile->uobj.uo_lock), RW_WRITER);
93 	if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) {
94 		rw_exit(&(ufile->uobj.uo_lock));
95 		sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file);
96 	} else {
97 		rw_exit(&(ufile->uobj.uo_lock));
98 	}
99 }
100 
101 /*
102  * Function:
103  *      sol_uverbs_event_file_read
104  * Input:
105  *      ufile	- The user file pointer of the event channel.
106  *      uiop	- The user I/O pointer in which to place the event.
107  *	cred	- Pointer to the callers credentials.
108  * Output:
109  *      uiop	- Upon success the caller's buffer has been updated with
110  *		  the event details.
111  * Returns:
112  *      Zero on success, else error code.
113  *		EAGAIN	- No event available and caller is non-
114  *			  blocking.
115  *		ERESTART- A signal was received.
116  *		EINVAL	- Caller parameter/user buffer invalid.
117  *		EFAULT	- Failure copying data to user.
118  * Description:
119  *      Perfrom a blocking read to retrieve an event (asynchronous or
120  *	completion) for the user file specified.  If an event is available it
121  *	is immediately returned. If an event is not available, then the
122  *	caller will block unless the open flags indicate it is a non-
123  *	blocking open.  If the caller does block it does so interruptable
124  *	and therefore will return upon an event or receipt of a signal.
125  */
126 /* ARGSUSED */
127 int
sol_uverbs_event_file_read(uverbs_ufile_uobj_t * ufile,struct uio * uiop,cred_t * cred)128 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop,
129     cred_t *cred)
130 {
131 	int			rc = 0;
132 	uverbs_event_t		*evt;
133 	llist_head_t		*entry;
134 	int			eventsz;
135 	int			ioflag;
136 
137 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), "
138 	    "ufile = %p, is_async =%d, uio_resid=%d",
139 	    ufile, ufile->is_async, uiop->uio_resid);
140 
141 	ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY);
142 
143 	mutex_enter(&ufile->lock);
144 
145 	/*
146 	 * If Event list not empty and CQ event notification is disabled
147 	 * by sol_ucma, do not  return events. Either return EAGAIN (if
148 	 * flag is O_NONBLOCK, or wait using cv_wait_sig().
149 	 */
150 	if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE &&
151 	    llist_empty(&ufile->event_list) != 0) {
152 		if (ioflag) {
153 			mutex_exit(&ufile->lock);
154 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
155 			    "event_file_read - notify disabled, no block");
156 			return (EAGAIN);
157 		}
158 
159 		if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
160 			mutex_exit(&ufile->lock);
161 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
162 			    "event_file_read - sig_wakeup");
163 			return (ERESTART);
164 		}
165 	}
166 
167 	while (llist_empty(&ufile->event_list)) {
168 		if (ioflag) {
169 			mutex_exit(&ufile->lock);
170 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
171 			    "event_file_read - no events, no block");
172 			return (EAGAIN);
173 		}
174 
175 		if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
176 			mutex_exit(&ufile->lock);
177 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
178 			    "event_file_read - sig_wakeup");
179 			return (ERESTART);
180 		}
181 	}
182 
183 	entry = ufile->event_list.nxt;
184 	evt   = entry->ptr;
185 
186 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: "
187 	    "Event entry found: entry:%p, event:%p, evt_list %p",
188 	    entry, evt, &evt->ev_list);
189 
190 	if (ufile->is_async) {
191 		eventsz = sizeof (struct ib_uverbs_async_event_desc);
192 	} else {
193 		eventsz = sizeof (struct ib_uverbs_comp_event_desc);
194 	}
195 
196 	if (eventsz > uiop->uio_resid) {
197 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
198 		    "event_file_read - Event too big");
199 		rc  = EINVAL;
200 		evt = NULL;
201 	} else {
202 		llist_del(ufile->event_list.nxt);
203 		if (evt->ev_counter) {
204 			++(*evt->ev_counter);
205 			llist_del(&evt->ev_obj_list);
206 		}
207 	}
208 
209 	mutex_exit(&ufile->lock);
210 
211 	if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) {
212 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
213 		    "EVENT FILE READ: Error writing ev");
214 		rc = EFAULT;
215 	}
216 
217 	if (evt) {
218 		kmem_free(evt, sizeof (*evt));
219 	}
220 
221 	return (rc);
222 }
223 
224 /*
225  * Function:
226  *      sol_uverbs_event_file_poll
227  * Input:
228  *      ufile	- user file for desired completion channel event file
229  *      events	- The events that may occur.
230  *	anyyet	- A flag that is non-zero if any files in the set
231  *		  of descriptors has an event waiting.
232  *	ct	- Pointer to the callers context.
233  * Output:
234  *	reventssp - A pointer updated to return a bitmask of events specified.
235  *      phpp      - A pointer to a pollhead pointer, updated to reflect the
236  *		    the event file's pollhead used for synchronization.
237  * Returns:
238  *      Zero on success, else error code.
239  *		EINVAL    - Vnode does not point to valid event file.
240  * Description:
241  *	Support for event channel polling interface, allows use of completion
242  *	channel in asynchronous type environment.  If events may be read
243  *      without blocking indicate a POLLIN | POLLRDNORM event; otherwise if
244  *	no other descriptors in the set have data waiting, set the pollhead
245  *	pointer to our the associated completion event file's pollhead.
246  */
247 int
sol_uverbs_event_file_poll(uverbs_ufile_uobj_t * ufile,short events,int anyyet,short * reventsp,pollhead_t ** phpp)248 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events,
249     int anyyet, short *reventsp, pollhead_t **phpp)
250 {
251 	short	revent = 0;
252 
253 #ifdef DEBUG
254 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)",
255 	    ufile, events);
256 #endif
257 
258 	if (!ufile) {
259 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ",
260 		    "ufile %p", ufile);
261 		return (EINVAL);
262 	}
263 
264 #ifdef DEBUG
265 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
266 	    "ufile = %p, is_async =%d", ufile, ufile->is_async);
267 #endif
268 
269 	mutex_enter(&ufile->lock);
270 
271 	/*
272 	 * If poll request and event is ready.
273 	 */
274 	if ((events & (POLLIN | POLLRDNORM)) &&
275 	    !llist_empty(&ufile->event_list)) {
276 
277 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
278 		    "Event entry available");
279 
280 		revent |= POLLIN | POLLRDNORM;
281 	}
282 
283 	/*
284 	 * If we didn't get an event or are edge-triggered
285 	 */
286 	if ((revent == 0 && !anyyet) || (events & POLLET)) {
287 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
288 		    "Event entry NOT available");
289 
290 		*phpp = &ufile->poll_head;
291 	}
292 
293 	mutex_exit(&ufile->lock);
294 	*reventsp = revent;
295 	return (0);
296 }
297 
298 /*
299  * Function:
300  *      uverbs_alloc_event_file
301  * Input:
302  *      uctxt	 - The Solaris User Verbs user context associated with the
303  *		   event channel.
304  *      is_async - Indicates the file is for asynchronous events if non-zero;
305  *                 other wise it is for completion events.
306  * Output:
307  *      None.
308  * Returns:
309  *      New user verb event file object or NULL on error.
310  * Description:
311  *	Allocate an asynchronous or completion event file
312  */
313 uverbs_ufile_uobj_t *
uverbs_alloc_event_file(uverbs_uctxt_uobj_t * uctxt,int is_async)314 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async)
315 {
316 	uverbs_ufile_uobj_t	*ufile;
317 
318 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)",
319 	    uctxt, is_async);
320 
321 	ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP);
322 	if (!ufile) {
323 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
324 		    "alloc_event_file: mem alloc fail");
325 		return (NULL);
326 	}
327 	ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE;
328 	sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE);
329 	rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
330 
331 	if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) {
332 		/*
333 		 * The initialization routine set's the initial reference,
334 		 * we dereference the object here to clean it up.
335 		 */
336 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
337 		    "ALLOC UFILE: Object add failed");
338 		rw_exit(&ufile->uobj.uo_lock);
339 		ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t);
340 		sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free);
341 		return (NULL);
342 	}
343 
344 	ufile->is_async		= is_async ? 1 : 0;
345 	llist_head_init(&ufile->event_list, NULL);
346 
347 	mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL);
348 	cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL);
349 
350 	ufile->uctxt		= uctxt;
351 	ufile->uobj.uo_live	= 1;
352 	rw_exit(&ufile->uobj.uo_lock);
353 	return (ufile);
354 }
355 
356 /*
357  * Function:
358  *      uverbs_release_event_file
359  * Input:
360  *      ufile	 - Pointer to the ufile user object that is being freed.
361  * Output:
362  *      None.
363  * Returns:
364  *      None.
365  * Description:
366  *	Release/destroy event file resources before freeing memory.  This
367  *	routine should only be used if the event file is successfully
368  *	created.
369  */
370 void
uverbs_release_event_file(sol_ofs_uobj_t * uobj)371 uverbs_release_event_file(sol_ofs_uobj_t *uobj)
372 {
373 	uverbs_ufile_uobj_t	*ufile = (uverbs_ufile_uobj_t *)uobj;
374 	uverbs_event_t		*evt;
375 	llist_head_t		*entry;
376 	llist_head_t		*list_tmp;
377 
378 	if (!ufile) {
379 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
380 		    "UFILE RELEASE: Ufile NULL\n");
381 		return;
382 	}
383 
384 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
385 	    "UFILE RELEASE: Event file=%p, is async = %s",
386 	    ufile, ufile->is_async ? "yes" : "no");
387 
388 	/*
389 	 * Release any events still queued to the event file.
390 	 */
391 	mutex_enter(&ufile->lock);
392 
393 	entry = ufile->event_list.nxt;
394 	list_tmp = entry->nxt;
395 	while (entry != &ufile->event_list) {
396 		ASSERT(entry);
397 		evt = (uverbs_event_t *)entry->ptr;
398 
399 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
400 		    "UFILE RELEASE: Deleting event %p on event file %p",
401 		    evt, ufile);
402 
403 		llist_del(&evt->ev_list);
404 		kmem_free(evt, sizeof (*evt));
405 		entry = list_tmp;
406 		list_tmp = entry->nxt;
407 	}
408 
409 	mutex_exit(&ufile->lock);
410 
411 	pollhead_clean(&ufile->poll_head);
412 	cv_destroy(&ufile->poll_wait);
413 	mutex_destroy(&ufile->lock);
414 	sol_ofs_uobj_free(uobj);
415 }
416 
417 /*
418  * Function:
419  *      uverbs_ibt_to_ofa_event_code
420  * Input:
421  *      code	- The OFA event code.
422  * Output:
423  *      The OFED event code.
424  * Returns:
425  *      Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if
426  *	a valid translation does not exist.
427  * Description:
428  *      Map an IBT asynchronous event code to an OFED event code.
429  */
430 enum ib_event_type
uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)431 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)
432 {
433 	enum ib_event_type    ofa_code;
434 
435 	switch (code) {
436 	case IBT_EVENT_PATH_MIGRATED:
437 		ofa_code = IB_EVENT_PATH_MIG;
438 		break;
439 
440 	case IBT_EVENT_SQD:
441 		ofa_code = IB_EVENT_SQ_DRAINED;
442 		break;
443 
444 	case IBT_EVENT_COM_EST:
445 		ofa_code = IB_EVENT_COMM_EST;
446 		break;
447 
448 	case IBT_ERROR_CATASTROPHIC_CHAN:
449 	case IBT_ERROR_LOCAL_CATASTROPHIC:
450 		ofa_code = IB_EVENT_QP_FATAL;
451 		break;
452 
453 	case IBT_ERROR_INVALID_REQUEST_CHAN:
454 		ofa_code = IB_EVENT_QP_REQ_ERR;
455 		break;
456 
457 	case IBT_ERROR_ACCESS_VIOLATION_CHAN:
458 		ofa_code = IB_EVENT_QP_ACCESS_ERR;
459 		break;
460 
461 	case IBT_ERROR_PATH_MIGRATE_REQ:
462 		ofa_code = IB_EVENT_PATH_MIG_ERR;
463 		break;
464 
465 	case IBT_ERROR_CQ:
466 		ofa_code = IB_EVENT_CQ_ERR;
467 		break;
468 
469 	case IBT_EVENT_PORT_UP:
470 		ofa_code = IB_EVENT_PORT_ACTIVE;
471 		break;
472 
473 	case IBT_ERROR_PORT_DOWN:
474 		ofa_code = IB_EVENT_PORT_ERR;
475 		break;
476 
477 	case IBT_HCA_ATTACH_EVENT:
478 		ofa_code = IB_EVENT_CLIENT_REREGISTER;
479 		break;
480 
481 	case IBT_EVENT_LIMIT_REACHED_SRQ:
482 		ofa_code = IB_EVENT_SRQ_LIMIT_REACHED;
483 		break;
484 
485 	case IBT_ERROR_CATASTROPHIC_SRQ:
486 		ofa_code = IB_EVENT_SRQ_ERR;
487 		break;
488 
489 	case IBT_EVENT_EMPTY_CHAN:
490 		ofa_code = IB_EVENT_QP_LAST_WQE_REACHED;
491 		break;
492 
493 	/*
494 	 * No mapping exists.
495 	 */
496 	case IBT_ASYNC_OPAQUE1:
497 	case IBT_ASYNC_OPAQUE2:
498 	case IBT_ASYNC_OPAQUE3:
499 	case IBT_ASYNC_OPAQUE4:
500 	case IBT_HCA_DETACH_EVENT:
501 	default:
502 		ofa_code = -1;
503 	}
504 
505 	return (ofa_code);
506 }
507 
508 /*
509  * Function:
510  *      uverbs_async_qp_event_handler
511  * Input:
512  *      clnt_private	- The IBT attach client handle.
513  *	hca_hdl		- The IBT hca handle associated with the notification.
514  *	code		- The OFED event identifier.
515  *	event		- The IBT event.
516  * Output:
517  *      None
518  * Returns:
519  *      None
520  * Description:
521  *      Handle QP affiliated asynchronous event noficiation.
522  */
523 /* ARGSUSED */
524 void
uverbs_async_qp_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)525 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
526     enum ib_event_type code, ibt_async_event_t *event)
527 {
528 	uverbs_uqp_uobj_t	*uqp;
529 
530 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
531 	    "async_qp_event_handler()");
532 
533 	if (event->ev_chan_hdl == NULL) {
534 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
535 		    "async_qp_event_handler: event handle NULL");
536 		return;
537 	}
538 	uqp = ibt_get_qp_private(event->ev_chan_hdl);
539 	ASSERT(uqp);
540 	if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) {
541 		SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
542 		    "async_qp_event_handler: User QP context has been freed");
543 		return;
544 	}
545 	if (uqp->qp != event->ev_chan_hdl) {
546 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
547 		    "async_qp_event_handler: QP handle mismatch");
548 		return;
549 	}
550 
551 	uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle,
552 	    code, &uqp->async_list, &uqp->async_events_reported);
553 
554 }
555 
556 /*
557  * Function:
558  *      uverbs_async_cq_event_handler
559  * Input:
560  *      clnt_private	- The IBT attach client handle.
561  *	hca_hdl		- The IBT hca handle associated with the notification.
562  *	code		- The OFED event identifier.
563  *	event		- The IBT event.
564  * Output:
565  *      None
566  * Returns:
567  *      None
568  * Description:
569  *      Handle a CQ affiliated asynchronous event notification.
570  */
571 /* ARGSUSED */
572 void
uverbs_async_cq_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)573 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
574     enum ib_event_type code, ibt_async_event_t *event)
575 {
576 	uverbs_ucq_uobj_t	*ucq;
577 
578 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
579 	    "ASYNC CQ EVENT HANDLER:");
580 
581 	if (event->ev_cq_hdl == NULL) {
582 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
583 		    "ASYNC CQ EVENT HANDLER: event handle is NULL");
584 		return;
585 	}
586 
587 	ucq = ibt_get_cq_private(event->ev_cq_hdl);
588 	if (ucq->cq != event->ev_cq_hdl) {
589 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
590 		    "ASYNC CQ EVENT HANDLER: CQ handle mismatch");
591 		return;
592 	}
593 
594 	uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle,
595 	    code, &ucq->async_list, &ucq->async_events_reported);
596 }
597 
598 /*
599  * Function:
600  *      uverbs_async_srq_event_handler
601  * Input:
602  *      clnt_private	- The IBT attach client handle.
603  *	hca_hdl		- The IBT hca handle associated with the notification.
604  *	code		- The OFED event identifier.
605  *	event		- The IBT event.
606  * Output:
607  *      None
608  * Returns:
609  *      None
610  * Description:
611  *      Handle a shared receive queue asynchronous event notification.
612  */
613 /* ARGSUSED */
614 void
uverbs_async_srq_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)615 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
616     enum ib_event_type code, ibt_async_event_t *event)
617 {
618 	uverbs_usrq_uobj_t	*usrq;
619 
620 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
621 	    "ASYNC SRQ EVENT HANDLER:");
622 
623 	if (event->ev_srq_hdl == NULL) {
624 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
625 		    "ASYNC SRQ EVENT HANDLER: event handle is NULL");
626 		return;
627 	}
628 
629 	usrq = ibt_get_srq_private(event->ev_srq_hdl);
630 	if (usrq->srq != event->ev_srq_hdl) {
631 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
632 		    "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch");
633 		return;
634 	}
635 
636 	uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle,
637 	    code, &usrq->async_list, &usrq->async_events_reported);
638 }
639 
640 /*
641  * Function:
642  *      uverbs_async_unaff_event_handler
643  * Input:
644  *      clnt_private	- The IBT attach client handle.
645  *	hca_hdl		- The IBT hca handle associated with the notification.
646  *	code		- The OFED event identifier.
647  *	event		- The IBT event.
648  * Output:
649  *      None
650  * Returns:
651  *      None
652  * Description:
653  *      Handle an unaffiliated asynchronous event notification.
654  */
655 /* ARGSUSED */
656 void
uverbs_async_unaff_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)657 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
658     enum ib_event_type code, ibt_async_event_t *event)
659 {
660 	sol_ofs_uobj_table_t	*uo_tbl = &uverbs_uctxt_uo_tbl;
661 	sol_ofs_uobj_blk_t	*blk;
662 	uverbs_uctxt_uobj_t	*uctxt;
663 	int			i, j;
664 
665 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
666 	    "ASYNC UNAFF EVENT HANDLER:");
667 
668 	/*
669 	 * Unaffiliated events are returned at the IBT client level.  We must
670 	 * return the event to all user context allocated for the specific
671 	 * HCA device specified.
672 	 */
673 	rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER);
674 
675 	for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) {
676 		blk = uo_tbl->uobj_tbl_uo_root[i];
677 		if (blk == NULL) {
678 			continue;
679 		}
680 		for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) {
681 			uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j];
682 			if (uctxt == NULL) {
683 				continue;
684 			}
685 			/*
686 			 * OK, check to see if this user context belongs
687 			 * to the idicated hca.
688 			 */
689 			if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) {
690 				uverbs_async_event_common(uctxt,
691 				    event->ev_port, code, NULL, NULL);
692 			}
693 		}
694 	}
695 	rw_exit(&uo_tbl->uobj_tbl_lock);
696 }
697 
698 /*
699  * Function:
700  *      uverbs_async_event_handler
701  * Input:
702  *      clnt_private	- The IBT attach client handle.
703  *	hca_hdl		- The IBT hca handle associated with the notification.
704  *	code		- The OFED event identifier.
705  *	event		- The IBT event.
706  * Output:
707  *      None
708  * Returns:
709  *      None
710  * Description:
711  *	Main IBT asynchronous event handler registered at ibt_attach.
712  *      Convert to OFA event type and forward to the appropriate
713  *      asynchronous handler.
714  */
715 void
uverbs_async_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,ibt_async_code_t code,ibt_async_event_t * event)716 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
717     ibt_async_code_t code, ibt_async_event_t *event)
718 {
719 	enum ib_event_type		ofa_type;
720 	sol_uverbs_ib_event_handler_t	*handler;
721 	llist_head_t			*entry;
722 	sol_uverbs_hca_t		*hca;
723 
724 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
725 	    "ASYNNC EVENT HANDLER: entry, code=%d", code);
726 
727 	ofa_type = uverbs_ibt_to_ofa_event_code(code);
728 
729 	if ((int)ofa_type < 0) {
730 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
731 		    "ASYNC EVENT HANDLER:Event %d did not map to OFA "
732 		    "Event", code);
733 		return;
734 	}
735 
736 	switch (ofa_type) {
737 		case IB_EVENT_QP_FATAL:
738 		case IB_EVENT_QP_REQ_ERR:
739 		case IB_EVENT_QP_ACCESS_ERR:
740 		case IB_EVENT_QP_LAST_WQE_REACHED:
741 		case IB_EVENT_SQ_DRAINED:
742 		case IB_EVENT_PATH_MIG:
743 			/*
744 			 * These events are related with a QP
745 			 */
746 			uverbs_async_qp_event_handler(clnt_private, hca_hdl,
747 			    ofa_type, event);
748 			break;
749 
750 		case IB_EVENT_CQ_ERR:
751 			/*
752 			 * These events are related with a CQ
753 			 */
754 			uverbs_async_cq_event_handler(clnt_private, hca_hdl,
755 			    ofa_type, event);
756 			break;
757 
758 		case IB_EVENT_SRQ_ERR:
759 		case IB_EVENT_SRQ_LIMIT_REACHED:
760 			/*
761 			 * These events are related with a SRQ
762 			 */
763 			uverbs_async_srq_event_handler(clnt_private, hca_hdl,
764 			    ofa_type, event);
765 			break;
766 
767 
768 		case IB_EVENT_PORT_ERR:
769 		case IB_EVENT_PORT_ACTIVE:
770 		case IB_EVENT_LID_CHANGE:
771 		case IB_EVENT_PKEY_CHANGE:
772 		case IB_EVENT_SM_CHANGE:
773 		case IB_EVENT_CLIENT_REREGISTER:
774 		case IB_EVENT_DEVICE_FATAL:
775 		case IB_EVENT_PATH_MIG_ERR:
776 			/*
777 			 * Unaffiliated asynchronous notifications.
778 			 */
779 			uverbs_async_unaff_event_handler(clnt_private, hca_hdl,
780 			    ofa_type, event);
781 			break;
782 
783 		default:
784 		break;
785 	}
786 
787 	/*
788 	 * Give other kernel agents a notification.
789 	 */
790 	hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl);
791 	if (hca) {
792 		mutex_enter(&hca->event_handler_lock);
793 		list_for_each(entry, &hca->event_handler_list) {
794 			handler = (sol_uverbs_ib_event_handler_t *)entry->ptr;
795 
796 			ASSERT(handler != NULL);
797 			handler->handler(handler, hca_hdl, code, event);
798 		}
799 		mutex_exit(&hca->event_handler_lock);
800 	}
801 }
802 
803 /*
804  * Function:
805  *      uverbs_async_event_common
806  * Input:
807  *      uctxt	- Pointer to the user context associated with the
808  *			affiliated event.
809  *	element		- The users handle to the associated object.
810  *	event           - The event type.
811  *	uobj_list       - The list to enqueue the asynchronous event.
812  *      counter         - The counter to track the event delivery.
813  * Output:
814  *      None
815  * Returns:
816  *      None
817  * Description:
818  *      Create an asyncronous event and enqueue it on the specified list.
819  *      Then wake callers that may  be blocked on the list.
820  */
821 static void
uverbs_async_event_common(uverbs_uctxt_uobj_t * uctxt,uint64_t element,uint32_t event,llist_head_t * obj_list,uint32_t * counter)822 uverbs_async_event_common(uverbs_uctxt_uobj_t  *uctxt, uint64_t element,
823     uint32_t event, llist_head_t *obj_list, uint32_t *counter)
824 {
825 	uverbs_ufile_uobj_t	*ufile = uctxt->async_evfile;
826 	uverbs_event_t		*entry;
827 
828 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, "
829 	    "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter);
830 
831 	if (!ufile) {
832 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
833 		    "ufile %p", ufile);
834 		return;
835 	}
836 
837 	mutex_enter(&ufile->lock);
838 	entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
839 	if (!entry) {
840 		mutex_exit(&ufile->lock);
841 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
842 		    "kmem_zalloc failed");
843 		return;
844 	}
845 
846 	entry->ev_desc.async.element    = element;
847 	entry->ev_desc.async.event_type = event;
848 	entry->ev_counter		= counter;
849 
850 	llist_head_init(&entry->ev_list, entry);
851 	llist_head_init(&entry->ev_obj_list, entry);
852 
853 	llist_add_tail(&entry->ev_list, &ufile->event_list);
854 
855 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
856 	    "adding ASYNC entry-ev_list=%p, entry %p",
857 	    &entry->ev_list, entry);
858 
859 	if (obj_list) {
860 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
861 		    "adding ASYNC entry-ev_obj_list=%p, entry=%p",
862 		    &entry->ev_obj_list, entry);
863 		llist_add_tail(&entry->ev_obj_list, obj_list);
864 	}
865 
866 	mutex_exit(&ufile->lock);
867 	cv_signal(&ufile->poll_wait);
868 	pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
869 }
870 
871 /*
872  * Function:
873  *      uverbs_release_ucq_channel
874  * Input:
875  *	uctxt	- A pointer to the callers user context.
876  *	ufile - A pointer to the event file associated with a CQ.
877  *	ucq	- A pointer to the user CQ object.
878  * Output:
879  *      None
880  * Returns:
881  *      None
882  * Description:
883  *	Release any completion and asynchronous events that may
884  *	be queued to the specified completion channel/UCQ but not
885  *	yet reaped.
886  */
887 void
uverbs_release_ucq_channel(uverbs_uctxt_uobj_t * uctxt,uverbs_ufile_uobj_t * ufile,uverbs_ucq_uobj_t * ucq)888 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt,
889     uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t   *ucq)
890 {
891 	uverbs_event_t	*evt;
892 	llist_head_t	*entry;
893 	llist_head_t	*list_tmp;
894 
895 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
896 	    "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p",
897 	    uctxt, ufile, ucq);
898 
899 	/*
900 	 * Release completion events that have been queued on the CQ completion
901 	 * eventlist.
902 	 */
903 	if (ufile) {
904 		rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
905 		ufile->ufile_cq_cnt--;
906 		if (ufile->ufile_cq_cnt) {
907 			rw_exit(&ufile->uobj.uo_lock);
908 			uverbs_release_ucq_uevents(ufile, ucq);
909 			return;
910 		}
911 		rw_exit(&ufile->uobj.uo_lock);
912 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
913 		    "release_ucq_chan : comp_list %p, prv %p, nxt %p",
914 		    &ucq->comp_list, ucq->comp_list.prv,
915 		    ucq->comp_list.nxt);
916 		mutex_enter(&ufile->lock);
917 
918 		entry = ucq->comp_list.nxt;
919 		list_tmp = entry->nxt;
920 		while (entry != &ucq->comp_list) {
921 			ASSERT(entry);
922 			evt = (uverbs_event_t *)entry->ptr;
923 
924 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
925 			    "RELEASE UCQ CHANNEL:Deleting event "
926 			    "on CQ comp list: %p", evt);
927 			llist_del(&evt->ev_list);
928 			llist_del(&evt->ev_obj_list);
929 			kmem_free(evt, sizeof (*evt));
930 			entry = list_tmp;
931 			list_tmp = entry->nxt;
932 		}
933 		mutex_exit(&ufile->lock);
934 
935 		uverbs_release_ucq_uevents(ufile, ucq);
936 	}
937 
938 }
939 
940 /*
941  * Function:
942  *      uverbs_release_ucq_uevents
943  * Input:
944  *	ufile	- A pointer to the asynchronous event file associated with a QP.
945  *	ucq	- A pointer to the user CQ object.
946  * Output:
947  *      None
948  * Returns:
949  *      None
950  * Description:
951  *      Free any user asynchronous events that have been queued for the
952  *	user CQ object specified.
953  */
954 void
uverbs_release_ucq_uevents(uverbs_ufile_uobj_t * ufile,uverbs_ucq_uobj_t * ucq)955 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq)
956 {
957 	uverbs_event_t	*evt;
958 	llist_head_t	*entry;
959 	llist_head_t	*list_tmp;
960 
961 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
962 	    "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq);
963 
964 	if (ufile) {
965 		mutex_enter(&ufile->lock);
966 
967 		entry = ucq->async_list.nxt;
968 		list_tmp = entry->nxt;
969 		while (entry != &ucq->async_list) {
970 			ASSERT(entry);
971 			evt = (uverbs_event_t *)entry->ptr;
972 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
973 			    "RELEASE UCQ EVENTS: Deleting event "
974 			    "on CQ async list: %p", evt);
975 			llist_del(&evt->ev_list);
976 			llist_del(&evt->ev_obj_list);
977 			kmem_free(evt, sizeof (*evt));
978 			entry = list_tmp;
979 			list_tmp = entry->nxt;
980 		}
981 		mutex_exit(&ufile->lock);
982 	}
983 }
984 
985 /*
986  * Function:
987  *      uverbs_release_uqp_uevents
988  * Input:
989  *	ufile	- A pointer to the asynchronous event file associated with a QP.
990  *	uqp	- A pointer to the user QP object.
991  * Output:
992  *      None
993  * Returns:
994  *      None
995  * Description:
996  *      Free any user asynchronous events that have been queued for the
997  *	user QP object specified.
998  */
999 void
uverbs_release_uqp_uevents(uverbs_ufile_uobj_t * ufile,uverbs_uqp_uobj_t * uqp)1000 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp)
1001 {
1002 	uverbs_event_t	*evt;
1003 	llist_head_t	*entry;
1004 	llist_head_t	*list_tmp;
1005 
1006 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1007 	    "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp);
1008 
1009 	if (ufile) {
1010 		mutex_enter(&ufile->lock);
1011 		entry = uqp->async_list.nxt;
1012 		list_tmp = entry->nxt;
1013 		while (entry != &uqp->async_list) {
1014 			ASSERT(entry);
1015 			evt = (uverbs_event_t *)entry->ptr;
1016 			ASSERT(evt);
1017 
1018 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1019 			    "RELEASE UQP EVENTS: Deleting event "
1020 			    "ON qp async list: %p", evt);
1021 			llist_del(&evt->ev_list);
1022 			llist_del(&evt->ev_obj_list);
1023 			kmem_free(evt, sizeof (*evt));
1024 			entry = list_tmp;
1025 			list_tmp = entry->nxt;
1026 		}
1027 		mutex_exit(&ufile->lock);
1028 	}
1029 }
1030 
1031 /*
1032  * Function:
1033  *      uverbs_release_usrq_uevents
1034  * Input:
1035  *	ufile	- A pointer to the asynchronous event file associated with a
1036  *		  SRQ.
1037  *	uqp	- A pointer to the user SRQ object.
1038  * Output:
1039  *      None
1040  * Returns:
1041  *      None
1042  * Description:
1043  *      Free any user asynchronous events that have been queued for the
1044  *	user SRQ object specified.
1045  */
1046 void
uverbs_release_usrq_uevents(uverbs_ufile_uobj_t * ufile,uverbs_usrq_uobj_t * usrq)1047 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile,
1048     uverbs_usrq_uobj_t *usrq)
1049 {
1050 	uverbs_event_t	*evt;
1051 	llist_head_t	*entry;
1052 	llist_head_t	*list_tmp;
1053 
1054 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1055 	    "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq);
1056 
1057 	if (ufile) {
1058 		mutex_enter(&ufile->lock);
1059 
1060 		entry = usrq->async_list.nxt;
1061 		list_tmp = entry->nxt;
1062 		while (entry != &usrq->async_list) {
1063 			ASSERT(entry);
1064 			evt = (uverbs_event_t *)entry->ptr;
1065 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1066 			    "RELEASE SRQ EVENTS: Deleting event "
1067 			    "on SRQ async list: %p", evt);
1068 			llist_del(&evt->ev_list);
1069 			llist_del(&evt->ev_obj_list);
1070 			kmem_free(evt, sizeof (*evt));
1071 			entry = list_tmp;
1072 			list_tmp = entry->nxt;
1073 		}
1074 		mutex_exit(&ufile->lock);
1075 	}
1076 }
1077