1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2017 Joyent, Inc.
25  */
26 
27 /*
28  * sol_uverbs_event.c
29  *
30  * OFED User Verbs Kernel Async Event funtions
31  *
32  */
33 #include <sys/file.h>
34 #include <sys/fcntl.h>
35 #include <sys/vfs.h>
36 #include <sys/errno.h>
37 #include <sys/cred.h>
38 #include <sys/uio.h>
39 #include <sys/semaphore.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42 
43 #include <sys/ib/ibtl/ibvti.h>
44 #include <sys/ib/clients/of/ofa_solaris.h>
45 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
46 #include <sys/ib/clients/of/ofed_kernel.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
49 
50 extern char	*sol_uverbs_dbg_str;
51 
52 static void
53 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t,
54     llist_head_t *, uint32_t *);
55 
56 /*
57  * Function:
58  *      sol_uverbs_event_file_close
59  * Input:
60  *      ufile	- Pointer to the event ufile
61  *
62  * Output:
63  *      None
64  * Returns:
65  *      Zero on success, else error code.
66  * Description:
67  *	Called when all kernel references to the event file have been
68  *	removed and the kernel (asynchronous) or user library (completion)
69  *	have closed the file.
70  */
71 void
72 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile)
73 {
74 	if (!ufile) {
75 		SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
76 		    "UFILE CLOSE: Ufile NULL\n");
77 		return;
78 	}
79 
80 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
81 	    "UFILE CLOSE: Is async? %s",
82 	    ufile->is_async ? "yes" : "no");
83 
84 	/*
85 	 * Remove the user file from the user object table and
86 	 * releases appropriate references.  The object resources
87 	 * are freed when it is no longer referenced.
88 	 *
89 	 * If sol_ofs_uobj_remove() returns NULL then the obj was already
90 	 * removed.
91 	 */
92 	rw_enter(&(ufile->uobj.uo_lock), RW_WRITER);
93 	if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) {
94 		rw_exit(&(ufile->uobj.uo_lock));
95 		sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file);
96 	} else {
97 		rw_exit(&(ufile->uobj.uo_lock));
98 	}
99 }
100 
101 /*
102  * Function:
103  *      sol_uverbs_event_file_read
104  * Input:
105  *      ufile	- The user file pointer of the event channel.
106  *      uiop	- The user I/O pointer in which to place the event.
107  *	cred	- Pointer to the callers credentials.
108  * Output:
109  *      uiop	- Upon success the caller's buffer has been updated with
110  *		  the event details.
111  * Returns:
112  *      Zero on success, else error code.
113  *		EAGAIN	- No event available and caller is non-
114  *			  blocking.
115  *		ERESTART- A signal was received.
116  *		EINVAL	- Caller parameter/user buffer invalid.
117  *		EFAULT	- Failure copying data to user.
118  * Description:
119  *      Perfrom a blocking read to retrieve an event (asynchronous or
120  *	completion) for the user file specified.  If an event is available it
121  *	is immediately returned. If an event is not available, then the
122  *	caller will block unless the open flags indicate it is a non-
123  *	blocking open.  If the caller does block it does so interruptable
124  *	and therefore will return upon an event or receipt of a signal.
125  */
126 /* ARGSUSED */
127 int
128 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop,
129     cred_t *cred)
130 {
131 	int			rc = 0;
132 	uverbs_event_t		*evt;
133 	llist_head_t		*entry;
134 	int			eventsz;
135 	int			ioflag;
136 
137 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), "
138 	    "ufile = %p, is_async =%d, uio_resid=%d",
139 	    ufile, ufile->is_async, uiop->uio_resid);
140 
141 	ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY);
142 
143 	mutex_enter(&ufile->lock);
144 
145 	/*
146 	 * If Event list not empty and CQ event notification is disabled
147 	 * by sol_ucma, do not  return events. Either return EAGAIN (if
148 	 * flag is O_NONBLOCK, or wait using cv_wait_sig().
149 	 */
150 	if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE &&
151 	    llist_empty(&ufile->event_list) != 0) {
152 		if (ioflag) {
153 			mutex_exit(&ufile->lock);
154 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
155 			    "event_file_read - notify disabled, no block");
156 			return (EAGAIN);
157 		}
158 
159 		if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
160 			mutex_exit(&ufile->lock);
161 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
162 			    "event_file_read - sig_wakeup");
163 			return (ERESTART);
164 		}
165 	}
166 
167 	while (llist_empty(&ufile->event_list)) {
168 		if (ioflag) {
169 			mutex_exit(&ufile->lock);
170 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
171 			    "event_file_read - no events, no block");
172 			return (EAGAIN);
173 		}
174 
175 		if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
176 			mutex_exit(&ufile->lock);
177 			SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
178 			    "event_file_read - sig_wakeup");
179 			return (ERESTART);
180 		}
181 	}
182 
183 	entry = ufile->event_list.nxt;
184 	evt   = entry->ptr;
185 
186 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: "
187 	    "Event entry found: entry:%p, event:%p, evt_list %p",
188 	    entry, evt, &evt->ev_list);
189 
190 	if (ufile->is_async) {
191 		eventsz = sizeof (struct ib_uverbs_async_event_desc);
192 	} else {
193 		eventsz = sizeof (struct ib_uverbs_comp_event_desc);
194 	}
195 
196 	if (eventsz > uiop->uio_resid) {
197 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
198 		    "event_file_read - Event too big");
199 		rc  = EINVAL;
200 		evt = NULL;
201 	} else {
202 		llist_del(ufile->event_list.nxt);
203 		if (evt->ev_counter) {
204 			++(*evt->ev_counter);
205 			llist_del(&evt->ev_obj_list);
206 		}
207 	}
208 
209 	mutex_exit(&ufile->lock);
210 
211 	if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) {
212 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
213 		    "EVENT FILE READ: Error writing ev");
214 		rc = EFAULT;
215 	}
216 
217 	if (evt) {
218 		kmem_free(evt, sizeof (*evt));
219 	}
220 
221 	return (rc);
222 }
223 
224 /*
225  * Function:
226  *      sol_uverbs_event_file_poll
227  * Input:
228  *      ufile	- user file for desired completion channel event file
229  *      events 	- The events that may occur.
230  *	anyyet	- A flag that is non-zero if any files in the set
231  *		  of descriptors has an event waiting.
232  *	ct	- Pointer to the callers context.
233  * Output:
234  *	reventssp - A pointer updated to return a bitmask of events specified.
235  *      phpp      - A pointer to a pollhead pointer, updated to reflect the
236  *		    the event file's pollhead used for synchronization.
237  * Returns:
238  *      Zero on success, else error code.
239  *		EINVAL    - Vnode does not point to valid event file.
240  * Description:
241  * 	Support for event channel polling interface, allows use of completion
242  *	channel in asynchronous type environment.  If events may be read
243  *      without blocking indicate a POLLIN | POLLRDNORM event; otherwise if
244  *	no other descriptors in the set have data waiting, set the pollhead
245  *	pointer to our the associated completion event file's pollhead.
246  */
247 int
248 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events,
249     int anyyet, short *reventsp, pollhead_t **phpp)
250 {
251 	short	revent = 0;
252 
253 #ifdef DEBUG
254 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)",
255 	    ufile, events);
256 #endif
257 
258 	if (!ufile) {
259 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ",
260 		    "ufile %p", ufile);
261 		return (EINVAL);
262 	}
263 
264 #ifdef DEBUG
265 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
266 	    "ufile = %p, is_async =%d", ufile, ufile->is_async);
267 #endif
268 
269 	mutex_enter(&ufile->lock);
270 
271 	/*
272 	 * If poll request and event is ready.
273 	 */
274 	if ((events & (POLLIN | POLLRDNORM)) &&
275 	    !llist_empty(&ufile->event_list)) {
276 
277 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
278 		    "Event entry available");
279 
280 		revent |= POLLIN | POLLRDNORM;
281 	}
282 
283 	/*
284 	 * If we didn't get an event or are edge-triggered
285 	 */
286 	if ((revent == 0 && !anyyet) || (events & POLLET)) {
287 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
288 		    "Event entry NOT available");
289 
290 		*phpp = &ufile->poll_head;
291 	}
292 
293 	mutex_exit(&ufile->lock);
294 	*reventsp = revent;
295 	return (0);
296 }
297 
298 /*
299  * Function:
300  *      uverbs_alloc_event_file
301  * Input:
302  *      uctxt	 - The Solaris User Verbs user context associated with the
303  *		   event channel.
304  *      is_async - Indicates the file is for asynchronous events if non-zero;
305  *                 other wise it is for completion events.
306  * Output:
307  *      None.
308  * Returns:
309  *      New user verb event file object or NULL on error.
310  * Description:
311  *	Allocate an asynchronous or completion event file
312  */
313 uverbs_ufile_uobj_t *
314 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async)
315 {
316 	uverbs_ufile_uobj_t	*ufile;
317 
318 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)",
319 	    uctxt, is_async);
320 
321 	ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP);
322 	if (!ufile) {
323 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
324 		    "alloc_event_file: mem alloc fail");
325 		return (NULL);
326 	}
327 	ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE;
328 	sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE);
329 	rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
330 
331 	if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) {
332 		/*
333 		 * The initialization routine set's the initial reference,
334 		 * we dereference the object here to clean it up.
335 		 */
336 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
337 		    "ALLOC UFILE: Object add failed");
338 		rw_exit(&ufile->uobj.uo_lock);
339 		ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t);
340 		sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free);
341 		return (NULL);
342 	}
343 
344 	ufile->is_async		= is_async ? 1 : 0;
345 	llist_head_init(&ufile->event_list, NULL);
346 
347 	mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL);
348 	cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL);
349 
350 	ufile->uctxt		= uctxt;
351 	ufile->uobj.uo_live	= 1;
352 	rw_exit(&ufile->uobj.uo_lock);
353 	return (ufile);
354 }
355 
356 /*
357  * Function:
358  *      uverbs_release_event_file
359  * Input:
360  *      ufile	 - Pointer to the ufile user object that is being freed.
361  * Output:
362  *      None.
363  * Returns:
364  *      None.
365  * Description:
366  *	Release/destroy event file resources before freeing memory.  This
367  *	routine should only be used if the event file is successfully
368  *	created.
369  */
370 void
371 uverbs_release_event_file(sol_ofs_uobj_t *uobj)
372 {
373 	uverbs_ufile_uobj_t	*ufile = (uverbs_ufile_uobj_t *)uobj;
374 	uverbs_event_t		*evt;
375 	llist_head_t		*entry;
376 	llist_head_t		*list_tmp;
377 
378 	if (!ufile) {
379 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
380 		    "UFILE RELEASE: Ufile NULL\n");
381 		return;
382 	}
383 
384 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
385 	    "UFILE RELEASE: Event file=%p, is async = %s",
386 	    ufile, ufile->is_async ? "yes" : "no");
387 
388 	/*
389 	 * Release any events still queued to the event file.
390 	 */
391 	mutex_enter(&ufile->lock);
392 
393 	entry = ufile->event_list.nxt;
394 	list_tmp = entry->nxt;
395 	while (entry != &ufile->event_list) {
396 		ASSERT(entry);
397 		evt = (uverbs_event_t *)entry->ptr;
398 
399 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
400 		    "UFILE RELEASE: Deleting event %p on event file %p",
401 		    evt, ufile);
402 
403 		llist_del(&evt->ev_list);
404 		kmem_free(evt, sizeof (*evt));
405 		entry = list_tmp;
406 		list_tmp = entry->nxt;
407 	}
408 
409 	mutex_exit(&ufile->lock);
410 
411 	cv_destroy(&ufile->poll_wait);
412 	mutex_destroy(&ufile->lock);
413 	sol_ofs_uobj_free(uobj);
414 }
415 
416 /*
417  * Function:
418  *      uverbs_ibt_to_ofa_event_code
419  * Input:
420  *      code 	- The OFA event code.
421  * Output:
422  *      The OFED event code.
423  * Returns:
424  *      Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if
425  *	a valid translation does not exist.
426  * Description:
427  *      Map an IBT asynchronous event code to an OFED event code.
428  */
429 enum ib_event_type
430 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)
431 {
432 	enum ib_event_type    ofa_code;
433 
434 	switch (code) {
435 	case IBT_EVENT_PATH_MIGRATED:
436 		ofa_code = IB_EVENT_PATH_MIG;
437 		break;
438 
439 	case IBT_EVENT_SQD:
440 		ofa_code = IB_EVENT_SQ_DRAINED;
441 		break;
442 
443 	case IBT_EVENT_COM_EST:
444 		ofa_code = IB_EVENT_COMM_EST;
445 		break;
446 
447 	case IBT_ERROR_CATASTROPHIC_CHAN:
448 	case IBT_ERROR_LOCAL_CATASTROPHIC:
449 		ofa_code = IB_EVENT_QP_FATAL;
450 		break;
451 
452 	case IBT_ERROR_INVALID_REQUEST_CHAN:
453 		ofa_code = IB_EVENT_QP_REQ_ERR;
454 		break;
455 
456 	case IBT_ERROR_ACCESS_VIOLATION_CHAN:
457 		ofa_code = IB_EVENT_QP_ACCESS_ERR;
458 		break;
459 
460 	case IBT_ERROR_PATH_MIGRATE_REQ:
461 		ofa_code = IB_EVENT_PATH_MIG_ERR;
462 		break;
463 
464 	case IBT_ERROR_CQ:
465 		ofa_code = IB_EVENT_CQ_ERR;
466 		break;
467 
468 	case IBT_EVENT_PORT_UP:
469 		ofa_code = IB_EVENT_PORT_ACTIVE;
470 		break;
471 
472 	case IBT_ERROR_PORT_DOWN:
473 		ofa_code = IB_EVENT_PORT_ERR;
474 		break;
475 
476 	case IBT_HCA_ATTACH_EVENT:
477 		ofa_code = IB_EVENT_CLIENT_REREGISTER;
478 		break;
479 
480 	case IBT_EVENT_LIMIT_REACHED_SRQ:
481 		ofa_code = IB_EVENT_SRQ_LIMIT_REACHED;
482 		break;
483 
484 	case IBT_ERROR_CATASTROPHIC_SRQ:
485 		ofa_code = IB_EVENT_SRQ_ERR;
486 		break;
487 
488 	case IBT_EVENT_EMPTY_CHAN:
489 		ofa_code = IB_EVENT_QP_LAST_WQE_REACHED;
490 		break;
491 
492 	/*
493 	 * No mapping exists.
494 	 */
495 	case IBT_ASYNC_OPAQUE1:
496 	case IBT_ASYNC_OPAQUE2:
497 	case IBT_ASYNC_OPAQUE3:
498 	case IBT_ASYNC_OPAQUE4:
499 	case IBT_HCA_DETACH_EVENT:
500 	default:
501 		ofa_code = -1;
502 	}
503 
504 	return (ofa_code);
505 }
506 
507 /*
508  * Function:
509  *      uverbs_async_qp_event_handler
510  * Input:
511  *      clnt_private	- The IBT attach client handle.
512  *	hca_hdl		- The IBT hca handle associated with the notification.
513  *	code		- The OFED event identifier.
514  *	event		- The IBT event.
515  * Output:
516  *      None
517  * Returns:
518  *      None
519  * Description:
520  *      Handle QP affiliated asynchronous event noficiation.
521  */
522 /* ARGSUSED */
523 void
524 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
525     enum ib_event_type code, ibt_async_event_t *event)
526 {
527 	uverbs_uqp_uobj_t	*uqp;
528 
529 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
530 	    "async_qp_event_handler()");
531 
532 	if (event->ev_chan_hdl == NULL) {
533 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
534 		    "async_qp_event_handler: event handle NULL");
535 		return;
536 	}
537 	uqp = ibt_get_qp_private(event->ev_chan_hdl);
538 	ASSERT(uqp);
539 	if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) {
540 		SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
541 		    "async_qp_event_handler: User QP context has been freed");
542 		return;
543 	}
544 	if (uqp->qp != event->ev_chan_hdl) {
545 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
546 		    "async_qp_event_handler: QP handle mismatch");
547 		return;
548 	}
549 
550 	uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle,
551 	    code, &uqp->async_list, &uqp->async_events_reported);
552 
553 }
554 
555 /*
556  * Function:
557  *      uverbs_async_cq_event_handler
558  * Input:
559  *      clnt_private	- The IBT attach client handle.
560  *	hca_hdl		- The IBT hca handle associated with the notification.
561  *	code		- The OFED event identifier.
562  *	event		- The IBT event.
563  * Output:
564  *      None
565  * Returns:
566  *      None
567  * Description:
568  *      Handle a CQ affiliated asynchronous event notification.
569  */
570 /* ARGSUSED */
571 void
572 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
573     enum ib_event_type code, ibt_async_event_t *event)
574 {
575 	uverbs_ucq_uobj_t	*ucq;
576 
577 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
578 	    "ASYNC CQ EVENT HANDLER:");
579 
580 	if (event->ev_cq_hdl == NULL) {
581 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
582 		    "ASYNC CQ EVENT HANDLER: event handle is NULL");
583 		return;
584 	}
585 
586 	ucq = ibt_get_cq_private(event->ev_cq_hdl);
587 	if (ucq->cq != event->ev_cq_hdl) {
588 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
589 		    "ASYNC CQ EVENT HANDLER: CQ handle mismatch");
590 		return;
591 	}
592 
593 	uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle,
594 	    code, &ucq->async_list, &ucq->async_events_reported);
595 }
596 
597 /*
598  * Function:
599  *      uverbs_async_srq_event_handler
600  * Input:
601  *      clnt_private	- The IBT attach client handle.
602  *	hca_hdl		- The IBT hca handle associated with the notification.
603  *	code		- The OFED event identifier.
604  *	event		- The IBT event.
605  * Output:
606  *      None
607  * Returns:
608  *      None
609  * Description:
610  *      Handle a shared receive queue asynchronous event notification.
611  */
612 /* ARGSUSED */
613 void
614 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
615     enum ib_event_type code, ibt_async_event_t *event)
616 {
617 	uverbs_usrq_uobj_t	*usrq;
618 
619 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
620 	    "ASYNC SRQ EVENT HANDLER:");
621 
622 	if (event->ev_srq_hdl == NULL) {
623 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
624 		    "ASYNC SRQ EVENT HANDLER: event handle is NULL");
625 		return;
626 	}
627 
628 	usrq = ibt_get_srq_private(event->ev_srq_hdl);
629 	if (usrq->srq != event->ev_srq_hdl) {
630 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
631 		    "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch");
632 		return;
633 	}
634 
635 	uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle,
636 	    code, &usrq->async_list, &usrq->async_events_reported);
637 }
638 
639 /*
640  * Function:
641  *      uverbs_async_unaff_event_handler
642  * Input:
643  *      clnt_private	- The IBT attach client handle.
644  *	hca_hdl		- The IBT hca handle associated with the notification.
645  *	code		- The OFED event identifier.
646  *	event		- The IBT event.
647  * Output:
648  *      None
649  * Returns:
650  *      None
651  * Description:
652  *      Handle an unaffiliated asynchronous event notification.
653  */
654 /* ARGSUSED */
655 void
656 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
657     enum ib_event_type code, ibt_async_event_t *event)
658 {
659 	sol_ofs_uobj_table_t	*uo_tbl = &uverbs_uctxt_uo_tbl;
660 	sol_ofs_uobj_blk_t	*blk;
661 	uverbs_uctxt_uobj_t	*uctxt;
662 	int			i, j;
663 
664 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
665 	    "ASYNC UNAFF EVENT HANDLER:");
666 
667 	/*
668 	 * Unaffiliated events are returned at the IBT client level.  We must
669 	 * return the event to all user context allocated for the specific
670 	 * HCA device specified.
671 	 */
672 	rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER);
673 
674 	for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) {
675 		blk = uo_tbl->uobj_tbl_uo_root[i];
676 		if (blk == NULL) {
677 			continue;
678 		}
679 		for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) {
680 			uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j];
681 			if (uctxt == NULL) {
682 				continue;
683 			}
684 			/*
685 			 * OK, check to see if this user context belongs
686 			 * to the idicated hca.
687 			 */
688 			if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) {
689 				uverbs_async_event_common(uctxt,
690 				    event->ev_port, code, NULL, NULL);
691 			}
692 		}
693 	}
694 	rw_exit(&uo_tbl->uobj_tbl_lock);
695 }
696 
697 /*
698  * Function:
699  *      uverbs_async_event_handler
700  * Input:
701  *      clnt_private	- The IBT attach client handle.
702  *	hca_hdl		- The IBT hca handle associated with the notification.
703  *	code		- The OFED event identifier.
704  *	event		- The IBT event.
705  * Output:
706  *      None
707  * Returns:
708  *      None
709  * Description:
710  *	Main IBT asynchronous event handler registered at ibt_attach.
711  *      Convert to OFA event type and forward to the appropriate
712  *      asynchronous handler.
713  */
714 void
715 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
716     ibt_async_code_t code, ibt_async_event_t *event)
717 {
718 	enum ib_event_type		ofa_type;
719 	sol_uverbs_ib_event_handler_t	*handler;
720 	llist_head_t			*entry;
721 	sol_uverbs_hca_t		*hca;
722 
723 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
724 	    "ASYNNC EVENT HANDLER: entry, code=%d", code);
725 
726 	ofa_type = uverbs_ibt_to_ofa_event_code(code);
727 
728 	if ((int)ofa_type < 0) {
729 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
730 		    "ASYNC EVENT HANDLER:Event %d did not map to OFA "
731 		    "Event", code);
732 		return;
733 	}
734 
735 	switch (ofa_type) {
736 		case IB_EVENT_QP_FATAL:
737 		case IB_EVENT_QP_REQ_ERR:
738 		case IB_EVENT_QP_ACCESS_ERR:
739 		case IB_EVENT_QP_LAST_WQE_REACHED:
740 		case IB_EVENT_SQ_DRAINED:
741 		case IB_EVENT_PATH_MIG:
742 			/*
743 			 * These events are related with a QP
744 			 */
745 			uverbs_async_qp_event_handler(clnt_private, hca_hdl,
746 			    ofa_type, event);
747 			break;
748 
749 		case IB_EVENT_CQ_ERR:
750 			/*
751 			 * These events are related with a CQ
752 			 */
753 			uverbs_async_cq_event_handler(clnt_private, hca_hdl,
754 			    ofa_type, event);
755 			break;
756 
757 		case IB_EVENT_SRQ_ERR:
758 		case IB_EVENT_SRQ_LIMIT_REACHED:
759 			/*
760 			 * These events are related with a SRQ
761 			 */
762 			uverbs_async_srq_event_handler(clnt_private, hca_hdl,
763 			    ofa_type, event);
764 			break;
765 
766 
767 		case IB_EVENT_PORT_ERR:
768 		case IB_EVENT_PORT_ACTIVE:
769 		case IB_EVENT_LID_CHANGE:
770 		case IB_EVENT_PKEY_CHANGE:
771 		case IB_EVENT_SM_CHANGE:
772 		case IB_EVENT_CLIENT_REREGISTER:
773 		case IB_EVENT_DEVICE_FATAL:
774 		case IB_EVENT_PATH_MIG_ERR:
775 			/*
776 			 * Unaffiliated asynchronous notifications.
777 			 */
778 			uverbs_async_unaff_event_handler(clnt_private, hca_hdl,
779 			    ofa_type, event);
780 			break;
781 
782 		default:
783 		break;
784 	}
785 
786 	/*
787 	 * Give other kernel agents a notification.
788 	 */
789 	hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl);
790 	if (hca) {
791 		mutex_enter(&hca->event_handler_lock);
792 		list_for_each(entry, &hca->event_handler_list) {
793 			handler = (sol_uverbs_ib_event_handler_t *)entry->ptr;
794 
795 			ASSERT(handler != NULL);
796 			handler->handler(handler, hca_hdl, code, event);
797 		}
798 		mutex_exit(&hca->event_handler_lock);
799 	}
800 }
801 
802 /*
803  * Function:
804  *      uverbs_async_event_common
805  * Input:
806  *      uctxt       	- Pointer to the user context associated with the
807  *			affiliated event.
808  *	element 	- The users handle to the associated object.
809  *	event           - The event type.
810  *	uobj_list       - The list to enqueue the asynchronous event.
811  *      counter         - The counter to track the event delivery.
812  * Output:
813  *      None
814  * Returns:
815  *      None
816  * Description:
817  *      Create an asyncronous event and enqueue it on the specified list.
818  *      Then wake callers that may  be blocked on the list.
819  */
820 static void
821 uverbs_async_event_common(uverbs_uctxt_uobj_t  *uctxt, uint64_t element,
822     uint32_t event, llist_head_t *obj_list, uint32_t *counter)
823 {
824 	uverbs_ufile_uobj_t	*ufile = uctxt->async_evfile;
825 	uverbs_event_t		*entry;
826 
827 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, "
828 	    "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter);
829 
830 	if (!ufile) {
831 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
832 		    "ufile %p", ufile);
833 		return;
834 	}
835 
836 	mutex_enter(&ufile->lock);
837 	entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
838 	if (!entry) {
839 		mutex_exit(&ufile->lock);
840 		SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
841 		    "kmem_zalloc failed");
842 		return;
843 	}
844 
845 	entry->ev_desc.async.element    = element;
846 	entry->ev_desc.async.event_type = event;
847 	entry->ev_counter		= counter;
848 
849 	llist_head_init(&entry->ev_list, entry);
850 	llist_head_init(&entry->ev_obj_list, entry);
851 
852 	llist_add_tail(&entry->ev_list, &ufile->event_list);
853 
854 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
855 	    "adding ASYNC entry-ev_list=%p, entry %p",
856 	    &entry->ev_list, entry);
857 
858 	if (obj_list) {
859 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
860 		    "adding ASYNC entry-ev_obj_list=%p, entry=%p",
861 		    &entry->ev_obj_list, entry);
862 		llist_add_tail(&entry->ev_obj_list, obj_list);
863 	}
864 
865 	mutex_exit(&ufile->lock);
866 	cv_signal(&ufile->poll_wait);
867 	pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
868 }
869 
870 /*
871  * Function:
872  *      uverbs_release_ucq_channel
873  * Input:
874  *	uctxt	- A pointer to the callers user context.
875  *	ufile - A pointer to the event file associated with a CQ.
876  *	ucq	- A pointer to the user CQ object.
877  * Output:
878  *      None
879  * Returns:
880  *      None
881  * Description:
882  * 	Release any completion and asynchronous events that may
883  *	be queued to the specified completion channel/UCQ but not
884  *	yet reaped.
885  */
886 void
887 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt,
888     uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t   *ucq)
889 {
890 	uverbs_event_t	*evt;
891 	llist_head_t	*entry;
892 	llist_head_t	*list_tmp;
893 
894 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
895 	    "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p",
896 	    uctxt, ufile, ucq);
897 
898 	/*
899 	 * Release completion events that have been queued on the CQ completion
900 	 * eventlist.
901 	 */
902 	if (ufile) {
903 		rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
904 		ufile->ufile_cq_cnt--;
905 		if (ufile->ufile_cq_cnt) {
906 			rw_exit(&ufile->uobj.uo_lock);
907 			uverbs_release_ucq_uevents(ufile, ucq);
908 			return;
909 		}
910 		rw_exit(&ufile->uobj.uo_lock);
911 		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
912 		    "release_ucq_chan : comp_list %p, prv %p, nxt %p",
913 		    &ucq->comp_list, ucq->comp_list.prv,
914 		    ucq->comp_list.nxt);
915 		mutex_enter(&ufile->lock);
916 
917 		entry = ucq->comp_list.nxt;
918 		list_tmp = entry->nxt;
919 		while (entry != &ucq->comp_list) {
920 			ASSERT(entry);
921 			evt = (uverbs_event_t *)entry->ptr;
922 
923 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
924 			    "RELEASE UCQ CHANNEL:Deleting event "
925 			    "on CQ comp list: %p", evt);
926 			llist_del(&evt->ev_list);
927 			llist_del(&evt->ev_obj_list);
928 			kmem_free(evt, sizeof (*evt));
929 			entry = list_tmp;
930 			list_tmp = entry->nxt;
931 		}
932 		mutex_exit(&ufile->lock);
933 
934 		uverbs_release_ucq_uevents(ufile, ucq);
935 	}
936 
937 }
938 
939 /*
940  * Function:
941  *      uverbs_release_ucq_uevents
942  * Input:
943  *	ufile	- A pointer to the asynchronous event file associated with a QP.
944  *	ucq	- A pointer to the user CQ object.
945  * Output:
946  *      None
947  * Returns:
948  *      None
949  * Description:
950  *      Free any user asynchronous events that have been queued for the
951  *	user CQ object specified.
952  */
953 void
954 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq)
955 {
956 	uverbs_event_t	*evt;
957 	llist_head_t	*entry;
958 	llist_head_t	*list_tmp;
959 
960 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
961 	    "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq);
962 
963 	if (ufile) {
964 		mutex_enter(&ufile->lock);
965 
966 		entry = ucq->async_list.nxt;
967 		list_tmp = entry->nxt;
968 		while (entry != &ucq->async_list) {
969 			ASSERT(entry);
970 			evt = (uverbs_event_t *)entry->ptr;
971 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
972 			    "RELEASE UCQ EVENTS: Deleting event "
973 			    "on CQ async list: %p", evt);
974 			llist_del(&evt->ev_list);
975 			llist_del(&evt->ev_obj_list);
976 			kmem_free(evt, sizeof (*evt));
977 			entry = list_tmp;
978 			list_tmp = entry->nxt;
979 		}
980 		mutex_exit(&ufile->lock);
981 	}
982 }
983 
984 /*
985  * Function:
986  *      uverbs_release_uqp_uevents
987  * Input:
988  *	ufile	- A pointer to the asynchronous event file associated with a QP.
989  *	uqp	- A pointer to the user QP object.
990  * Output:
991  *      None
992  * Returns:
993  *      None
994  * Description:
995  *      Free any user asynchronous events that have been queued for the
996  *	user QP object specified.
997  */
998 void
999 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp)
1000 {
1001 	uverbs_event_t	*evt;
1002 	llist_head_t	*entry;
1003 	llist_head_t	*list_tmp;
1004 
1005 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1006 	    "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp);
1007 
1008 	if (ufile) {
1009 		mutex_enter(&ufile->lock);
1010 		entry = uqp->async_list.nxt;
1011 		list_tmp = entry->nxt;
1012 		while (entry != &uqp->async_list) {
1013 			ASSERT(entry);
1014 			evt = (uverbs_event_t *)entry->ptr;
1015 			ASSERT(evt);
1016 
1017 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1018 			    "RELEASE UQP EVENTS: Deleting event "
1019 			    "ON qp async list: %p", evt);
1020 			llist_del(&evt->ev_list);
1021 			llist_del(&evt->ev_obj_list);
1022 			kmem_free(evt, sizeof (*evt));
1023 			entry = list_tmp;
1024 			list_tmp = entry->nxt;
1025 		}
1026 		mutex_exit(&ufile->lock);
1027 	}
1028 }
1029 
1030 /*
1031  * Function:
1032  *      uverbs_release_usrq_uevents
1033  * Input:
1034  *	ufile	- A pointer to the asynchronous event file associated with a
1035  *		  SRQ.
1036  *	uqp	- A pointer to the user SRQ object.
1037  * Output:
1038  *      None
1039  * Returns:
1040  *      None
1041  * Description:
1042  *      Free any user asynchronous events that have been queued for the
1043  *	user SRQ object specified.
1044  */
1045 void
1046 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile,
1047     uverbs_usrq_uobj_t *usrq)
1048 {
1049 	uverbs_event_t	*evt;
1050 	llist_head_t	*entry;
1051 	llist_head_t	*list_tmp;
1052 
1053 	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1054 	    "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq);
1055 
1056 	if (ufile) {
1057 		mutex_enter(&ufile->lock);
1058 
1059 		entry = usrq->async_list.nxt;
1060 		list_tmp = entry->nxt;
1061 		while (entry != &usrq->async_list) {
1062 			ASSERT(entry);
1063 			evt = (uverbs_event_t *)entry->ptr;
1064 			SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1065 			    "RELEASE SRQ EVENTS: Deleting event "
1066 			    "on SRQ async list: %p", evt);
1067 			llist_del(&evt->ev_list);
1068 			llist_del(&evt->ev_obj_list);
1069 			kmem_free(evt, sizeof (*evt));
1070 			entry = list_tmp;
1071 			list_tmp = entry->nxt;
1072 		}
1073 		mutex_exit(&ufile->lock);
1074 	}
1075 }
1076