1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * sol_uverbs_event.c 28 * 29 * OFED User Verbs Kernel Async Event funtions 30 * 31 */ 32 #include <sys/file.h> 33 #include <sys/fcntl.h> 34 #include <sys/vfs.h> 35 #include <sys/errno.h> 36 #include <sys/cred.h> 37 #include <sys/uio.h> 38 #include <sys/semaphore.h> 39 #include <sys/ddi.h> 40 #include <sys/sunddi.h> 41 42 #include <sys/ib/ibtl/ibvti.h> 43 #include <sys/ib/clients/of/ofa_solaris.h> 44 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 45 #include <sys/ib/clients/of/ofed_kernel.h> 46 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h> 47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h> 48 49 extern char *sol_uverbs_dbg_str; 50 51 static void 52 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t, 53 llist_head_t *, uint32_t *); 54 55 /* 56 * Function: 57 * sol_uverbs_event_file_close 58 * Input: 59 * ufile - Pointer to the event ufile 60 * 61 * Output: 62 * None 63 * Returns: 64 * Zero on success, else error code. 65 * Description: 66 * Called when all kernel references to the event file have been 67 * removed and the kernel (asynchronous) or user library (completion) 68 * have closed the file. 69 */ 70 void 71 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile) 72 { 73 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 74 "UFILE CLOSE: Is async? %s", 75 ufile->is_async ? "yes" : "no"); 76 77 if (!ufile) { 78 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 79 "UFILE CLOSE: Ufile NULL\n"); 80 return; 81 } 82 83 /* 84 * Remove the user file from the user object table and 85 * releases appropriate references. The object resources 86 * are freed when it is no longer referenced. 87 * 88 * If sol_ofs_uobj_remove() returns NULL then the obj was already 89 * removed. 90 */ 91 rw_enter(&(ufile->uobj.uo_lock), RW_WRITER); 92 if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) { 93 rw_exit(&(ufile->uobj.uo_lock)); 94 sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file); 95 } else { 96 rw_exit(&(ufile->uobj.uo_lock)); 97 } 98 } 99 100 /* 101 * Function: 102 * sol_uverbs_event_file_read 103 * Input: 104 * ufile - The user file pointer of the event channel. 105 * uiop - The user I/O pointer in which to place the event. 106 * cred - Pointer to the callers credentials. 107 * Output: 108 * uiop - Upon success the caller's buffer has been updated with 109 * the event details. 110 * Returns: 111 * Zero on success, else error code. 112 * EAGAIN - No event available and caller is non- 113 * blocking. 114 * ERESTART- A signal was received. 115 * EINVAL - Caller parameter/user buffer invalid. 116 * EFAULT - Failure copying data to user. 117 * Description: 118 * Perfrom a blocking read to retrieve an event (asynchronous or 119 * completion) for the user file specified. If an event is available it 120 * is immediately returned. If an event is not available, then the 121 * caller will block unless the open flags indicate it is a non- 122 * blocking open. If the caller does block it does so interruptable 123 * and therefore will return upon an event or receipt of a signal. 124 */ 125 /* ARGSUSED */ 126 int 127 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop, 128 cred_t *cred) 129 { 130 int rc = 0; 131 uverbs_event_t *evt; 132 llist_head_t *entry; 133 int eventsz; 134 int ioflag; 135 136 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), " 137 "ufile = %p, is_async =%d, uio_resid=%d", 138 ufile, ufile->is_async, uiop->uio_resid); 139 140 ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY); 141 142 mutex_enter(&ufile->lock); 143 144 /* 145 * If Event list not empty and CQ event notification is disabled 146 * by sol_ucma, do not return events. Either return EAGAIN (if 147 * flag is O_NONBLOCK, or wait using cv_wait_sig(). 148 */ 149 if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE && 150 llist_empty(&ufile->event_list) != 0) { 151 if (ioflag) { 152 mutex_exit(&ufile->lock); 153 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 154 "event_file_read - notify disabled, no block"); 155 return (EAGAIN); 156 } 157 158 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 159 mutex_exit(&ufile->lock); 160 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 161 "event_file_read - sig_wakeup"); 162 return (ERESTART); 163 } 164 } 165 166 while (llist_empty(&ufile->event_list)) { 167 if (ioflag) { 168 mutex_exit(&ufile->lock); 169 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 170 "event_file_read - no events, no block"); 171 return (EAGAIN); 172 } 173 174 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 175 mutex_exit(&ufile->lock); 176 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 177 "event_file_read - sig_wakeup"); 178 return (ERESTART); 179 } 180 } 181 182 entry = ufile->event_list.nxt; 183 evt = entry->ptr; 184 185 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: " 186 "Event entry found: entry:%p, event:%p, evt_list %p", 187 entry, evt, &evt->ev_list); 188 189 if (ufile->is_async) { 190 eventsz = sizeof (struct ib_uverbs_async_event_desc); 191 } else { 192 eventsz = sizeof (struct ib_uverbs_comp_event_desc); 193 } 194 195 if (eventsz > uiop->uio_resid) { 196 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 197 "event_file_read - Event too big"); 198 rc = EINVAL; 199 evt = NULL; 200 } else { 201 llist_del(ufile->event_list.nxt); 202 if (evt->ev_counter) { 203 ++(*evt->ev_counter); 204 llist_del(&evt->ev_obj_list); 205 } 206 } 207 208 mutex_exit(&ufile->lock); 209 210 if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) { 211 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 212 "EVENT FILE READ: Error writing ev"); 213 rc = EFAULT; 214 } 215 216 if (evt) { 217 kmem_free(evt, sizeof (*evt)); 218 } 219 220 return (rc); 221 } 222 223 /* 224 * Function: 225 * sol_uverbs_event_file_poll 226 * Input: 227 * ufile - user file for desired completion channel event file 228 * events - The events that may occur. 229 * anyyet - A flag that is non-zero if any files in the set 230 * of descriptors has an event waiting. 231 * ct - Pointer to the callers context. 232 * Output: 233 * reventssp - A pointer updated to return a bitmask of events specified. 234 * phpp - A pointer to a pollhead pointer, updated to reflect the 235 * the event file's pollhead used for synchronization. 236 * Returns: 237 * Zero on success, else error code. 238 * EINVAL - Vnode does not point to valid event file. 239 * Description: 240 * Support for event channel polling interface, allows use of completion 241 * channel in asynchronous type environment. If events may be read 242 * without blocking indicate a POLLIN | POLLRDNORM event; otherwise if 243 * no other descriptors in the set have data waiting, set the pollhead 244 * pointer to our the associated completion event file's pollhead. 245 */ 246 int 247 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events, 248 int anyyet, short *reventsp, pollhead_t **phpp) 249 { 250 short revent = 0; 251 252 #ifdef DEBUG 253 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)", 254 ufile, events); 255 #endif 256 257 if (!ufile) { 258 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ", 259 "ufile %p", ufile); 260 return (EINVAL); 261 } 262 263 #ifdef DEBUG 264 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 265 "ufile = %p, is_async =%d", ufile, ufile->is_async); 266 #endif 267 268 mutex_enter(&ufile->lock); 269 270 /* 271 * If poll request and event is ready. 272 */ 273 if ((events & (POLLIN | POLLRDNORM)) && 274 !llist_empty(&ufile->event_list)) { 275 276 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 277 "Event entry available"); 278 279 revent |= POLLIN | POLLRDNORM; 280 } 281 282 /* 283 * If we didn't get an event 284 */ 285 if (revent == 0 && !anyyet) { 286 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 287 "Event entry NOT available"); 288 289 *phpp = &ufile->poll_head; 290 } 291 292 mutex_exit(&ufile->lock); 293 *reventsp = revent; 294 return (0); 295 } 296 297 /* 298 * Function: 299 * uverbs_alloc_event_file 300 * Input: 301 * uctxt - The Solaris User Verbs user context associated with the 302 * event channel. 303 * is_async - Indicates the file is for asynchronous events if non-zero; 304 * other wise it is for completion events. 305 * Output: 306 * None. 307 * Returns: 308 * New user verb event file object or NULL on error. 309 * Description: 310 * Allocate an asynchronous or completion event file 311 */ 312 uverbs_ufile_uobj_t * 313 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async) 314 { 315 uverbs_ufile_uobj_t *ufile; 316 317 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)", 318 uctxt, is_async); 319 320 ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP); 321 if (!ufile) { 322 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 323 "alloc_event_file: mem alloc fail"); 324 return (NULL); 325 } 326 ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE; 327 sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE); 328 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 329 330 if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) { 331 /* 332 * The initialization routine set's the initial reference, 333 * we dereference the object here to clean it up. 334 */ 335 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 336 "ALLOC UFILE: Object add failed"); 337 rw_exit(&ufile->uobj.uo_lock); 338 ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t); 339 sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free); 340 return (NULL); 341 } 342 343 ufile->is_async = is_async ? 1 : 0; 344 llist_head_init(&ufile->event_list, NULL); 345 346 mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL); 347 cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL); 348 349 ufile->uctxt = uctxt; 350 ufile->uobj.uo_live = 1; 351 rw_exit(&ufile->uobj.uo_lock); 352 return (ufile); 353 354 free_uobj: 355 /* 356 * Need to set uo_live, so sol_ofs_uobj_remove() will 357 * remove the object from the object table. 358 */ 359 ufile->uobj.uo_live = 1; 360 (void) sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj); 361 rw_exit(&ufile->uobj.uo_lock); 362 sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free); 363 364 return (NULL); 365 } 366 367 /* 368 * Function: 369 * uverbs_release_event_file 370 * Input: 371 * ufile - Pointer to the ufile user object that is being freed. 372 * Output: 373 * None. 374 * Returns: 375 * None. 376 * Description: 377 * Release/destroy event file resources before freeing memory. This 378 * routine should only be used if the event file is successfully 379 * created. 380 */ 381 void 382 uverbs_release_event_file(sol_ofs_uobj_t *uobj) 383 { 384 uverbs_ufile_uobj_t *ufile = (uverbs_ufile_uobj_t *)uobj; 385 uverbs_event_t *evt; 386 llist_head_t *entry; 387 llist_head_t *list_tmp; 388 389 if (!ufile) { 390 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 391 "UFILE RELEASE: Ufile NULL\n"); 392 return; 393 } 394 395 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 396 "UFILE RELEASE: Event file=%p, is async = %s", 397 ufile, ufile->is_async ? "yes" : "no"); 398 399 /* 400 * Release any events still queued to the event file. 401 */ 402 mutex_enter(&ufile->lock); 403 404 entry = ufile->event_list.nxt; 405 list_tmp = entry->nxt; 406 while (entry != &ufile->event_list) { 407 ASSERT(entry); 408 evt = (uverbs_event_t *)entry->ptr; 409 410 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 411 "UFILE RELEASE: Deleting event %p on event file %p", 412 evt, ufile); 413 414 llist_del(&evt->ev_list); 415 kmem_free(evt, sizeof (*evt)); 416 entry = list_tmp; 417 list_tmp = entry->nxt; 418 } 419 420 mutex_exit(&ufile->lock); 421 422 cv_destroy(&ufile->poll_wait); 423 mutex_destroy(&ufile->lock); 424 sol_ofs_uobj_free(uobj); 425 } 426 427 /* 428 * Function: 429 * uverbs_ibt_to_ofa_event_code 430 * Input: 431 * code - The OFA event code. 432 * Output: 433 * The OFED event code. 434 * Returns: 435 * Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if 436 * a valid translation does not exist. 437 * Description: 438 * Map an IBT asynchronous event code to an OFED event code. 439 */ 440 enum ib_event_type 441 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code) 442 { 443 enum ib_event_type ofa_code; 444 445 switch (code) { 446 case IBT_EVENT_PATH_MIGRATED: 447 ofa_code = IB_EVENT_PATH_MIG; 448 break; 449 450 case IBT_EVENT_SQD: 451 ofa_code = IB_EVENT_SQ_DRAINED; 452 break; 453 454 case IBT_EVENT_COM_EST: 455 break; 456 457 case IBT_ERROR_CATASTROPHIC_CHAN: 458 case IBT_ERROR_LOCAL_CATASTROPHIC: 459 ofa_code = IB_EVENT_QP_FATAL; 460 break; 461 462 case IBT_ERROR_INVALID_REQUEST_CHAN: 463 ofa_code = IB_EVENT_QP_REQ_ERR; 464 break; 465 466 case IBT_ERROR_ACCESS_VIOLATION_CHAN: 467 ofa_code = IB_EVENT_QP_ACCESS_ERR; 468 break; 469 470 case IBT_ERROR_PATH_MIGRATE_REQ: 471 ofa_code = IB_EVENT_PATH_MIG_ERR; 472 break; 473 474 case IBT_ERROR_CQ: 475 ofa_code = IB_EVENT_CQ_ERR; 476 break; 477 478 case IBT_EVENT_PORT_UP: 479 ofa_code = IB_EVENT_PORT_ACTIVE; 480 break; 481 482 case IBT_ERROR_PORT_DOWN: 483 ofa_code = IB_EVENT_PORT_ERR; 484 break; 485 486 case IBT_HCA_ATTACH_EVENT: 487 ofa_code = IB_EVENT_CLIENT_REREGISTER; 488 break; 489 490 case IBT_EVENT_LIMIT_REACHED_SRQ: 491 ofa_code = IB_EVENT_SRQ_LIMIT_REACHED; 492 break; 493 494 case IBT_ERROR_CATASTROPHIC_SRQ: 495 ofa_code = IB_EVENT_SRQ_ERR; 496 break; 497 498 case IBT_EVENT_EMPTY_CHAN: 499 ofa_code = IB_EVENT_QP_LAST_WQE_REACHED; 500 break; 501 502 /* 503 * No mapping exists. 504 */ 505 case IBT_ASYNC_OPAQUE1: 506 case IBT_ASYNC_OPAQUE2: 507 case IBT_ASYNC_OPAQUE3: 508 case IBT_ASYNC_OPAQUE4: 509 case IBT_HCA_DETACH_EVENT: 510 default: 511 ofa_code = -1; 512 } 513 514 return (ofa_code); 515 } 516 517 /* 518 * Function: 519 * uverbs_async_qp_event_handler 520 * Input: 521 * clnt_private - The IBT attach client handle. 522 * hca_hdl - The IBT hca handle associated with the notification. 523 * code - The OFED event identifier. 524 * event - The IBT event. 525 * Output: 526 * None 527 * Returns: 528 * None 529 * Description: 530 * Handle QP affiliated asynchronous event noficiation. 531 */ 532 /* ARGSUSED */ 533 void 534 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 535 enum ib_event_type code, ibt_async_event_t *event) 536 { 537 uverbs_uqp_uobj_t *uqp; 538 539 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 540 "async_qp_event_handler()"); 541 542 if (event->ev_chan_hdl == NULL) { 543 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 544 "async_qp_event_handler: event handle NULL"); 545 return; 546 } 547 uqp = ibt_get_qp_private(event->ev_chan_hdl); 548 ASSERT(uqp); 549 if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) { 550 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 551 "async_qp_event_handler: User QP context has been freed"); 552 return; 553 } 554 if (uqp->qp != event->ev_chan_hdl) { 555 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 556 "async_qp_event_handler: QP handle mismatch"); 557 return; 558 } 559 560 uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle, 561 code, &uqp->async_list, &uqp->async_events_reported); 562 563 } 564 565 /* 566 * Function: 567 * uverbs_async_cq_event_handler 568 * Input: 569 * clnt_private - The IBT attach client handle. 570 * hca_hdl - The IBT hca handle associated with the notification. 571 * code - The OFED event identifier. 572 * event - The IBT event. 573 * Output: 574 * None 575 * Returns: 576 * None 577 * Description: 578 * Handle a CQ affiliated asynchronous event notification. 579 */ 580 /* ARGSUSED */ 581 void 582 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 583 enum ib_event_type code, ibt_async_event_t *event) 584 { 585 uverbs_ucq_uobj_t *ucq; 586 587 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 588 "ASYNC CQ EVENT HANDLER:"); 589 590 if (event->ev_cq_hdl == NULL) { 591 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 592 "ASYNC CQ EVENT HANDLER: event handle is NULL"); 593 return; 594 } 595 596 ucq = ibt_get_cq_private(event->ev_cq_hdl); 597 if (ucq->cq != event->ev_cq_hdl) { 598 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 599 "ASYNC CQ EVENT HANDLER: CQ handle mismatch"); 600 return; 601 } 602 603 uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle, 604 code, &ucq->async_list, &ucq->async_events_reported); 605 } 606 607 /* 608 * Function: 609 * uverbs_async_srq_event_handler 610 * Input: 611 * clnt_private - The IBT attach client handle. 612 * hca_hdl - The IBT hca handle associated with the notification. 613 * code - The OFED event identifier. 614 * event - The IBT event. 615 * Output: 616 * None 617 * Returns: 618 * None 619 * Description: 620 * Handle a shared receive queue asynchronous event notification. 621 */ 622 /* ARGSUSED */ 623 void 624 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 625 enum ib_event_type code, ibt_async_event_t *event) 626 { 627 uverbs_usrq_uobj_t *usrq; 628 629 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 630 "ASYNC SRQ EVENT HANDLER:"); 631 632 if (event->ev_srq_hdl == NULL) { 633 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 634 "ASYNC SRQ EVENT HANDLER: event handle is NULL"); 635 return; 636 } 637 638 usrq = ibt_get_srq_private(event->ev_srq_hdl); 639 if (usrq->srq != event->ev_srq_hdl) { 640 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 641 "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch"); 642 return; 643 } 644 645 uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle, 646 code, &usrq->async_list, &usrq->async_events_reported); 647 } 648 649 /* 650 * Function: 651 * uverbs_async_unaff_event_handler 652 * Input: 653 * clnt_private - The IBT attach client handle. 654 * hca_hdl - The IBT hca handle associated with the notification. 655 * code - The OFED event identifier. 656 * event - The IBT event. 657 * Output: 658 * None 659 * Returns: 660 * None 661 * Description: 662 * Handle an unaffiliated asynchronous event notification. 663 */ 664 /* ARGSUSED */ 665 void 666 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 667 enum ib_event_type code, ibt_async_event_t *event) 668 { 669 sol_ofs_uobj_table_t *uo_tbl = &uverbs_uctxt_uo_tbl; 670 sol_ofs_uobj_blk_t *blk; 671 uverbs_uctxt_uobj_t *uctxt; 672 int i, j; 673 674 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 675 "ASYNC UNAFF EVENT HANDLER:"); 676 677 /* 678 * Unaffiliated events are returned at the IBT client level. We must 679 * return the event to all user context allocated for the specific 680 * HCA device specified. 681 */ 682 rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER); 683 684 for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) { 685 blk = uo_tbl->uobj_tbl_uo_root[i]; 686 if (blk == NULL) { 687 continue; 688 } 689 for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) { 690 uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j]; 691 if (uctxt == NULL) { 692 continue; 693 } 694 /* 695 * OK, check to see if this user context belongs 696 * to the idicated hca. 697 */ 698 if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) { 699 uverbs_async_event_common(uctxt, 700 event->ev_port, code, NULL, NULL); 701 } 702 } 703 } 704 rw_exit(&uo_tbl->uobj_tbl_lock); 705 } 706 707 /* 708 * Function: 709 * uverbs_async_event_handler 710 * Input: 711 * clnt_private - The IBT attach client handle. 712 * hca_hdl - The IBT hca handle associated with the notification. 713 * code - The OFED event identifier. 714 * event - The IBT event. 715 * Output: 716 * None 717 * Returns: 718 * None 719 * Description: 720 * Main IBT asynchronous event handler registered at ibt_attach. 721 * Convert to OFA event type and forward to the appropriate 722 * asynchronous handler. 723 */ 724 void 725 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 726 ibt_async_code_t code, ibt_async_event_t *event) 727 { 728 enum ib_event_type ofa_type; 729 sol_uverbs_ib_event_handler_t *handler; 730 llist_head_t *entry; 731 sol_uverbs_hca_t *hca; 732 733 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 734 "ASYNNC EVENT HANDLER: entry, code=%d", code); 735 736 ofa_type = uverbs_ibt_to_ofa_event_code(code); 737 738 if ((int)ofa_type < 0) { 739 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 740 "ASYNC EVENT HANDLER:Event %d did not map to OFA " 741 "Event", code); 742 return; 743 } 744 745 switch (ofa_type) { 746 case IB_EVENT_QP_FATAL: 747 case IB_EVENT_QP_REQ_ERR: 748 case IB_EVENT_QP_ACCESS_ERR: 749 case IB_EVENT_QP_LAST_WQE_REACHED: 750 case IB_EVENT_SQ_DRAINED: 751 case IB_EVENT_PATH_MIG: 752 /* 753 * These events are related with a QP 754 */ 755 uverbs_async_qp_event_handler(clnt_private, hca_hdl, 756 ofa_type, event); 757 break; 758 759 case IB_EVENT_CQ_ERR: 760 /* 761 * These events are related with a CQ 762 */ 763 uverbs_async_cq_event_handler(clnt_private, hca_hdl, 764 ofa_type, event); 765 break; 766 767 case IB_EVENT_SRQ_ERR: 768 case IB_EVENT_SRQ_LIMIT_REACHED: 769 /* 770 * These events are related with a SRQ 771 */ 772 uverbs_async_srq_event_handler(clnt_private, hca_hdl, 773 ofa_type, event); 774 break; 775 776 777 case IB_EVENT_PORT_ERR: 778 case IB_EVENT_PORT_ACTIVE: 779 case IB_EVENT_LID_CHANGE: 780 case IB_EVENT_PKEY_CHANGE: 781 case IB_EVENT_SM_CHANGE: 782 case IB_EVENT_CLIENT_REREGISTER: 783 case IB_EVENT_DEVICE_FATAL: 784 case IB_EVENT_PATH_MIG_ERR: 785 /* 786 * Unaffiliated asynchronous notifications. 787 */ 788 uverbs_async_unaff_event_handler(clnt_private, hca_hdl, 789 ofa_type, event); 790 break; 791 792 default: 793 break; 794 } 795 796 /* 797 * Give other kernel agents a notification. 798 */ 799 hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl); 800 if (hca) { 801 mutex_enter(&hca->event_handler_lock); 802 list_for_each(entry, &hca->event_handler_list) { 803 handler = (sol_uverbs_ib_event_handler_t *)entry->ptr; 804 805 ASSERT(handler != NULL); 806 handler->handler(handler, hca_hdl, code, event); 807 } 808 mutex_exit(&hca->event_handler_lock); 809 } 810 } 811 812 /* 813 * Function: 814 * uverbs_async_event_common 815 * Input: 816 * uctxt - Pointer to the user context associated with the 817 * affiliated event. 818 * element - The users handle to the associated object. 819 * event - The event type. 820 * uobj_list - The list to enqueue the asynchronous event. 821 * counter - The counter to track the event delivery. 822 * Output: 823 * None 824 * Returns: 825 * None 826 * Description: 827 * Create an asyncronous event and enqueue it on the specified list. 828 * Then wake callers that may be blocked on the list. 829 */ 830 static void 831 uverbs_async_event_common(uverbs_uctxt_uobj_t *uctxt, uint64_t element, 832 uint32_t event, llist_head_t *obj_list, uint32_t *counter) 833 { 834 uverbs_ufile_uobj_t *ufile = uctxt->async_evfile; 835 uverbs_event_t *entry; 836 837 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, " 838 "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter); 839 840 if (!ufile) { 841 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 842 "ufile %p", ufile); 843 return; 844 } 845 846 mutex_enter(&ufile->lock); 847 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP); 848 if (!entry) { 849 mutex_exit(&ufile->lock); 850 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 851 "kmem_zalloc failed"); 852 return; 853 } 854 855 entry->ev_desc.async.element = element; 856 entry->ev_desc.async.event_type = event; 857 entry->ev_counter = counter; 858 859 llist_head_init(&entry->ev_list, entry); 860 llist_head_init(&entry->ev_obj_list, entry); 861 862 llist_add_tail(&entry->ev_list, &ufile->event_list); 863 864 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 865 "adding ASYNC entry-ev_list=%p, entry %p", 866 &entry->ev_list, entry); 867 868 if (obj_list) { 869 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 870 "adding ASYNC entry-ev_obj_list=%p, entry=%p", 871 &entry->ev_obj_list, entry); 872 llist_add_tail(&entry->ev_obj_list, obj_list); 873 } 874 875 mutex_exit(&ufile->lock); 876 cv_signal(&ufile->poll_wait); 877 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM); 878 } 879 880 /* 881 * Function: 882 * uverbs_release_ucq_channel 883 * Input: 884 * uctxt - A pointer to the callers user context. 885 * ufile - A pointer to the event file associated with a CQ. 886 * ucq - A pointer to the user CQ object. 887 * Output: 888 * None 889 * Returns: 890 * None 891 * Description: 892 * Release any completion and asynchronous events that may 893 * be queued to the specified completion channel/UCQ but not 894 * yet reaped. 895 */ 896 void 897 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt, 898 uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 899 { 900 uverbs_event_t *evt; 901 llist_head_t *entry; 902 llist_head_t *list_tmp; 903 904 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 905 "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p", 906 uctxt, ufile, ucq); 907 908 /* 909 * Release completion events that have been queued on the CQ completion 910 * eventlist. 911 */ 912 if (ufile) { 913 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 914 ufile->ufile_cq_cnt--; 915 if (ufile->ufile_cq_cnt) { 916 rw_exit(&ufile->uobj.uo_lock); 917 uverbs_release_ucq_uevents(ufile, ucq); 918 return; 919 } 920 rw_exit(&ufile->uobj.uo_lock); 921 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 922 "release_ucq_chan : comp_list %p, prv %p, nxt %p", 923 &ucq->comp_list, ucq->comp_list.prv, 924 ucq->comp_list.nxt); 925 mutex_enter(&ufile->lock); 926 927 entry = ucq->comp_list.nxt; 928 list_tmp = entry->nxt; 929 while (entry != &ucq->comp_list) { 930 ASSERT(entry); 931 evt = (uverbs_event_t *)entry->ptr; 932 933 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 934 "RELEASE UCQ CHANNEL:Deleting event " 935 "on CQ comp list: %p", evt); 936 llist_del(&evt->ev_list); 937 llist_del(&evt->ev_obj_list); 938 kmem_free(evt, sizeof (*evt)); 939 entry = list_tmp; 940 list_tmp = entry->nxt; 941 } 942 mutex_exit(&ufile->lock); 943 944 uverbs_release_ucq_uevents(ufile, ucq); 945 } 946 947 } 948 949 /* 950 * Function: 951 * uverbs_release_ucq_uevents 952 * Input: 953 * ufile - A pointer to the asynchronous event file associated with a QP. 954 * ucq - A pointer to the user CQ object. 955 * Output: 956 * None 957 * Returns: 958 * None 959 * Description: 960 * Free any user asynchronous events that have been queued for the 961 * user CQ object specified. 962 */ 963 void 964 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 965 { 966 uverbs_event_t *evt; 967 llist_head_t *entry; 968 llist_head_t *list_tmp; 969 970 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 971 "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq); 972 973 if (ufile) { 974 mutex_enter(&ufile->lock); 975 976 entry = ucq->async_list.nxt; 977 list_tmp = entry->nxt; 978 while (entry != &ucq->async_list) { 979 ASSERT(entry); 980 evt = (uverbs_event_t *)entry->ptr; 981 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 982 "RELEASE UCQ EVENTS: Deleting event " 983 "on CQ async list: %p", evt); 984 llist_del(&evt->ev_list); 985 llist_del(&evt->ev_obj_list); 986 kmem_free(evt, sizeof (*evt)); 987 entry = list_tmp; 988 list_tmp = entry->nxt; 989 } 990 mutex_exit(&ufile->lock); 991 } 992 } 993 994 /* 995 * Function: 996 * uverbs_release_uqp_uevents 997 * Input: 998 * ufile - A pointer to the asynchronous event file associated with a QP. 999 * uqp - A pointer to the user QP object. 1000 * Output: 1001 * None 1002 * Returns: 1003 * None 1004 * Description: 1005 * Free any user asynchronous events that have been queued for the 1006 * user QP object specified. 1007 */ 1008 void 1009 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp) 1010 { 1011 uverbs_event_t *evt; 1012 llist_head_t *entry; 1013 llist_head_t *list_tmp; 1014 1015 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1016 "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp); 1017 1018 if (ufile) { 1019 mutex_enter(&ufile->lock); 1020 entry = uqp->async_list.nxt; 1021 list_tmp = entry->nxt; 1022 while (entry != &uqp->async_list) { 1023 ASSERT(entry); 1024 evt = (uverbs_event_t *)entry->ptr; 1025 ASSERT(evt); 1026 1027 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1028 "RELEASE UQP EVENTS: Deleting event " 1029 "ON qp async list: %p", evt); 1030 llist_del(&evt->ev_list); 1031 llist_del(&evt->ev_obj_list); 1032 kmem_free(evt, sizeof (*evt)); 1033 entry = list_tmp; 1034 list_tmp = entry->nxt; 1035 } 1036 mutex_exit(&ufile->lock); 1037 } 1038 } 1039 1040 /* 1041 * Function: 1042 * uverbs_release_usrq_uevents 1043 * Input: 1044 * ufile - A pointer to the asynchronous event file associated with a 1045 * SRQ. 1046 * uqp - A pointer to the user SRQ object. 1047 * Output: 1048 * None 1049 * Returns: 1050 * None 1051 * Description: 1052 * Free any user asynchronous events that have been queued for the 1053 * user SRQ object specified. 1054 */ 1055 void 1056 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile, 1057 uverbs_usrq_uobj_t *usrq) 1058 { 1059 uverbs_event_t *evt; 1060 llist_head_t *entry; 1061 llist_head_t *list_tmp; 1062 1063 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1064 "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq); 1065 1066 if (ufile) { 1067 mutex_enter(&ufile->lock); 1068 1069 entry = usrq->async_list.nxt; 1070 list_tmp = entry->nxt; 1071 while (entry != &usrq->async_list) { 1072 ASSERT(entry); 1073 evt = (uverbs_event_t *)entry->ptr; 1074 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1075 "RELEASE SRQ EVENTS: Deleting event " 1076 "on SRQ async list: %p", evt); 1077 llist_del(&evt->ev_list); 1078 llist_del(&evt->ev_obj_list); 1079 kmem_free(evt, sizeof (*evt)); 1080 entry = list_tmp; 1081 list_tmp = entry->nxt; 1082 } 1083 mutex_exit(&ufile->lock); 1084 } 1085 } 1086