1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * The sol_ucma driver provides the API for librdmacm library for RDMACM 28 * functionality. 29 * 30 * sol_uverbs will create a minor node with prefix ":ucma", 31 * which can be opened only by the kernel (cred == kcred). 32 * 33 * sol_cma driver will open and close the sol_uverb minor 34 * device using the Layered Driver Interfaces (See PSARC 35 * 2001/769). 36 */ 37 38 /* Standard driver includes */ 39 #include <sys/types.h> 40 #include <sys/modctl.h> 41 #include <sys/ddi.h> 42 #include <sys/sunddi.h> 43 #include <sys/file.h> 44 #include <sys/errno.h> 45 #include <sys/open.h> 46 #include <sys/cred.h> 47 #include <sys/stat.h> 48 #include <sys/ddi.h> 49 #include <sys/sunddi.h> 50 #include <sys/conf.h> 51 #include <sys/uio.h> 52 #include <sys/sunldi.h> 53 #include <sys/modctl.h> 54 55 /* Common header files */ 56 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 57 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs2ucma.h> 58 #include <sys/ib/clients/of/ofed_kernel.h> 59 60 /* Kernel Headers for User rdma_cm API */ 61 #include <sys/ib/clients/of/rdma/ib_addr.h> 62 #include <sys/ib/clients/of/rdma/rdma_user_cm.h> 63 64 /* Kernel rdma_cm API */ 65 #include <sys/ib/clients/of/rdma/rdma_cm.h> 66 67 /* sol_ucma internal Header files */ 68 #include <sys/ib/clients/of/sol_ucma/sol_ucma.h> 69 70 /* entry point function prototype declarations */ 71 static int sol_ucma_attach(dev_info_t *, ddi_attach_cmd_t); 72 static int sol_ucma_detach(dev_info_t *, ddi_detach_cmd_t); 73 static int sol_ucma_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 74 static int sol_ucma_open(dev_t *, int, int, cred_t *); 75 static int sol_ucma_close(dev_t, int, int, cred_t *); 76 static int sol_ucma_write(dev_t, struct uio *, cred_t *); 77 static int sol_ucma_poll(dev_t, short, int, short *, struct pollhead **); 78 79 /* Driver entry points */ 80 static struct cb_ops sol_ucma_cb_ops = { 81 sol_ucma_open, /* open */ 82 sol_ucma_close, /* close */ 83 nodev, /* strategy (block) */ 84 nodev, /* print (block) */ 85 nodev, /* dump (block) */ 86 nodev, /* read */ 87 sol_ucma_write, /* write */ 88 nodev, /* ioctl */ 89 nodev, /* devmap */ 90 nodev, /* mmap */ 91 nodev, /* segmap */ 92 sol_ucma_poll, /* chpoll */ 93 ddi_prop_op, /* prop_op */ 94 NULL, /* streams */ 95 D_NEW | D_MP | D_64BIT, /* flags */ 96 CB_REV /* rev */ 97 }; 98 99 /* Driver operations */ 100 static struct dev_ops sol_ucma_dev_ops = { 101 DEVO_REV, /* struct rev */ 102 0, /* refcnt */ 103 sol_ucma_getinfo, /* getinfo */ 104 nulldev, /* identify */ 105 nulldev, /* probe */ 106 sol_ucma_attach, /* attach */ 107 sol_ucma_detach, /* detach */ 108 nodev, /* reset */ 109 &sol_ucma_cb_ops, /* cb_ops */ 110 NULL, /* bus_ops */ 111 nodev, /* power */ 112 ddi_quiesce_not_needed /* quiesce */ 113 }; 114 115 /* Module Driver Info */ 116 static struct modldrv sol_ucma_modldrv = { 117 &mod_driverops, 118 "Solaris User RDMACM driver", 119 &sol_ucma_dev_ops 120 }; 121 122 /* Module Linkage */ 123 static struct modlinkage sol_ucma_modlinkage = { 124 MODREV_1, 125 &sol_ucma_modldrv, 126 NULL, 127 }; 128 129 static char *sol_ucma_dbg_str = "sol_ucma"; 130 sol_ofs_uobj_table_t ucma_file_uo_tbl; 131 sol_ofs_uobj_table_t ucma_ctx_uo_tbl; 132 sol_ofs_uobj_table_t ucma_mcast_uo_tbl; 133 134 /* Function pointers for uverbs functions */ 135 static uverbs_get_clnt_hdl_t uverbs_get_hdl_fp = NULL; 136 static uverbs_qpnum2qphdl_t uverbs_qpnum2qphdl_fp = NULL; 137 static uverbs_disable_uqpn_mod_t uverbs_disable_uqpn_modify_fp = NULL; 138 static uverbs_uqpn_cq_ctrl_t uverbs_uqpn_cq_ctrl_fp = NULL; 139 static uverbs_set_qp_free_state_t uverbs_set_qp_free_state_fp = NULL; 140 static uverbs_flush_qp_t uverbs_flush_qp_fp = NULL; 141 142 /* Global Variables */ 143 sol_ucma_t sol_ucma; 144 145 /* RDMACM Functions */ 146 static int sol_ucma_create_id(dev_t, void *, struct uio *); 147 static int sol_ucma_destroy_id(dev_t, void *, struct uio *); 148 static int sol_ucma_bind_addr(dev_t, void *, struct uio *); 149 static int sol_ucma_resolve_addr(dev_t, void *, struct uio *); 150 static int sol_ucma_resolve_route(dev_t, void *, struct uio *); 151 static int sol_ucma_query_route(dev_t, void *, struct uio *); 152 static int sol_ucma_connect(dev_t, void *, struct uio *); 153 static int sol_ucma_listen(dev_t, void *, struct uio *); 154 static int sol_ucma_accept(dev_t, void *, struct uio *); 155 static int sol_ucma_reject(dev_t, void *, struct uio *); 156 static int sol_ucma_disconnect(dev_t, void *, struct uio *); 157 static int sol_ucma_init_qp_attr(dev_t, void *, struct uio *); 158 static int sol_ucma_get_event(dev_t, void *, struct uio *); 159 static int sol_ucma_set_option(dev_t, void *, struct uio *); 160 static int sol_ucma_notify(dev_t, void *, struct uio *); 161 static int sol_ucma_join_mcast(dev_t, void *, struct uio *); 162 static int sol_ucma_leave_mcast(dev_t, void *, struct uio *); 163 164 /* 165 * Event callback from sol_cma 166 */ 167 int sol_ucma_evt_hdlr(struct rdma_cm_id *, struct rdma_cm_event *); 168 169 /* 170 * Internal functions. 171 */ 172 static sol_ucma_file_t * 173 ucma_alloc_file(minor_t *); 174 175 static sol_ucma_chan_t * 176 ucma_alloc_chan(sol_ucma_file_t *, sol_ucma_create_id_t *); 177 178 static void 179 ucma_free_chan(sol_ucma_chan_t *, int); 180 181 static int 182 get_file_chan(uint32_t, sol_ucma_file_t **, sol_ucma_chan_t **, char *, int); 183 184 static void 185 rdma2usr_route(struct rdma_cm_id *, sol_ucma_query_route_resp_t *); 186 187 static void 188 usr2rdma_conn_param(struct rdma_ucm_conn_param *, struct rdma_conn_param *); 189 190 static void 191 rdma2usr_conn_param(struct rdma_conn_param *, struct rdma_ucm_conn_param *); 192 193 static void 194 rdma2usr_ud_param(struct rdma_ud_param *, sol_ucma_ud_param_t *); 195 196 static void sol_ucma_user_objs_init(); 197 static void sol_ucma_user_objs_fini(); 198 199 int 200 _init(void) 201 { 202 int error; 203 204 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_init()"); 205 sol_ucma_user_objs_init(); 206 mutex_init(&sol_ucma.ucma_mutex, NULL, MUTEX_DRIVER, NULL); 207 cv_init(&sol_ucma.ucma_open_cv, NULL, CV_DRIVER, NULL); 208 209 if ((error = ldi_ident_from_mod(&sol_ucma_modlinkage, 210 &sol_ucma.ucma_ldi_ident)) != 0) { 211 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 212 "ldi_ident_from_mod() failed"); 213 mutex_destroy(&sol_ucma.ucma_mutex); 214 cv_destroy(&sol_ucma.ucma_open_cv); 215 sol_ucma_user_objs_fini(); 216 return (error); 217 } 218 sol_ucma.ucma_clnt_hdl_flag = SOL_UCMA_CLNT_HDL_UNINITIALIZED; 219 error = mod_install(&sol_ucma_modlinkage); 220 if (error) { 221 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "mod_install() failed"); 222 ldi_ident_release(sol_ucma.ucma_ldi_ident); 223 mutex_destroy(&sol_ucma.ucma_mutex); 224 cv_destroy(&sol_ucma.ucma_open_cv); 225 sol_ucma_user_objs_fini(); 226 return (error); 227 } 228 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_init(): ret"); 229 return (error); 230 } 231 232 int 233 _info(struct modinfo *modinfop) 234 { 235 return (mod_info(&sol_ucma_modlinkage, modinfop)); 236 } 237 238 int 239 _fini(void) 240 { 241 int ret; 242 243 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_fini()"); 244 if ((ret = mod_remove(&sol_ucma_modlinkage)) != 0) { 245 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 246 "sol_ucma, _fini : mod_remove failed"); 247 return (ret); 248 } 249 ldi_ident_release(sol_ucma.ucma_ldi_ident); 250 mutex_destroy(&sol_ucma.ucma_mutex); 251 cv_destroy(&sol_ucma.ucma_open_cv); 252 sol_ucma_user_objs_fini(); 253 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_fini(): ret"); 254 return (DDI_SUCCESS); 255 } 256 257 static int 258 sol_ucma_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 259 { 260 int rval; 261 262 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "attach(%p, %x)", dip, cmd); 263 264 switch (cmd) { 265 case DDI_ATTACH: 266 mutex_enter(&sol_ucma.ucma_mutex); 267 if (sol_ucma.ucma_dip != NULL) { 268 mutex_exit(&sol_ucma.ucma_mutex); 269 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 270 "attach: failed, > 1 instance"); 271 return (DDI_FAILURE); 272 } 273 sol_ucma.ucma_dip = dip; 274 mutex_exit(&sol_ucma.ucma_mutex); 275 276 rval = ddi_create_minor_node(dip, "sol_ucma", S_IFCHR, 277 0, DDI_PSEUDO, 0); 278 if (rval != DDI_SUCCESS) { 279 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 280 "attach: ddi_create_minor_node failed"); 281 mutex_enter(&sol_ucma.ucma_mutex); 282 sol_ucma.ucma_dip = NULL; 283 mutex_exit(&sol_ucma.ucma_mutex); 284 return (DDI_FAILURE); 285 } 286 287 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 288 "attach : DDI_ATTACH success"); 289 return (DDI_SUCCESS); 290 case DDI_RESUME: 291 return (DDI_SUCCESS); 292 default: 293 return (DDI_FAILURE); 294 } 295 } 296 297 static int 298 sol_ucma_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 299 { 300 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "detach(%p, %x)", dip, cmd); 301 302 switch (cmd) { 303 case DDI_DETACH: 304 mutex_enter(&sol_ucma.ucma_mutex); 305 if (sol_ucma.ucma_num_file) { 306 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 307 "detach : %x files not closed", 308 sol_ucma.ucma_num_file); 309 mutex_exit(&sol_ucma.ucma_mutex); 310 return (DDI_FAILURE); 311 } 312 sol_ucma.ucma_dip = NULL; 313 mutex_exit(&sol_ucma.ucma_mutex); 314 315 ddi_remove_minor_node(dip, "sol_ucma"); 316 317 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 318 "detach : DDI_DETACH success"); 319 return (DDI_SUCCESS); 320 case DDI_SUSPEND: 321 return (DDI_SUCCESS); 322 default: 323 return (DDI_FAILURE); 324 } 325 } 326 327 /*ARGSUSED*/ 328 static int 329 sol_ucma_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 330 void **resultp) 331 { 332 switch (cmd) { 333 case DDI_INFO_DEVT2DEVINFO: 334 *resultp = (void *)sol_ucma.ucma_dip; 335 return (DDI_SUCCESS); 336 case DDI_INFO_DEVT2INSTANCE: 337 *resultp = (void *)0; 338 return (DDI_SUCCESS); 339 default : 340 return (DDI_FAILURE); 341 } 342 } 343 344 static int 345 sol_ucma_open(dev_t *devp, int flag, int otype, cred_t *credp) 346 { 347 sol_ucma_file_t *new_filep; 348 minor_t new_minor; 349 350 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "open(%p, %x, %x, %p)", 351 devp, flag, otype, credp); 352 353 new_filep = ucma_alloc_file(&new_minor); 354 if (new_filep == NULL) 355 return (EAGAIN); 356 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "sol_ucma new minor %x", 357 new_minor); 358 359 /* 360 * For the first open, ensure that the sol_uverbs driver is attached. 361 * Also get the function pointers for uverbs API functions using 362 * ddi_modopen() and ddi_modsym() for the sol_uverbs driver. 363 * 364 * ldi_open() is done to ensure that sol_uverbs driver is attached, 365 * even though ddi_modopen is sufficient to get the function pointers 366 * for the uverbs APIs 367 */ 368 mutex_enter(&sol_ucma.ucma_mutex); 369 if (sol_ucma.ucma_clnt_hdl_flag == SOL_UCMA_CLNT_HDL_UNINITIALIZED) { 370 int rval, ret_errno; 371 372 sol_ucma.ucma_clnt_hdl_flag = 373 SOL_UCMA_CLNT_HDL_INITIALIZING; 374 if ((rval = ldi_open_by_name(SOL_UCMA_UVERBS_PATH, 375 FREAD | FWRITE, kcred, &sol_ucma.ucma_ldi_hdl, 376 sol_ucma.ucma_ldi_ident)) != 0) { 377 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 378 "ldi_open_by_name(%s, ...) failed with rval %x", 379 SOL_UCMA_UVERBS_PATH, rval); 380 sol_ofs_uobj_free(&new_filep->file_uobj); 381 sol_ucma.ucma_clnt_hdl_flag = 382 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 383 mutex_exit(&sol_ucma.ucma_mutex); 384 return (ENODEV); 385 } 386 if ((sol_ucma.ucma_mod_hdl = ddi_modopen("drv/sol_uverbs", 387 KRTLD_MODE_FIRST, &ret_errno)) == NULL) { 388 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 389 "ddi_modopen(%s, ...) failed", "drv/sol_uverbs"); 390 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 391 FREAD | FWRITE, kcred); 392 sol_ofs_uobj_free(&new_filep->file_uobj); 393 sol_ucma.ucma_clnt_hdl_flag = 394 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 395 mutex_exit(&sol_ucma.ucma_mutex); 396 return (ret_errno); 397 } 398 if ((uverbs_get_hdl_fp = (uverbs_get_clnt_hdl_t)ddi_modsym( 399 sol_ucma.ucma_mod_hdl, SOL_UVERBS_GET_CLNT_HDL, &ret_errno)) 400 == NULL) { 401 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 402 "ddi_modsym(%s, ...) failed", 403 SOL_UVERBS_GET_CLNT_HDL); 404 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 405 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 406 FREAD | FWRITE, kcred); 407 sol_ofs_uobj_free(&new_filep->file_uobj); 408 sol_ucma.ucma_clnt_hdl_flag = 409 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 410 mutex_exit(&sol_ucma.ucma_mutex); 411 return (ret_errno); 412 } 413 if ((uverbs_qpnum2qphdl_fp = (uverbs_qpnum2qphdl_t)ddi_modsym( 414 sol_ucma.ucma_mod_hdl, SOL_UVERBS_QPNUM2QPHDL, &ret_errno)) 415 == NULL) { 416 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 417 "ddi_modsym(%s, ...) failed", 418 SOL_UVERBS_QPNUM2QPHDL); 419 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 420 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 421 FREAD | FWRITE, kcred); 422 sol_ofs_uobj_free(&new_filep->file_uobj); 423 sol_ucma.ucma_clnt_hdl_flag = 424 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 425 mutex_exit(&sol_ucma.ucma_mutex); 426 return (ret_errno); 427 } 428 if ((uverbs_disable_uqpn_modify_fp = 429 (uverbs_disable_uqpn_mod_t)ddi_modsym( 430 sol_ucma.ucma_mod_hdl, SOL_UVERBS_DISABLE_UQPN_MODIFY, 431 &ret_errno)) == NULL) { 432 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 433 "ddi_modsym(%s, ...) failed", 434 SOL_UVERBS_DISABLE_UQPN_MODIFY); 435 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 436 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 437 FREAD | FWRITE, kcred); 438 sol_ofs_uobj_free(&new_filep->file_uobj); 439 sol_ucma.ucma_clnt_hdl_flag = 440 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 441 mutex_exit(&sol_ucma.ucma_mutex); 442 return (ret_errno); 443 } 444 if ((uverbs_uqpn_cq_ctrl_fp = 445 (uverbs_uqpn_cq_ctrl_t)ddi_modsym( 446 sol_ucma.ucma_mod_hdl, SOL_UVERBS_UQPN_CQ_CTRL, 447 &ret_errno)) == NULL) { 448 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 449 "ddi_modsym(%s, ...) failed", 450 SOL_UVERBS_UQPN_CQ_CTRL); 451 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 452 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 453 FREAD | FWRITE, kcred); 454 sol_ofs_uobj_free(&new_filep->file_uobj); 455 sol_ucma.ucma_clnt_hdl_flag = 456 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 457 mutex_exit(&sol_ucma.ucma_mutex); 458 return (ret_errno); 459 } 460 if ((uverbs_set_qp_free_state_fp = 461 (uverbs_set_qp_free_state_t)ddi_modsym( 462 sol_ucma.ucma_mod_hdl, SOL_UVERBS_SET_QPFREE_STATE, 463 &ret_errno)) == NULL) { 464 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 465 "ddi_modsym(%s, ...) failed", 466 SOL_UVERBS_SET_QPFREE_STATE); 467 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 468 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 469 FREAD | FWRITE, kcred); 470 sol_ofs_uobj_free(&new_filep->file_uobj); 471 sol_ucma.ucma_clnt_hdl_flag = 472 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 473 mutex_exit(&sol_ucma.ucma_mutex); 474 return (ret_errno); 475 } 476 if ((uverbs_flush_qp_fp = 477 (uverbs_flush_qp_t)ddi_modsym( 478 sol_ucma.ucma_mod_hdl, SOL_UVERBS_FLUSH_QP, 479 &ret_errno)) == NULL) { 480 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 481 "ddi_modsym(%s, ...) failed", 482 SOL_UVERBS_FLUSH_QP); 483 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 484 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 485 FREAD | FWRITE, kcred); 486 sol_ofs_uobj_free(&new_filep->file_uobj); 487 sol_ucma.ucma_clnt_hdl_flag = 488 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 489 mutex_exit(&sol_ucma.ucma_mutex); 490 return (ret_errno); 491 } 492 493 (*uverbs_get_hdl_fp) (&sol_ucma.ucma_ib_clnt_hdl, 494 &sol_ucma.ucma_iw_clnt_hdl); 495 if (sol_ucma.ucma_ib_clnt_hdl == NULL && 496 sol_ucma.ucma_iw_clnt_hdl == NULL) { 497 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 498 "uverbs_get_clnt_hdl failed"); 499 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 500 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 501 FREAD | FWRITE, kcred); 502 sol_ofs_uobj_free(&new_filep->file_uobj); 503 sol_ucma.ucma_clnt_hdl_flag = 504 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 505 mutex_exit(&sol_ucma.ucma_mutex); 506 return (ENODEV); 507 } 508 sol_ucma.ucma_clnt_hdl_flag = 509 SOL_UCMA_CLNT_HDL_INITIALIZED; 510 cv_broadcast(&sol_ucma.ucma_open_cv); 511 } else if (sol_ucma.ucma_clnt_hdl_flag == 512 SOL_UCMA_CLNT_HDL_INITIALIZING) { 513 cv_wait(&sol_ucma.ucma_open_cv, &sol_ucma.ucma_mutex); 514 } 515 mutex_exit(&sol_ucma.ucma_mutex); 516 *devp = makedevice(getmajor(*devp), new_minor); 517 518 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "open Success"); 519 return (0); 520 } 521 522 static int 523 sol_ucma_close(dev_t dev, int flag, int otype, cred_t *credp) 524 { 525 minor_t minor; 526 sol_ucma_file_t *filep; 527 genlist_entry_t *entry; 528 529 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "close(%x, %x, %x, %p)", 530 dev, flag, otype, credp); 531 532 minor = getminor(dev); 533 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 534 &ucma_file_uo_tbl, minor); 535 if (!filep) { 536 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "close, no dev_t %x", 537 dev); 538 return (0); 539 } 540 541 /* Disable further event handling for this CM event channel */ 542 mutex_enter(&filep->file_mutex); 543 if (filep->file_evt_close_flag == SOL_UCMA_EVT_PROGRESS) { 544 cv_wait(&filep->file_evt_close_cv, &filep->file_mutex); 545 } 546 filep->file_evt_close_flag = SOL_UCMA_EVT_DISABLED; 547 mutex_exit(&filep->file_mutex); 548 549 /* 550 * Destroy CM IDs which have not been destroyed. 551 * For CMIDs which have been connected, call 552 * uverbs_set_qp_free_state(SOL_UVERBS2UCMA_ENABLE_QP_FREE) 553 * so that QP free will be done when appropriate, 554 */ 555 entry = remove_genlist_head(&filep->file_id_list); 556 while (entry) { 557 sol_ucma_chan_t *chanp; 558 void *qphdl; 559 560 chanp = (sol_ucma_chan_t *)entry->data; 561 mutex_enter(&chanp->chan_mutex); 562 if (chanp->chan_rdma_id) 563 (chanp->chan_rdma_id)->context = NULL; 564 mutex_exit(&chanp->chan_mutex); 565 rdma_destroy_id(chanp->chan_rdma_id); 566 567 mutex_enter(&chanp->chan_mutex); 568 qphdl = chanp->chan_qp_hdl; 569 chanp->chan_qp_hdl = NULL; 570 mutex_exit(&chanp->chan_mutex); 571 if (qphdl) 572 (*uverbs_set_qp_free_state_fp) ( 573 SOL_UVERBS2UCMA_ENABLE_QP_FREE, 0, qphdl); 574 ucma_free_chan(chanp, 1); 575 576 entry = remove_genlist_head(&filep->file_id_list); 577 } 578 579 /* Flush out any events that have not been acknowledged. */ 580 mutex_enter(&filep->file_mutex); 581 if (filep->file_pending_evt_cnt) { 582 sol_ucma_event_t *evtp; 583 584 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 585 "close : %d Events not reported to userland", 586 filep->file_pending_evt_cnt); 587 entry = remove_genlist_head(&filep->file_evt_list); 588 while (entry) { 589 evtp = (sol_ucma_event_t *)entry->data; 590 kmem_free(evtp, sizeof (sol_ucma_event_t)); 591 kmem_free(entry, sizeof (genlist_entry_t)); 592 entry = remove_genlist_head(&filep->file_evt_list); 593 }; 594 mutex_exit(&filep->file_mutex); 595 } 596 597 /* 598 * Module close for sol_uverbs when the last file is closed. 599 * Set the function pointers to sol_uverbs API to NULL 600 * ddi_modclose() and ldi_close() - sol_uverbs driver 601 */ 602 mutex_enter(&sol_ucma.ucma_mutex); 603 if (sol_ucma.ucma_num_file == 1) { 604 sol_ucma.ucma_clnt_hdl_flag = 605 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 606 uverbs_get_hdl_fp = NULL; 607 uverbs_qpnum2qphdl_fp = NULL; 608 uverbs_disable_uqpn_modify_fp = NULL; 609 uverbs_uqpn_cq_ctrl_fp = NULL; 610 uverbs_uqpn_cq_ctrl_fp = NULL; 611 uverbs_set_qp_free_state_fp = NULL; 612 uverbs_flush_qp_fp = NULL; 613 sol_ucma.ucma_ib_clnt_hdl = NULL; 614 sol_ucma.ucma_iw_clnt_hdl = NULL; 615 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 616 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 617 FREAD | FWRITE, kcred); 618 } 619 sol_ucma.ucma_num_file--; 620 mutex_exit(&sol_ucma.ucma_mutex); 621 622 kmem_free(filep->file_pollhead, sizeof (struct pollhead)); 623 sol_ofs_uobj_put(&filep->file_uobj); 624 mutex_destroy(&filep->file_mutex); 625 cv_destroy(&filep->file_evt_cv); 626 cv_destroy(&filep->file_evt_close_cv); 627 rw_enter(&(filep->file_uobj.uo_lock), RW_WRITER); 628 (void) sol_ofs_uobj_remove(&ucma_file_uo_tbl, &(filep->file_uobj)); 629 rw_exit(&(filep->file_uobj.uo_lock)); 630 sol_ofs_uobj_free(&(filep->file_uobj)); 631 return (0); 632 } 633 634 typedef struct sol_ucma_cmd_table_s { 635 int (*sol_ucma_cmd_fnc) (dev_t, void *, struct uio *); 636 uint16_t sol_ucma_in_len; 637 uint16_t sol_ucma_out_len; 638 } sol_ucma_cmd_table_t; 639 640 static sol_ucma_cmd_table_t sol_ucma_cmd_table[] = { 641 [RDMA_USER_CM_CMD_CREATE_ID] = sol_ucma_create_id, 642 sizeof (sol_ucma_create_id_t), 643 sizeof (sol_ucma_create_id_resp_t), 644 [RDMA_USER_CM_CMD_DESTROY_ID] = sol_ucma_destroy_id, 645 sizeof (sol_ucma_destroy_id_t), 646 sizeof (sol_ucma_destroy_id_resp_t), 647 [RDMA_USER_CM_CMD_BIND_ADDR] = sol_ucma_bind_addr, 648 sizeof (sol_ucma_bind_addr_t), 649 0, 650 [RDMA_USER_CM_CMD_RESOLVE_ADDR] = sol_ucma_resolve_addr, 651 sizeof (sol_ucma_resolve_addr_t), 652 0, 653 [RDMA_USER_CM_CMD_RESOLVE_ROUTE] = sol_ucma_resolve_route, 654 sizeof (sol_ucma_resolve_route_t), 655 0, 656 [RDMA_USER_CM_CMD_QUERY_ROUTE] = sol_ucma_query_route, 657 sizeof (sol_ucma_query_route_t), 658 sizeof (sol_ucma_query_route_resp_t), 659 [RDMA_USER_CM_CMD_CONNECT] = sol_ucma_connect, 660 sizeof (sol_ucma_connect_t), 661 0, 662 [RDMA_USER_CM_CMD_LISTEN] = sol_ucma_listen, 663 sizeof (sol_ucma_listen_t), 664 0, 665 [RDMA_USER_CM_CMD_ACCEPT] = sol_ucma_accept, 666 sizeof (sol_ucma_accept_t), 667 0, 668 [RDMA_USER_CM_CMD_REJECT] = sol_ucma_reject, 669 sizeof (sol_ucma_reject_t), 670 0, 671 [RDMA_USER_CM_CMD_DISCONNECT] = sol_ucma_disconnect, 672 sizeof (sol_ucma_disconnect_t), 673 0, 674 [RDMA_USER_CM_CMD_INIT_QP_ATTR] = sol_ucma_init_qp_attr, 675 sizeof (sol_ucma_init_qp_attr_t), 676 sizeof (struct ib_uverbs_qp_attr), 677 [RDMA_USER_CM_CMD_GET_EVENT] = sol_ucma_get_event, 678 sizeof (sol_ucma_get_event_t), 679 sizeof (sol_ucma_event_resp_t), 680 [RDMA_USER_CM_CMD_GET_OPTION] = NULL, 681 0, 682 0, 683 [RDMA_USER_CM_CMD_SET_OPTION] = sol_ucma_set_option, 684 sizeof (sol_ucma_set_option_t), 685 0, 686 [RDMA_USER_CM_CMD_NOTIFY] = sol_ucma_notify, 687 sizeof (sol_ucma_notify_t), 688 0, 689 [RDMA_USER_CM_CMD_JOIN_MCAST] = sol_ucma_join_mcast, 690 sizeof (sol_ucma_join_mcast_t), 691 sizeof (sol_ucma_create_id_resp_t), 692 [RDMA_USER_CM_CMD_LEAVE_MCAST] = sol_ucma_leave_mcast, 693 sizeof (sol_ucma_destroy_id_t), 694 sizeof (sol_ucma_destroy_id_resp_t) 695 }; 696 697 #define SOL_UCMA_MAX_CMD_DATA 512 698 static int 699 sol_ucma_write(dev_t dev, struct uio *uio, cred_t *credp) 700 { 701 sol_ucma_cmd_hdr_t *user_hdrp; 702 int ret; 703 void *data_buf = NULL; 704 char uio_data[SOL_UCMA_MAX_CMD_DATA]; 705 size_t uio_data_len = uio->uio_resid; 706 707 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "write(%x, %p, %p)", 708 dev, uio, credp); 709 710 ret = uiomove((caddr_t)&uio_data, uio_data_len, UIO_WRITE, uio); 711 user_hdrp = (sol_ucma_cmd_hdr_t *)uio_data; 712 713 if (ret != 0) { 714 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "write: uiomove failed"); 715 return (ret); 716 } 717 718 if (user_hdrp->cmd >= 719 sizeof (sol_ucma_cmd_table) / sizeof (sol_ucma_cmd_table_t)) { 720 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 721 "open : cmd out of bound 0x%x", user_hdrp->cmd); 722 return (EINVAL); 723 } 724 if (!(sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_cmd_fnc)) { 725 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 726 "open : Unsupported cmd 0x%x", user_hdrp->cmd); 727 return (EINVAL); 728 } 729 730 /* 731 * Check the user passed IN-OUT buffer length, with expected lengths 732 */ 733 if (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_in_len != 734 (user_hdrp->in)) { 735 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 736 "write : Invalid Input length cmd %x, in %x expected %x", 737 user_hdrp->cmd, user_hdrp->in, 738 sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_in_len); 739 return (EINVAL); 740 } 741 742 if (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_out_len != 743 (user_hdrp->out)) { 744 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 745 "write : Invalid Output length cmd %x, in %x expected %x", 746 user_hdrp->cmd, user_hdrp->out, 747 sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_out_len); 748 return (EINVAL); 749 } 750 751 752 if (user_hdrp->in) { 753 data_buf = (void *)((char *)uio_data + 754 sizeof (sol_ucma_cmd_hdr_t)); 755 } 756 757 ret = (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_cmd_fnc) 758 (dev, data_buf, uio); 759 760 /* If the command fails, set back the uio_resid */ 761 if (ret) 762 uio->uio_resid += uio_data_len; 763 764 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "write : ret %x", ret); 765 return (ret); 766 } 767 768 static int 769 sol_ucma_poll(dev_t dev, short events, int anyyet, 770 short *reventsp, struct pollhead **phpp) 771 { 772 minor_t minor = getminor(dev); 773 sol_ucma_file_t *filep; 774 775 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "poll(%x, %x)", 776 dev, events); 777 if (!(events & (POLLIN | POLLRDNORM))) 778 return (EINVAL); 779 780 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 781 &ucma_file_uo_tbl, minor); 782 ASSERT(filep); 783 784 if (filep->file_pending_evt_cnt) { 785 *reventsp = POLLIN | POLLRDNORM; 786 } else { 787 *reventsp = 0; 788 if (!anyyet) 789 *phpp = filep->file_pollhead; 790 } 791 sol_ofs_uobj_put(&filep->file_uobj); 792 793 return (0); 794 } 795 796 /* 797 * RDMACM functions. 798 */ 799 /*ARGSUSED*/ 800 static int 801 sol_ucma_create_id(dev_t dev, void *io_buf, struct uio *uio) 802 { 803 minor_t minor = getminor(dev); 804 sol_ucma_file_t *filep; 805 sol_ucma_chan_t *chanp; 806 sol_ucma_create_id_t *ucma_id_inp; 807 sol_ucma_create_id_resp_t ucma_id_resp; 808 809 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "create_id(%x, %p), minor %x", 810 dev, io_buf, minor); 811 812 ucma_id_inp = (sol_ucma_create_id_t *)io_buf; 813 ASSERT(ucma_id_inp); 814 ASSERT(ucma_id_inp->response.r_laddr); 815 816 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read(&ucma_file_uo_tbl, 817 minor); 818 ASSERT(filep); 819 820 chanp = ucma_alloc_chan(filep, ucma_id_inp); 821 if (chanp == NULL) { 822 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 823 "create_id: No free Channel"); 824 sol_ofs_uobj_put(&filep->file_uobj); 825 return (ENODEV); 826 } 827 ucma_id_resp.id = chanp->chan_id; 828 829 #ifdef _LP64 830 if (copyout(&ucma_id_resp, (void *)(ucma_id_inp->response.r_laddr), 831 sizeof (sol_ucma_create_id_resp_t))) { 832 #else 833 if (copyout(&ucma_id_resp, (void *)(ucma_id_inp->response.r_addr), 834 sizeof (sol_ucma_create_id_resp_t))) { 835 #endif 836 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 837 "create_id: copyout fault"); 838 ucma_free_chan(chanp, 1); 839 sol_ofs_uobj_put(&filep->file_uobj); 840 return (EFAULT); 841 } 842 /* */ 843 844 chanp->chan_rdma_id = rdma_create_id(sol_ucma_evt_hdlr, 845 chanp, ucma_id_inp->ps); 846 if (chanp->chan_rdma_id == NULL) { 847 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 848 "create_id: rdma_create_id failed"); 849 ucma_free_chan(chanp, 1); 850 sol_ofs_uobj_put(&filep->file_uobj); 851 return (EINVAL); 852 } 853 mutex_enter(&chanp->chan_mutex); 854 (chanp->chan_rdma_id)->context = chanp; 855 mutex_exit(&chanp->chan_mutex); 856 rdma_map_id2clnthdl(chanp->chan_rdma_id, sol_ucma.ucma_ib_clnt_hdl, 857 sol_ucma.ucma_iw_clnt_hdl); 858 859 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "create_id: Return SUCCESS"); 860 sol_ofs_uobj_put(&filep->file_uobj); 861 return (0); 862 } 863 864 /*ARGSUSED*/ 865 static int 866 sol_ucma_destroy_id(dev_t dev, void *io_buf, struct uio *uio) 867 { 868 sol_ucma_chan_t *chanp; 869 uint32_t ucma_id; 870 sol_ucma_file_t *filep; 871 sol_ucma_destroy_id_t *id_inp; 872 minor_t minor; 873 genlist_entry_t *entry; 874 sol_ucma_destroy_id_resp_t id_resp; 875 876 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id(%x, %p)", 877 dev, io_buf); 878 879 id_inp = (sol_ucma_destroy_id_t *)io_buf; 880 ucma_id = id_inp->id; 881 if (!get_file_chan(ucma_id, &filep, &chanp, "destroy_id", 0)) { 882 minor = getminor(dev); 883 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 884 &ucma_file_uo_tbl, minor); 885 if (!filep) { 886 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 887 "destroy_id : filep NULL"); 888 return (EINVAL); 889 } 890 } else { 891 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "destroy_id : " 892 "ucma_id %x invalid", ucma_id); 893 return (0); 894 } 895 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id: chanp %p", chanp); 896 897 /* 898 * Event handling, Flush out events pending 899 * return the number of events that were acked. Free events not acked. 900 */ 901 ASSERT(filep); 902 mutex_enter(&filep->file_mutex); 903 if (filep->file_pending_evt_cnt != 0) { 904 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, 905 "destroy_id: pending events"); 906 entry = remove_genlist_head(&filep->file_evt_list); 907 while (entry) { 908 kmem_free((void *) (entry->data), 909 sizeof (sol_ucma_event_t)); 910 kmem_free(entry, sizeof (genlist_entry_t)); 911 entry = remove_genlist_head(&filep->file_evt_list); 912 }; 913 filep->file_pending_evt_cnt = 0; 914 } 915 if (chanp) { 916 mutex_enter(&chanp->chan_mutex); 917 id_resp.events_reported = chanp->chan_evt_cnt; 918 mutex_exit(&chanp->chan_mutex); 919 } else { 920 id_resp.events_reported = 0; 921 } 922 mutex_exit(&filep->file_mutex); 923 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id : chanp %p, " 924 "evts %x", chanp, id_resp.events_reported); 925 926 #ifdef _LP64 927 if (copyout(&id_resp, (void *) (id_inp->response.r_laddr), 928 sizeof (sol_ucma_destroy_id_resp_t))) { 929 #else 930 if (copyout(&id_resp, (void *) (id_inp->response.r_addr), 931 sizeof (sol_ucma_destroy_id_resp_t))) { 932 #endif 933 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 934 "destroy_id: copyout fault"); 935 sol_ofs_uobj_put(&filep->file_uobj); 936 return (EFAULT); 937 } 938 /* */ 939 940 if (chanp) { 941 mutex_enter(&chanp->chan_mutex); 942 if (chanp->chan_rdma_id) 943 (chanp->chan_rdma_id)->context = NULL; 944 mutex_exit(&chanp->chan_mutex); 945 rdma_destroy_id(chanp->chan_rdma_id); 946 ucma_free_chan(chanp, 1); 947 } 948 949 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id: Success"); 950 sol_ofs_uobj_put(&filep->file_uobj); 951 return (0); 952 } 953 954 /*ARGSUSED*/ 955 static int 956 sol_ucma_bind_addr(dev_t dev, void *io_buf, struct uio *uio) 957 { 958 int ret; 959 sol_ucma_chan_t *chanp; 960 uint32_t ucma_id; 961 sol_ucma_bind_addr_t *bind_addrp; 962 963 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr(%x, %p)", 964 dev, io_buf); 965 966 bind_addrp = (sol_ucma_bind_addr_t *)io_buf; 967 ucma_id = bind_addrp->id; 968 if (get_file_chan(ucma_id, NULL, &chanp, "bind_addr", 1)) 969 return (EINVAL); 970 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr - chanp %p", chanp); 971 972 ret = rdma_bind_addr(chanp->chan_rdma_id, 973 (struct sockaddr *)&bind_addrp->addr); 974 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr: ret %x", ret); 975 return (ret); 976 } 977 978 /*ARGSUSED*/ 979 static int 980 sol_ucma_resolve_addr(dev_t dev, void *io_buf, struct uio *uio) 981 { 982 sol_ucma_chan_t *chanp; 983 uint32_t ucma_id; 984 int ret; 985 sol_ucma_resolve_addr_t *resolve_addrp; 986 987 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr(%x, %p)", 988 dev, io_buf); 989 990 resolve_addrp = (sol_ucma_resolve_addr_t *)io_buf; 991 ucma_id = resolve_addrp->id; 992 if (get_file_chan(ucma_id, NULL, &chanp, "resolve_addr", 1)) { 993 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 994 "resolve_addr: ucma_id %x invalid", ucma_id); 995 return (EINVAL); 996 } 997 ASSERT(chanp); 998 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr - chanp %p", chanp); 999 1000 ret = rdma_resolve_addr(chanp->chan_rdma_id, 1001 (struct sockaddr *)&resolve_addrp->src_addr, 1002 (struct sockaddr *)&resolve_addrp->dst_addr, 1003 resolve_addrp->timeout_ms); 1004 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr: ret %x", ret); 1005 return (ret); 1006 } 1007 1008 /*ARGSUSED*/ 1009 static int 1010 sol_ucma_resolve_route(dev_t dev, void *io_buf, struct uio *uio) 1011 { 1012 sol_ucma_chan_t *chanp; 1013 uint32_t ucma_id; 1014 int ret; 1015 sol_ucma_resolve_route_t *resolve_routep; 1016 1017 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 1018 "resolve_route(%x, %p)", dev, io_buf); 1019 1020 resolve_routep = (sol_ucma_resolve_route_t *)io_buf; 1021 ucma_id = resolve_routep->id; 1022 if (get_file_chan(ucma_id, NULL, &chanp, "resolve_route", 1)) 1023 return (EINVAL); 1024 ASSERT(chanp); 1025 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_route - chanp %p", 1026 chanp); 1027 1028 ret = rdma_resolve_route(chanp->chan_rdma_id, 1029 resolve_routep->timeout_ms); 1030 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_route: ret %x", ret); 1031 return (ret); 1032 } 1033 1034 /*ARGSUSED*/ 1035 static int 1036 sol_ucma_query_route(dev_t dev, void *io_buf, struct uio *uio) 1037 { 1038 sol_ucma_chan_t *chanp; 1039 uint32_t ucma_id; 1040 struct rdma_cm_id *idp; 1041 sol_ucma_query_route_t *query_routep; 1042 sol_ucma_query_route_resp_t route_info; 1043 1044 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route(%x, %p)", 1045 dev, io_buf); 1046 1047 query_routep = (sol_ucma_query_route_t *)io_buf; 1048 ucma_id = query_routep->id; 1049 if (get_file_chan(ucma_id, NULL, &chanp, "query_route", 1)) 1050 return (EINVAL); 1051 ASSERT(chanp); 1052 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route - chanp %p", chanp); 1053 idp = chanp->chan_rdma_id; 1054 1055 bzero(&route_info, sizeof (route_info)); 1056 rdma2usr_route(idp, &route_info); 1057 1058 #ifdef _LP64 1059 if (copyout(&route_info, (void *) (query_routep->response.r_laddr), 1060 sizeof (sol_ucma_query_route_resp_t))) { 1061 #else 1062 if (copyout(&route_info, (void *) (query_routep->response.r_addr), 1063 sizeof (sol_ucma_query_route_resp_t))) { 1064 #endif 1065 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1066 "query_route: copyout fault"); 1067 return (EFAULT); 1068 } 1069 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route: Succcess"); 1070 return (0); 1071 } 1072 1073 /*ARGSUSED*/ 1074 static int 1075 sol_ucma_connect(dev_t dev, void *io_buf, struct uio *uio) 1076 { 1077 sol_ucma_chan_t *chanp; 1078 uint32_t ucma_id; 1079 int ret; 1080 void *qphdl; 1081 sol_ucma_connect_t *connectp; 1082 struct rdma_conn_param conn_param; 1083 struct rdma_cm_id *idp; 1084 1085 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect(%x, %p)", 1086 dev, io_buf); 1087 1088 connectp = (sol_ucma_connect_t *)io_buf; 1089 ucma_id = connectp->id; 1090 if (get_file_chan(ucma_id, NULL, &chanp, "connect", 1)) 1091 return (EINVAL); 1092 ASSERT(chanp); 1093 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect - chanp %p", chanp); 1094 1095 usr2rdma_conn_param(&(connectp->conn_param), &conn_param); 1096 ASSERT(uverbs_qpnum2qphdl_fp); 1097 ASSERT(uverbs_disable_uqpn_modify_fp); 1098 ASSERT(uverbs_uqpn_cq_ctrl_fp); 1099 qphdl = (*uverbs_qpnum2qphdl_fp) (conn_param.qp_num); 1100 if (qphdl == NULL) { 1101 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "connect: " 1102 "invalid QPNum %x", conn_param.qp_num); 1103 return (EINVAL); 1104 } 1105 (*uverbs_disable_uqpn_modify_fp) (conn_param.qp_num); 1106 rdma_map_id2qphdl(chanp->chan_rdma_id, qphdl); 1107 idp = chanp->chan_rdma_id; 1108 if (idp->ps == RDMA_PS_TCP) 1109 (void) (*uverbs_uqpn_cq_ctrl_fp) (conn_param.qp_num, 1110 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE); 1111 chanp->chan_qp_num = conn_param.qp_num; 1112 ret = rdma_connect(chanp->chan_rdma_id, &conn_param); 1113 1114 /* 1115 * rdma_connect() initiated for this CMID, disable sol_uverbs to 1116 * free the QP assosiated with this CM ID. 1117 */ 1118 if (ret == 0 && idp->ps == RDMA_PS_TCP) { 1119 mutex_enter(&chanp->chan_mutex); 1120 chanp->chan_qp_hdl = qphdl; 1121 chanp->chan_flags |= SOL_UCMA_CHAN_CONNECT_FLAG; 1122 mutex_exit(&chanp->chan_mutex); 1123 (*uverbs_set_qp_free_state_fp) ( 1124 SOL_UVERBS2UCMA_DISABLE_QP_FREE, conn_param.qp_num, 1125 NULL); 1126 } 1127 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect: ret %x", ret); 1128 return (ret); 1129 } 1130 1131 /*ARGSUSED*/ 1132 static int 1133 sol_ucma_listen(dev_t dev, void *io_buf, struct uio *uio) 1134 { 1135 sol_ucma_chan_t *chanp; 1136 uint32_t ucma_id; 1137 int ret; 1138 sol_ucma_listen_t *listenp; 1139 1140 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen(%x, %p)", 1141 dev, io_buf); 1142 1143 listenp = (sol_ucma_listen_t *)io_buf; 1144 ucma_id = listenp->id; 1145 if (get_file_chan(ucma_id, NULL, &chanp, "listen", 1)) 1146 return (EINVAL); 1147 ASSERT(chanp); 1148 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen - chanp %p", chanp); 1149 1150 listenp->backlog = (listenp->backlog == 0 || 1151 listenp->backlog > SOL_UCMA_MAX_LISTEN) ? 1152 SOL_UCMA_MAX_LISTEN : listenp->backlog; 1153 chanp->chan_backlog = listenp->backlog; 1154 1155 ret = rdma_listen(chanp->chan_rdma_id, listenp->backlog); 1156 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen: ret %x", ret); 1157 return (ret); 1158 } 1159 1160 /*ARGSUSED*/ 1161 static int 1162 sol_ucma_accept(dev_t dev, void *io_buf, struct uio *uio) 1163 { 1164 int ret; 1165 uint32_t ucma_id; 1166 sol_ucma_chan_t *chanp; 1167 void *qphdl; 1168 sol_ucma_accept_t *acpt; 1169 struct rdma_conn_param conn_param; 1170 1171 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept(%x, %p)", 1172 dev, io_buf); 1173 1174 acpt = (sol_ucma_accept_t *)io_buf; 1175 ucma_id = acpt->id; 1176 if (get_file_chan(ucma_id, NULL, &chanp, "accept", 1)) 1177 return (EINVAL); 1178 ASSERT(chanp); 1179 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept - chanp %p", chanp); 1180 1181 if ((acpt->conn_param).valid) { 1182 struct rdma_cm_id *idp; 1183 1184 chanp->chan_user_id = acpt->uid; 1185 usr2rdma_conn_param(&acpt->conn_param, &conn_param); 1186 1187 ASSERT(uverbs_qpnum2qphdl_fp); 1188 qphdl = (*uverbs_qpnum2qphdl_fp) (conn_param.qp_num); 1189 if (qphdl == NULL) { 1190 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "accept: " 1191 "invalid QPNum %x", conn_param.qp_num); 1192 return (EINVAL); 1193 } 1194 (*uverbs_disable_uqpn_modify_fp) (conn_param.qp_num); 1195 rdma_map_id2qphdl(chanp->chan_rdma_id, qphdl); 1196 idp = chanp->chan_rdma_id; 1197 if (idp->ps == RDMA_PS_TCP) 1198 (void) (*uverbs_uqpn_cq_ctrl_fp) (conn_param.qp_num, 1199 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE); 1200 chanp->chan_qp_num = conn_param.qp_num; 1201 ret = rdma_accept(chanp->chan_rdma_id, &conn_param); 1202 } else 1203 ret = rdma_accept(chanp->chan_rdma_id, NULL); 1204 1205 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept: ret %x", ret); 1206 return (ret); 1207 } 1208 1209 /*ARGSUSED*/ 1210 static int 1211 sol_ucma_reject(dev_t dev, void *io_buf, struct uio *uio) 1212 { 1213 int ret; 1214 uint32_t ucma_id; 1215 sol_ucma_chan_t *chanp; 1216 sol_ucma_reject_t *rjct; 1217 1218 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject(%x, %p)", dev, io_buf); 1219 1220 rjct = (sol_ucma_reject_t *)io_buf; 1221 ucma_id = rjct->id; 1222 if (get_file_chan(ucma_id, NULL, &chanp, "reject", 1)) 1223 return (EINVAL); 1224 ASSERT(chanp); 1225 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject - chanp %p", chanp); 1226 1227 ret = rdma_reject(chanp->chan_rdma_id, rjct->private_data, 1228 rjct->private_data_len); 1229 1230 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject: ret %x", ret); 1231 return (ret); 1232 } 1233 1234 /*ARGSUSED*/ 1235 static int 1236 sol_ucma_init_qp_attr(dev_t dev, void *io_buf, struct uio *uio) 1237 { 1238 int ret; 1239 uint32_t ucma_id; 1240 uint32_t qp_attr_mask; 1241 sol_ucma_chan_t *chanp; 1242 sol_ucma_init_qp_attr_t *qp_attr_inp; 1243 struct ib_uverbs_qp_attr uverbs_qp_attr; 1244 struct ib_qp_attr qp_attr; 1245 1246 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr(%x, %p)", 1247 dev, io_buf); 1248 1249 qp_attr_inp = (sol_ucma_init_qp_attr_t *)io_buf; 1250 ucma_id = qp_attr_inp->id; 1251 if (get_file_chan(ucma_id, NULL, &chanp, "init_qp_attr", 1)) 1252 return (EINVAL); 1253 ASSERT(chanp); 1254 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr - chanp %p", chanp); 1255 1256 qp_attr.qp_state = qp_attr_inp->qp_state; 1257 if ((ret = rdma_init_qp_attr(chanp->chan_rdma_id, &qp_attr, 1258 (int *)&qp_attr_mask)) != 0) { 1259 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "init_qp_attr: ret %x, " 1260 "mask %x", ret, qp_attr_mask); 1261 return (EINVAL); 1262 } 1263 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr: ret %x, mask %x", 1264 ret, qp_attr_mask); 1265 1266 bzero(&uverbs_qp_attr, sizeof (uverbs_qp_attr)); 1267 uverbs_qp_attr.qp_attr_mask = qp_attr_mask; 1268 uverbs_qp_attr.qp_state = qp_attr.qp_state; 1269 uverbs_qp_attr.pkey_index = qp_attr.pkey_index; 1270 uverbs_qp_attr.port_num = qp_attr.port_num; 1271 uverbs_qp_attr.qp_access_flags = qp_attr.qp_access_flags; 1272 uverbs_qp_attr.qkey = qp_attr.qkey; 1273 uverbs_qp_attr.path_mtu = qp_attr.path_mtu; 1274 uverbs_qp_attr.dest_qp_num = qp_attr.dest_qp_num; 1275 uverbs_qp_attr.rq_psn = qp_attr.rq_psn; 1276 uverbs_qp_attr.max_dest_rd_atomic = qp_attr.max_dest_rd_atomic; 1277 uverbs_qp_attr.min_rnr_timer = qp_attr.min_rnr_timer; 1278 uverbs_qp_attr.ah_attr.dlid = qp_attr.ah_attr.dlid; 1279 if (qp_attr.ah_attr.ah_flags) { 1280 uverbs_qp_attr.ah_attr.is_global = 1; 1281 bcopy(&(qp_attr.ah_attr.grh.dgid), 1282 &(uverbs_qp_attr.ah_attr.grh.dgid), 16); 1283 uverbs_qp_attr.ah_attr.grh.flow_label = 1284 qp_attr.ah_attr.grh.flow_label; 1285 uverbs_qp_attr.ah_attr.grh.sgid_index = 1286 qp_attr.ah_attr.grh.sgid_index; 1287 uverbs_qp_attr.ah_attr.grh.hop_limit = 1288 qp_attr.ah_attr.grh.hop_limit; 1289 uverbs_qp_attr.ah_attr.grh.traffic_class = 1290 qp_attr.ah_attr.grh.traffic_class; 1291 } 1292 uverbs_qp_attr.ah_attr.sl = qp_attr.ah_attr.sl; 1293 uverbs_qp_attr.ah_attr.src_path_bits = qp_attr.ah_attr.src_path_bits; 1294 uverbs_qp_attr.ah_attr.static_rate = qp_attr.ah_attr.static_rate; 1295 uverbs_qp_attr.ah_attr.port_num = qp_attr.ah_attr.port_num; 1296 1297 #ifdef _LP64 1298 if (copyout(&uverbs_qp_attr, (void *) (qp_attr_inp->response.r_laddr), 1299 sizeof (uverbs_qp_attr))) { 1300 #else 1301 if (copyout(&uverbs_qp_attr, (void *) (qp_attr_inp->response.r_addr), 1302 sizeof (uverbs_qp_attr))) { 1303 #endif 1304 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "init_qp_attr : copyout " 1305 "failed"); 1306 return (EFAULT); 1307 } 1308 return (0); 1309 } 1310 1311 static int 1312 sol_ucma_get_event(dev_t dev, void *io_buf, struct uio *uio) 1313 { 1314 minor_t minor; 1315 sol_ucma_file_t *filep; 1316 sol_ucma_chan_t *evt_chanp; 1317 genlist_entry_t *entry; 1318 struct rdma_ucm_get_event *user_evt_inp; 1319 sol_ucma_event_t *queued_evt; 1320 struct rdma_ucm_event_resp *user_evt_resp; 1321 1322 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event(%x, %p)", dev, io_buf); 1323 user_evt_inp = (struct rdma_ucm_get_event *)io_buf; 1324 1325 minor = getminor(dev); 1326 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read(&ucma_file_uo_tbl, 1327 minor); 1328 ASSERT(filep); 1329 1330 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event fmode %x", 1331 uio->uio_fmode); 1332 1333 mutex_enter(&filep->file_mutex); 1334 while (filep->file_pending_evt_cnt == 0) { 1335 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "get_event: No events"); 1336 if (uio->uio_fmode & (FNONBLOCK | FNDELAY)) { 1337 mutex_exit(&filep->file_mutex); 1338 sol_ofs_uobj_put(&filep->file_uobj); 1339 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, 1340 "get_event: No events, nonblocking"); 1341 return (EAGAIN); 1342 } 1343 if (!cv_wait_sig(&filep->file_evt_cv, &filep->file_mutex)) { 1344 mutex_exit(&filep->file_mutex); 1345 sol_ofs_uobj_put(&filep->file_uobj); 1346 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 1347 "get_event: Got Sig"); 1348 return (EINTR); 1349 } 1350 } 1351 1352 entry = remove_genlist_head(&filep->file_evt_list); 1353 mutex_exit(&filep->file_mutex); 1354 ASSERT(entry); 1355 queued_evt = (sol_ucma_event_t *)entry->data; 1356 ASSERT(queued_evt); 1357 user_evt_resp = &queued_evt->event_resp; 1358 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "event2usr " 1359 "uid %llx, id %x, event %x, status %x", user_evt_resp->uid, 1360 user_evt_resp->id, user_evt_resp->event, user_evt_resp->status); 1361 #ifdef _LP64 1362 if (copyout((void *)user_evt_resp, 1363 (void *)(user_evt_inp->response.r_laddr), 1364 sizeof (sol_ucma_event_resp_t))) { 1365 #else 1366 if (copyout((void *)user_evt_resp, 1367 (void *)(user_evt_inp->response.r_addr), 1368 sizeof (sol_ucma_event_resp_t))) { 1369 #endif 1370 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event: copyout " 1371 "failed"); 1372 sol_ofs_uobj_put(&filep->file_uobj); 1373 kmem_free(entry, sizeof (genlist_entry_t)); 1374 return (EFAULT); 1375 } 1376 mutex_enter(&filep->file_mutex); 1377 filep->file_pending_evt_cnt--; 1378 if (queued_evt->event_mcast) 1379 (queued_evt->event_mcast)->mcast_events++; 1380 evt_chanp = queued_evt->event_chan; 1381 if (evt_chanp) { 1382 /* 1383 * If the event is RDMA_CM_EVENT_CONNECT_RESPONSE or 1384 * RDMA_CM_EVENT_ESTABLISHED and the CM ID is for RC, 1385 * enable completion notifications for the QP. 1386 */ 1387 if (user_evt_resp->event == RDMA_CM_EVENT_CONNECT_RESPONSE || 1388 user_evt_resp->event == RDMA_CM_EVENT_ESTABLISHED) { 1389 struct rdma_cm_id *idp; 1390 int rc; 1391 1392 idp = evt_chanp->chan_rdma_id; 1393 if (idp->ps == RDMA_PS_TCP) { 1394 ASSERT(uverbs_uqpn_cq_ctrl_fp); 1395 rc = (*uverbs_uqpn_cq_ctrl_fp)( 1396 evt_chanp->chan_qp_num, 1397 SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE); 1398 if (rc) { 1399 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1400 "uverbs_uqpn_cq_ctrl_fp(%X) " 1401 "failed!!", 1402 evt_chanp->chan_qp_num); 1403 mutex_exit(&filep->file_mutex); 1404 filep->file_pending_evt_cnt++; 1405 return (EIO); 1406 } 1407 } 1408 } 1409 1410 /* Bump up backlog for CONNECT_REQUEST events */ 1411 mutex_enter(&evt_chanp->chan_mutex); 1412 if (user_evt_resp->event == RDMA_CM_EVENT_CONNECT_REQUEST) 1413 evt_chanp->chan_backlog++; 1414 1415 evt_chanp->chan_evt_cnt++; 1416 mutex_exit(&evt_chanp->chan_mutex); 1417 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event : " 1418 "chan %p, cnt %x", evt_chanp, evt_chanp->chan_evt_cnt); 1419 } 1420 mutex_exit(&filep->file_mutex); 1421 kmem_free(entry, sizeof (genlist_entry_t)); 1422 kmem_free(queued_evt, sizeof (sol_ucma_event_t)); 1423 1424 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event: Success"); 1425 sol_ofs_uobj_put(&filep->file_uobj); 1426 return (0); 1427 } 1428 1429 /* 1430 * This is used when ULP wants to set the QOS option. This is *not* 1431 * supported by Solaris IB stack, return failure. 1432 */ 1433 /*ARGSUSED*/ 1434 static int 1435 sol_ucma_set_option(dev_t dev, void *io_buf, struct uio *uio) 1436 { 1437 return (EINVAL); 1438 } 1439 1440 /* 1441 * This is used when ULP uses librdmacm but uses out of band connection for CM. 1442 */ 1443 /*ARGSUSED*/ 1444 static int 1445 sol_ucma_notify(dev_t dev, void *io_buf, struct uio *uio) 1446 { 1447 sol_ucma_notify_t *notifyp; 1448 uint32_t ucma_id; 1449 sol_ucma_chan_t *chan; 1450 int ret; 1451 1452 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "notify(%x, %p)", dev, io_buf); 1453 notifyp = (sol_ucma_notify_t *)io_buf; 1454 ucma_id = notifyp->id; 1455 if (get_file_chan(ucma_id, NULL, &chan, "notify", 1)) 1456 return (EINVAL); 1457 ASSERT(chan); 1458 1459 ret = rdma_notify(chan->chan_rdma_id, notifyp->event); 1460 if (ret) 1461 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "notify failed %x", ret); 1462 else 1463 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "notify Success"); 1464 return (ret); 1465 } 1466 1467 /*ARGSUSED*/ 1468 static int 1469 sol_ucma_join_mcast(dev_t dev, void *io_buf, struct uio *uio) 1470 { 1471 sol_ucma_join_mcast_t *join_buf; 1472 sol_ucma_create_id_resp_t join_resp; 1473 sol_ucma_chan_t *chanp; 1474 sol_ucma_mcast_t *mcastp; 1475 int rc; 1476 uint32_t ucma_id; 1477 1478 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast(%x, %p)", 1479 dev, io_buf); 1480 join_buf = (sol_ucma_join_mcast_t *)io_buf; 1481 ucma_id = join_buf->id; 1482 if (get_file_chan(ucma_id, NULL, &chanp, "join_mcast", 1)) 1483 return (EINVAL); 1484 1485 mcastp = kmem_zalloc(sizeof (sol_ucma_mcast_t), KM_SLEEP); 1486 bcopy((void *)(&(join_buf->addr)), (void *)(&(mcastp->mcast_addr)), 1487 sizeof (struct sockaddr)); 1488 mcastp->mcast_chan = chanp; 1489 sol_ofs_uobj_init(&mcastp->mcast_uobj, NULL, SOL_UCMA_MCAST_TYPE); 1490 if (sol_ofs_uobj_add(&ucma_mcast_uo_tbl, &mcastp->mcast_uobj) != 0) { 1491 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1492 return (ENOMEM); 1493 } 1494 mcastp->mcast_uobj.uo_live = 1; 1495 mcastp->mcast_id = join_resp.id = mcastp->mcast_uobj.uo_id; 1496 mcastp->mcast_uid = join_buf->uid; 1497 1498 rc = rdma_join_multicast(chanp->chan_rdma_id, 1499 (struct sockaddr *)(&(join_buf->addr)), mcastp); 1500 if (rc) { 1501 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1502 "join_mcast: rdma_join_multicast ret %x", rc); 1503 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1504 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, 1505 &mcastp->mcast_uobj); 1506 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1507 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1508 return (rc); 1509 } 1510 1511 #ifdef _LP64 1512 if (copyout(&join_resp, (void *) (join_buf->response.r_laddr), 1513 sizeof (sol_ucma_create_id_resp_t))) { 1514 #else 1515 if (copyout(&join_resp, (void *) (join_buf->response.r_addr), 1516 sizeof (sol_ucma_create_id_resp_t))) { 1517 #endif 1518 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast: copyout " 1519 "failed"); 1520 rdma_leave_multicast(chanp->chan_rdma_id, 1521 (struct sockaddr *)(&(join_buf->addr))); 1522 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1523 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, 1524 &mcastp->mcast_uobj); 1525 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1526 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1527 return (EFAULT); 1528 } 1529 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast: Return Success"); 1530 return (0); 1531 } 1532 1533 /*ARGSUSED*/ 1534 static int 1535 sol_ucma_leave_mcast(dev_t dev, void *io_buf, struct uio *uio) 1536 { 1537 sol_ucma_destroy_id_t *id_inp; 1538 sol_ucma_destroy_id_resp_t id_resp; 1539 sol_ucma_mcast_t *mcastp; 1540 sol_ucma_chan_t *chanp; 1541 uint32_t ucma_id; 1542 1543 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "leave_mcast(%x, %p)", 1544 dev, io_buf); 1545 id_inp = (sol_ucma_destroy_id_t *)io_buf; 1546 ucma_id = id_inp->id; 1547 mcastp = (sol_ucma_mcast_t *)sol_ofs_uobj_get_read(&ucma_mcast_uo_tbl, 1548 ucma_id); 1549 if (mcastp == NULL) { 1550 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "leave_mcast: invalid " 1551 "ID %x", ucma_id); 1552 return (EINVAL); 1553 } 1554 chanp = mcastp->mcast_chan; 1555 1556 rdma_leave_multicast(chanp->chan_rdma_id, &mcastp->mcast_addr); 1557 id_resp.events_reported = mcastp->mcast_events; 1558 1559 #ifdef _LP64 1560 if (copyout(&id_resp, (void *) (id_inp->response.r_laddr), 1561 sizeof (sol_ucma_destroy_id_resp_t))) { 1562 #else 1563 if (copyout(&id_resp, (void *) (id_inp->response.r_addr), 1564 sizeof (sol_ucma_destroy_id_resp_t))) { 1565 #endif 1566 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "leave_mcast: copyout " 1567 "fault"); 1568 sol_ofs_uobj_put(&mcastp->mcast_uobj); 1569 return (EFAULT); 1570 } 1571 sol_ofs_uobj_put(&mcastp->mcast_uobj); 1572 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1573 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, &mcastp->mcast_uobj); 1574 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1575 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1576 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "leave_mcast: ret 0"); 1577 return (0); 1578 } 1579 1580 /*ARGSUSED*/ 1581 static int 1582 sol_ucma_disconnect(dev_t dev, void *io_buf, struct uio *uio) 1583 { 1584 sol_ucma_disconnect_t *disconnectp; 1585 uint32_t ucma_id; 1586 sol_ucma_chan_t *chan; 1587 int ret; 1588 1589 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "disconnect(%x, %p)", 1590 dev, io_buf); 1591 disconnectp = (sol_ucma_disconnect_t *)io_buf; 1592 ucma_id = disconnectp->id; 1593 if (get_file_chan(ucma_id, NULL, &chan, "disconnect", 1)) 1594 return (EINVAL); 1595 ASSERT(chan); 1596 1597 /* 1598 * For a TCP CMID, which has got the DISCONNECT event, call 1599 * ibt_flush_qp(), to transition QP to error state. 1600 */ 1601 mutex_enter(&chan->chan_mutex); 1602 if (chan->chan_flush_qp_flag == SOL_UCMA_FLUSH_QP_PENDING) { 1603 chan->chan_flush_qp_flag = SOL_UCMA_FLUSH_QP_DONE; 1604 mutex_exit(&chan->chan_mutex); 1605 (*uverbs_flush_qp_fp)(chan->chan_qp_num); 1606 } else 1607 mutex_exit(&chan->chan_mutex); 1608 1609 ret = rdma_disconnect(chan->chan_rdma_id); 1610 mutex_enter(&chan->chan_mutex); 1611 chan->chan_flush_qp_flag = SOL_UCMA_FLUSH_QP_DONE; 1612 mutex_exit(&chan->chan_mutex); 1613 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "disconnect: ret %x", ret); 1614 return (ret); 1615 } 1616 1617 /* 1618 * RDMA ID Event handler 1619 */ 1620 int 1621 sol_ucma_evt_hdlr(struct rdma_cm_id *idp, struct rdma_cm_event *eventp) 1622 { 1623 sol_ucma_chan_t *chan, *req_chan; 1624 sol_ucma_file_t *file; 1625 sol_ucma_event_t *ucma_evt; 1626 sol_ucma_create_id_t ucma_create_id; 1627 1628 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "ucma_evt_hdlr(%p, %p), " 1629 "event %x, status %x", idp, eventp, eventp->event, 1630 eventp->status); 1631 chan = (sol_ucma_chan_t *)idp->context; 1632 if (!chan) { 1633 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1634 "after destroy - %p", idp); 1635 return (0); 1636 } 1637 mutex_enter(&chan->chan_mutex); 1638 file = chan->chan_file; 1639 if (!file) { 1640 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1641 "after file destroy - idp %p", idp); 1642 mutex_exit(&chan->chan_mutex); 1643 return (0); 1644 } 1645 mutex_exit(&chan->chan_mutex); 1646 1647 mutex_enter(&file->file_mutex); 1648 if (file->file_evt_close_flag == SOL_UCMA_EVT_DISABLED) { 1649 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1650 "after file close - idp %p", idp); 1651 mutex_exit(&file->file_mutex); 1652 return (0); 1653 } 1654 file->file_evt_close_flag = SOL_UCMA_EVT_PROGRESS; 1655 mutex_exit(&file->file_mutex); 1656 1657 /* 1658 * If the event is RDMA_CM_EVENT_CONNECT_REQUEST, allocate a 1659 * new chan. The rdma_cm_id for this chan has already been 1660 * allocated by sol_ofs. 1661 */ 1662 ucma_evt = kmem_zalloc(sizeof (sol_ucma_event_t), KM_SLEEP); 1663 ucma_evt->event_chan = chan; 1664 if (eventp->event == RDMA_CM_EVENT_CONNECT_REQUEST) { 1665 mutex_enter(&chan->chan_mutex); 1666 if (!chan->chan_backlog) { 1667 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 1668 "backlog exceeded"); 1669 mutex_exit(&chan->chan_mutex); 1670 mutex_enter(&file->file_mutex); 1671 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1672 cv_broadcast(&file->file_evt_close_cv); 1673 mutex_exit(&file->file_mutex); 1674 kmem_free(ucma_evt, sizeof (sol_ucma_event_t)); 1675 return (-1); 1676 } 1677 chan->chan_backlog--; 1678 mutex_exit(&chan->chan_mutex); 1679 ucma_create_id.uid = chan->chan_user_id; 1680 req_chan = ucma_alloc_chan(file, &ucma_create_id); 1681 if (req_chan == NULL) { 1682 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1683 "evt hdlr: No free Channel"); 1684 sol_ofs_uobj_put(&file->file_uobj); 1685 mutex_enter(&file->file_mutex); 1686 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1687 cv_broadcast(&file->file_evt_close_cv); 1688 mutex_exit(&file->file_mutex); 1689 return (-1); 1690 } 1691 req_chan->chan_rdma_id = idp; 1692 mutex_enter(&req_chan->chan_mutex); 1693 idp->context = req_chan; 1694 mutex_exit(&req_chan->chan_mutex); 1695 chan = req_chan; 1696 } else if (eventp->event == RDMA_CM_EVENT_DISCONNECTED || 1697 eventp->event == RDMA_CM_EVENT_REJECTED) { 1698 void *qphdl; 1699 1700 /* 1701 * Connection has been rejected or disconnected, 1702 * Enable uverbs to free QP, if it had been disabled 1703 * before. sol_uverbs will free the QP appropriately. 1704 */ 1705 mutex_enter(&chan->chan_mutex); 1706 qphdl = chan->chan_qp_hdl; 1707 chan->chan_qp_hdl = NULL; 1708 if (idp->ps == RDMA_PS_TCP && 1709 chan->chan_flush_qp_flag != SOL_UCMA_FLUSH_QP_DONE && 1710 eventp->event == RDMA_CM_EVENT_DISCONNECTED) { 1711 chan->chan_flush_qp_flag = 1712 SOL_UCMA_FLUSH_QP_PENDING; 1713 } 1714 mutex_exit(&chan->chan_mutex); 1715 1716 if (idp->ps == RDMA_PS_TCP && qphdl) 1717 (*uverbs_set_qp_free_state_fp) ( 1718 SOL_UVERBS2UCMA_ENABLE_QP_FREE, 0, qphdl); 1719 } else if (eventp->event == RDMA_CM_EVENT_ESTABLISHED && 1720 chan->chan_flags & SOL_UCMA_CHAN_CONNECT_FLAG) 1721 eventp->event = RDMA_CM_EVENT_CONNECT_RESPONSE; 1722 1723 ucma_evt->event_resp.event = eventp->event; 1724 ucma_evt->event_resp.status = eventp->status; 1725 if (idp->ps == RDMA_PS_UDP || idp->ps == RDMA_PS_IPOIB) 1726 rdma2usr_ud_param(&(eventp->param.ud), 1727 &(ucma_evt->event_resp.param.ud)); 1728 else 1729 rdma2usr_conn_param(&(eventp->param.conn), 1730 &(ucma_evt->event_resp.param.conn)); 1731 1732 if (eventp->event == RDMA_CM_EVENT_MULTICAST_JOIN || eventp->event == 1733 RDMA_CM_EVENT_MULTICAST_ERROR) { 1734 ucma_evt->event_mcast = (sol_ucma_mcast_t *) 1735 eventp->param.ud.private_data; 1736 ucma_evt->event_resp.uid = (ucma_evt->event_mcast)->mcast_uid; 1737 ucma_evt->event_resp.id = (ucma_evt->event_mcast)->mcast_id; 1738 } else { 1739 ucma_evt->event_resp.uid = chan->chan_user_id; 1740 ucma_evt->event_resp.id = chan->chan_id; 1741 } 1742 1743 mutex_enter(&file->file_mutex); 1744 (void) add_genlist(&file->file_evt_list, (uintptr_t)ucma_evt, NULL); 1745 file->file_pending_evt_cnt++; 1746 mutex_exit(&file->file_mutex); 1747 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "ucma_evt_hdlr-pollwakeup"); 1748 pollwakeup(file->file_pollhead, POLLIN | POLLRDNORM); 1749 mutex_enter(&file->file_mutex); 1750 cv_broadcast(&file->file_evt_cv); 1751 mutex_exit(&file->file_mutex); 1752 1753 mutex_enter(&file->file_mutex); 1754 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1755 cv_broadcast(&file->file_evt_close_cv); 1756 mutex_exit(&file->file_mutex); 1757 return (0); 1758 } 1759 1760 /* 1761 * Local Functions 1762 */ 1763 static sol_ucma_file_t * 1764 ucma_alloc_file(minor_t *new_minorp) 1765 { 1766 sol_ucma_file_t *new_file; 1767 1768 new_file = kmem_zalloc(sizeof (sol_ucma_file_t), KM_SLEEP); 1769 sol_ofs_uobj_init(&new_file->file_uobj, NULL, SOL_UCMA_EVT_FILE_TYPE); 1770 if (sol_ofs_uobj_add(&ucma_file_uo_tbl, &new_file->file_uobj) != 0) { 1771 sol_ofs_uobj_free(&new_file->file_uobj); 1772 return (NULL); 1773 } 1774 new_file->file_uobj.uo_live = 1; 1775 init_genlist(&new_file->file_id_list); 1776 init_genlist(&new_file->file_evt_list); 1777 1778 mutex_enter(&sol_ucma.ucma_mutex); 1779 sol_ucma.ucma_num_file++; 1780 mutex_exit(&sol_ucma.ucma_mutex); 1781 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "new file num %x, %p", 1782 (new_file->file_uobj).uo_id, new_file); 1783 1784 mutex_init(&new_file->file_mutex, NULL, 1785 MUTEX_DRIVER, NULL); 1786 cv_init(&new_file->file_evt_cv, NULL, CV_DRIVER, 1787 NULL); 1788 cv_init(&new_file->file_evt_close_cv, NULL, CV_DRIVER, 1789 NULL); 1790 new_file->file_pollhead = kmem_zalloc(sizeof (struct pollhead), 1791 KM_SLEEP); 1792 1793 *new_minorp = (minor_t)((new_file->file_uobj).uo_id); 1794 return (new_file); 1795 } 1796 1797 static sol_ucma_chan_t * 1798 ucma_alloc_chan(sol_ucma_file_t *filep, sol_ucma_create_id_t *create_id_inp) 1799 { 1800 sol_ucma_chan_t *new_chanp; 1801 1802 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_alloc_chan(%p, %p)", 1803 filep, create_id_inp); 1804 1805 new_chanp = kmem_zalloc(sizeof (sol_ucma_chan_t), KM_SLEEP); 1806 sol_ofs_uobj_init(&new_chanp->chan_uobj, NULL, SOL_UCMA_CM_ID_TYPE); 1807 if (sol_ofs_uobj_add(&ucma_ctx_uo_tbl, &new_chanp->chan_uobj) != 0) { 1808 sol_ofs_uobj_free(&new_chanp->chan_uobj); 1809 return (NULL); 1810 } 1811 mutex_init(&new_chanp->chan_mutex, NULL, MUTEX_DRIVER, NULL); 1812 1813 new_chanp->chan_uobj.uo_live = 1; 1814 mutex_enter(&filep->file_mutex); 1815 new_chanp->chan_list_ent = add_genlist(&filep->file_id_list, 1816 (uintptr_t)new_chanp, NULL); 1817 mutex_exit(&filep->file_mutex); 1818 1819 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_alloc_chan - filep %p, " 1820 "chan_num %x, new_chan %p", filep, (new_chanp->chan_uobj).uo_id, 1821 new_chanp); 1822 1823 new_chanp->chan_file = filep; 1824 new_chanp->chan_user_id = create_id_inp->uid; 1825 new_chanp->chan_id = (new_chanp->chan_uobj).uo_id; 1826 1827 return (new_chanp); 1828 } 1829 1830 static void 1831 ucma_free_chan(sol_ucma_chan_t *chanp, int delete_list) 1832 { 1833 sol_ucma_file_t *filep; 1834 1835 ASSERT(chanp); 1836 if (delete_list) { 1837 filep = chanp->chan_file; 1838 ASSERT(filep); 1839 mutex_enter(&filep->file_mutex); 1840 delete_genlist(&filep->file_id_list, chanp->chan_list_ent); 1841 mutex_exit(&filep->file_mutex); 1842 } 1843 1844 mutex_destroy(&chanp->chan_mutex); 1845 rw_enter(&(chanp->chan_uobj.uo_lock), RW_WRITER); 1846 (void) sol_ofs_uobj_remove(&ucma_ctx_uo_tbl, &(chanp->chan_uobj)); 1847 rw_exit(&(chanp->chan_uobj.uo_lock)); 1848 sol_ofs_uobj_free(&(chanp->chan_uobj)); 1849 } 1850 1851 static int 1852 get_file_chan(uint32_t ucma_id, sol_ucma_file_t **filep, 1853 sol_ucma_chan_t **chanp, char *caller, int flag_err) 1854 { 1855 sol_ucma_chan_t *chan; 1856 1857 if (filep) 1858 *filep = NULL; 1859 if (chanp) 1860 *chanp = NULL; 1861 1862 chan = (sol_ucma_chan_t *)sol_ofs_uobj_get_read(&ucma_ctx_uo_tbl, 1863 ucma_id); 1864 if (chan == NULL) { 1865 if (flag_err) 1866 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1867 "%s, ucma_id %x invalid", caller, ucma_id); 1868 else 1869 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 1870 "%s, ucma_id %x invalid", caller, ucma_id); 1871 return (-1); 1872 } 1873 1874 if (filep) 1875 *filep = chan->chan_file; 1876 if (chanp) 1877 *chanp = chan; 1878 1879 sol_ofs_uobj_put(&chan->chan_uobj); 1880 return (0); 1881 } 1882 1883 static void 1884 rdma2usr_pathrec(struct ib_sa_path_rec *kern_path, 1885 struct ib_user_path_rec *usr_path) 1886 { 1887 bcopy(&kern_path->dgid, &usr_path->dgid, 16); 1888 bcopy(&kern_path->sgid, &usr_path->sgid, 16); 1889 usr_path->dlid = kern_path->dlid; 1890 usr_path->slid = kern_path->slid; 1891 usr_path->raw_traffic = kern_path->raw_traffic; 1892 usr_path->flow_label = kern_path->flow_label; 1893 usr_path->reversible = kern_path->reversible; 1894 usr_path->mtu = kern_path->mtu; 1895 usr_path->pkey = kern_path->pkey; 1896 usr_path->hop_limit = kern_path->hop_limit; 1897 usr_path->traffic_class = kern_path->traffic_class; 1898 usr_path->sl = kern_path->sl; 1899 usr_path->mtu_selector = kern_path->mtu_selector; 1900 usr_path->rate_selector = kern_path->rate_selector; 1901 usr_path->rate = kern_path->rate; 1902 usr_path->packet_life_time_selector = 1903 kern_path->packet_life_time_selector; 1904 usr_path->packet_life_time = kern_path->packet_life_time; 1905 usr_path->preference = kern_path->preference; 1906 usr_path->numb_path = kern_path->numb_path; 1907 } 1908 1909 static void 1910 rdma2usr_route(struct rdma_cm_id *idp, sol_ucma_query_route_resp_t *resp) 1911 { 1912 struct rdma_route *routep; 1913 int i; 1914 1915 routep = &(idp->route); 1916 if (idp->device) { 1917 resp->node_guid = idp->device->node_guid; 1918 resp->port_num = idp->port_num; 1919 } 1920 bcopy(&(routep->addr.src_addr), &resp->src_addr, 1921 sizeof (struct sockaddr_in6)); 1922 bcopy(&(routep->addr.dst_addr), &resp->dst_addr, 1923 sizeof (struct sockaddr_in6)); 1924 resp->num_paths = routep->num_paths; 1925 for (i = 0; i < resp->num_paths; i++) { 1926 rdma2usr_pathrec(&(routep->path_rec[i]), 1927 &(resp->ib_route[i])); 1928 } 1929 } 1930 1931 static void 1932 usr2rdma_conn_param(struct rdma_ucm_conn_param *usr_conn_paramp, 1933 struct rdma_conn_param *conn_paramp) 1934 { 1935 conn_paramp->private_data = usr_conn_paramp->private_data; 1936 conn_paramp->private_data_len = usr_conn_paramp->private_data_len; 1937 conn_paramp->responder_resources = usr_conn_paramp->responder_resources; 1938 conn_paramp->initiator_depth = usr_conn_paramp->initiator_depth; 1939 conn_paramp->flow_control = usr_conn_paramp->flow_control; 1940 conn_paramp->retry_count = usr_conn_paramp->retry_count; 1941 conn_paramp->rnr_retry_count = usr_conn_paramp->rnr_retry_count; 1942 conn_paramp->srq = usr_conn_paramp->srq; 1943 conn_paramp->qp_num = usr_conn_paramp->qp_num; 1944 } 1945 1946 static void 1947 rdma2usr_conn_param(struct rdma_conn_param *conn_paramp, 1948 struct rdma_ucm_conn_param *usr_conn_paramp) 1949 { 1950 usr_conn_paramp->private_data_len = conn_paramp->private_data_len; 1951 1952 bzero(usr_conn_paramp->private_data, RDMA_MAX_PRIVATE_DATA); 1953 if (conn_paramp->private_data) 1954 bcopy(conn_paramp->private_data, 1955 usr_conn_paramp->private_data, 1956 usr_conn_paramp->private_data_len); 1957 usr_conn_paramp->responder_resources = conn_paramp->responder_resources; 1958 usr_conn_paramp->initiator_depth = conn_paramp->initiator_depth; 1959 usr_conn_paramp->flow_control = conn_paramp->flow_control; 1960 usr_conn_paramp->retry_count = conn_paramp->retry_count; 1961 usr_conn_paramp->rnr_retry_count = conn_paramp->rnr_retry_count; 1962 usr_conn_paramp->srq = conn_paramp->srq; 1963 usr_conn_paramp->qp_num = conn_paramp->qp_num; 1964 } 1965 1966 static void 1967 rdma2usr_ud_param(struct rdma_ud_param *ud_paramp, 1968 sol_ucma_ud_param_t *usr_ud_paramp) 1969 { 1970 struct ib_ah_attr *ah_attrp; 1971 struct ib_uverbs_ah_attr *usr_ah_attrp; 1972 1973 usr_ud_paramp->private_data_len = ud_paramp->private_data_len; 1974 1975 bzero(usr_ud_paramp->private_data, RDMA_MAX_PRIVATE_DATA); 1976 if (ud_paramp->private_data) 1977 bcopy(ud_paramp->private_data, 1978 usr_ud_paramp->private_data, 1979 usr_ud_paramp->private_data_len); 1980 usr_ud_paramp->qp_num = ud_paramp->qp_num; 1981 usr_ud_paramp->qkey = ud_paramp->qkey; 1982 1983 ah_attrp = &(ud_paramp->ah_attr); 1984 usr_ah_attrp = &(usr_ud_paramp->ah_attr); 1985 bcopy(&(ah_attrp->grh.dgid), &(usr_ah_attrp->grh.dgid[0]), 16); 1986 usr_ah_attrp->grh.flow_label = ah_attrp->grh.flow_label; 1987 usr_ah_attrp->grh.sgid_index = ah_attrp->grh.sgid_index; 1988 usr_ah_attrp->grh.hop_limit = ah_attrp->grh.hop_limit; 1989 usr_ah_attrp->grh.traffic_class = ah_attrp->grh.traffic_class; 1990 usr_ah_attrp->dlid = ah_attrp->dlid; 1991 usr_ah_attrp->sl = ah_attrp->sl; 1992 usr_ah_attrp->src_path_bits = ah_attrp->src_path_bits; 1993 usr_ah_attrp->static_rate = ah_attrp->static_rate; 1994 usr_ah_attrp->is_global = ah_attrp->ah_flags; 1995 usr_ah_attrp->port_num = ah_attrp->port_num; 1996 } 1997 1998 static void 1999 sol_ucma_user_objs_init() 2000 { 2001 sol_ofs_uobj_tbl_init(&ucma_file_uo_tbl, sizeof (sol_ucma_file_t)); 2002 sol_ofs_uobj_tbl_init(&ucma_ctx_uo_tbl, sizeof (sol_ucma_chan_t)); 2003 sol_ofs_uobj_tbl_init(&ucma_mcast_uo_tbl, sizeof (sol_ucma_mcast_t)); 2004 } 2005 2006 static void 2007 sol_ucma_user_objs_fini() 2008 { 2009 sol_ofs_uobj_tbl_fini(&ucma_file_uo_tbl); 2010 sol_ofs_uobj_tbl_fini(&ucma_ctx_uo_tbl); 2011 sol_ofs_uobj_tbl_fini(&ucma_mcast_uo_tbl); 2012 } 2013