1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * IB specific routines for RDMA CM functionality 28 */ 29 /* Standard driver includes */ 30 #include <sys/types.h> 31 #include <sys/modctl.h> 32 #include <sys/errno.h> 33 #include <sys/stat.h> 34 35 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 36 #include <sys/ib/clients/of/ofed_kernel.h> 37 #include <sys/ib/clients/of/rdma/ib_addr.h> 38 #include <sys/ib/clients/of/rdma/rdma_cm.h> 39 40 #include <sys/ib/clients/of/sol_ofs/sol_cma.h> 41 #include <sys/ib/clients/of/sol_ofs/sol_ib_cma.h> 42 43 extern char *sol_rdmacm_dbg_str; 44 45 /* Delay of 5 secs */ 46 #define SOL_OFS_REQ_DELAY 5000000 47 48 /* Solaris CM Event Callbacks */ 49 static ibt_cm_status_t ibcma_rc_hdlr(void *, ibt_cm_event_t *, 50 ibt_cm_return_args_t *, void *, ibt_priv_data_len_t); 51 static ibt_cm_status_t ibcma_ud_hdlr(void *, ibt_cm_ud_event_t *, 52 ibt_cm_ud_return_args_t *, void *, ibt_priv_data_len_t); 53 static void ibcma_multicast_hdlr(void *, ibt_status_t, ibt_mcg_info_t *); 54 55 /* Local functions */ 56 static int ibcma_tcp_connect(struct rdma_cm_id *, ibcma_chan_t *, 57 struct rdma_conn_param *); 58 static int ibcma_udp_connect(struct rdma_cm_id *, ibcma_chan_t *, 59 struct rdma_conn_param *); 60 static struct rdma_cm_id *ibcma_create_new_id(struct rdma_cm_id *); 61 static int ibcma_query_local_ip(struct rdma_cm_id *, sol_cma_chan_t *, 62 ibcma_chan_t *); 63 static int ibcma_get_paths(struct rdma_cm_id *, sol_cma_chan_t *, 64 ibcma_chan_t *); 65 static void ibcma_get_devlist(sol_cma_chan_t *, ib_guid_t *, int, 66 genlist_t *); 67 static int ibcma_any_addr(ibt_ip_addr_t *); 68 static int ibcma_get_first_ib_ipaddr(struct rdma_cm_id *); 69 70 /* Utility Conversion Routines */ 71 static void ipaddr2mgid(struct sockaddr *, ib_gid_t *, ib_pkey_t); 72 static void ibt_path2ah(ibt_path_info_t *, struct ib_ah_attr *); 73 static void ibt_addsvect2ah(ibt_adds_vect_t *, struct ib_ah_attr *); 74 static void ibt_addsvect2sa_path(ibt_adds_vect_t *, 75 struct ib_sa_path_rec *, ib_lid_t); 76 static void ibt_path2sa_path(ibt_path_info_t *, struct ib_sa_path_rec *, 77 ib_lid_t); 78 static void mcginfo2ah(ibt_mcg_info_t *, struct ib_ah_attr *); 79 static void sockaddr2ibtaddr_port(struct rdma_cm_id *, struct sockaddr *, 80 ibt_ip_addr_t *, in_port_t *); 81 static void ipaddr2sockaddr(ibt_ip_addr_t *, struct sockaddr *, 82 in_port_t *); 83 84 #ifdef QP_DEBUG 85 static void dump_qp_info(ibt_qp_hdl_t); 86 #endif 87 static void dump_priv_data(void *, ibt_priv_data_len_t, 88 ibt_priv_data_len_t, char *); 89 90 extern cma_chan_state_t cma_get_chan_state(sol_cma_chan_t *); 91 92 /* 93 * RDMA CM API - Transport specific functions 94 */ 95 void 96 rdma_ib_destroy_id(struct rdma_cm_id *idp) 97 { 98 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 99 ibcma_chan_t *ibchanp; 100 ibt_status_t status; 101 102 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_destroy_id(%p)", idp); 103 ASSERT(chanp); 104 ibchanp = &(chanp->chan_ib); 105 106 if (ibchanp->chan_mcast_cnt) { 107 genlist_entry_t *entry; 108 ibcma_mcast_t *ibmcastp; 109 ib_gid_t zero_gid; 110 111 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 112 "rdma_ib_destroy_id: pending mcast!!"); 113 entry = remove_genlist_head(&ibchanp->chan_mcast_list); 114 while (entry) { 115 ibmcastp = (ibcma_mcast_t *)entry->data; 116 117 bzero(&zero_gid, sizeof (ib_gid_t)); 118 status = ibt_leave_mcg(ibchanp->chan_devp->dev_sgid, 119 ibmcastp->mcast_gid, zero_gid, IB_MC_JSTATE_FULL); 120 if (status != IBT_SUCCESS) 121 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 122 "destroy_id: ibt_leave_mcg failed %d", 123 status); 124 kmem_free(ibmcastp, sizeof (ibcma_mcast_t)); 125 126 entry = remove_genlist_head(&ibchanp->chan_mcast_list); 127 } 128 } 129 if (ibchanp->chan_devp) { 130 kmem_free(ibchanp->chan_devp, sizeof (ibcma_dev_t)); 131 ibchanp->chan_devp = NULL; 132 } 133 if (ibchanp->chan_pathp) { 134 kmem_free(ibchanp->chan_pathp, ibchanp->chan_path_size); 135 ibchanp->chan_pathp = NULL; 136 } 137 138 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_destroy_id: return"); 139 } 140 141 int 142 rdma_ib_bind_addr(struct rdma_cm_id *idp, struct sockaddr *addr) 143 { 144 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 145 ibcma_chan_t *ibchanp; 146 int ret; 147 in_port_t port; 148 149 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_bind_addr(%p, %p)", 150 idp, addr); 151 ASSERT(chanp); 152 ibchanp = &(chanp->chan_ib); 153 154 sockaddr2ibtaddr_port(idp, addr, &ibchanp->chan_local_addr, &port); 155 ibchanp->chan_addr_flag = IBCMA_LOCAL_ADDR_SET_FLAG; 156 157 /* 158 * If this is IF_ADDR_ANY, get info of IB port with IP @. 159 * Return Failure, if there are no IB ports with IP @. 160 */ 161 if (sol_cma_any_addr(addr)) { 162 ibchanp->chan_port = port; 163 ibchanp->chan_addr_flag |= IBCMA_LOCAL_ADDR_IFADDRANY; 164 return (ibcma_get_first_ib_ipaddr(idp)); 165 } 166 167 ret = ibcma_query_local_ip(idp, chanp, ibchanp); 168 if (ret == 0) { 169 init_genlist(&ibchanp->chan_mcast_list); 170 ibchanp->chan_sid = ibt_get_ip_sid(idp->ps, port); 171 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 172 "chan SID %llx , ps %x, port %x", 173 ibchanp->chan_sid, idp->ps, port); 174 ibchanp->chan_port = port; 175 chanp->chan_xport_type = SOL_CMA_XPORT_IB; 176 } 177 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_bind_addr: return %x", 178 ret); 179 return (ret); 180 } 181 182 int 183 rdma_ib_resolve_addr(struct rdma_cm_id *idp, struct sockaddr *src_addr, 184 struct sockaddr *dst_addr, int timeout_ms) 185 { 186 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 187 ibcma_chan_t *ibchanp; 188 int ret; 189 in_port_t port; 190 in_addr_t remote_addr; 191 192 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_resolve_addr(" 193 "%p, %p, %p, %x)", idp, src_addr, dst_addr, timeout_ms); 194 ASSERT(chanp); 195 ibchanp = &(chanp->chan_ib); 196 197 /* 198 * Copy src_addr if the passed src @ is valid IP address and 199 * the local @ has not been set for this CMID. 200 */ 201 if ((ibchanp->chan_addr_flag & IBCMA_LOCAL_ADDR_SET_FLAG) == 0 && 202 IS_VALID_SOCKADDR(src_addr)) { 203 sockaddr2ibtaddr_port(idp, src_addr, &ibchanp->chan_local_addr, 204 &port); 205 ibchanp->chan_addr_flag |= IBCMA_LOCAL_ADDR_SET_FLAG; 206 if (port) { 207 ibchanp->chan_sid = ibt_get_ip_sid(idp->ps, port); 208 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_addr, " 209 "local @ SID %llx, ps %x, port %x", 210 ibchanp->chan_sid, idp->ps, port); 211 ibchanp->chan_port = port; 212 } 213 } 214 215 sockaddr2ibtaddr_port(idp, dst_addr, &ibchanp->chan_remote_addr, 216 &port); 217 ibchanp->chan_addr_flag |= IBCMA_REMOTE_ADDR_SET_FLAG; 218 if (ibchanp->chan_sid == 0) { 219 ASSERT(!sol_cma_any_addr(dst_addr)); 220 ibchanp->chan_sid = ibt_get_ip_sid(idp->ps, port); 221 ibchanp->chan_port = port; 222 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_addr, remote @ " 223 "SID %llx , ps %x, port %x", ibchanp->chan_sid, 224 idp->ps, port); 225 init_genlist(&ibchanp->chan_mcast_list); 226 } 227 228 /* 229 * Return SUCCESS if remote address is a MCAST address 230 * and local address is not IF_ADDR_ANY. If local_addr 231 * is IF_ADDR_ANY and remote is MCAST, return FAILURE. 232 */ 233 remote_addr = htonl((ibchanp->chan_remote_addr).un.ip4addr); 234 if ((ibchanp->chan_remote_addr).family == AF_INET && 235 (remote_addr >= 0xE0000000 && remote_addr <= 0xEFFFFFFF)) { 236 if (ibchanp->chan_devp) { 237 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 238 "ib_resolve_addr - mcast dest @, local IP"); 239 (idp->route).num_paths = 0; 240 ret = 0; 241 } else { 242 ret = ibcma_get_first_ib_ipaddr(idp); 243 } 244 245 if (ret == 0 && idp->device == NULL) 246 idp->device = sol_cma_acquire_device(ntohll( 247 ibchanp->chan_devp->dev_node_guid)); 248 return (0); 249 } 250 251 if ((ret = ibcma_get_paths(idp, chanp, ibchanp)) == 0) 252 chanp->chan_xport_type = SOL_CMA_XPORT_IB; 253 254 return (ret); 255 } 256 257 /* 258 * Linux OFED implementation is as below : 259 * 1. librdmacm sends INIT_QP_ATTR command to get QP attributes 260 * which the kernel CM expects QP attribute to be in. Kernel 261 * CM sets the QP attribute to be set and passes it back to 262 * user library. 263 * 2. librdmacm calls ibv_modify_qp() to modify the QP attribute. 264 * The QP attribute used is the same as the that passed by 265 * kernel sol_ucma. 266 * 267 * For RC connections, Solaris ibcm manages the QP state after : 268 * CM Event Handler is called - Passive side 269 * ibv_open_rc_channel(9f) - Active Side 270 * The client will *not* have to do an explcit modify_qp(). To fit this 271 * INIT_QP_ATTR commands *marks* the QP to fake it's attributes and 272 * ignore ibv_modify_qp() for this QP. Solaris ibcm manages QP state. 273 * 274 * Before the above calls, the client will have to maintain the QP state. 275 * The sol_ucma driver will pass the appropriate QP atrributes, for the 276 * clients to pass to ibv_modify_qp(). 277 * 278 * For UD, OFED model is adhered to till the QP is transitioned to RTS. 279 * Any transitions after the QP has transitioned to RTS are ignored. 280 */ 281 int 282 rdma_ib_init_qp_attr(struct rdma_cm_id *idp, struct ib_qp_attr *qpattr, 283 int *qp_attr_mask) 284 { 285 sol_cma_chan_t *chanp; 286 ibcma_chan_t *ibchanp; 287 ibcma_dev_t *devp; 288 uint32_t qpstate; 289 290 ASSERT(idp); 291 chanp = (sol_cma_chan_t *)idp; 292 ibchanp = &chanp->chan_ib; 293 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_init_qp_attr(" 294 "%p, %p, %p)", idp, qpattr, qp_attr_mask); 295 296 if (ibchanp->chan_qpmodifyflag == 1) { 297 SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, 298 "Ignoring Init QP Attr"); 299 return (0); 300 } 301 302 qpstate = qpattr->qp_state; 303 bzero(qpattr, sizeof (struct ib_qp_attr)); 304 qpattr->qp_state = qpstate; 305 306 devp = ibchanp->chan_devp; 307 if (devp == NULL) { 308 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 309 "init_qp_attr, devp NULL"); 310 return (EINVAL); 311 } 312 qpattr->pkey_index = devp->dev_pkey_ix; 313 qpattr->port_num = devp->dev_port_num; 314 315 if (idp->ps == RDMA_PS_TCP && qpstate == IB_QPS_INIT) { 316 qpattr->qp_access_flags = IB_ACCESS_REMOTE_WRITE | 317 IB_ACCESS_REMOTE_READ; 318 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | 319 IB_QP_ACCESS_FLAGS; 320 return (0); 321 } else if (idp->ps == RDMA_PS_TCP && 322 qpstate == IB_QPS_RTR) { 323 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | 324 IB_QP_DEST_QPN | IB_QP_RQ_PSN | 325 IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 326 /* 327 * Fill in valid values for address vector & Remote QPN. 328 * Fill in MTU as MTU_256 & PSN as 0. This QP will be 329 * reset anyway. 330 */ 331 ibt_addsvect2ah(&ibchanp->chan_rcreq_addr, &qpattr->ah_attr); 332 qpattr->path_mtu = (uint32_t) 333 ((ibchanp->chan_rtr_data).req_path_mtu); 334 qpattr->dest_qp_num = ibchanp->chan_rcreq_qpn; 335 qpattr->rq_psn = (ibchanp->chan_rtr_data).req_rq_psn; 336 qpattr->max_dest_rd_atomic = ibchanp->chan_rcreq_ra_in; 337 qpattr->min_rnr_timer = 338 (ibchanp->chan_rtr_data).req_rnr_nak_time; 339 return (0); 340 } else if (IS_UDP_CMID(idp)) { 341 qpattr->qkey = RDMA_UDP_QKEY; 342 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | 343 IB_QP_PORT | IB_QP_QKEY; 344 return (0); 345 } else 346 return (EINVAL); 347 } 348 349 int 350 rdma_ib_connect(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param) 351 { 352 sol_cma_chan_t *chanp; 353 ibcma_chan_t *ibchanp; 354 int ret; 355 356 ASSERT(idp); 357 chanp = (sol_cma_chan_t *)idp; 358 ibchanp = &chanp->chan_ib; 359 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_connect(%p, %p)", idp, 360 conn_param); 361 362 ASSERT(chanp->chan_xport_type == SOL_CMA_XPORT_IB); 363 if (ibchanp->chan_devp == NULL || ibchanp->chan_pathp == NULL) { 364 SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str, "rdma_ib_connect : " 365 "invalid IP @"); 366 return (EINVAL); 367 } 368 ASSERT(ibchanp->chan_devp); 369 370 ibchanp->chan_qpmodifyflag = 1; 371 if (idp->ps == RDMA_PS_TCP) 372 ret = ibcma_tcp_connect(idp, ibchanp, conn_param); 373 else 374 ret = ibcma_udp_connect(idp, ibchanp, conn_param); 375 376 return (ret); 377 } 378 379 extern void sol_cma_add_hca_list(sol_cma_chan_t *, ib_guid_t); 380 void 381 ibcma_append_listen_list(struct rdma_cm_id *root_idp) 382 { 383 int num_hcas; 384 ib_guid_t *hca_guidp; 385 struct rdma_cm_id *ep_idp; 386 sol_cma_chan_t *root_chanp, *ep_chanp; 387 ibcma_chan_t *root_ibchanp, *ep_ibchanp; 388 genlist_t dev_genlist; 389 genlist_entry_t *entry; 390 391 sol_cma_listen_info_t *listenp; 392 ibcma_dev_t *devp; 393 394 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "append_listen(%p)", root_idp); 395 root_chanp = (sol_cma_chan_t *)root_idp; 396 root_ibchanp = &root_chanp->chan_ib; 397 398 /* 399 * Address other than IF_ADDR_ANY bound to this channel. Listen on 400 * this IP address alone. 401 */ 402 if (root_ibchanp->chan_devp && 403 (root_ibchanp->chan_addr_flag & IBCMA_LOCAL_ADDR_IFADDRANY) == 0) { 404 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "Create listen chan"); 405 ep_idp = ibcma_create_new_id(root_idp); 406 ASSERT(ep_idp); 407 408 ep_chanp = (sol_cma_chan_t *)ep_idp; 409 listenp = kmem_zalloc(sizeof (sol_cma_listen_info_t), 410 KM_SLEEP); 411 ep_chanp->chan_listenp = listenp; 412 413 ep_ibchanp = &ep_chanp->chan_ib; 414 ep_ibchanp->chan_port = root_ibchanp->chan_port; 415 listenp->listen_ep_root_entry = add_genlist( 416 &(CHAN_LISTEN_LIST(root_chanp)), 417 (uintptr_t)ep_idp, root_idp); 418 devp = ep_ibchanp->chan_devp; 419 sol_cma_add_hca_list(ep_chanp, ntohll(devp->dev_node_guid)); 420 return; 421 } 422 423 /* 424 * Get the list of IB devs with valid IP addresses 425 * Append to the list of listeners for root_idp 426 */ 427 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "Search IP @"); 428 num_hcas = ibt_get_hca_list(&hca_guidp); 429 ibcma_get_devlist(root_chanp, hca_guidp, num_hcas, 430 &dev_genlist); 431 entry = remove_genlist_head(&dev_genlist); 432 while (entry) { 433 devp = (ibcma_dev_t *)(entry->data); 434 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 435 "Create listen chan- ALL"); 436 ep_idp = ibcma_create_new_id(root_idp); 437 ASSERT(ep_idp); 438 439 ep_chanp = (sol_cma_chan_t *)ep_idp; 440 ipaddr2sockaddr(&devp->dev_ipaddr, 441 &(ep_idp->route.addr.src_addr), NULL); 442 listenp = kmem_zalloc(sizeof (sol_cma_listen_info_t), 443 KM_SLEEP); 444 ep_chanp->chan_listenp = listenp; 445 446 ep_ibchanp = &ep_chanp->chan_ib; 447 kmem_free(ep_ibchanp->chan_devp, sizeof (ibcma_dev_t)); 448 ep_ibchanp->chan_devp = devp; 449 ep_ibchanp->chan_port = root_ibchanp->chan_port; 450 451 listenp->listen_ep_root_entry = add_genlist( 452 &(CHAN_LISTEN_LIST(root_chanp)), 453 (uintptr_t)ep_idp, root_idp); 454 sol_cma_add_hca_list(ep_chanp, ntohll(devp->dev_node_guid)); 455 kmem_free(entry, sizeof (genlist_entry_t)); 456 entry = remove_genlist_head(&dev_genlist); 457 } 458 ibt_free_hca_list(hca_guidp, num_hcas); 459 } 460 461 int 462 ibcma_init_root_chan(sol_cma_chan_t *root_chanp, sol_cma_glbl_listen_t *listenp) 463 { 464 ibcma_chan_t *root_ibchanp; 465 ibt_srv_desc_t service; 466 ibt_status_t status; 467 struct rdma_cm_id *root_idp; 468 469 root_idp = &(root_chanp->chan_rdma_cm); 470 root_ibchanp = &root_chanp->chan_ib; 471 472 if (root_idp->ps == RDMA_PS_TCP) 473 service.sd_handler = ibcma_rc_hdlr; 474 else 475 service.sd_ud_handler = ibcma_ud_hdlr; 476 service.sd_flags = IBT_SRV_NO_FLAGS; 477 status = ibt_register_service(root_chanp->chan_ib_client_hdl, 478 &service, root_ibchanp->chan_sid, 479 root_ibchanp->chan_port ? 1 : 0xffff, 480 &((root_chanp->chan_listenp)->listen_ib_srv_hdl), 481 NULL); 482 if (status != IBT_SUCCESS) { 483 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 484 "init_root_chan: ibt_register_service ret %x" 485 "SID %x, port %x", status, root_ibchanp->chan_sid, 486 root_ibchanp->chan_port); 487 return (EINVAL); 488 } 489 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "init_root_chan: " 490 "ibt_register_service: SID %x, port %x: done", 491 root_ibchanp->chan_sid, root_ibchanp->chan_port); 492 listenp->cma_listen_svc_hdl = 493 (void *)(root_chanp->chan_listenp)->listen_ib_srv_hdl; 494 return (0); 495 } 496 497 int 498 ibcma_fini_root_chan(sol_cma_chan_t *rchanp) 499 { 500 ibt_status_t status; 501 502 status = ibt_deregister_service(rchanp->chan_ib_client_hdl, 503 (rchanp->chan_listenp)->listen_ib_srv_hdl); 504 if (status != IBT_SUCCESS) { 505 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 506 "fini_root_chan: ibt_deregister_service ret %x", 507 status); 508 return (EINVAL); 509 } 510 return (0); 511 } 512 513 void 514 ibcma_copy_srv_hdl(sol_cma_chan_t *root_chanp, sol_cma_glbl_listen_t *listenp) 515 { 516 (root_chanp->chan_listenp)->listen_ib_srv_hdl = 517 (ibt_srv_hdl_t)listenp->cma_listen_svc_hdl; 518 } 519 520 int 521 ibcma_fini_ep_chan(sol_cma_chan_t *ep_chanp) 522 { 523 struct rdma_cm_id *root_idp; 524 sol_cma_chan_t *root_chanp; 525 sol_cma_listen_info_t *root_listenp, *ep_listenp; 526 ibt_status_t status; 527 ibcma_chan_t *ep_ibchanp = &ep_chanp->chan_ib; 528 529 ASSERT(ep_chanp); 530 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 531 "fini_ep_chan(%p)", ep_chanp); 532 root_idp = CHAN_LISTEN_ROOT(ep_chanp); 533 root_chanp = (sol_cma_chan_t *)root_idp; 534 root_listenp = root_chanp->chan_listenp; 535 ep_listenp = ep_chanp->chan_listenp; 536 537 if (ep_ibchanp->chan_devp) 538 kmem_free(ep_ibchanp->chan_devp, sizeof (ibcma_dev_t)); 539 if (ep_ibchanp->chan_pathp) 540 kmem_free(ep_ibchanp->chan_pathp, 541 ep_ibchanp->chan_path_size); 542 543 if (!ep_listenp->listen_ib_sbind_hdl) 544 return (0); 545 status = ibt_unbind_service(root_listenp->listen_ib_srv_hdl, 546 ep_listenp->listen_ib_sbind_hdl); 547 if (status != IBT_SUCCESS) { 548 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 549 "fini_ep_chan(%p) : ibt_unbind_service() ret %d", 550 status); 551 return (-1); 552 } 553 554 return (0); 555 } 556 557 uint64_t 558 ibcma_init_root_sid(sol_cma_chan_t *root_chanp) 559 { 560 ibcma_chan_t *root_ibchanp; 561 struct rdma_cm_id *root_idp; 562 563 root_ibchanp = &root_chanp->chan_ib; 564 root_idp = (struct rdma_cm_id *)root_chanp; 565 if (root_ibchanp->chan_sid == 0) { 566 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "listen No SID : ps %x", 567 root_idp->ps); 568 root_ibchanp->chan_sid = ibt_get_ip_sid(root_idp->ps, 569 root_ibchanp->chan_port); 570 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "chan SID %llx , ps %x, " 571 "port %x", root_ibchanp->chan_sid, root_idp->ps, 572 root_ibchanp->chan_port); 573 } 574 return ((uint64_t)root_ibchanp->chan_sid); 575 } 576 577 /*ARGSUSED*/ 578 int 579 rdma_ib_listen(struct rdma_cm_id *ep_idp, int bklog) 580 { 581 struct rdma_cm_id *root_idp; 582 ibcma_chan_t *ep_ibchanp; 583 sol_cma_chan_t *root_chanp, *ep_chanp; 584 ibcma_dev_t *ep_devp; 585 ibt_status_t status; 586 587 ASSERT(ep_idp); 588 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "ib_listen(%p)", ep_idp); 589 ep_chanp = (sol_cma_chan_t *)ep_idp; 590 root_idp = CHAN_LISTEN_ROOT(ep_chanp); 591 root_chanp = (sol_cma_chan_t *)root_idp; 592 ep_ibchanp = &ep_chanp->chan_ib; 593 594 ep_devp = ep_ibchanp->chan_devp; 595 ASSERT(ep_devp); 596 status = ibt_bind_service( 597 (root_chanp->chan_listenp)->listen_ib_srv_hdl, 598 ep_devp->dev_sgid, NULL, ep_idp, 599 &((ep_chanp->chan_listenp)->listen_ib_sbind_hdl)); 600 if (status != IBT_SUCCESS) { 601 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_listen_ep: " 602 "ibt_bind_service failed with %x", status); 603 return (EINVAL); 604 } 605 return (0); 606 } 607 608 #define SOL_REP_PRIV_DATA_SZ 208 609 int 610 rdma_ib_accept(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param) 611 { 612 sol_cma_chan_t *chanp; 613 ibcma_chan_t *ibchanp; 614 ibt_status_t status; 615 void *privp = NULL; 616 uint8_t priv_len; 617 618 ASSERT(idp); 619 chanp = (sol_cma_chan_t *)idp; 620 ibchanp = &chanp->chan_ib; 621 622 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_accept(%p, %p)", 623 idp, conn_param); 624 if (chanp->chan_session_id == NULL) { 625 SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, 626 "Active side, cm_proceed not needed"); 627 return (0); 628 } 629 630 if (!conn_param) { 631 SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, "conn_param NULL"); 632 return (0); 633 } 634 635 ibchanp->chan_qpmodifyflag = 1; 636 if (idp->ps == RDMA_PS_TCP) { 637 ibt_cm_proceed_reply_t cm_reply; 638 639 /* Fill cm_reply */ 640 cm_reply.rep.cm_channel = 641 (ibt_channel_hdl_t)chanp->chan_qp_hdl; 642 cm_reply.rep.cm_rdma_ra_out = conn_param->initiator_depth; 643 cm_reply.rep.cm_rdma_ra_in = conn_param->responder_resources; 644 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "accept: " 645 "init_dept %x, resp_res %x", conn_param->initiator_depth, 646 conn_param->responder_resources); 647 cm_reply.rep.cm_rnr_retry_cnt = conn_param->rnr_retry_count; 648 priv_len = conn_param->private_data_len; 649 if (priv_len) { 650 privp = (void *)kmem_zalloc( 651 SOL_REP_PRIV_DATA_SZ, KM_SLEEP); 652 bcopy((void *)conn_param->private_data, 653 privp, priv_len); 654 #ifdef DEBUG 655 dump_priv_data(privp, SOL_REP_PRIV_DATA_SZ, 656 conn_param->private_data_len, "ib_accept"); 657 #endif 658 } 659 660 status = ibt_ofuvcm_proceed(IBT_CM_EVENT_REQ_RCV, 661 chanp->chan_session_id, IBT_CM_ACCEPT, &cm_reply, 662 privp, priv_len); 663 if (status != IBT_SUCCESS) { 664 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ib_accept: " 665 "ibt_ofuvcm_proceed failed %x", status); 666 if (privp) 667 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 668 return (EINVAL); 669 } 670 if (privp) 671 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 672 chanp->chan_session_id = NULL; 673 } else { 674 ibt_qp_hdl_t qphdl = chanp->chan_qp_hdl; 675 676 priv_len = conn_param->private_data_len; 677 if (priv_len) { 678 privp = (void *)kmem_zalloc( 679 SOL_REP_PRIV_DATA_SZ, KM_SLEEP); 680 bcopy((void *)conn_param->private_data, 681 privp, priv_len); 682 #ifdef DEBUG 683 dump_priv_data(privp, SOL_REP_PRIV_DATA_SZ, 684 conn_param->private_data_len, "ib_accept"); 685 #endif 686 } 687 688 status = ibt_cm_ud_proceed(chanp->chan_session_id, qphdl, 689 IBT_CM_ACCEPT, NULL, privp, priv_len); 690 if (status != IBT_SUCCESS) { 691 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ib_accept: " 692 "ibt_cm_ud_proceed failed %x", status); 693 if (privp) 694 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 695 return (EINVAL); 696 } 697 chanp->chan_connect_flag = SOL_CMA_CONNECT_SERVER_DONE; 698 699 if (privp) 700 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 701 } 702 return (0); 703 } 704 705 int 706 rdma_ib_reject(struct rdma_cm_id *idp, const void *private_data, 707 uint8_t private_data_len) 708 { 709 sol_cma_chan_t *chanp; 710 ibt_status_t status; 711 void *privp = NULL; 712 713 ASSERT(idp); 714 chanp = (sol_cma_chan_t *)idp; 715 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 716 "rdma_ib_reject(%p, %p, %x)", idp, 717 private_data, private_data_len); 718 719 if (chanp->chan_session_id == NULL) { 720 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_ib_reject :" 721 "chan_session_id NULL"); 722 return (EINVAL); 723 } 724 725 if (private_data_len) { 726 privp = (void *)kmem_zalloc(SOL_REP_PRIV_DATA_SZ, 727 KM_SLEEP); 728 bcopy((void *)private_data, privp, 729 private_data_len); 730 #ifdef DEBUG 731 dump_priv_data(privp, SOL_REP_PRIV_DATA_SZ, 732 private_data_len, "ib_reject"); 733 #endif 734 } 735 736 if (idp->ps == RDMA_PS_TCP) { 737 ibt_cm_proceed_reply_t cm_reply; 738 739 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_reject :" 740 "calling ibt_cm_proceed"); 741 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, 742 chanp->chan_session_id, IBT_CM_REJECT, &cm_reply, 743 privp, private_data_len); 744 if (status != IBT_SUCCESS) { 745 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ib_reject: " 746 "ibt_cm_proceed failed %x", status); 747 if (privp) 748 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 749 return (EINVAL); 750 } 751 mutex_enter(&chanp->chan_mutex); 752 chanp->chan_connect_flag = SOL_CMA_CONNECT_SERVER_DONE; 753 mutex_exit(&chanp->chan_mutex); 754 } else { 755 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_ib_reject :" 756 "calling ibt_cm_ud_proceed"); 757 status = ibt_cm_ud_proceed(chanp->chan_session_id, NULL, 758 IBT_CM_REJECT, NULL, privp, private_data_len); 759 if (status != IBT_SUCCESS) { 760 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ib_reject: " 761 "ibt_cm_ud_proceed failed %x", status); 762 if (privp) 763 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 764 return (EINVAL); 765 } 766 mutex_enter(&chanp->chan_mutex); 767 chanp->chan_connect_flag = SOL_CMA_CONNECT_SERVER_DONE; 768 mutex_exit(&chanp->chan_mutex); 769 } 770 771 if (privp) 772 kmem_free(privp, SOL_REP_PRIV_DATA_SZ); 773 return (0); 774 } 775 776 int 777 rdma_ib_disconnect(struct rdma_cm_id *idp) 778 { 779 sol_cma_chan_t *root_chanp, *chanp; 780 ibt_status_t status; 781 782 ASSERT(idp); 783 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "ib_disconnect(%p)", idp); 784 chanp = (sol_cma_chan_t *)idp; 785 mutex_enter(&chanp->chan_mutex); 786 787 root_chanp = (sol_cma_chan_t *)CHAN_LISTEN_ROOT(chanp); 788 if (IS_UDP_CMID(idp) && chanp->chan_connect_flag == 789 SOL_CMA_CONNECT_SERVER_RCVD && root_chanp) { 790 CHAN_LISTEN_ROOT(chanp) = NULL; 791 mutex_exit(&chanp->chan_mutex); 792 793 status = ibt_cm_ud_proceed(chanp->chan_session_id, 794 NULL, IBT_CM_NO_CHANNEL, NULL, NULL, 0); 795 if (status != IBT_SUCCESS) { 796 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 797 "ib_disconnect(%p) Reject for incoming REQ " 798 "failed, status %d", status); 799 return (EINVAL); 800 } 801 mutex_enter(&chanp->chan_mutex); 802 if (chanp->chan_req_state == REQ_CMID_CREATED || 803 chanp->chan_req_state == REQ_CMID_NOTIFIED) { 804 mutex_enter(&root_chanp->chan_mutex); 805 avl_remove(&root_chanp->chan_req_avl_tree, idp); 806 mutex_exit(&root_chanp->chan_mutex); 807 } 808 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 809 } 810 if (idp->ps == RDMA_PS_TCP && chanp->chan_connect_flag == 811 SOL_CMA_CONNECT_SERVER_RCVD && chanp->chan_session_id) { 812 ibt_cm_proceed_reply_t cm_reply; 813 814 mutex_exit(&chanp->chan_mutex); 815 bzero(&cm_reply, sizeof (cm_reply)); 816 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, 817 chanp->chan_session_id, IBT_CM_REJECT, &cm_reply, 818 NULL, 0); 819 if (status != IBT_SUCCESS) { 820 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 821 "ib_disconnect(%p) Reject for incoming REQ " 822 "failed, status %d", status); 823 return (EINVAL); 824 } 825 mutex_enter(&chanp->chan_mutex); 826 if (chanp->chan_req_state == REQ_CMID_CREATED || 827 chanp->chan_req_state == REQ_CMID_NOTIFIED) { 828 mutex_enter(&root_chanp->chan_mutex); 829 avl_remove(&root_chanp->chan_req_avl_tree, idp); 830 mutex_exit(&root_chanp->chan_mutex); 831 } 832 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 833 } 834 835 /* 836 * Close RC channel for RC. 837 * No explicit Disconnect required for UD 838 */ 839 if (idp->ps == RDMA_PS_TCP && chanp->chan_qp_hdl && 840 chanp->chan_connect_flag != SOL_CMA_CONNECT_NONE) { 841 ibt_execution_mode_t mode; 842 void *qp_hdl = chanp->chan_qp_hdl; 843 844 845 /* 846 * No callbacks for CMIDs for which CONNECT has been 847 * initiated but not completed. 848 */ 849 mode = (SOL_CMAID_IS_CONNECTED(chanp)) ? IBT_BLOCKING : 850 IBT_NOCALLBACKS; 851 mutex_exit(&chanp->chan_mutex); 852 status = ibt_close_rc_channel(qp_hdl, 853 mode, NULL, 0, NULL, NULL, NULL); 854 if (status != IBT_SUCCESS) { 855 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 856 "disconnect: close_rc_channel failed %x", 857 status); 858 return (EINVAL); 859 } 860 } else 861 mutex_exit(&chanp->chan_mutex); 862 863 return (0); 864 } 865 866 int 867 rdma_ib_join_multicast(struct rdma_cm_id *idp, struct sockaddr *addr, 868 void *context) 869 { 870 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 871 ibcma_chan_t *ibchanp; 872 ibt_mcg_attr_t mcg_attr; 873 ibt_ip_addr_t mcast_addr; 874 ibt_mcg_info_t *mcg_infop; 875 ibt_status_t status; 876 ib_gid_t mcast_gid, mcast_gid_horder; 877 ibcma_dev_t *devp; 878 ibcma_mcast_t *ibmcastp = NULL; 879 880 ibchanp = &chanp->chan_ib; 881 devp = ibchanp->chan_devp; 882 if (devp == NULL) { 883 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "join_mcast: devp NULL"); 884 return (EINVAL); 885 } 886 887 ibmcastp = kmem_zalloc(sizeof (ibcma_mcast_t), KM_SLEEP); 888 ibmcastp->mcast_idp = idp; 889 ibmcastp->mcast_ctx = context; 890 bcopy(addr, &ibmcastp->mcast_addr, sizeof (struct sockaddr)); 891 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "join_mcast: ibmcastp %p", 892 ibmcastp); 893 894 sockaddr2ibtaddr_port(idp, addr, &mcast_addr, NULL); 895 896 /* Check if input @ to rdma_join_mcast is multicast IP @ */ 897 if (!(mcast_addr.family == AF_INET && 898 ((htonl(mcast_addr.un.ip4addr) & 0xE0000000) == 899 0xE0000000))) { 900 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 901 "Invalid IP addr specified"); 902 kmem_free(ibmcastp, sizeof (ibcma_mcast_t)); 903 return (EINVAL); 904 } 905 906 bzero(&mcg_attr, sizeof (mcg_attr)); 907 if (sol_cma_any_addr(addr)) { 908 bzero(&mcast_gid, sizeof (mcast_gid)); 909 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ANY mcast addr input"); 910 } else { 911 ipaddr2mgid(addr, &mcast_gid_horder, devp->dev_pkey); 912 913 mcast_gid.gid_prefix = htonll( 914 mcast_gid_horder.gid_prefix); 915 mcast_gid.gid_guid = htonll( 916 mcast_gid_horder.gid_guid); 917 } 918 bcopy(&mcast_gid, &(mcg_attr.mc_mgid), sizeof (ib_gid_t)); 919 mcg_attr.mc_mtu_req.r_selector = IBT_BEST; 920 mcg_attr.mc_flow = 0; 921 mcg_attr.mc_hop = 0xFF; 922 mcg_attr.mc_tclass = 0; 923 mcg_attr.mc_sl = 0; 924 mcg_attr.mc_pkt_lt_req.p_selector = IBT_BEST; 925 mcg_attr.mc_pkey = devp->dev_pkey; 926 mcg_attr.mc_rate_req.r_selector = IBT_BEST; 927 mcg_attr.mc_join_state = IB_MC_JSTATE_FULL; 928 mcg_attr.mc_qkey = RDMA_UDP_QKEY; 929 mcg_infop = kmem_zalloc(sizeof (ibt_mcg_info_t), KM_SLEEP); 930 931 status = ibt_join_mcg(ibchanp->chan_devp->dev_sgid, 932 &mcg_attr, mcg_infop, ibcma_multicast_hdlr, ibmcastp); 933 if (status != IBT_SUCCESS) { 934 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "\t join_mcast : " 935 "ibt_join_mcg failed with status %d", status); 936 kmem_free(ibmcastp, sizeof (ibcma_mcast_t)); 937 return (EINVAL); 938 } 939 940 (void) add_genlist(&ibchanp->chan_mcast_list, (uintptr_t)ibmcastp, 941 NULL); 942 ibchanp->chan_mcast_cnt++; 943 944 return (0); 945 } 946 947 void 948 rdma_ib_leave_multicast(struct rdma_cm_id *idp, struct sockaddr *addr) 949 { 950 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 951 ibcma_chan_t *ibchanp; 952 ibcma_mcast_t *ibmcastp = NULL; 953 genlist_entry_t *entry; 954 ib_gid_t zero_gid; 955 ibt_status_t status; 956 957 ibchanp = &chanp->chan_ib; 958 genlist_for_each(entry, &ibchanp->chan_mcast_list) { 959 ibmcastp = (ibcma_mcast_t *)entry->data; 960 ASSERT(ibmcastp); 961 if (bcmp(&ibmcastp->mcast_addr, addr, 962 sizeof (struct sockaddr)) == 0) { 963 delete_genlist(&ibchanp->chan_mcast_list, entry); 964 break; 965 } 966 ibmcastp = NULL; 967 } 968 if (ibmcastp == NULL) { 969 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 970 "leave_mcast: No matching @"); 971 return; 972 } 973 ibchanp->chan_mcast_cnt--; 974 bzero(&zero_gid, sizeof (ib_gid_t)); 975 status = ibt_leave_mcg(ibchanp->chan_devp->dev_sgid, 976 ibmcastp->mcast_gid, zero_gid, IB_MC_JSTATE_FULL); 977 if (status != IBT_SUCCESS) 978 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "leave_mcast: " 979 "ibt_leave_mcg failed %d", status); 980 kmem_free(ibmcastp, sizeof (ibcma_mcast_t)); 981 } 982 983 /* Local Functions */ 984 #define SOL_REQ_PRIV_DATA_SZ 96 985 static int 986 ibcma_tcp_connect(struct rdma_cm_id *idp, ibcma_chan_t *ibchanp, 987 struct rdma_conn_param *conn_paramp) 988 { 989 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 990 ibt_chan_open_flags_t flags; 991 ibt_chan_open_args_t args; 992 ibt_status_t status; 993 ibt_ip_cm_info_t ipcm_info; 994 995 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "tcp_connect(%p, %p, %p)", idp, 996 ibchanp, conn_paramp); 997 bzero(&args, sizeof (args)); 998 args.oc_path_retry_cnt = conn_paramp->retry_count; 999 args.oc_path_rnr_retry_cnt = conn_paramp->rnr_retry_count; 1000 flags = IBT_OCHAN_OFUV; 1001 args.oc_path = ibchanp->chan_pathp; 1002 (args.oc_path)->pi_sid = ibchanp->chan_sid; 1003 args.oc_cm_handler = ibcma_rc_hdlr; 1004 args.oc_cm_clnt_private = idp; 1005 args.oc_rdma_ra_out = conn_paramp->initiator_depth; 1006 args.oc_rdma_ra_in = conn_paramp->responder_resources; 1007 args.oc_priv_data_len = IBT_IP_HDR_PRIV_DATA_SZ + 1008 conn_paramp->private_data_len; 1009 args.oc_priv_data = kmem_zalloc(SOL_REQ_PRIV_DATA_SZ, KM_SLEEP); 1010 1011 bcopy(&ibchanp->chan_local_addr, &ipcm_info.src_addr, 1012 sizeof (ibt_ip_addr_t)); 1013 bcopy(&ibchanp->chan_remote_addr, &ipcm_info.dst_addr, 1014 sizeof (ibt_ip_addr_t)); 1015 ipcm_info.src_port = ibchanp->chan_port; 1016 status = ibt_format_ip_private_data(&ipcm_info, args.oc_priv_data_len, 1017 args.oc_priv_data); 1018 if (status != IBT_SUCCESS) { 1019 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1020 "ibt_format_ip_private_data failed!!"); 1021 kmem_free(args.oc_priv_data, SOL_REQ_PRIV_DATA_SZ); 1022 return (EINVAL); 1023 } 1024 1025 if (conn_paramp->private_data_len) { 1026 void *dest; 1027 1028 dest = (void *)((uint8_t *)args.oc_priv_data + 1029 IBT_IP_HDR_PRIV_DATA_SZ); 1030 bcopy(conn_paramp->private_data, dest, 1031 conn_paramp->private_data_len); 1032 } 1033 1034 /* 1035 * Set the RDMA related flags for this QP, if required. 1036 */ 1037 if (conn_paramp->initiator_depth || conn_paramp->responder_resources) { 1038 ibt_cep_modify_flags_t cep_flags = IBT_CEP_SET_NOTHING; 1039 ibt_cep_flags_t flags = IBT_CEP_NO_FLAGS; 1040 1041 if (conn_paramp->initiator_depth) { 1042 cep_flags |= IBT_CEP_SET_RDMA_R; 1043 flags |= IBT_CEP_RDMA_RD; 1044 } 1045 if (conn_paramp->responder_resources) { 1046 cep_flags |= IBT_CEP_SET_RDMA_W; 1047 flags |= IBT_CEP_RDMA_WR; 1048 } 1049 1050 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1051 "tcp_connect: Calling ibt_modify_rdma(%p, %x)", 1052 chanp->chan_qp_hdl, cep_flags); 1053 status = ibt_modify_rdma(chanp->chan_qp_hdl, 1054 cep_flags, flags); 1055 if (status != IBT_SUCCESS) { 1056 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "tcp_connect: " 1057 "ibt_open_rdma failed %x", status); 1058 kmem_free(args.oc_priv_data, SOL_REQ_PRIV_DATA_SZ); 1059 return (EINVAL); 1060 } 1061 } 1062 1063 dump_priv_data(args.oc_priv_data, SOL_REQ_PRIV_DATA_SZ, 1064 args.oc_priv_data_len, "tcp_connect"); 1065 chanp->chan_connect_flag = SOL_CMA_CONNECT_INITIATED; 1066 status = ibt_open_rc_channel(chanp->chan_qp_hdl, flags, 1067 IBT_NONBLOCKING, &args, NULL); 1068 if (status != IBT_SUCCESS) { 1069 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1070 "tcp_connect: ibv_open_rc_channel failed %x", 1071 status); 1072 kmem_free(args.oc_priv_data, SOL_REQ_PRIV_DATA_SZ); 1073 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1074 return (EINVAL); 1075 } 1076 kmem_free(args.oc_priv_data, SOL_REQ_PRIV_DATA_SZ); 1077 1078 return (0); 1079 } 1080 1081 static int 1082 ibcma_udp_connect(struct rdma_cm_id *idp, ibcma_chan_t *ibchanp, 1083 struct rdma_conn_param *conn_paramp) 1084 { 1085 ibt_status_t status; 1086 ibt_ud_dest_attr_t attr; 1087 ibt_path_info_t *pathp; 1088 ibt_adds_vect_t *addr_vect; 1089 ibcma_dev_t *devp; 1090 ibt_ip_cm_info_t ipcm_info; 1091 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 1092 1093 devp = ibchanp->chan_devp; 1094 ASSERT(devp); 1095 1096 /* We always select the first path */ 1097 pathp = ibchanp->chan_pathp; 1098 addr_vect = &((pathp->pi_prim_cep_path).cep_adds_vect); 1099 1100 bzero(&attr, sizeof (attr)); 1101 attr.ud_pkey_ix = devp->dev_pkey_ix; 1102 attr.ud_cm_handler = ibcma_ud_hdlr; 1103 attr.ud_cm_private = idp; 1104 attr.ud_priv_data_len = IBT_IP_HDR_PRIV_DATA_SZ + 1105 conn_paramp->private_data_len; 1106 attr.ud_priv_data = kmem_zalloc(attr.ud_priv_data_len, KM_SLEEP); 1107 if (conn_paramp->private_data_len) { 1108 bcopy(conn_paramp->private_data, 1109 (void *)(((char *)attr.ud_priv_data) + 1110 IBT_IP_HDR_PRIV_DATA_SZ), 1111 conn_paramp->private_data_len); 1112 } 1113 1114 bcopy((void *)&ibchanp->chan_local_addr, &ipcm_info.src_addr, 1115 sizeof (ibt_ip_addr_t)); 1116 bcopy((void *)&ibchanp->chan_remote_addr, &ipcm_info.dst_addr, 1117 sizeof (ibt_ip_addr_t)); 1118 ipcm_info.src_port = ibchanp->chan_port; 1119 status = ibt_format_ip_private_data(&ipcm_info, attr.ud_priv_data_len, 1120 attr.ud_priv_data); 1121 if (status != IBT_SUCCESS) { 1122 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "\tibudp_connect: " 1123 "ibt_format_ip_private_data() failed with status %d", 1124 status); 1125 kmem_free(attr.ud_priv_data, attr.ud_priv_data_len); 1126 return (EINVAL); 1127 } 1128 attr.ud_sid = ibchanp->chan_sid; 1129 attr.ud_addr = addr_vect; 1130 1131 chanp->chan_connect_flag = SOL_CMA_CONNECT_INITIATED; 1132 status = ibt_ud_get_dqpn(&attr, IBT_NONBLOCKING, NULL); 1133 1134 if (status != IBT_SUCCESS) { 1135 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "\tibudp_connect: " 1136 "ibt_ud_get_dqpn failed with status %x", status); 1137 kmem_free(attr.ud_priv_data, attr.ud_priv_data_len); 1138 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1139 return (EINVAL); 1140 } 1141 1142 kmem_free(attr.ud_priv_data, attr.ud_priv_data_len); 1143 1144 return (0); 1145 } 1146 1147 static int 1148 ibcma_init_devinfo(struct rdma_cm_id *idp, ibcma_chan_t *ibchanp, 1149 ibt_path_info_t *pathp) 1150 { 1151 ibcma_dev_t *devp; 1152 ibt_status_t status; 1153 uint_t nports, psize; 1154 ib_pkey_t pkey; 1155 ibt_hca_portinfo_t *pinfop; 1156 1157 if (ibchanp->chan_devp) 1158 return (-1); 1159 1160 /* Get the port_info and the pkey */ 1161 status = ibt_query_hca_ports_byguid(pathp->pi_hca_guid, 1162 pathp->pi_prim_cep_path.cep_hca_port_num, 1163 &pinfop, &nports, &psize); 1164 if (status != IBT_SUCCESS) { 1165 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "init_devinfo - " 1166 "query_hca_port failed rc %d", status); 1167 return (-1); 1168 } else { 1169 int index; 1170 1171 index = pathp->pi_prim_cep_path.cep_pkey_ix; 1172 pkey = (pinfop->p_pkey_tbl)[index]; 1173 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "init_devinfo - pkey %x", 1174 pkey); 1175 ibt_free_portinfo(pinfop, psize); 1176 } 1177 devp = kmem_zalloc(sizeof (ibcma_dev_t), KM_SLEEP); 1178 ibchanp->chan_devp = devp; 1179 devp->dev_node_guid = pathp->pi_hca_guid; 1180 devp->dev_port_num = pathp->pi_prim_cep_path.cep_hca_port_num; 1181 devp->dev_pkey_ix = pathp->pi_prim_cep_path.cep_pkey_ix; 1182 devp->dev_pkey = pkey; 1183 devp->dev_sgid = pathp->pi_prim_cep_path.cep_adds_vect.av_sgid; 1184 1185 idp->device = sol_cma_acquire_device(ntohll(devp->dev_node_guid)); 1186 idp->port_num = devp->dev_port_num; 1187 return (0); 1188 } 1189 1190 static int 1191 ibcma_query_local_ip(struct rdma_cm_id *idp, sol_cma_chan_t *chanp, 1192 ibcma_chan_t *ibchanp) 1193 { 1194 ibt_status_t status; 1195 ibt_ip_addr_t *local_addrp; 1196 ibt_ip_path_attr_t path_attr; 1197 ibt_path_info_t local_path; 1198 1199 if (ibchanp->chan_pathp != NULL) { 1200 return (0); 1201 } 1202 local_addrp = &ibchanp->chan_local_addr; 1203 bzero(&path_attr, sizeof (path_attr)); 1204 path_attr.ipa_dst_ip = local_addrp; 1205 bcopy(local_addrp, &path_attr.ipa_src_ip, sizeof (ibt_ip_addr_t)); 1206 path_attr.ipa_ndst = 1; 1207 path_attr.ipa_max_paths = 1; 1208 1209 if ((status = ibt_get_ip_paths(chanp->chan_ib_client_hdl, 1210 IBT_PATH_NO_FLAGS, &path_attr, &local_path, NULL, NULL)) != 1211 IBT_SUCCESS) { 1212 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1213 "ib_cma_get_devinfo:status %d, %p not IB IP @", 1214 status, local_addrp); 1215 return (EINVAL); 1216 } 1217 if (ibcma_init_devinfo(idp, ibchanp, &local_path)) { 1218 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1219 "ib_cma_get_devinfo:init_devinfo failed"); 1220 return (EINVAL); 1221 } 1222 1223 return (0); 1224 } 1225 1226 static int 1227 ibcma_get_paths(struct rdma_cm_id *idp, sol_cma_chan_t *chanp, 1228 ibcma_chan_t *ibchanp) 1229 { 1230 ibt_ip_path_attr_t path_attr; 1231 ibt_status_t status; 1232 ibt_path_ip_src_t *src_ip_p = NULL; 1233 uint8_t max_paths; 1234 ibcma_dev_t *devp; 1235 ibt_ip_addr_t *dst_addrp; 1236 ib_lid_t base_lid; 1237 int i; 1238 1239 ASSERT(ibchanp); 1240 1241 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "ibcma_get_paths(%p, %p)", idp, 1242 ibchanp); 1243 max_paths = 2; 1244 ibchanp->chan_path_size = max_paths * sizeof (ibt_path_info_t); 1245 ibchanp->chan_pathp = kmem_zalloc(ibchanp->chan_path_size, KM_SLEEP); 1246 1247 devp = ibchanp->chan_devp; 1248 if (devp == NULL) { 1249 src_ip_p = kmem_zalloc(sizeof (ibt_path_ip_src_t) * max_paths, 1250 KM_SLEEP); 1251 } 1252 bzero(&path_attr, sizeof (ibt_ip_path_attr_t)); 1253 dst_addrp = kmem_zalloc(sizeof (ibt_ip_addr_t), KM_SLEEP); 1254 bcopy(&ibchanp->chan_remote_addr, dst_addrp, sizeof (ibt_ip_addr_t)); 1255 path_attr.ipa_dst_ip = dst_addrp; 1256 bcopy(&ibchanp->chan_local_addr, &path_attr.ipa_src_ip, 1257 sizeof (ibt_ip_addr_t)); 1258 path_attr.ipa_ndst = 1; 1259 path_attr.ipa_max_paths = max_paths; 1260 if (ibcma_any_addr(&path_attr.ipa_src_ip)) 1261 path_attr.ipa_src_ip.family = AF_UNSPEC; 1262 1263 status = ibt_get_ip_paths(chanp->chan_ib_client_hdl, IBT_PATH_NO_FLAGS, 1264 &path_attr, ibchanp->chan_pathp, &ibchanp->chan_numpaths, 1265 src_ip_p); 1266 if (status != IBT_SUCCESS && status != IBT_INSUFF_DATA) { 1267 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1268 "cma_get_paths : failed %d", status); 1269 kmem_free(dst_addrp, sizeof (ibt_ip_addr_t)); 1270 if (src_ip_p) 1271 kmem_free(src_ip_p, 1272 sizeof (ibt_path_ip_src_t) * max_paths); 1273 kmem_free(ibchanp->chan_pathp, ibchanp->chan_path_size); 1274 ibchanp->chan_pathp = NULL; 1275 return (EINVAL); 1276 } 1277 1278 if (src_ip_p) { 1279 ipaddr2sockaddr(&(src_ip_p[0].ip_primary), 1280 &(idp->route.addr.src_addr), NULL); 1281 bcopy(&(src_ip_p[0].ip_primary), &ibchanp->chan_local_addr, 1282 sizeof (ibt_ip_addr_t)); 1283 if (ibcma_init_devinfo(idp, ibchanp, ibchanp->chan_pathp)) { 1284 kmem_free(src_ip_p, sizeof (ibt_path_ip_src_t) * 1285 max_paths); 1286 kmem_free(dst_addrp, sizeof (ibt_ip_addr_t)); 1287 kmem_free(ibchanp->chan_pathp, 1288 ibchanp->chan_path_size); 1289 return (EINVAL); 1290 } 1291 kmem_free(src_ip_p, sizeof (ibt_path_ip_src_t) * max_paths); 1292 } 1293 if (!ibchanp->chan_devp) { 1294 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1295 "cma_get_paths : devp ERROR"); 1296 kmem_free(dst_addrp, sizeof (ibt_ip_addr_t)); 1297 return (EINVAL); 1298 } 1299 devp = ibchanp->chan_devp; 1300 (idp->route).num_paths = ibchanp->chan_numpaths; 1301 idp->route.path_rec = kmem_zalloc(sizeof (struct ib_sa_path_rec) * 1302 ibchanp->chan_numpaths, KM_SLEEP); 1303 base_lid = ibt_get_port_state_byguid(devp->dev_node_guid, 1304 devp->dev_port_num, NULL, &base_lid); 1305 for (i = 0; i < ibchanp->chan_numpaths; i++) 1306 ibt_path2sa_path(&((ibchanp->chan_pathp)[i]), 1307 &((idp->route.path_rec)[i]), base_lid); 1308 1309 kmem_free(dst_addrp, sizeof (ibt_ip_addr_t)); 1310 return (0); 1311 } 1312 1313 /* 1314 * Solaris Event Handlers 1315 */ 1316 1317 /* UD Event Handler */ 1318 /*ARGSUSED*/ 1319 static ibt_cm_status_t 1320 ibcma_ud_hdlr(void *inp, ibt_cm_ud_event_t *eventp, 1321 ibt_cm_ud_return_args_t *ret_args, void *priv_data, 1322 ibt_priv_data_len_t priv_datalen) 1323 { 1324 struct rdma_cm_id *root_idp, *event_idp, *idp; 1325 sol_cma_chan_t *root_chanp, *chanp, *event_chanp; 1326 ibcma_chan_t *ibchanp, *event_ibchanp; 1327 struct rdma_ud_param ud_param, *ud_paramp = &ud_param; 1328 enum rdma_cm_event_type event; 1329 int evt_status = -1; 1330 ibt_priv_data_len_t cm_privlen; 1331 void *cm_priv; 1332 ibt_status_t ibt_status; 1333 ibt_ip_cm_info_t info; 1334 cma_chan_state_t chan_state; 1335 1336 event_idp = idp = (struct rdma_cm_id *)inp; 1337 chanp = (sol_cma_chan_t *)idp; 1338 ibchanp = &chanp->chan_ib; 1339 root_idp = CHAN_LISTEN_ROOT(chanp); 1340 root_chanp = (sol_cma_chan_t *)root_idp; 1341 SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str, "cma_ud_hdlr(%p, %p)", 1342 inp, eventp); 1343 1344 bzero(&ud_param, sizeof (struct rdma_ud_param)); 1345 cm_privlen = eventp->cm_priv_data_len; 1346 cm_priv = eventp->cm_priv_data; 1347 if (eventp->cm_type == IBT_CM_UD_EVENT_SIDR_REQ) { 1348 ibt_cm_sidr_req_t *sidr_req; 1349 void *find_ret; 1350 avl_index_t where; 1351 1352 ASSERT(root_chanp); 1353 1354 /* 1355 * Reject further REQs if destroy of listen CMID 1356 * has been called. 1357 */ 1358 mutex_enter(&root_chanp->chan_mutex); 1359 chan_state = cma_get_chan_state(root_chanp); 1360 mutex_exit(&root_chanp->chan_mutex); 1361 if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING || 1362 chan_state == SOL_CMA_CHAN_DESTROY_WAIT) { 1363 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "UD Req Hdlr, " 1364 "listen CMID destroy called"); 1365 return (IBT_CM_REJECT); 1366 } 1367 1368 sidr_req = &((eventp->cm_event).sidr_req); 1369 SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, "SIDR REQ"); 1370 1371 if (cm_privlen < IBT_IP_HDR_PRIV_DATA_SZ) { 1372 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "UD Req Hdlr, " 1373 "Priv data len %x < %x", cm_privlen, 1374 IBT_IP_HDR_PRIV_DATA_SZ); 1375 return (IBT_CM_REJECT); 1376 } 1377 ibt_status = ibt_get_ip_data(cm_privlen, cm_priv, &info); 1378 if (ibt_status != IBT_SUCCESS) { 1379 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "UD Req Hdlr, " 1380 "ibt_get_ip_data failed, %x", ibt_status); 1381 return (IBT_CM_REJECT); 1382 } 1383 cm_privlen -= IBT_IP_HDR_PRIV_DATA_SZ; 1384 cm_priv = (void *)(((uchar_t *)cm_priv) + 1385 IBT_IP_HDR_PRIV_DATA_SZ); 1386 1387 event_idp = ibcma_create_new_id(idp); 1388 if (event_idp == NULL) { 1389 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1390 "create_new_id failed!!"); 1391 return (IBT_CM_REJECT); 1392 } 1393 event_idp->device = sol_cma_acquire_device(ntohll( 1394 sidr_req->sreq_hca_guid)); 1395 event_idp->port_num = sidr_req->sreq_hca_port; 1396 (event_idp->route).num_paths = 0; 1397 1398 event_chanp = (sol_cma_chan_t *)event_idp; 1399 event_chanp->chan_req_state = REQ_CMID_CREATED; 1400 event_ibchanp = &event_chanp->chan_ib; 1401 event_chanp->chan_session_id = eventp->cm_session_id; 1402 event_chanp->chan_connect_flag = 1403 SOL_CMA_CONNECT_SERVER_RCVD; 1404 bcopy(&info.src_addr, &event_ibchanp->chan_remote_addr, 1405 sizeof (ibt_ip_addr_t)); 1406 ipaddr2sockaddr(&info.src_addr, 1407 &(event_idp->route.addr.dst_addr), &info.src_port); 1408 1409 /* 1410 * Increment number of Reqs for listening CMID, 1411 * so that listening CMID is not deleted, till this 1412 * connection expects no more events. 1413 * chan_req_cnt is decremented connection is 1414 * notified to the consumer. 1415 * 1416 * Insert the CMID into the REQ_AVL_TREE. This is 1417 * deleted when the connection is accepted or rejected. 1418 */ 1419 mutex_enter(&root_chanp->chan_mutex); 1420 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1421 "Add to REQ AVL of %p IDP, idp %p, session_id %p", 1422 root_idp, event_idp, event_chanp->chan_session_id); 1423 find_ret = avl_find(&root_chanp->chan_req_avl_tree, 1424 (void *)event_chanp->chan_session_id, &where); 1425 if (find_ret) { 1426 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1427 "DUPLICATE ENTRY in REQ AVL : root %p, " 1428 "idp %p, session_id %p", 1429 root_idp, event_idp, 1430 event_chanp->chan_session_id); 1431 mutex_exit(&root_chanp->chan_mutex); 1432 rdma_destroy_id(event_idp); 1433 return (IBT_CM_REJECT); 1434 } 1435 root_chanp->chan_req_cnt++; 1436 root_chanp->chan_req_state = REQ_CMID_CREATED; 1437 root_chanp->chan_req_total_cnt++; 1438 avl_insert(&root_chanp->chan_req_avl_tree, 1439 (void *)event_idp, where); 1440 mutex_exit(&root_chanp->chan_mutex); 1441 1442 event = RDMA_CM_EVENT_CONNECT_REQUEST; 1443 evt_status = 0; 1444 } else if (eventp->cm_type == IBT_CM_UD_EVENT_SIDR_REP) { 1445 ibt_cm_sidr_rep_t *sidr_rep; 1446 1447 ASSERT(chanp->chan_connect_flag == SOL_CMA_CONNECT_INITIATED); 1448 mutex_enter(&chanp->chan_mutex); 1449 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1450 mutex_exit(&chanp->chan_mutex); 1451 sidr_rep = &((eventp->cm_event).sidr_rep); 1452 if (sidr_rep->srep_status == IBT_CM_SREP_CHAN_VALID) { 1453 evt_status = 0; 1454 event = RDMA_CM_EVENT_ESTABLISHED; 1455 ud_paramp->qp_num = sidr_rep->srep_remote_qpn; 1456 ud_paramp->qkey = sidr_rep->srep_remote_qkey; 1457 ibt_path2ah(ibchanp->chan_pathp, &ud_paramp->ah_attr); 1458 } else { 1459 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1460 "SIDR Response err with status %x", 1461 sidr_rep->srep_status); 1462 event = RDMA_CM_EVENT_UNREACHABLE; 1463 evt_status = sidr_rep->srep_status; 1464 goto ud_gen_event; 1465 } 1466 } 1467 1468 ud_paramp->private_data_len = cm_privlen; 1469 if (evt_status == 0 && cm_privlen) { 1470 ud_paramp->private_data = kmem_zalloc(cm_privlen, KM_SLEEP); 1471 bcopy(cm_priv, (void *)ud_paramp->private_data, 1472 cm_privlen); 1473 #ifdef DEBUG 1474 dump_priv_data((void *)ud_paramp->private_data, 1475 SOL_REP_PRIV_DATA_SZ, cm_privlen, "ibcma_ud_hdlr"); 1476 #endif 1477 } 1478 1479 ud_gen_event: 1480 /* Pass back the event to sol_cma consumer */ 1481 cma_generate_event(event_idp, event, evt_status, NULL, ud_paramp); 1482 1483 if (ud_paramp->private_data) 1484 kmem_free((void *)ud_paramp->private_data, cm_privlen); 1485 1486 if (eventp->cm_type == IBT_CM_UD_EVENT_SIDR_REQ) 1487 return (IBT_CM_DEFER); 1488 else 1489 return (IBT_CM_DEFAULT); 1490 } 1491 1492 static ibt_cm_status_t 1493 ibcma_handle_req(struct rdma_cm_id *idp, struct rdma_cm_id **event_id_ptr, 1494 ibt_cm_event_t *eventp, struct rdma_conn_param *paramp, 1495 enum rdma_cm_event_type *event, int *evt_status) 1496 { 1497 struct rdma_cm_id *root_idp, *event_idp; 1498 sol_cma_chan_t *root_chanp, *event_chanp, *chanp; 1499 ibcma_chan_t *event_ibchanp, *ibchanp; 1500 ibt_status_t ibt_status; 1501 ibt_cm_req_rcv_t *reqp; 1502 ibt_priv_data_len_t cm_privlen; 1503 ibt_ofuvcm_req_data_t rtr_data; 1504 ibt_ip_cm_info_t info; 1505 void *cm_priv, *priv_data; 1506 ib_lid_t base_lid; 1507 void *find_ret; 1508 avl_index_t where; 1509 cma_chan_state_t chan_state; 1510 #ifdef DEBUG 1511 void *dump_priv; 1512 #endif 1513 1514 chanp = (sol_cma_chan_t *)idp; 1515 ibchanp = &chanp->chan_ib; 1516 root_idp = CHAN_LISTEN_ROOT(chanp); 1517 root_chanp = (sol_cma_chan_t *)root_idp; 1518 ASSERT(chanp->chan_listenp); 1519 ASSERT(root_idp); 1520 1521 /* 1522 * Reject further REQs if destroy of listen CMID 1523 * has been called. 1524 */ 1525 mutex_enter(&root_chanp->chan_mutex); 1526 chan_state = cma_get_chan_state(root_chanp); 1527 mutex_exit(&root_chanp->chan_mutex); 1528 if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING || 1529 chan_state == SOL_CMA_CHAN_DESTROY_WAIT) { 1530 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "RC Req Hdlr, " 1531 "listen CMID destroy called"); 1532 return (IBT_CM_REJECT); 1533 } 1534 1535 *event = RDMA_CM_EVENT_CONNECT_REQUEST; 1536 *evt_status = 0; 1537 reqp = &(eventp->cm_event.req); 1538 paramp->qp_num = reqp->req_remote_qpn; 1539 paramp->srq = (reqp->req_flags & IBT_CM_SRQ_EXISTS) ? 1 : 0; 1540 paramp->responder_resources = reqp->req_rdma_ra_in; 1541 paramp->initiator_depth = reqp->req_rdma_ra_out; 1542 paramp->flow_control = (reqp->req_flags & IBT_CM_FLOW_CONTROL) 1543 ? 1 : 0; 1544 paramp->retry_count = reqp->req_retry_cnt; 1545 paramp->rnr_retry_count = reqp->req_rnr_retry_cnt; 1546 1547 #ifdef DEBUG 1548 dump_priv = kmem_zalloc(SOL_REQ_PRIV_DATA_SZ, KM_SLEEP); 1549 bcopy(eventp->cm_priv_data, dump_priv, eventp->cm_priv_data_len); 1550 dump_priv_data(dump_priv, SOL_REQ_PRIV_DATA_SZ, 1551 eventp->cm_priv_data_len, "handle_req"); 1552 kmem_free(dump_priv, SOL_REQ_PRIV_DATA_SZ); 1553 #endif /* DEBUG */ 1554 1555 cm_privlen = eventp->cm_priv_data_len; 1556 cm_priv = eventp->cm_priv_data; 1557 if (cm_privlen < IBT_IP_HDR_PRIV_DATA_SZ) { 1558 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "RC Req Hdlr, " 1559 "Priv data len %x < %x", cm_privlen, 1560 IBT_IP_HDR_PRIV_DATA_SZ); 1561 return (IBT_CM_REJECT); 1562 } 1563 ibt_status = ibt_get_ip_data(cm_privlen, cm_priv, &info); 1564 if (ibt_status != IBT_SUCCESS) { 1565 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "RC Req Hdlr, " 1566 "ibt_get_ip_data failed, %x", ibt_status); 1567 return (IBT_CM_REJECT); 1568 } 1569 bcopy(&info.dst_addr, &ibchanp->chan_remote_addr, 1570 sizeof (ibt_ip_addr_t)); 1571 1572 ibt_status = ibt_ofuvcm_get_req_data(eventp->cm_session_id, &rtr_data); 1573 if (ibt_status != IBT_SUCCESS) { 1574 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "RC Req Hdlr, " 1575 "ibt_ofuvcm_get_req_data failed, %x", ibt_status); 1576 return (IBT_CM_REJECT); 1577 } 1578 1579 paramp->private_data_len = cm_privlen - IBT_IP_HDR_PRIV_DATA_SZ; 1580 if (paramp->private_data_len) { 1581 priv_data = (void *)((uint8_t *)cm_priv + 1582 IBT_IP_HDR_PRIV_DATA_SZ); 1583 paramp->private_data = kmem_zalloc(paramp->private_data_len, 1584 KM_SLEEP); 1585 bcopy(priv_data, (void *)paramp->private_data, 1586 paramp->private_data_len); 1587 } 1588 event_idp = ibcma_create_new_id(idp); 1589 if (event_idp == NULL) { 1590 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1591 "create_new_id failed!!"); 1592 if (paramp->private_data) 1593 kmem_free((void *)paramp->private_data, 1594 paramp->private_data_len); 1595 return (IBT_CM_REJECT); 1596 } 1597 1598 /* 1599 * Fill the route, device and port_num. 1600 * TBD - Fill up packet_life_time 1601 */ 1602 event_idp->device = sol_cma_acquire_device(ntohll( 1603 reqp->req_hca_guid)); 1604 event_idp->port_num = reqp->req_prim_hca_port; 1605 (event_idp->route).num_paths = reqp->req_alt_hca_port ? 2 : 1; 1606 event_idp->route.path_rec = kmem_zalloc( 1607 sizeof (struct ib_sa_path_rec) * ((event_idp->route).num_paths), 1608 KM_SLEEP); 1609 base_lid = ibt_get_port_state_byguid(reqp->req_hca_guid, 1610 reqp->req_prim_hca_port, NULL, &base_lid); 1611 ibt_addsvect2sa_path(&reqp->req_prim_addr, 1612 &(event_idp->route.path_rec[0]), base_lid); 1613 (event_idp->route.path_rec[0]).mtu = (uint8_t)rtr_data.req_path_mtu; 1614 if (reqp->req_alt_hca_port) { 1615 base_lid = ibt_get_port_state_byguid( 1616 reqp->req_hca_guid, reqp->req_alt_hca_port, 1617 NULL, &base_lid); 1618 ibt_addsvect2sa_path(&reqp->req_alt_addr, 1619 &(event_idp->route.path_rec[1]), base_lid); 1620 (event_idp->route.path_rec[1]).mtu = 1621 (uint8_t)rtr_data.req_path_mtu; 1622 } 1623 1624 *event_id_ptr = event_idp; 1625 1626 event_chanp = (sol_cma_chan_t *)event_idp; 1627 event_chanp->chan_req_state = REQ_CMID_CREATED; 1628 event_ibchanp = &event_chanp->chan_ib; 1629 event_chanp->chan_session_id = eventp->cm_session_id; 1630 event_chanp->chan_connect_flag = SOL_CMA_CONNECT_SERVER_RCVD; 1631 bcopy((void *)(&reqp->req_prim_addr), 1632 (void *)(&event_ibchanp->chan_rcreq_addr), 1633 sizeof (ibt_adds_vect_t)); 1634 bcopy(&rtr_data, &(event_ibchanp->chan_rtr_data), 1635 sizeof (ibt_ofuvcm_req_data_t)); 1636 event_ibchanp->chan_rcreq_qpn = reqp->req_remote_qpn; 1637 event_ibchanp->chan_rcreq_ra_in = reqp->req_rdma_ra_in; 1638 1639 /* 1640 * Increment number of Reqs for listening CMID, so that 1641 * listening CMID is not deleted, till this connection 1642 * expects no more events. chan_req_cnt is decremented 1643 * when connection is notified to the consumer. 1644 * 1645 * Insert the CMID into the REQ_AVL_TREE. This is 1646 * deleted when the connection is accepted or rejected. 1647 */ 1648 mutex_enter(&root_chanp->chan_mutex); 1649 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1650 "Add to REQ AVL of %p IDP, idp %p, session_id %p", 1651 root_idp, event_idp, event_chanp->chan_session_id); 1652 find_ret = avl_find(&root_chanp->chan_req_avl_tree, 1653 (void *)event_chanp->chan_session_id, &where); 1654 if (find_ret) { 1655 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1656 "DUPLICATE ENTRY in REQ AVL : root %p, " 1657 "idp %p, session_id %p", 1658 root_idp, event_idp, 1659 event_chanp->chan_session_id); 1660 mutex_exit(&root_chanp->chan_mutex); 1661 rdma_destroy_id(event_idp); 1662 return (IBT_CM_REJECT); 1663 } 1664 root_chanp->chan_req_cnt++; 1665 root_chanp->chan_req_state = REQ_CMID_CREATED; 1666 root_chanp->chan_req_total_cnt++; 1667 avl_insert(&root_chanp->chan_req_avl_tree, (void *)event_idp, where); 1668 mutex_exit(&root_chanp->chan_mutex); 1669 1670 return (IBT_CM_DEFER); 1671 } 1672 1673 static void 1674 ibcma_handle_rep(struct rdma_cm_id *idp, ibt_cm_event_t *eventp) 1675 { 1676 sol_cma_chan_t *chanp; 1677 ibt_cm_rep_rcv_t *repp; 1678 struct rdma_conn_param *paramp; 1679 1680 chanp = (sol_cma_chan_t *)idp; 1681 1682 paramp = &chanp->chan_param; 1683 bzero(paramp, sizeof (chanp->chan_param)); 1684 repp = &((eventp->cm_event).rep); 1685 paramp->srq = (repp->rep_flags & IBT_CM_SRQ_EXISTS) ? 1 : 0; 1686 paramp->responder_resources = repp->rep_rdma_ra_in; 1687 paramp->initiator_depth = repp->rep_rdma_ra_out; 1688 paramp->flow_control = (repp->rep_flags & IBT_CM_FLOW_CONTROL) ? 1 : 0; 1689 1690 #ifdef DEBUG 1691 dump_priv_data(eventp->cm_priv_data, SOL_REP_PRIV_DATA_SZ, 1692 eventp->cm_priv_data_len, "handle_rep"); 1693 #endif 1694 paramp->private_data_len = eventp->cm_priv_data_len; 1695 if (paramp->private_data_len) { 1696 paramp->private_data = kmem_zalloc(paramp->private_data_len, 1697 KM_SLEEP); 1698 bcopy((void *)eventp->cm_priv_data, 1699 (void *)paramp->private_data, paramp->private_data_len); 1700 } 1701 } 1702 1703 static ibt_cm_status_t 1704 ibcma_handle_est(struct rdma_cm_id *idp, struct rdma_cm_id **event_id_ptr, 1705 ibt_cm_event_t *eventp, struct rdma_conn_param *paramp, 1706 enum rdma_cm_event_type *event, int *evt_status) 1707 { 1708 struct rdma_cm_id *event_idp, *root_idp; 1709 sol_cma_chan_t *event_chanp, *chanp, *root_chanp; 1710 ibcma_chan_t *event_ibchanp; 1711 1712 /* Established event on active / client side */ 1713 chanp = (sol_cma_chan_t *)idp; 1714 if (chanp->chan_listenp == NULL) { 1715 ASSERT(chanp->chan_connect_flag == SOL_CMA_CONNECT_INITIATED); 1716 chanp->chan_connect_flag = SOL_CMA_CONNECT_CLIENT_DONE; 1717 bcopy(&chanp->chan_param, paramp, 1718 sizeof (struct rdma_conn_param)); 1719 if (paramp->private_data_len) { 1720 paramp->private_data = kmem_zalloc( 1721 paramp->private_data_len, KM_SLEEP); 1722 bcopy((void *)((chanp->chan_param).private_data), 1723 (void *)paramp->private_data, 1724 paramp->private_data_len); 1725 kmem_free((void *)((chanp->chan_param).private_data), 1726 paramp->private_data_len); 1727 } 1728 event_chanp = chanp; 1729 goto est_common; 1730 } 1731 1732 root_idp = CHAN_LISTEN_ROOT((chanp)); 1733 ASSERT(root_idp); 1734 root_chanp = (sol_cma_chan_t *)root_idp; 1735 event_chanp = NULL; 1736 1737 event_idp = cma_get_acpt_idp(root_idp, eventp->cm_channel); 1738 if (event_idp == NULL) { 1739 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "ibcma_handle_est: " 1740 "No matching CMID for qp_hdl %p in ACPT AVL of CMID %p", 1741 eventp->cm_channel, root_chanp); 1742 return (IBT_CM_REJECT); 1743 } 1744 *event_id_ptr = event_idp; 1745 event_chanp = (sol_cma_chan_t *)event_idp; 1746 event_chanp->chan_connect_flag = SOL_CMA_CONNECT_SERVER_DONE; 1747 1748 est_common: 1749 #ifdef QP_DEBUG 1750 dump_qp_info(event_chanp->chan_qp_hdl); 1751 #endif 1752 1753 /* 1754 * Pass back CONNECT_ESTABLISHED event to consumer. 1755 */ 1756 *event = RDMA_CM_EVENT_ESTABLISHED; 1757 event_ibchanp = &event_chanp->chan_ib; 1758 event_ibchanp->chan_qpmodifyflag = 1; 1759 1760 *evt_status = 0; 1761 return (IBT_CM_DEFAULT); 1762 } 1763 1764 static ibt_cm_status_t 1765 ibcma_handle_closed(struct rdma_cm_id *idp, struct rdma_cm_id **event_id_ptr, 1766 ibt_cm_event_t *eventp, enum rdma_cm_event_type *event, int *evt_status) 1767 { 1768 struct rdma_cm_id *root_idp, *event_idp; 1769 sol_cma_chan_t *chanp, *event_chanp; 1770 1771 *event = RDMA_CM_EVENT_DISCONNECTED; 1772 *evt_status = 0; 1773 chanp = (sol_cma_chan_t *)idp; 1774 mutex_enter(&chanp->chan_mutex); 1775 root_idp = CHAN_LISTEN_ROOT((chanp)); 1776 chanp->chan_qp_hdl = NULL; 1777 if (!root_idp) { 1778 chanp->chan_connect_flag = 0; 1779 mutex_exit(&chanp->chan_mutex); 1780 return (IBT_CM_DEFAULT); 1781 } 1782 mutex_exit(&chanp->chan_mutex); 1783 1784 /* On the passive side, search ACPT AVL Tree */ 1785 event_idp = cma_get_acpt_idp(root_idp, eventp->cm_channel); 1786 if (event_idp == NULL) { 1787 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1788 "ibcma_handle_closed: " 1789 "No matching CMID for qp hdl %p in EST AVL of CMID %p", 1790 eventp->cm_channel, root_idp); 1791 return (IBT_CM_DEFAULT); 1792 } 1793 event_chanp = (sol_cma_chan_t *)event_idp; 1794 mutex_enter(&event_chanp->chan_mutex); 1795 event_chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1796 mutex_exit(&event_chanp->chan_mutex); 1797 *event_id_ptr = event_idp; 1798 return (IBT_CM_DEFAULT); 1799 } 1800 1801 static ibt_cm_status_t 1802 ibcma_handle_failed(struct rdma_cm_id *idp, struct rdma_cm_id **event_id_ptr, 1803 ibt_cm_event_t *eventp, struct rdma_conn_param *paramp, 1804 enum rdma_cm_event_type *event, int *evt_status) 1805 { 1806 1807 struct rdma_cm_id *root_idp, *event_idp; 1808 sol_cma_chan_t *event_chanp, *chanp, *root_chanp; 1809 ibt_cm_conn_failed_t *failedp; 1810 1811 failedp = &(eventp->cm_event.failed); 1812 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "ibcma_handle_failed - idp %p, " 1813 "cf_code %x, cf_msg %x, cf_arej_info_valid %x, cf_reason %x", 1814 idp, failedp->cf_code, failedp->cf_msg, 1815 failedp->cf_arej_info_valid, failedp->cf_reason); 1816 chanp = (sol_cma_chan_t *)idp; 1817 root_idp = CHAN_LISTEN_ROOT((chanp)); 1818 root_chanp = (sol_cma_chan_t *)root_idp; 1819 1820 *evt_status = 0; 1821 switch (failedp->cf_code) { 1822 case IBT_CM_FAILURE_REJ_SENT : 1823 /* Reject sent. No event to userland. */ 1824 break; 1825 1826 case IBT_CM_FAILURE_REJ_RCV : 1827 /* 1828 * Reject recieved. If this is a consumer reject, copy the 1829 * private * data. Send RDMA_CM_EVENT_REJECTED to user land. 1830 */ 1831 if (failedp->cf_reason == IBT_CM_CONSUMER && 1832 eventp->cm_priv_data_len) { 1833 paramp->private_data_len = eventp->cm_priv_data_len; 1834 paramp->private_data = kmem_zalloc( 1835 paramp->private_data_len, KM_SLEEP); 1836 bcopy(eventp->cm_priv_data, 1837 (void *)paramp->private_data, 1838 paramp->private_data_len); 1839 } 1840 1841 /* 1842 * If this an REJECT for an accepted CMID, pass the 1843 * event to accepted CMID. 1844 */ 1845 if (root_idp) { 1846 ASSERT(eventp->cm_channel); 1847 event_idp = cma_get_acpt_idp(root_idp, 1848 eventp->cm_channel); 1849 if (event_idp == NULL) { 1850 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1851 "ibcma_handle_failed: No matching CMID " 1852 "for qp_hdl %p in ACPT AVL of CMID %p", 1853 eventp->cm_channel, idp); 1854 break; 1855 } 1856 1857 event_chanp = (sol_cma_chan_t *)event_idp; 1858 mutex_enter(&event_chanp->chan_mutex); 1859 event_chanp->chan_connect_flag = 1860 SOL_CMA_CONNECT_NONE; 1861 event_chanp->chan_qp_hdl = NULL; 1862 mutex_exit(&event_chanp->chan_mutex); 1863 *event_id_ptr = event_idp; 1864 mutex_enter(&root_chanp->chan_mutex); 1865 avl_remove(&root_chanp->chan_acpt_avl_tree, 1866 event_idp); 1867 mutex_exit(&root_chanp->chan_mutex); 1868 } else 1869 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1870 *evt_status = failedp->cf_reason; 1871 *event = RDMA_CM_EVENT_REJECTED; 1872 break; 1873 1874 case IBT_CM_FAILURE_TIMEOUT : 1875 /* 1876 * Connection Timeout, Send RDMA_CM_EVENT_REJECTED event and 1877 * status as IBT_CM_TIMEOUT. 1878 */ 1879 if (eventp->cm_session_id && root_idp) { 1880 event_idp = cma_get_req_idp(root_idp, 1881 eventp->cm_session_id); 1882 if (event_idp == NULL) { 1883 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1884 "ibcma_handle_failed: No matching CMID " 1885 "for qp_hdl %p in REQ AVL of CMID %p", 1886 eventp->cm_session_id, idp); 1887 break; 1888 } 1889 1890 event_chanp = (sol_cma_chan_t *)event_idp; 1891 mutex_enter(&event_chanp->chan_mutex); 1892 event_chanp->chan_connect_flag = 1893 SOL_CMA_CONNECT_NONE; 1894 event_chanp->chan_qp_hdl = NULL; 1895 mutex_exit(&event_chanp->chan_mutex); 1896 *event_id_ptr = event_idp; 1897 mutex_enter(&root_chanp->chan_mutex); 1898 avl_remove(&root_chanp->chan_req_avl_tree, 1899 event_idp); 1900 root_chanp->chan_req_cnt--; 1901 mutex_exit(&root_chanp->chan_mutex); 1902 1903 *evt_status = IBT_CM_TIMEOUT; 1904 *event = RDMA_CM_EVENT_REJECTED; 1905 } 1906 if (!eventp->cm_session_id && root_idp) { 1907 SOL_OFS_DPRINTF_L0(sol_rdmacm_dbg_str, 1908 "ibcma_handle_failed: timeout " 1909 "session_id NULL"); 1910 } 1911 if (!root_idp) { 1912 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1913 *evt_status = IBT_CM_TIMEOUT; 1914 *event = RDMA_CM_EVENT_REJECTED; 1915 } 1916 chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE; 1917 chanp->chan_qp_hdl = NULL; 1918 break; 1919 1920 case IBT_CM_FAILURE_STALE : 1921 /* Stale connection, ignore */ 1922 break; 1923 } 1924 return (IBT_CM_DEFAULT); 1925 } 1926 1927 static ibt_cm_status_t 1928 ibcma_rc_hdlr(void *inp, ibt_cm_event_t *eventp, 1929 ibt_cm_return_args_t *ret_args, void *priv_data, 1930 ibt_priv_data_len_t priv_datalen) 1931 { 1932 struct rdma_cm_id *idp, *event_idp; 1933 sol_cma_chan_t *chanp; 1934 ibt_cm_status_t status; 1935 ibt_status_t ibt_status; 1936 enum rdma_cm_event_type event; 1937 struct rdma_conn_param conn_param, *paramp = &conn_param; 1938 int event_status; 1939 1940 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "ib_cma_rc_hdlr(%p, %p, %p, " 1941 "%p, %x)", inp, eventp, ret_args, priv_data, priv_datalen); 1942 idp = event_idp = (struct rdma_cm_id *)inp; 1943 chanp = (sol_cma_chan_t *)idp; 1944 chanp->chan_session_id = NULL; 1945 1946 bzero(paramp, sizeof (struct rdma_conn_param)); 1947 switch (eventp->cm_type) { 1948 1949 case IBT_CM_EVENT_REQ_RCV : 1950 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1951 "ibcma_rc_hdlr : REQ Event"); 1952 1953 /* 1954 * We need to do a round trip to userland. Send a MRA 1955 * so that the client does not send multiple REQs. Then 1956 * continue the processing of REQs. 1957 */ 1958 ibt_status = ibt_cm_delay(IBT_CM_DELAY_REQ, 1959 eventp->cm_session_id, SOL_OFS_REQ_DELAY, NULL, 0); 1960 if (ibt_status != IBT_SUCCESS) { 1961 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 1962 "ibcma_rc_hdlr : ibt_cma_delay failed %x", 1963 ibt_status); 1964 return (IBT_CM_REJECT); 1965 } 1966 status = ibcma_handle_req(idp, &event_idp, eventp, paramp, 1967 &event, &event_status); 1968 if (status == IBT_CM_REJECT) 1969 return (status); 1970 break; 1971 case IBT_CM_EVENT_REP_RCV : 1972 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1973 "ibcma_rc_hdlr : REP Event"); 1974 1975 ibcma_handle_rep(idp, eventp); 1976 return (IBT_CM_DEFAULT); 1977 /* NOTREACHED */ 1978 /* break; */ 1979 case IBT_CM_EVENT_LAP_RCV : 1980 case IBT_CM_EVENT_APR_RCV : 1981 /* 1982 * Alternate Paths not supported from userland. Return 1983 * IBT_CM_REJECT. 1984 */ 1985 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1986 "ibcma_rc_hdlr : AP Event"); 1987 return (IBT_CM_REJECT); 1988 /* NOTREACHED */ 1989 /* break; */ 1990 case IBT_CM_EVENT_MRA_RCV : 1991 /* Let Solaris ibcm take default action for MRA */ 1992 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1993 "ibcma_rc_hdlr : MRA Event"); 1994 return (IBT_CM_DEFAULT); 1995 /* NOTREACHED */ 1996 /* break; */ 1997 case IBT_CM_EVENT_CONN_EST : 1998 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 1999 "ibcma_rc_hdlr : EST Event"); 2000 status = ibcma_handle_est(idp, &event_idp, eventp, paramp, 2001 &event, &event_status); 2002 break; 2003 case IBT_CM_EVENT_CONN_CLOSED : 2004 /* 2005 * Pass on RDMA_CM_EVENT_DISCONNECTED to consumer 2006 */ 2007 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2008 "ibcma_rc_hdlr : CLOSED Event"); 2009 status = ibcma_handle_closed(idp, &event_idp, eventp, 2010 &event, &event_status); 2011 break; 2012 2013 case IBT_CM_EVENT_FAILURE : 2014 /* Handle Failure Event */ 2015 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2016 "ibcma_rc_hdlr : FAIL Event"); 2017 status = ibcma_handle_failed(idp, &event_idp, eventp, paramp, 2018 &event, &event_status); 2019 2020 /* 2021 * Check if there is an event to be send to the userland. 2022 * Return if there are none. 2023 */ 2024 if (event_status == 0) 2025 return (status); 2026 break; 2027 } 2028 2029 /* Pass back the event to sol_cma consumer */ 2030 if (event_idp) { 2031 cma_generate_event(event_idp, event, event_status, 2032 paramp, NULL); 2033 } else 2034 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 2035 "No Event to userland!!"); 2036 if (paramp->private_data) 2037 kmem_free((void *)paramp->private_data, 2038 paramp->private_data_len); 2039 2040 return (status); 2041 } 2042 2043 static void 2044 ibcma_multicast_hdlr(void *arg, ibt_status_t status, ibt_mcg_info_t *mcg_infop) 2045 { 2046 struct rdma_cm_id *idp; 2047 ibcma_mcast_t *ib_mcastp = (ibcma_mcast_t *)arg; 2048 int evt_status; 2049 struct rdma_ud_param uddata, *ud_param = &uddata; 2050 enum rdma_cm_event_type event; 2051 2052 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "multicast_hdlr(%p, %x, %p)", 2053 arg, status, mcg_infop); 2054 idp = ib_mcastp->mcast_idp; 2055 2056 bzero(ud_param, sizeof (struct rdma_ud_param)); 2057 bcopy(&(mcg_infop->mc_adds_vect.av_dgid), 2058 &(ib_mcastp->mcast_gid), sizeof (ib_gid_t)); 2059 ud_param->private_data = ib_mcastp->mcast_ctx; 2060 2061 event = (status == IBT_SUCCESS) ? 2062 RDMA_CM_EVENT_MULTICAST_JOIN : RDMA_CM_EVENT_MULTICAST_ERROR; 2063 evt_status = (status == IBT_SUCCESS) ? 0 : -1; 2064 if (status == IBT_SUCCESS) { 2065 mcginfo2ah(mcg_infop, &ud_param->ah_attr); 2066 ud_param->qp_num = IB_MC_QPN; 2067 ud_param->qkey = RDMA_UDP_QKEY; 2068 } 2069 2070 /* Send the event to consumer of sol_cma. */ 2071 cma_generate_event(idp, event, evt_status, NULL, ud_param); 2072 kmem_free(mcg_infop, sizeof (ibt_mcg_info_t)); 2073 } 2074 2075 static int 2076 ibcma_get_first_ib_ipaddr(struct rdma_cm_id *idp) 2077 { 2078 sol_cma_chan_t *chanp = (sol_cma_chan_t *)idp; 2079 ibcma_chan_t *ibchanp; 2080 int num_hcas, info_inited = 0; 2081 ib_guid_t *hca_guidp; 2082 genlist_t devlist; 2083 genlist_entry_t *entry; 2084 ibcma_dev_t *devp; 2085 2086 ASSERT(idp); 2087 ibchanp = &(chanp->chan_ib); 2088 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "get_first_ib_ipaddr(%p)", idp); 2089 2090 num_hcas = ibt_get_hca_list(&hca_guidp); 2091 ibcma_get_devlist(chanp, hca_guidp, num_hcas, &devlist); 2092 entry = remove_genlist_head(&devlist); 2093 while (entry) { 2094 devp = (ibcma_dev_t *)entry->data; 2095 if (info_inited == 0) { 2096 (idp->route).num_paths = 0; 2097 idp->port_num = devp->dev_port_num; 2098 chanp->chan_xport_type = SOL_CMA_XPORT_IB; 2099 ibchanp->chan_devp = devp; 2100 info_inited = 1; 2101 } else { 2102 kmem_free(devp, sizeof (ibcma_dev_t)); 2103 } 2104 kmem_free(entry, sizeof (genlist_entry_t)); 2105 entry = remove_genlist_head(&devlist); 2106 } 2107 ibt_free_hca_list(hca_guidp, num_hcas); 2108 2109 if (info_inited) 2110 return (0); 2111 else 2112 return (ENODEV); 2113 } 2114 2115 /* Utility Conversion functions */ 2116 static void 2117 ipaddr2sockaddr(ibt_ip_addr_t *ibt_addrp, struct sockaddr *sock_addrp, 2118 in_port_t *portp) 2119 { 2120 sock_addrp->sa_family = ibt_addrp->family; 2121 if (ibt_addrp->family == AF_INET) { 2122 struct sockaddr_in *sock_in4p; 2123 sock_in4p = (struct sockaddr_in *)sock_addrp; 2124 2125 sock_in4p->sin_addr.s_addr = ibt_addrp->un.ip4addr; 2126 if (portp) 2127 sock_in4p->sin_port = ntohs(*portp); 2128 } else { 2129 struct sockaddr_in6 *in6_addr; 2130 in6_addr = (struct sockaddr_in6 *)sock_addrp; 2131 2132 bcopy(&(ibt_addrp->un.ip6addr), &(in6_addr->sin6_addr), 2133 sizeof (in6_addr_t)); 2134 if (portp) 2135 in6_addr->sin6_port = *portp; 2136 } 2137 } 2138 2139 static void 2140 sockaddr2ibtaddr_port(struct rdma_cm_id *idp, struct sockaddr *sock_addrp, 2141 ibt_ip_addr_t *ibt_addrp, in_port_t *portp) 2142 { 2143 in_port_t ip_port; 2144 2145 ibt_addrp->family = sock_addrp->sa_family; 2146 if (sock_addrp->sa_family == AF_INET) { 2147 struct sockaddr_in *sock_in4p; 2148 sock_in4p = (struct sockaddr_in *)sock_addrp; 2149 2150 ibt_addrp->un.ip4addr = sock_in4p->sin_addr.s_addr; 2151 if (IS_UDP_CMID(idp)) 2152 ip_port = ddi_swap16(sock_in4p->sin_port); 2153 else 2154 ip_port = htons(sock_in4p->sin_port); 2155 2156 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "sockaddr2ibtaddr : " 2157 "AF_INET addr %x, port %x, %x", ibt_addrp->un.ip4addr, 2158 sock_in4p->sin_port, ip_port); 2159 2160 if (portp) 2161 *portp = ip_port; 2162 2163 } else { 2164 struct sockaddr_in6 *in6_addr; 2165 in6_addr = (struct sockaddr_in6 *)sock_addrp; 2166 bcopy(&(in6_addr->sin6_addr), &(ibt_addrp->un.ip6addr), 2167 sizeof (in6_addr_t)); 2168 if (portp) 2169 *portp = in6_addr->sin6_port; 2170 } 2171 } 2172 2173 static void 2174 mcginfo2ah(ibt_mcg_info_t *mcgp, struct ib_ah_attr *ah_attr) 2175 { 2176 ibt_adds_vect_t *adds_vectp; 2177 ib_gid_t dgid_nworder; 2178 2179 adds_vectp = &(mcgp->mc_adds_vect); 2180 2181 /* 2182 * Libraries expect the GID to be in network order. Convert 2183 * to network order before passing it to the library. 2184 */ 2185 dgid_nworder.gid_prefix = htonll( 2186 (adds_vectp->av_dgid).gid_prefix); 2187 dgid_nworder.gid_guid = htonll( 2188 (adds_vectp->av_dgid).gid_guid); 2189 bcopy(&dgid_nworder, &((ah_attr->grh).dgid), sizeof (ib_gid_t)); 2190 2191 (ah_attr->grh).flow_label = adds_vectp->av_flow; 2192 (ah_attr->grh).sgid_index = adds_vectp->av_sgid_ix; 2193 (ah_attr->grh).hop_limit = adds_vectp->av_hop; 2194 (ah_attr->grh).traffic_class = adds_vectp->av_tclass; 2195 2196 ah_attr->dlid = adds_vectp->av_dlid; 2197 ah_attr->sl = adds_vectp->av_srvl; 2198 ah_attr->src_path_bits = adds_vectp->av_src_path; 2199 ah_attr->static_rate = adds_vectp->av_srate; 2200 ah_attr->ah_flags = (adds_vectp->av_send_grh) ? 1 : 0; 2201 ah_attr->port_num = adds_vectp->av_port_num; 2202 } 2203 2204 static void 2205 ibt_path2ah(ibt_path_info_t *pathp, struct ib_ah_attr *ah_attr) 2206 { 2207 2208 ibt_addsvect2ah(&((pathp->pi_prim_cep_path).cep_adds_vect), ah_attr); 2209 } 2210 2211 static void 2212 ibt_addsvect2ah(ibt_adds_vect_t *adds_vectp, struct ib_ah_attr *ah_attr) 2213 { 2214 ib_gid_t dgid_nworder; 2215 2216 /* 2217 * Libraries expect the GID to be in network order. Convert 2218 * to network order before passing it to the library. 2219 */ 2220 dgid_nworder.gid_prefix = htonll( 2221 (adds_vectp->av_dgid).gid_prefix); 2222 dgid_nworder.gid_guid = htonll( 2223 (adds_vectp->av_dgid).gid_guid); 2224 bcopy(&dgid_nworder, &((ah_attr->grh).dgid), sizeof (ib_gid_t)); 2225 (ah_attr->grh).flow_label = adds_vectp->av_flow; 2226 (ah_attr->grh).sgid_index = adds_vectp->av_sgid_ix; 2227 (ah_attr->grh).hop_limit = adds_vectp->av_hop; 2228 (ah_attr->grh).traffic_class = adds_vectp->av_tclass; 2229 2230 ah_attr->dlid = adds_vectp->av_dlid; 2231 ah_attr->sl = adds_vectp->av_srvl; 2232 ah_attr->src_path_bits = adds_vectp->av_src_path; 2233 ah_attr->static_rate = adds_vectp->av_srate; 2234 ah_attr->ah_flags = (adds_vectp->av_send_grh) ? 1 : 0; 2235 ah_attr->port_num = adds_vectp->av_port_num; 2236 } 2237 2238 static void 2239 ibt_path2sa_path(ibt_path_info_t *pathp, struct ib_sa_path_rec *sa_pathp, 2240 ib_lid_t base_lid) 2241 { 2242 ibt_adds_vect_t *adds_vectp; 2243 2244 adds_vectp = &((pathp->pi_prim_cep_path).cep_adds_vect); 2245 ibt_addsvect2sa_path(adds_vectp, sa_pathp, base_lid); 2246 sa_pathp->mtu = pathp->pi_path_mtu; 2247 sa_pathp->packet_life_time = pathp->pi_prim_pkt_lt; 2248 } 2249 2250 static void 2251 ibt_addsvect2sa_path(ibt_adds_vect_t *adds_vectp, 2252 struct ib_sa_path_rec *sa_pathp, ib_lid_t base_lid) 2253 { 2254 bcopy(&(adds_vectp->av_dgid), &(sa_pathp->dgid), 16); 2255 bcopy(&(adds_vectp->av_sgid), &(sa_pathp->sgid), 16); 2256 sa_pathp->dlid = adds_vectp->av_dlid; 2257 sa_pathp->slid = base_lid + adds_vectp->av_src_path; 2258 sa_pathp->flow_label = adds_vectp->av_flow; 2259 sa_pathp->reversible = 1; 2260 sa_pathp->hop_limit = adds_vectp->av_hop; 2261 sa_pathp->traffic_class = adds_vectp->av_tclass; 2262 sa_pathp->sl = adds_vectp->av_srvl; 2263 sa_pathp->rate = adds_vectp->av_srate; 2264 sa_pathp->mtu_selector = IBT_EQU; 2265 sa_pathp->rate_selector = IBT_EQU; 2266 sa_pathp->packet_life_time_selector = IBT_EQU; 2267 } 2268 2269 /* 2270 * Map a multicast IP onto multicast MAC for type IP-over-InfiniBand. 2271 * Leave P_Key as 0 to be filled in by caller 2272 */ 2273 static void 2274 ip_ib_mc_map(uint32_t addr, char *buf) 2275 { 2276 buf[0] = 0; /* Reserved */ 2277 buf[1] = 0xff; /* Multicast QPN */ 2278 buf[2] = 0xff; 2279 buf[3] = 0xff; 2280 addr = ntohl(addr); 2281 buf[4] = 0xff; 2282 buf[5] = 0x12; /* link local scope */ 2283 buf[6] = 0x40; /* IPv4 signature */ 2284 buf[7] = 0x1b; 2285 buf[8] = 0; /* P_Key */ 2286 buf[9] = 0; 2287 buf[10] = 0; 2288 buf[11] = 0; 2289 buf[12] = 0; 2290 buf[13] = 0; 2291 buf[14] = 0; 2292 buf[15] = 0; 2293 buf[19] = addr & 0xff; 2294 addr >>= 8; 2295 buf[18] = addr & 0xff; 2296 addr >>= 8; 2297 buf[17] = addr & 0xff; 2298 addr >>= 8; 2299 buf[16] = addr & 0x0f; 2300 } 2301 2302 static void 2303 ipaddr2mgid(struct sockaddr *addrp, ib_gid_t *mgidp, ib_pkey_t pkey) 2304 { 2305 char mc_map[32]; /* Max H/W addr len */ 2306 struct sockaddr_in *sin = (struct sockaddr_in *)addrp; 2307 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addrp; 2308 2309 if ((addrp->sa_family == AF_INET6) && 2310 b2h32((sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) == 2311 0xFF10A01B) { 2312 bcopy(&sin6->sin6_addr, mgidp, sizeof (ib_gid_t)); 2313 } else { 2314 ip_ib_mc_map(sin->sin_addr.s_addr, mc_map); 2315 mc_map[7] = 0x01; /* Use RDMA CM signature */ 2316 mc_map[8] = (char)(pkey >> 8); 2317 mc_map[9] = (char)(pkey); 2318 bcopy(mc_map+4, mgidp, sizeof (ib_gid_t)); 2319 } 2320 } 2321 2322 static int 2323 ibcma_any_addr(ibt_ip_addr_t *addr) 2324 { 2325 ASSERT(addr); 2326 if (addr->family == AF_INET) 2327 return (addr->un.ip4addr == INADDR_ANY); 2328 else if (addr->family == AF_INET6) 2329 return (IN6_IS_ADDR_UNSPECIFIED(&(addr->un.ip6addr))); 2330 return (0); 2331 } 2332 2333 static struct rdma_cm_id * 2334 ibcma_create_new_id(struct rdma_cm_id *idp) 2335 { 2336 struct rdma_cm_id *new_idp; 2337 sol_cma_chan_t *chanp, *new_chanp; 2338 ibcma_chan_t *ibchanp, *new_ibchanp; 2339 2340 new_idp = cma_create_new_id(idp); 2341 if (new_idp == NULL) 2342 return (new_idp); 2343 new_chanp = (sol_cma_chan_t *)new_idp; 2344 new_ibchanp = &new_chanp->chan_ib; 2345 chanp = (sol_cma_chan_t *)idp; 2346 ibchanp = &chanp->chan_ib; 2347 if (ibchanp->chan_devp) { 2348 ibcma_dev_t *devp; 2349 2350 devp = (ibcma_dev_t *)kmem_zalloc(sizeof (ibcma_dev_t), 2351 KM_SLEEP); 2352 new_ibchanp->chan_devp = devp; 2353 bcopy(ibchanp->chan_devp, devp, sizeof (ibcma_dev_t)); 2354 } 2355 2356 if (ibchanp->chan_pathp && ibchanp->chan_numpaths && 2357 ibchanp->chan_path_size) { 2358 new_ibchanp->chan_pathp = (ibt_path_info_t *)kmem_zalloc( 2359 ibchanp->chan_path_size, KM_SLEEP); 2360 bcopy(ibchanp->chan_pathp, new_ibchanp->chan_pathp, 2361 ibchanp->chan_path_size); 2362 new_ibchanp->chan_path_size = ibchanp->chan_path_size; 2363 new_ibchanp->chan_numpaths = ibchanp->chan_numpaths; 2364 } 2365 bcopy(&ibchanp->chan_local_addr, &new_ibchanp->chan_local_addr, 2366 sizeof (ibt_ip_addr_t)); 2367 bcopy(&ibchanp->chan_remote_addr, &new_ibchanp->chan_remote_addr, 2368 sizeof (ibt_ip_addr_t)); 2369 new_ibchanp->chan_port = ibchanp->chan_port; 2370 new_ibchanp->chan_sid = ibchanp->chan_sid; 2371 2372 return (new_idp); 2373 } 2374 2375 static void 2376 ibcma_get_devlist(sol_cma_chan_t *root_chanp, ib_guid_t *hca_guidp, 2377 int num_hcas, genlist_t *ret_devlist) 2378 { 2379 int i; 2380 ibt_status_t status; 2381 ibcma_dev_t *devp; 2382 uint_t num_ports, p; 2383 uint_t port_size; 2384 ibt_hca_portinfo_t *port_info, *tmp; 2385 ibt_ip_addr_t hca_ipaddr; 2386 2387 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "get_devlist(%p, %p, %x, %p)", 2388 root_chanp, hca_guidp, num_hcas, ret_devlist); 2389 2390 init_genlist(ret_devlist); 2391 for (i = 0; i < num_hcas; i++) { 2392 status = ibt_query_hca_ports_byguid(hca_guidp[i], 0, &port_info, 2393 &num_ports, &port_size); 2394 if (status != IBT_SUCCESS) { 2395 SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, 2396 "ibt_query_hca_ports_byguid failed %d", status); 2397 continue; 2398 } 2399 2400 for (p = 0, tmp = port_info; p < num_ports; p++, tmp++) { 2401 uint_t s, num_sgids; 2402 uint16_t pk; 2403 uint_t num_pkeys; 2404 2405 if (tmp->p_linkstate != IBT_PORT_ACTIVE) 2406 continue; 2407 2408 num_sgids = tmp->p_sgid_tbl_sz / sizeof (ib_gid_t); 2409 num_pkeys = tmp->p_pkey_tbl_sz / sizeof (ib_pkey_t); 2410 2411 for (s = 0; s < num_sgids; s++) { 2412 /* Skip holes in sgid table */ 2413 if (tmp->p_sgid_tbl[s].gid_guid == 0x0LL) 2414 continue; 2415 for (pk = 0; pk < num_pkeys; pk++) { 2416 /* Skip holes in pkey table */ 2417 if (tmp->p_pkey_tbl[pk] == 0) 2418 continue; 2419 status = ibt_get_src_ip( 2420 tmp->p_sgid_tbl[s], 2421 tmp->p_pkey_tbl[pk], 2422 &hca_ipaddr); 2423 if (status != IBT_SUCCESS) 2424 continue; 2425 2426 /* allocate devinfo & fill in info */ 2427 devp = kmem_zalloc( 2428 sizeof (ibcma_dev_t), KM_SLEEP); 2429 devp->dev_node_guid = hca_guidp[i]; 2430 devp->dev_port_num = p + 1; 2431 devp->dev_pkey_ix = pk; 2432 devp->dev_pkey = tmp->p_pkey_tbl[pk]; 2433 devp->dev_sgid = tmp->p_sgid_tbl[s]; 2434 bcopy(&hca_ipaddr, &devp->dev_ipaddr, 2435 sizeof (ibt_ip_addr_t)); 2436 2437 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2438 "get_devlist: add2devlist " 2439 "node_guid %llx", hca_guidp[i]); 2440 (void) add_genlist(ret_devlist, 2441 (uintptr_t)devp, NULL); 2442 } 2443 } 2444 } 2445 ibt_free_portinfo(port_info, port_size); 2446 } 2447 } 2448 2449 2450 #ifdef QP_DEBUG 2451 static void 2452 dump_qp_info(ibt_qp_hdl_t qphdl) 2453 { 2454 ibt_qp_query_attr_t qp_query; 2455 ibt_qp_info_t *qp_info; 2456 ibt_status_t status; 2457 ibt_qp_rc_attr_t *rcp; 2458 2459 bzero(&qp_query, sizeof (qp_query)); 2460 status = ibt_query_qp(qphdl, &qp_query); 2461 if (status != IBT_SUCCESS) { 2462 cmn_err(CE_WARN, "query_qp failed!!"); 2463 return; 2464 } 2465 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2466 "QP HDL : %p, qp_sq_cq %p, qp_rq_cq %p, " 2467 "qp_rdd_hdl %p, qp_qpn %x, qp_sq_sgl %x, qp_rq_sgl %x, " 2468 "qp_srq %p, quer_attr.qp_flags %x", 2469 qphdl, qp_query.qp_sq_cq, qp_query.qp_rq_cq, 2470 qp_query.qp_rdd_hdl, qp_query.qp_qpn, 2471 qp_query.qp_sq_sgl, qp_query.qp_rq_sgl, 2472 qp_query.qp_srq, qp_query.qp_flags); 2473 qp_info = &(qp_query.qp_info); 2474 rcp = &((qp_info->qp_transport).rc); 2475 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2476 "qp_sq_sz %x, qp_rq_sz %x, qp_state %x, " 2477 "qp_current_state %x, qp_info.qp_flags %x, qp_trans %x", 2478 qp_info->qp_sq_sz, qp_info->qp_rq_sz, qp_info->qp_state, 2479 qp_info->qp_current_state, qp_info->qp_flags, 2480 qp_info->qp_trans); 2481 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2482 "rc_sq_psn %x, rc_rq_psn %x, rc_dst_qpn %x, " 2483 "rc_mig_state %x, rc_rnr_retry_cnt %x, rc_retry_cnt %x, " 2484 "rc_rdma_ra_out %x, rc_rdma_ra_in %x, rc_min_rnr_nak %x, " 2485 "rc_path_mtu %x", rcp->rc_sq_psn, rcp->rc_rq_psn, 2486 rcp->rc_dst_qpn, rcp->rc_mig_state, rcp->rc_rnr_retry_cnt, 2487 rcp->rc_retry_cnt, rcp->rc_rdma_ra_out, rcp->rc_rdma_ra_in, 2488 rcp->rc_min_rnr_nak, rcp->rc_path_mtu); 2489 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2490 "av_dgid %llx: %llx, av_sgid: %llx, " 2491 "srate %x, srvl %x, flow %x, tclass %x, hop %x, " 2492 "av_port_num %x, av_send_grh %x, av_dlid %x, " 2493 "av_src_path %x, av_sgid_ix %x, pkey_index %x, " 2494 "port_num %x", 2495 (rcp->rc_path).cep_adds_vect.av_sgid.gid_prefix, 2496 (rcp->rc_path).cep_adds_vect.av_sgid.gid_guid, 2497 (rcp->rc_path).cep_adds_vect.av_dgid.gid_prefix, 2498 (rcp->rc_path).cep_adds_vect.av_dgid.gid_guid, 2499 (rcp->rc_path).cep_adds_vect.av_srate, 2500 (rcp->rc_path).cep_adds_vect.av_srvl, 2501 (rcp->rc_path).cep_adds_vect.av_flow, 2502 (rcp->rc_path).cep_adds_vect.av_tclass, 2503 (rcp->rc_path).cep_adds_vect.av_hop, 2504 (rcp->rc_path).cep_adds_vect.av_port_num, 2505 (rcp->rc_path).cep_adds_vect.av_opaque1, 2506 (rcp->rc_path).cep_adds_vect.av_opaque2, 2507 (rcp->rc_path).cep_adds_vect.av_opaque3, 2508 (rcp->rc_path).cep_adds_vect.av_opaque4, 2509 (rcp->rc_path).cep_pkey_ix, 2510 (rcp->rc_path).cep_hca_port_num); 2511 } 2512 #endif 2513 2514 static void 2515 dump_priv_data(void *priv_data, ibt_priv_data_len_t arr_len, 2516 ibt_priv_data_len_t priv_len, char *caller) 2517 { 2518 uint8_t i; 2519 uchar_t *c = (uchar_t *)priv_data; 2520 2521 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "priv_data to %s: %p, len %d", 2522 caller, priv_data, priv_len); 2523 if (!priv_len || !priv_data) 2524 return; 2525 2526 /* Display in rows of 16 uchar_t */ 2527 for (i = 0; i < arr_len; i += 16) 2528 SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, 2529 "%x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x", 2530 c[i], c[i + 1], c[i + 2], c[i + 3], c[i + 4], c[i + 5], 2531 c[i + 6], c[i + 7], c[i + 8], c[i + 9], c[i + 10], 2532 c[i + 11], c[i + 12], c[i + 13], c[i + 14], c[i + 15]); 2533 2534 } 2535