1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 29 #include <sys/ib/ibtl/ibti.h> 30 31 /* 32 * ibcm_ti.c 33 * These routines implement the Communication Manager's interfaces to IBTL. 34 */ 35 36 /* CM rc recycle task args structure definition */ 37 typedef struct ibcm_taskq_recycle_arg_s { 38 ibt_channel_hdl_t rc_chan; 39 ibt_cep_flags_t control; 40 uint8_t hca_port_num; 41 ibt_recycle_handler_t func; 42 void *arg; 43 } ibcm_taskq_recycle_arg_t; 44 45 _NOTE(READ_ONLY_DATA(ibcm_taskq_recycle_arg_s)) 46 47 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap, 48 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args, 49 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid); 50 static void ibcm_process_abort_via_taskq(void *args); 51 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg); 52 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg); 53 static void ibcm_process_async_join_mcg(void *tq_arg); 54 55 static ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *, 56 uint64_t c_mask, void *, size_t *); 57 58 /* Address Record management definitions */ 59 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service" 60 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL 61 #define IBCM_DAPL_ATS_NBYTES 16 62 ibcm_svc_info_t *ibcm_ar_svcinfop; 63 ibcm_ar_t *ibcm_ar_list; 64 65 #ifdef DEBUG 66 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel, 67 ibcm_mad_addr_t *cm_reply_addr); 68 #endif 69 70 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_hdl})) 71 72 /* access is controlled between ibcm_sm.c and ibcm_ti.c by CVs */ 73 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by CV", {ibt_rc_returns_t 74 ibt_ud_returns_t ibt_ap_returns_t ibt_ar_t})) 75 76 /* 77 * Typically, clients initialize these args in one api call, and use in 78 * another api 79 */ 80 _NOTE(SCHEME_PROTECTS_DATA("Expected usage of ibtl api by client", 81 {ibt_path_info_s ibt_cep_path_s ibt_adds_vect_s ibt_mcg_info_s ib_gid_s 82 ibt_ud_dest_attr_s ibt_ud_dest_s ibt_srv_data_s ibt_redirect_info_s})) 83 84 /* 85 * ibt_open_rc_channel() 86 * ibt_open_rc_channel opens a communication channel on the specified 87 * channel to the specified service. For connection service type qp's 88 * the CM initiates the CEP to establish the connection and transitions 89 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's 90 * attributes as necessary. 91 * The implementation of this function assumes that alt path is different 92 * from primary path. It is assumed that the Path functions ensure that. 93 * 94 * RETURN VALUES: 95 * IBT_SUCCESS on success (or respective failure on error) 96 */ 97 ibt_status_t 98 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags, 99 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args, 100 ibt_rc_returns_t *ret_args) 101 { 102 /* all fields that are related to REQ MAD formation */ 103 104 ib_pkey_t prim_pkey; 105 ib_lid_t primary_slid, alternate_slid; 106 ib_qpn_t local_qpn = 0; 107 ib_guid_t hca_guid; 108 ib_qkey_t local_qkey = 0; 109 ib_eecn_t local_eecn = 0; 110 ib_eecn_t remote_eecn = 0; 111 boolean_t primary_grh; 112 boolean_t alternate_grh = B_FALSE; 113 ib_lid_t base_lid; 114 ib_com_id_t local_comid; 115 ibmf_msg_t *ibmf_msg; 116 ibcm_req_msg_t *req_msgp; 117 118 uint8_t rdma_in, rdma_out; 119 uint8_t cm_retries; 120 uint64_t local_cm_proc_time; /* In usec */ 121 uint8_t local_cm_resp_time; /* IB time */ 122 uint64_t remote_cm_resp_time; /* In usec */ 123 uint32_t starting_psn = 0; 124 125 /* CM path related fields */ 126 ibmf_handle_t ibmf_hdl; 127 ibcm_qp_list_t *cm_qp_entry; 128 ibcm_mad_addr_t cm_reply_addr; 129 130 uint8_t cm_pkt_lt; 131 132 /* Local args for ibtl/internal CM functions called within */ 133 ibt_status_t status; 134 ibcm_status_t lkup_status; 135 ibt_qp_query_attr_t qp_query_attr; 136 137 /* Other misc local args */ 138 ibt_priv_data_len_t len; 139 ibcm_hca_info_t *hcap; 140 ibcm_state_data_t *statep; 141 uint8_t port_no; 142 143 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)", 144 channel, flags, mode, chan_args, ret_args); 145 146 if (IBCM_INVALID_CHANNEL(channel)) { 147 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel"); 148 return (IBT_CHAN_HDL_INVALID); 149 } 150 151 /* cm handler should always be specified */ 152 if (chan_args->oc_cm_handler == NULL) { 153 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 154 "CM handler is not be specified", channel); 155 return (IBT_INVALID_PARAM); 156 } 157 158 if (mode == IBT_NONBLOCKING) { 159 if (ret_args != NULL) { 160 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 161 " ret_args should be NULL when called in " 162 "non-blocking mode", channel); 163 return (IBT_INVALID_PARAM); 164 } 165 } else if (mode == IBT_BLOCKING) { 166 if (ret_args == NULL) { 167 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 168 " ret_args should be Non-NULL when called in " 169 "blocking mode", channel); 170 return (IBT_INVALID_PARAM); 171 } 172 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) { 173 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 174 " private data length is too large", channel); 175 return (IBT_INVALID_PARAM); 176 } 177 if ((ret_args->rc_priv_data_len > 0) && 178 (ret_args->rc_priv_data == NULL)) { 179 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 180 " rc_priv_data_len > 0, but rc_priv_data NULL", 181 channel); 182 return (IBT_INVALID_PARAM); 183 } 184 } else { /* any other mode is not valid for ibt_open_rc_channel */ 185 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 186 "invalid mode %x specified", channel, mode); 187 return (IBT_INVALID_PARAM); 188 } 189 190 /* 191 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP 192 */ 193 if (flags & IBT_OCHAN_DUP) { 194 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 195 "Unsupported Flags specified: 0x%X", channel, flags); 196 return (IBT_INVALID_PARAM); 197 } 198 199 if ((flags & IBT_OCHAN_REDIRECTED) && 200 (flags & IBT_OCHAN_PORT_REDIRECTED)) { 201 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 202 "Illegal to specify IBT_OCHAN_REDIRECTED and " 203 "IBT_OCHAN_PORT_REDIRECTED flags together", channel); 204 return (IBT_INVALID_PARAM); 205 } 206 207 if (((flags & IBT_OCHAN_REDIRECTED) && 208 (chan_args->oc_cm_redirect_info == NULL)) || 209 ((flags & IBT_OCHAN_PORT_REDIRECTED) && 210 (chan_args->oc_cm_cep_path == NULL))) { 211 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 212 "Redirect flag specified, but respective arg is NULL", 213 channel); 214 return (IBT_INVALID_PARAM); 215 } 216 217 if ((flags & IBT_OCHAN_REDIRECTED) && 218 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) && 219 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) { 220 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 221 "Either rdi_dlid or rdi_gid must be specified for" 222 " IBT_OCHAN_REDIRECTED", channel); 223 return (IBT_INVALID_PARAM); 224 } 225 226 /* primary dlid and hca_port_num should never be zero */ 227 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 228 229 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) { 230 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 231 "Primary Path's information is not valid", channel); 232 return (IBT_INVALID_PARAM); 233 } 234 235 /* validate SID */ 236 if (chan_args->oc_path->pi_sid == 0) { 237 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 238 "ERROR: Service ID in path information is 0", channel); 239 return (IBT_INVALID_PARAM); 240 } 241 242 /* validate rnr_retry_cnt (enum has more than 3 bits) */ 243 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) { 244 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 245 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range", 246 channel, chan_args->oc_path_rnr_retry_cnt); 247 return (IBT_INVALID_PARAM); 248 } 249 250 /* 251 * Ensure that client is not re-using a QP that is still associated 252 * with a statep 253 */ 254 IBCM_GET_CHAN_PRIVATE(channel, statep); 255 if (statep != NULL) { 256 IBCM_RELEASE_CHAN_PRIVATE(channel); 257 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 258 "Channel being re-used on active side", channel); 259 return (IBT_CHAN_IN_USE); 260 } 261 262 /* Get GUID from Channel */ 263 hca_guid = ibt_channel_to_hca_guid(channel); 264 265 /* validate QP's hca guid with that from primary path */ 266 if (hca_guid != chan_args->oc_path->pi_hca_guid) { 267 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 268 "GUID from Channel and primary path don't match", channel); 269 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 270 "Channel GUID %llX primary path GUID %llX", channel, 271 hca_guid, chan_args->oc_path->pi_hca_guid); 272 return (IBT_CHAN_HDL_INVALID); 273 } 274 275 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 276 "Local HCA GUID %llX", channel, hca_guid); 277 278 status = ibt_query_qp(channel, &qp_query_attr); 279 if (status != IBT_SUCCESS) { 280 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 281 "ibt_query_qp failed %d", channel, status); 282 return (status); 283 } 284 285 /* If client specified "no port change on QP" */ 286 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 287 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) { 288 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 289 "chan port %d and path port %d does not match", channel, 290 qp_query_attr.qp_info.qp_transport.rc.rc_path. \ 291 cep_hca_port_num, port_no); 292 return (IBT_INVALID_PARAM); 293 } 294 295 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) { 296 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 297 "Invalid Channel type: Applicable only to RC Channel", 298 channel); 299 return (IBT_CHAN_SRV_TYPE_INVALID); 300 } 301 302 /* Check if QP is in INIT state or not */ 303 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) { 304 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 305 "QP is not in INIT state %x", channel, 306 qp_query_attr.qp_info.qp_state); 307 return (IBT_CHAN_STATE_INVALID); 308 } 309 310 local_qpn = qp_query_attr.qp_qpn; 311 312 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x", 313 channel, local_qpn); 314 315 #ifdef NO_EEC_SUPPORT_YET 316 317 if (flags & IBT_OCHAN_RDC_EXISTS) { 318 ibt_eec_query_attr_t eec_query_attr; 319 320 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey; 321 322 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD"); 323 324 status = ibt_query_eec(channel, &eec_query_attr); 325 if (status != IBT_SUCCESS) { 326 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 327 " ibt_query_eec failed %d", channel, status); 328 return (status); 329 } 330 local_eecn = eec_query_attr.eec_eecn; 331 } 332 333 #endif 334 335 /* If no HCA found return failure */ 336 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) { 337 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 338 "hcap is NULL. Probably hca is not in active state", 339 channel); 340 return (IBT_CHAN_HDL_INVALID); 341 } 342 343 rdma_out = chan_args->oc_rdma_ra_out; 344 rdma_in = chan_args->oc_rdma_ra_in; 345 346 if ((rdma_in > hcap->hca_max_rdma_in_qp) || 347 (rdma_out > hcap->hca_max_rdma_out_qp)) { 348 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 349 "rdma in %d/out %d values exceed hca limits", channel, 350 rdma_in, rdma_out); 351 ibcm_dec_hca_acc_cnt(hcap); 352 return (IBT_INVALID_PARAM); 353 } 354 355 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 356 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out); 357 358 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) { 359 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 360 "Huge Primary Pkt lt %d", channel, 361 chan_args->oc_path->pi_prim_pkt_lt); 362 ibcm_dec_hca_acc_cnt(hcap); 363 return (IBT_PATH_PKT_LT_TOO_HIGH); 364 } 365 366 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, 367 NULL, &base_lid); 368 if (status != IBT_SUCCESS) { 369 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 370 "primary port_num %d not active", channel, port_no); 371 ibcm_dec_hca_acc_cnt(hcap); 372 return (status); 373 } 374 375 /* Validate P_KEY Index */ 376 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no, 377 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey); 378 if (status != IBT_SUCCESS) { 379 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 380 "Invalid Primary PKeyIx %x", channel, 381 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix); 382 ibcm_dec_hca_acc_cnt(hcap); 383 return (status); 384 } 385 386 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 387 "primary_port_num %d primary_pkey 0x%x", channel, port_no, 388 prim_pkey); 389 390 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 391 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 392 != IBT_SUCCESS)) { 393 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 394 "ibmf reg or callback setup failed during re-initialize", 395 channel); 396 ibcm_dec_hca_acc_cnt(hcap); 397 return (status); 398 } 399 400 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl; 401 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 402 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl); 403 404 405 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 406 407 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p " 408 "primary SLID = %x", channel, primary_slid); 409 410 /* check first if alternate path exists or not as it is OPTIONAL */ 411 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) { 412 uint8_t alt_port_no; 413 414 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 415 416 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) { 417 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 418 "Huge Alt Pkt lt %d", channel, 419 chan_args->oc_path->pi_alt_pkt_lt); 420 ibcm_dec_hca_acc_cnt(hcap); 421 return (IBT_PATH_PKT_LT_TOO_HIGH); 422 } 423 424 if (port_no != alt_port_no) { 425 426 status = ibt_get_port_state_byguid(hcap->hca_guid, 427 alt_port_no, NULL, &base_lid); 428 if (status != IBT_SUCCESS) { 429 430 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 431 "chan 0x%p alt_port_num %d inactive %d", 432 channel, alt_port_no, status); 433 ibcm_dec_hca_acc_cnt(hcap); 434 return (status); 435 } 436 437 } 438 alternate_slid = 439 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 440 441 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan %0xp " 442 "alternate SLID = %x", channel, alternate_slid); 443 } 444 445 /* 446 * only pkey needs to be zero'ed, because all other fields are set in 447 * in ibcm_init_reply_addr. But, let's bzero the complete struct for 448 * any future modifications. 449 */ 450 bzero(&cm_reply_addr, sizeof (cm_reply_addr)); 451 452 /* Initialize the MAD destination address in stored_reply_addr */ 453 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args, 454 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) { 455 456 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 457 "ibcm_init_reply_addr failed status %d ", channel, status); 458 ibcm_dec_hca_acc_cnt(hcap); 459 return (status); 460 } 461 462 463 /* Initialize the pkey for CM MAD communication */ 464 if (cm_reply_addr.rcvd_addr.ia_p_key == 0) 465 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey; 466 467 #ifdef DEBUG 468 ibcm_print_reply_addr(channel, &cm_reply_addr); 469 #endif 470 471 /* Retrieve an ibmf qp for sending CM MADs */ 472 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no, 473 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) { 474 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 475 "unable to allocate ibmf qp for CM MADs", channel); 476 ibcm_dec_hca_acc_cnt(hcap); 477 return (IBT_INSUFF_RESOURCE); 478 } 479 480 481 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) { 482 ibcm_release_qp(cm_qp_entry); 483 ibcm_dec_hca_acc_cnt(hcap); 484 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 485 " Unable to allocate comid", channel); 486 return (IBT_INSUFF_KERNEL_RESOURCE); 487 } 488 489 /* allocate an IBMF mad buffer */ 490 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, 491 MAD_METHOD_SEND)) != IBT_SUCCESS) { 492 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 493 "chan 0x%p ibcm_alloc_out_msg failed", channel); 494 ibcm_release_qp(cm_qp_entry); 495 ibcm_free_comid(hcap, local_comid); 496 ibcm_dec_hca_acc_cnt(hcap); 497 return (status); 498 } 499 500 /* Init to Init, if QP's port does not match with path information */ 501 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 502 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) { 503 504 ibt_qp_info_t qp_info; 505 ibt_cep_modify_flags_t cep_flags; 506 507 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 508 "chan 0x%p chan port %d", channel, 509 qp_query_attr.qp_info.qp_transport.rc.rc_path.\ 510 cep_hca_port_num); 511 512 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 513 "chan 0x%p path port %d", channel, port_no); 514 515 bzero(&qp_info, sizeof (qp_info)); 516 /* For now, set it to RC type */ 517 518 qp_info.qp_trans = IBT_RC_SRV; 519 qp_info.qp_state = IBT_STATE_INIT; 520 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no; 521 522 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 523 524 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL); 525 526 if (status != IBT_SUCCESS) { 527 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 528 "chan 0x%p ibt_modify_qp() = %d", channel, status); 529 ibcm_release_qp(cm_qp_entry); 530 ibcm_free_comid(hcap, local_comid); 531 ibcm_dec_hca_acc_cnt(hcap); 532 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 533 return (status); 534 } else 535 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 536 "chan 0x%p ibt_modify_qp() = %d", channel, status); 537 } 538 539 /* allocate ibcm_state_data_t before grabbing the WRITER lock */ 540 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP); 541 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 542 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0, 543 hcap, &statep); 544 rw_exit(&hcap->hca_state_rwlock); 545 546 /* CM should be seeing this for the first time */ 547 ASSERT(lkup_status == IBCM_LOOKUP_NEW); 548 549 /* Increment the hca's resource count */ 550 ibcm_inc_hca_res_cnt(hcap); 551 552 /* Once a resource created on hca, no need to hold the acc cnt */ 553 ibcm_dec_hca_acc_cnt(hcap); 554 555 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 556 557 statep->timerid = 0; 558 statep->local_hca_guid = hca_guid; 559 statep->local_qpn = local_qpn; 560 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry; 561 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 562 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 563 564 565 /* Save "statep" as channel's CM private data. */ 566 statep->channel = channel; 567 IBCM_SET_CHAN_PRIVATE(statep->channel, statep); 568 569 statep->stored_msg = ibmf_msg; 570 571 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*req_msgp)) 572 573 /* Start filling in the REQ MAD */ 574 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg); 575 req_msgp->req_local_comm_id = h2b32(local_comid); 576 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid); 577 req_msgp->req_local_ca_guid = h2b64(hca_guid); 578 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */ 579 580 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */ 581 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in); 582 583 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */ 584 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out); 585 586 if (flags & IBT_OCHAN_REMOTE_CM_TM) 587 remote_cm_resp_time = chan_args->oc_remote_cm_time; 588 else 589 remote_cm_resp_time = ibcm_remote_response_time; 590 591 /* 592 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type, 593 * IBT_CM_FLOW_CONTROL is always set by default. 594 */ 595 req_msgp->req_remote_eecn_plus = h2b32( 596 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 | 597 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL); 598 599 if (flags & IBT_OCHAN_LOCAL_CM_TM) 600 local_cm_proc_time = chan_args->oc_local_cm_time; 601 else 602 local_cm_proc_time = ibcm_local_processing_time; 603 604 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time + 605 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) + 606 ibcm_sw_delay); 607 608 /* save retry count */ 609 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt; 610 611 if (flags & IBT_OCHAN_STARTING_PSN) 612 starting_psn = chan_args->oc_starting_psn; 613 614 if (local_cm_resp_time > 0x1f) 615 local_cm_resp_time = 0x1f; 616 617 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */ 618 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 | 619 local_cm_resp_time << 3 | statep->cep_retry_cnt); 620 621 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 622 "Prim Pkt lt (IB time) 0x%x", channel, 623 chan_args->oc_path->pi_prim_pkt_lt); 624 625 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 626 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time); 627 628 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 629 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time); 630 631 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 632 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time); 633 634 statep->starting_psn = starting_psn; 635 636 /* Pkey - bytes 48-49 */ 637 req_msgp->req_part_key = h2b16(prim_pkey); 638 639 if (flags & IBT_OCHAN_CM_RETRY) 640 cm_retries = chan_args->oc_cm_retry_cnt; 641 else 642 cm_retries = ibcm_max_retries; 643 644 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries; 645 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4; 646 647 /* 648 * Check whether SRQ is associated with this Channel, if yes, then 649 * set the SRQ Exists bit in the REQ. 650 */ 651 if (qp_query_attr.qp_srq != NULL) { 652 req_msgp->req_max_cm_retries_plus |= (1 << 3); 653 } 654 655 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 | 656 chan_args->oc_path_rnr_retry_cnt; 657 658 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d" 659 " staring PSN %x", channel, cm_retries, starting_psn); 660 661 662 #ifdef NO_EEC_SUPPORT_YET 663 if (flags & IBT_OCHAN_RDC_EXISTS) 664 req_msgp->req_mtu_plus |= 8; 665 #endif 666 667 /* Initialize the "primary" port stuff next - bytes 52-95 */ 668 req_msgp->req_primary_l_port_lid = h2b16(primary_slid); 669 req_msgp->req_primary_r_port_lid = 670 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 671 req_msgp->req_primary_l_port_gid.gid_prefix = 672 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix); 673 req_msgp->req_primary_l_port_gid.gid_guid = 674 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid); 675 req_msgp->req_primary_r_port_gid.gid_prefix = 676 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 677 req_msgp->req_primary_r_port_gid.gid_guid = 678 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 679 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh; 680 681 statep->remote_hca_guid = /* not correct, but helpful for debugging */ 682 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid; 683 684 /* Bytes 88-91 - primary_flowlbl, and primary_srate */ 685 req_msgp->req_primary_flow_label_plus = 686 h2b32(((primary_grh == B_TRUE) ? 687 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) | 688 IBCM_PRIM_ADDS_VECT(chan_args).av_srate); 689 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ? 690 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0; 691 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ? 692 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 0xff; 693 req_msgp->req_primary_sl_plus = 694 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 | 695 ((primary_grh == B_TRUE) ? 0 : 8); 696 697 req_msgp->req_primary_localtime_plus = 698 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) + 699 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 700 701 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p", 702 channel, statep); 703 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 704 "active hca_ack_delay (usec) %d", channel, 705 req_msgp->req_primary_localtime_plus); 706 707 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 708 "Sent primary cep timeout (IB Time) %d", channel, 709 hcap->hca_ack_delay); 710 711 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ", 712 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 713 714 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 715 "prim GID %llX:%llX", channel, 716 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix, 717 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 718 719 /* Initialize the "alternate" port stuff - optional */ 720 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) { 721 ib_gid_t tmp_gid; 722 723 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid); 724 req_msgp->req_alt_r_port_lid = 725 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 726 /* 727 * doing all this as req_alt_r/l_port_gid is at offset 728 * 100, 116 which is not divisible by 8 729 */ 730 731 tmp_gid.gid_prefix = 732 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 733 tmp_gid.gid_guid = 734 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 735 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0], 736 sizeof (ib_gid_t)); 737 tmp_gid.gid_prefix = 738 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix); 739 tmp_gid.gid_guid = 740 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid); 741 742 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0], 743 sizeof (ib_gid_t)); 744 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh; 745 746 /* Bytes 132-135 - alternate_flow_label, and alternate srate */ 747 req_msgp->req_alt_flow_label_plus = h2b32( 748 (((alternate_grh == B_TRUE) ? 749 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) | 750 IBCM_ALT_ADDS_VECT(chan_args).av_srate)); 751 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ? 752 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0; 753 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ? 754 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 0xff; 755 req_msgp->req_alt_sl_plus = 756 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 | 757 ((alternate_grh == B_TRUE) ? 0 : 8); 758 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 * 759 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) + 760 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 761 762 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 763 "alt_dlid %x ", channel, 764 IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 765 766 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 767 "alt GID %llX:%llX", channel, 768 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix, 769 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 770 } 771 772 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ); 773 if ((len > 0) && chan_args->oc_priv_data) 774 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len); 775 776 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*req_msgp)) 777 778 /* return_data is filled up in the state machine code */ 779 if (ret_args != NULL) { 780 statep->open_return_data = ret_args; 781 } 782 783 /* initialize some statep fields here */ 784 statep->mode = IBCM_ACTIVE_MODE; 785 statep->hcap = hcap; 786 787 statep->cm_handler = chan_args->oc_cm_handler; 788 statep->state_cm_private = chan_args->oc_cm_clnt_private; 789 790 statep->pkt_life_time = 791 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt); 792 793 statep->timer_value = ibt_ib2usec(ibt_usec2ib( 794 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time)); 795 796 /* Initialize statep->stored_reply_addr */ 797 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl; 798 799 /* Initialize stored reply addr fields */ 800 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr; 801 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr; 802 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists; 803 statep->stored_reply_addr.port_num = cm_reply_addr.port_num; 804 805 /* 806 * The IPD on local/active side is calculated by path functions, 807 * hence available in the args of ibt_open_rc_channel 808 */ 809 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate; 810 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate; 811 812 /* Store the source path bits for primary and alt paths */ 813 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 814 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 815 816 statep->open_flow = 1; 817 statep->open_done = B_FALSE; 818 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT; 819 IBCM_REF_CNT_INCR(statep); /* Decremented before return */ 820 IBCM_REF_CNT_INCR(statep); /* Decremented after REQ is posted */ 821 statep->send_mad_flags |= IBCM_REQ_POST_BUSY; 822 823 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID = 824 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID); 825 826 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID = 827 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid, 828 0)); 829 830 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 831 832 ibcm_open_enqueue(statep); 833 834 mutex_enter(&statep->state_mutex); 835 836 if (mode == IBT_BLOCKING) { 837 838 /* wait for REQ/REP/RTU */ 839 while (statep->open_done != B_TRUE) { 840 cv_wait(&statep->block_client_cv, &statep->state_mutex); 841 } 842 843 /* 844 * In the case that open_channel() fails because of a 845 * REJ or timeout, change retval to IBT_CM_FAILURE 846 */ 847 if (statep->open_return_data->rc_status != IBT_CM_ACCEPT) 848 status = IBT_CM_FAILURE; 849 850 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p " 851 "ret status %d cm status %d", channel, status, 852 statep->open_return_data->rc_status); 853 } 854 855 /* decrement the ref-count before leaving here */ 856 IBCM_REF_CNT_DECR(statep); 857 858 mutex_exit(&statep->state_mutex); 859 860 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel); 861 return (status); 862 } 863 864 /* 865 * ibcm_init_reply_addr: 866 * 867 * The brief description of functionality below. 868 * 869 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case): 870 * Build CM path from chan_args->oc_cm_cep_path 871 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt 872 * 873 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case): 874 * If Redirect LID is specified, 875 * If Redirect GID is not specified or specified to be on the same 876 * subnet, then 877 * Build CM path from chan_args->oc_cm_redirect_info 878 * Set CM pkt lt to subnet timeout 879 * Else (ie., GID specified, but on a different subnet) 880 * Do a path lookup to build CM Path and set CM pkt lt 881 * 882 */ 883 static ibt_status_t 884 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr, 885 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags, 886 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid) 887 { 888 ibt_adds_vect_t *cm_adds; 889 ibt_path_info_t path; 890 boolean_t cm_grh; 891 ibt_status_t status; 892 893 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:"); 894 895 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*reply_addr)) 896 897 /* 898 * sending side CM lid/gid/port num are not based on any redirect 899 * params. These values are set to primary RC path lid/gid/port num. 900 * In the future, these values can be set based on framework policy 901 * decisions ensuring reachability. 902 */ 903 reply_addr->grh_hdr.ig_sender_gid = 904 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid; 905 reply_addr->rcvd_addr.ia_local_lid = prim_slid; 906 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 907 908 if (flags & IBT_OCHAN_PORT_REDIRECTED) { 909 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 910 "IBT_OCHAN_PORT_REDIRECTED specified"); 911 912 status = ibt_index2pkey_byguid(hcap->hca_guid, 913 chan_args->oc_cm_cep_path->cep_hca_port_num, 914 chan_args->oc_cm_cep_path->cep_pkey_ix, 915 &reply_addr->rcvd_addr.ia_p_key); 916 917 if (status != IBT_SUCCESS) { 918 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid " 919 "CM PKeyIx %x port_num %x", 920 chan_args->oc_cm_cep_path->cep_pkey_ix, 921 chan_args->oc_cm_cep_path->cep_hca_port_num); 922 return (status); 923 } 924 925 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect); 926 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x", 927 cm_adds->av_dlid); 928 929 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 930 reply_addr->rcvd_addr.ia_remote_qno = 1; 931 *cm_pkt_lt = chan_args->oc_cm_pkt_lt; 932 933 } else if (flags & IBT_OCHAN_REDIRECTED) { 934 ibt_redirect_info_t *redirect_info; 935 ibt_hca_portinfo_t *port_infop; 936 uint_t psize, nports; 937 938 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 939 "IBT_OCHAN_REDIRECTED specified"); 940 941 redirect_info = chan_args->oc_cm_redirect_info; 942 943 if ((redirect_info->rdi_gid.gid_prefix == 0) || 944 (redirect_info->rdi_gid.gid_guid == 0)) { 945 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 946 "ERROR: Re-direct GID value NOT Provided."); 947 return (IBT_INVALID_PARAM); 948 } 949 950 /* As per spec definition 1.1, it's always IB_GSI_QKEY */ 951 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey; 952 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn; 953 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey; 954 955 /* 956 * if LID is non-zero in classportinfo then use classportinfo 957 * fields to form CM MAD destination address. 958 */ 959 if (redirect_info->rdi_dlid != 0) { 960 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid, 961 reply_addr->port_num, &port_infop, &nports, &psize); 962 if ((status != IBT_SUCCESS) || (nports == 0)) { 963 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 964 "Query Ports Failed: %d", status); 965 return (status); 966 } else if (port_infop->p_subnet_timeout > 967 IBCM_MAX_IB_PKT_LT) { 968 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 969 "large subnet timeout %x port_no %x", 970 port_infop->p_subnet_timeout, 971 reply_addr->port_num); 972 ibt_free_portinfo(port_infop, psize); 973 return (IBT_PATH_PKT_LT_TOO_HIGH); 974 } else { 975 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: " 976 "subnet timeout %x port_no %x", 977 port_infop->p_subnet_timeout, 978 reply_addr->port_num); 979 980 *cm_pkt_lt = 981 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt, 982 port_infop->p_subnet_timeout)); 983 984 ibt_free_portinfo(port_infop, psize); 985 } 986 987 reply_addr->rcvd_addr.ia_remote_lid = 988 redirect_info->rdi_dlid; 989 reply_addr->rcvd_addr.ia_service_level = 990 redirect_info->rdi_sl; 991 reply_addr->grh_exists = B_TRUE; 992 reply_addr->grh_hdr.ig_recver_gid = 993 redirect_info->rdi_gid; 994 reply_addr->grh_hdr.ig_tclass = 995 redirect_info->rdi_tclass; 996 reply_addr->grh_hdr.ig_flow_label = 997 redirect_info->rdi_flow; 998 999 /* Classportinfo doesn't have hoplimit field */ 1000 reply_addr->grh_hdr.ig_hop_limit = 0xff; 1001 return (IBT_SUCCESS); 1002 1003 } else { 1004 ibt_path_attr_t path_attr; 1005 ib_gid_t path_dgid[1]; 1006 1007 /* 1008 * If GID is specified, and LID is zero in classportinfo 1009 * do a path lookup using specified GID, Pkey, 1010 * in classportinfo 1011 */ 1012 1013 bzero(&path_attr, sizeof (path_attr)); 1014 1015 path_attr.pa_dgids = &path_dgid[0]; 1016 path_attr.pa_dgids[0] = redirect_info->rdi_gid; 1017 1018 /* 1019 * use reply_addr below, as sender_gid in reply_addr 1020 * may have been set above based on some policy decision 1021 * for originating end point for CM MADs above 1022 */ 1023 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid; 1024 path_attr.pa_num_dgids = 1; 1025 path_attr.pa_pkey = redirect_info->rdi_pkey; 1026 1027 if ((status = ibt_get_paths(ibcm_ibt_handle, 1028 IBT_PATH_PKEY, &path_attr, 1, &path, NULL)) != 1029 IBT_SUCCESS) 1030 return (status); 1031 1032 /* Initialize cm_adds */ 1033 cm_adds = &path.pi_prim_cep_path.cep_adds_vect; 1034 *cm_pkt_lt = path.pi_prim_pkt_lt; 1035 } 1036 1037 } else { /* cm_pkey initialized in ibt_open_rc_channel */ 1038 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 1039 reply_addr->rcvd_addr.ia_remote_qno = 1; 1040 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt; 1041 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args)); 1042 } 1043 1044 1045 cm_grh = cm_adds->av_send_grh; 1046 reply_addr->grh_exists = cm_grh; 1047 1048 reply_addr->rcvd_addr.ia_remote_lid = 1049 cm_adds->av_dlid; 1050 reply_addr->grh_hdr.ig_recver_gid = 1051 cm_adds->av_dgid; 1052 reply_addr->grh_hdr.ig_flow_label = 1053 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK; 1054 reply_addr->grh_hdr.ig_tclass = 1055 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0; 1056 reply_addr->grh_hdr.ig_hop_limit = 1057 (cm_grh == B_TRUE) ? cm_adds->av_hop : 0xff; 1058 reply_addr->rcvd_addr.ia_service_level = 1059 cm_adds->av_srvl; 1060 1061 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*reply_addr)) 1062 1063 return (IBT_SUCCESS); 1064 } 1065 1066 1067 /* 1068 * ibt_prime_close_rc_channel() 1069 * It allocates resources required for close channel operation, so 1070 * ibt_close_rc_channel can be called from interrupt routine. 1071 * 1072 * INPUTS: 1073 * channel The address of an ibt_channel_t struct that 1074 * specifies the channel to open. 1075 * 1076 * RETURN VALUES: 1077 * IBT_SUCCESS on success(or respective failure on error) 1078 * 1079 * Clients are typically expected to call this function in established state 1080 */ 1081 ibt_status_t 1082 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel) 1083 { 1084 ibcm_state_data_t *statep; 1085 ibt_status_t status = IBT_SUCCESS; 1086 1087 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel); 1088 1089 /* validate channel, first */ 1090 if (IBCM_INVALID_CHANNEL(channel)) { 1091 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1092 "invalid channel", channel); 1093 return (IBT_CHAN_HDL_INVALID); 1094 } 1095 1096 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1097 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1098 "Invalid Channel type: Applicable only to RC Channel", 1099 channel); 1100 return (IBT_CHAN_SRV_TYPE_INVALID); 1101 } 1102 1103 /* get the statep */ 1104 IBCM_GET_CHAN_PRIVATE(channel, statep); 1105 1106 /* 1107 * This can happen, if the statep is already gone by a DREQ from 1108 * the remote side 1109 */ 1110 1111 if (statep == NULL) { 1112 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1113 "statep NULL", channel); 1114 return (IBT_SUCCESS); 1115 } 1116 1117 mutex_enter(&statep->state_mutex); 1118 IBCM_RELEASE_CHAN_PRIVATE(channel); 1119 if (statep->state != IBCM_STATE_ESTABLISHED) { 1120 mutex_exit(&statep->state_mutex); 1121 return (IBT_CHAN_STATE_INVALID); 1122 } 1123 IBCM_REF_CNT_INCR(statep); 1124 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p" 1125 " state %x", channel, statep, statep->state); 1126 mutex_exit(&statep->state_mutex); 1127 1128 /* clients could pre-allocate dreq mad, even before connection est */ 1129 if (statep->dreq_msg == NULL) 1130 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, 1131 &statep->dreq_msg, MAD_METHOD_SEND); 1132 1133 mutex_enter(&statep->state_mutex); 1134 IBCM_REF_CNT_DECR(statep); 1135 mutex_exit(&statep->state_mutex); 1136 1137 if (status != IBT_SUCCESS) { 1138 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1139 "ibcm_alloc_out_msg failed ", channel); 1140 return (status); 1141 } 1142 1143 /* If this message isn't seen then ibt_prime_close_rc_channel failed */ 1144 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done", 1145 channel); 1146 1147 return (IBT_SUCCESS); 1148 } 1149 1150 /* 1151 * ibt_close_rc_channel() 1152 * It closes an established channel. 1153 * 1154 * RETURN VALUES: 1155 * IBT_SUCCESS on success(or respective failure on error) 1156 */ 1157 ibt_status_t 1158 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 1159 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status, 1160 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p) 1161 { 1162 ibcm_hca_info_t *hcap; 1163 ibcm_state_data_t *statep; 1164 ibt_status_t status; 1165 1166 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)", 1167 channel, mode, priv_data, priv_data_len, 1168 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p); 1169 1170 /* validate channel, first */ 1171 if (IBCM_INVALID_CHANNEL(channel)) { 1172 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1173 "invalid channel", channel); 1174 return (IBT_CHAN_HDL_INVALID); 1175 } 1176 1177 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1178 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1179 "Invalid Channel type: Applicable only to RC Channel", 1180 channel); 1181 return (IBT_CHAN_SRV_TYPE_INVALID); 1182 } 1183 1184 if (mode == IBT_BLOCKING) { 1185 /* valid only for BLOCKING MODE */ 1186 if ((ret_priv_data_len_p != NULL) && 1187 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) { 1188 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1189 " private data len %d is too large", channel, 1190 *ret_priv_data_len_p); 1191 return (IBT_INVALID_PARAM); 1192 } 1193 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) { 1194 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1195 "invalid mode %x specified", channel, mode); 1196 return (IBT_INVALID_PARAM); 1197 } 1198 1199 /* get the statep */ 1200 IBCM_GET_CHAN_PRIVATE(channel, statep); 1201 1202 if (statep == NULL) { 1203 1204 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1205 "statep NULL", channel); 1206 if (ibtl_cm_is_chan_closing(channel) || 1207 ibtl_cm_is_chan_closed(channel)) { 1208 if (ret_status) 1209 *ret_status = IBT_CM_CLOSED_ALREADY; 1210 1211 /* No private data to return to the client */ 1212 if (ret_priv_data_len_p != NULL) 1213 *ret_priv_data_len_p = 0; 1214 1215 return (IBT_SUCCESS); 1216 } 1217 return (IBT_CHAN_STATE_INVALID); 1218 } 1219 1220 mutex_enter(&statep->state_mutex); 1221 IBCM_RELEASE_CHAN_PRIVATE(channel); 1222 IBCM_REF_CNT_INCR(statep); 1223 mutex_exit(&statep->state_mutex); 1224 1225 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p statep %p", 1226 channel, statep); 1227 1228 mutex_enter(&statep->state_mutex); 1229 hcap = statep->hcap; 1230 1231 /* HCA must have been in active state. If not, it's a client bug */ 1232 if (!IBCM_ACCESS_HCA_OK(hcap)) { 1233 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1234 "hcap 0x%p not active", channel, hcap); 1235 IBCM_REF_CNT_DECR(statep); 1236 mutex_exit(&statep->state_mutex); 1237 return (IBT_CHAN_HDL_INVALID); 1238 } 1239 1240 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) { 1241 while (statep->cep_in_rts == IBCM_BLOCK) 1242 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1243 } 1244 1245 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */ 1246 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT) 1247 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1248 1249 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1250 "connection state is %x", channel, statep->state); 1251 1252 statep->close_ret_status = ret_status; 1253 1254 /* If state is in pre-established states, abort the connection est */ 1255 if (statep->state != IBCM_STATE_ESTABLISHED) { 1256 statep->cm_retries++; /* ensure connection trace is dumped */ 1257 1258 /* No DREP private data possible */ 1259 if (ret_priv_data_len_p != NULL) 1260 *ret_priv_data_len_p = 0; 1261 1262 /* 1263 * If waiting for a response mad, then cancel the timer, 1264 * and delete the connection 1265 */ 1266 if (statep->state == IBCM_STATE_REQ_SENT || 1267 statep->state == IBCM_STATE_REP_SENT || 1268 statep->state == IBCM_STATE_REP_WAIT || 1269 statep->state == IBCM_STATE_MRA_REP_RCVD) { 1270 timeout_id_t timer_val = statep->timerid; 1271 ibcm_conn_state_t old_state; 1272 1273 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1274 "chan 0x%p connection aborted in state %x", channel, 1275 statep->state); 1276 1277 old_state = statep->state; 1278 statep->state = IBCM_STATE_DELETE; 1279 1280 if (mode == IBT_NONBLOCKING) { 1281 if (taskq_dispatch(ibcm_taskq, 1282 ibcm_process_abort_via_taskq, statep, 1283 TQ_NOSLEEP) == 0) { 1284 1285 IBCM_REF_CNT_DECR(statep); 1286 statep->state = old_state; 1287 mutex_exit(&statep->state_mutex); 1288 return (IBT_INSUFF_KERNEL_RESOURCE); 1289 } /* if taskq_dispatch succeeds */ 1290 /* Cancel the timer */ 1291 statep->timerid = 0; 1292 mutex_exit(&statep->state_mutex); 1293 } else { 1294 /* Cancel the timer */ 1295 statep->timerid = 0; 1296 mutex_exit(&statep->state_mutex); 1297 (void) taskq_dispatch(ibcm_taskq, 1298 ibcm_process_abort_via_taskq, statep, 1299 TQ_SLEEP); 1300 } 1301 1302 /* cancel the currently running timer */ 1303 if (timer_val != 0) 1304 (void) untimeout(timer_val); 1305 1306 /* wait until cm handler returns for BLOCKING cases */ 1307 if ((mode == IBT_BLOCKING) || 1308 (mode == IBT_NOCALLBACKS)) { 1309 mutex_enter(&statep->state_mutex); 1310 while (statep->close_done != B_TRUE) 1311 cv_wait(&statep->block_client_cv, 1312 &statep->state_mutex); 1313 mutex_exit(&statep->state_mutex); 1314 } 1315 1316 if (ret_status) 1317 *ret_status = IBT_CM_CLOSED_ABORT; 1318 1319 /* 1320 * It would ideal to post a REJ MAD, but that would 1321 * be non-conformance to spec. Hence, delete the state 1322 * data. Assuming that happens quickly, any retransmits 1323 * from the remote are replied by CM with reject 1324 * reason " no valid com id". That would stop remote 1325 * sending any more MADs. 1326 */ 1327 ibcm_delete_state_data(statep); 1328 return (IBT_SUCCESS); 1329 1330 /* if CM busy in cm handler, wait until cm handler returns */ 1331 } else if (statep->state == IBCM_STATE_REQ_RCVD || 1332 statep->state == IBCM_STATE_REP_RCVD || 1333 statep->state == IBCM_STATE_MRA_SENT || 1334 statep->state == IBCM_STATE_MRA_REP_SENT) { 1335 1336 /* take control of statep */ 1337 statep->abort_flag |= IBCM_ABORT_CLIENT; 1338 1339 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1340 "chan 0x%p connection aborted in state = %x", 1341 channel, statep->state); 1342 1343 /* 1344 * wait until state machine modifies qp state to error, 1345 * including disassociating statep and QP 1346 */ 1347 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS)) 1348 while (statep->close_done != B_TRUE) 1349 cv_wait(&statep->block_client_cv, 1350 &statep->state_mutex); 1351 1352 /* a sanity setting */ 1353 if (mode == IBT_NOCALLBACKS) 1354 statep->cm_handler = NULL; 1355 IBCM_REF_CNT_DECR(statep); 1356 mutex_exit(&statep->state_mutex); 1357 1358 /* 1359 * In rare situations, connection attempt could be 1360 * terminated for some other reason, before abort is 1361 * processed, but CM still returns ret_status as abort 1362 */ 1363 if (ret_status) 1364 *ret_status = IBT_CM_CLOSED_ABORT; 1365 1366 /* 1367 * REJ MAD is posted by the CM state machine for this 1368 * case, hence state structure is deleted in the 1369 * state machine processing. 1370 */ 1371 return (IBT_SUCCESS); 1372 1373 } else if ((statep->state == IBCM_STATE_TIMEWAIT) || 1374 (statep->state == IBCM_STATE_DELETE)) { 1375 1376 /* State already in timewait, so no return priv data */ 1377 IBCM_REF_CNT_DECR(statep); 1378 mutex_exit(&statep->state_mutex); 1379 1380 /* The teardown has already been done */ 1381 if (ret_status) 1382 *ret_status = IBT_CM_CLOSED_ALREADY; 1383 1384 return (IBT_SUCCESS); 1385 1386 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) || 1387 (statep->state == IBCM_STATE_DREQ_SENT) || 1388 (statep->state == IBCM_STATE_DREP_RCVD) || 1389 ((statep->state == IBCM_STATE_TIMED_OUT) && 1390 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) { 1391 1392 /* 1393 * Either the remote or local client has already 1394 * initiated the teardown. IBCM_STATE_DREP_RCVD is 1395 * possible, if CM initiated teardown without client's 1396 * knowledge, for stale handling, etc., 1397 */ 1398 if (mode == IBT_NOCALLBACKS) { 1399 if (statep->close_nocb_state == IBCM_UNBLOCK) { 1400 statep->close_nocb_state = IBCM_FAIL; 1401 /* enable free qp after return */ 1402 ibtl_cm_chan_is_closing( 1403 statep->channel); 1404 } else while (statep->close_nocb_state == 1405 IBCM_BLOCK) 1406 cv_wait(&statep->block_client_cv, 1407 &statep->state_mutex); 1408 statep->cm_handler = NULL; /* sanity setting */ 1409 if (ret_status) 1410 *ret_status = IBT_CM_CLOSED_ALREADY; 1411 } else if (mode == IBT_BLOCKING) { 1412 /* wait until state is moved to timewait */ 1413 while (statep->close_done != B_TRUE) 1414 cv_wait(&statep->block_client_cv, 1415 &statep->state_mutex); 1416 } 1417 1418 IBCM_REF_CNT_DECR(statep); 1419 mutex_exit(&statep->state_mutex); 1420 1421 /* ret_status is set in state machine code */ 1422 return (IBT_SUCCESS); 1423 1424 } else if (statep->state == IBCM_STATE_TIMED_OUT) { 1425 1426 if ((mode == IBT_BLOCKING) || 1427 (mode == IBT_NOCALLBACKS)) { 1428 1429 /* 1430 * wait until cm handler invocation and 1431 * disassociation between statep and channel 1432 * is complete 1433 */ 1434 while (statep->close_done != B_TRUE) 1435 cv_wait(&statep->block_client_cv, 1436 &statep->state_mutex); 1437 } 1438 1439 if (ret_status) 1440 *ret_status = IBT_CM_CLOSED_ABORT; 1441 IBCM_REF_CNT_DECR(statep); 1442 mutex_exit(&statep->state_mutex); 1443 1444 return (IBT_SUCCESS); 1445 } else { 1446 IBCM_REF_CNT_DECR(statep); 1447 mutex_exit(&statep->state_mutex); 1448 1449 return (IBT_CM_FAILURE); 1450 } 1451 } 1452 1453 ASSERT(statep->close_nocb_state != IBCM_BLOCK); 1454 1455 if (mode == IBT_NOCALLBACKS) { 1456 statep->close_nocb_state = IBCM_FAIL; 1457 statep->cm_handler = NULL; 1458 ibtl_cm_chan_is_closing(statep->channel); 1459 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1460 "NOCALLBACKS on in statep = %p", statep); 1461 } 1462 mutex_exit(&statep->state_mutex); 1463 1464 mutex_enter(&statep->state_mutex); 1465 if (statep->state != IBCM_STATE_ESTABLISHED) { 1466 goto lost_race; 1467 } 1468 1469 /* 1470 * Cancel/wait for any pending ibt_set_alt_path, and 1471 * release state mutex 1472 */ 1473 ibcm_sync_lapr_idle(statep); 1474 1475 ibcm_close_enter(); 1476 1477 mutex_enter(&statep->state_mutex); 1478 if (statep->state != IBCM_STATE_ESTABLISHED) { 1479 ibcm_close_exit(); 1480 goto lost_race; 1481 } 1482 1483 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT; 1484 statep->timerid = 0; 1485 statep->close_done = B_FALSE; 1486 statep->close_flow = 1; 1487 mutex_exit(&statep->state_mutex); 1488 1489 if (statep->dreq_msg == NULL) { 1490 if ((status = ibcm_alloc_out_msg( 1491 statep->stored_reply_addr.ibmf_hdl, &statep->dreq_msg, 1492 MAD_METHOD_SEND)) != IBT_SUCCESS) { 1493 1494 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: " 1495 "chan 0x%p ibcm_alloc_out_msg failed ", channel); 1496 mutex_enter(&statep->state_mutex); 1497 ibcm_close_exit(); 1498 statep->state = IBCM_STATE_ESTABLISHED; 1499 IBCM_REF_CNT_DECR(statep); 1500 cv_broadcast(&statep->block_mad_cv); 1501 statep->close_flow = 0; 1502 mutex_exit(&statep->state_mutex); 1503 return (status); 1504 } 1505 } else 1506 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: " 1507 "DREQ MAD already allocated in statep %p", statep); 1508 1509 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) { 1510 statep->close_priv_data = NULL; 1511 statep->close_priv_data_len = NULL; 1512 } else { 1513 statep->close_priv_data = ret_priv_data; 1514 statep->close_priv_data_len = ret_priv_data_len_p; 1515 } 1516 1517 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ); 1518 if ((priv_data != NULL) && (priv_data_len > 0)) 1519 bcopy(priv_data, ((ibcm_dreq_msg_t *) 1520 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data, 1521 priv_data_len); 1522 1523 ibcm_post_dreq_mad(statep); 1524 1525 mutex_enter(&statep->state_mutex); 1526 1527 lost_race: 1528 if (mode == IBT_BLOCKING) { 1529 1530 /* wait for DREP */ 1531 while (statep->close_done != B_TRUE) 1532 cv_wait(&statep->block_client_cv, 1533 &statep->state_mutex); 1534 1535 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1536 "done blocking", channel); 1537 } 1538 1539 IBCM_REF_CNT_DECR(statep); 1540 mutex_exit(&statep->state_mutex); 1541 1542 /* If this message isn't seen then ibt_close_rc_channel failed */ 1543 IBTF_DPRINTF_L5(cmlog, "ibt_close_rc_channel: chan 0x%p done", 1544 channel); 1545 1546 return (IBT_SUCCESS); 1547 } 1548 1549 ibt_status_t 1550 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control, 1551 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg) 1552 { 1553 ibcm_state_data_t *statep; 1554 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg; 1555 ibt_qp_query_attr_t qp_attr; 1556 ibt_status_t retval; 1557 1558 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan, 1559 control, hca_port_num, func, arg); 1560 1561 if (IBCM_INVALID_CHANNEL(rc_chan)) { 1562 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel"); 1563 return (IBT_CHAN_HDL_INVALID); 1564 } 1565 1566 /* check qp state */ 1567 retval = ibt_query_qp(rc_chan, &qp_attr); 1568 1569 if (retval != IBT_SUCCESS) 1570 return (retval); 1571 1572 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV) 1573 return (IBT_CHAN_SRV_TYPE_INVALID); 1574 1575 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR) 1576 return (IBT_CHAN_STATE_INVALID); 1577 1578 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1579 1580 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t), 1581 KM_SLEEP); 1582 1583 ibcm_tq_recycle_arg->rc_chan = rc_chan; 1584 ibcm_tq_recycle_arg->control = control; 1585 ibcm_tq_recycle_arg->hca_port_num = hca_port_num; 1586 ibcm_tq_recycle_arg->func = func; 1587 ibcm_tq_recycle_arg->arg = arg; 1588 1589 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1590 1591 IBCM_GET_CHAN_PRIVATE(rc_chan, statep); 1592 1593 /* 1594 * If non-blocking ie., func specified and channel has not yet completed 1595 * the timewait, then schedule the work for later 1596 */ 1597 if ((func != NULL) && (statep != NULL)) { 1598 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1599 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1600 statep->recycle_arg = ibcm_tq_recycle_arg; 1601 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1602 return (IBT_SUCCESS); 1603 } 1604 1605 /* 1606 * if blocking ie., func specified, and channel has not yet completed 1607 * the timewait, then block until the channel completes the timewait 1608 */ 1609 if (statep != NULL) 1610 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1611 IBCM_WAIT_CHAN_PRIVATE(rc_chan); 1612 1613 if (func) { /* NON BLOCKING case. Taskq for QP state change */ 1614 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle, 1615 ibcm_tq_recycle_arg, TQ_SLEEP); 1616 return (IBT_SUCCESS); 1617 } else /* BLOCKING case */ 1618 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg)); 1619 } 1620 1621 void 1622 ibcm_process_rc_recycle(void *recycle_arg) 1623 { 1624 (void) ibcm_process_rc_recycle_ret(recycle_arg); 1625 } 1626 1627 static ibt_status_t 1628 ibcm_process_rc_recycle_ret(void *recycle_arg) 1629 { 1630 ibt_qp_info_t qp_info; 1631 ibt_status_t ibt_status = IBT_SUCCESS; 1632 ibt_cep_modify_flags_t cep_flags; 1633 ibt_qp_query_attr_t qp_attr; 1634 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg = 1635 (ibcm_taskq_recycle_arg_t *)recycle_arg; 1636 1637 /* QP must have been in error state */ 1638 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr); 1639 if (ibt_status != IBT_SUCCESS) 1640 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1641 "chanp %p ibt_query_qp() = %d", 1642 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1643 else { 1644 /* perform the QP state change from ERROR to RESET */ 1645 bzero(&qp_info, sizeof (qp_info)); 1646 1647 qp_info.qp_trans = IBT_RC_SRV; 1648 qp_info.qp_state = IBT_STATE_RESET; 1649 1650 /* Call modify_qp to move to RESET state */ 1651 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1652 IBT_CEP_SET_STATE, &qp_info, NULL); 1653 1654 if (ibt_status != IBT_SUCCESS) 1655 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1656 "chanp %p ibt_modify_qp() = %d for ERROR to RESET", 1657 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1658 } 1659 1660 if (ibt_status == IBT_SUCCESS) { 1661 1662 qp_info.qp_state = IBT_STATE_INIT; 1663 1664 /* set flags for all mandatory args from RESET to INIT */ 1665 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 1666 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W; 1667 cep_flags |= IBT_CEP_SET_ATOMIC; 1668 1669 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 1670 ibcm_tq_recycle_arg->hca_port_num; 1671 qp_info.qp_flags |= 1672 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD; 1673 qp_info.qp_flags |= 1674 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR; 1675 qp_info.qp_flags |= 1676 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC; 1677 1678 /* Always use the existing pkey */ 1679 qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 1680 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix; 1681 1682 /* Call modify_qp to move to INIT state */ 1683 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1684 cep_flags, &qp_info, NULL); 1685 1686 if (ibt_status != IBT_SUCCESS) 1687 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1688 "chanp %p ibt_modify_qp() = %d for RESET to INIT", 1689 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1690 } 1691 1692 /* Change the QP CM state to indicate QP being re-used */ 1693 if (ibt_status == IBT_SUCCESS) 1694 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan); 1695 1696 /* Call func, if defined */ 1697 if (ibcm_tq_recycle_arg->func) 1698 (*(ibcm_tq_recycle_arg->func))(ibt_status, 1699 ibcm_tq_recycle_arg->arg); 1700 1701 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t)); 1702 1703 return (ibt_status); 1704 } 1705 1706 static void 1707 ibcm_process_abort_via_taskq(void *args) 1708 { 1709 ibcm_state_data_t *statep = (ibcm_state_data_t *)args; 1710 1711 ibcm_process_abort(statep); 1712 mutex_enter(&statep->state_mutex); 1713 IBCM_REF_CNT_DECR(statep); 1714 mutex_exit(&statep->state_mutex); 1715 } 1716 1717 /* 1718 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in 1719 * Non-Blocking mode operations. 1720 */ 1721 typedef struct ibcm_local_handler_s { 1722 ibt_cm_ud_handler_t actual_cm_handler; 1723 void *actual_cm_private; 1724 ibt_ud_dest_t *dest_hdl; 1725 } ibcm_local_handler_t; 1726 1727 _NOTE(READ_ONLY_DATA(ibcm_local_handler_s)) 1728 1729 /* 1730 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in 1731 * NON-Blocking mode. 1732 * 1733 * Out here, we update the UD Destination handle with 1734 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client 1735 * handler that was specified by the client. 1736 */ 1737 static ibt_cm_status_t 1738 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event, 1739 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 1740 { 1741 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv; 1742 1743 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d", 1744 event->cm_type); 1745 1746 ASSERT(handler_priv != NULL); 1747 1748 switch (event->cm_type) { 1749 case IBT_CM_UD_EVENT_SIDR_REP: 1750 /* Update QPN & QKey from event into destination handle. */ 1751 if (handler_priv->dest_hdl != NULL) { 1752 handler_priv->dest_hdl->ud_dst_qpn = 1753 event->cm_event.sidr_rep.srep_remote_qpn; 1754 handler_priv->dest_hdl->ud_qkey = 1755 event->cm_event.sidr_rep.srep_remote_qkey; 1756 } 1757 1758 /* Invoke the client handler - inform only, so ignore retval */ 1759 (void) handler_priv->actual_cm_handler( 1760 handler_priv->actual_cm_private, event, ret_args, priv_data, 1761 len); 1762 1763 /* Free memory allocated for local handler's private data. */ 1764 if (handler_priv != NULL) 1765 kmem_free(handler_priv, sizeof (*handler_priv)); 1766 1767 break; 1768 default: 1769 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR"); 1770 break; 1771 } 1772 1773 return (IBT_CM_ACCEPT); 1774 } 1775 1776 1777 /* Validate the input UD destination attributes. */ 1778 static ibt_status_t 1779 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1780 ibt_ud_returns_t *ret_args) 1781 { 1782 /* cm handler must always be specified */ 1783 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) { 1784 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1785 "CM handler is not specified "); 1786 return (IBT_INVALID_PARAM); 1787 } 1788 1789 if (mode == IBT_NONBLOCKING) { 1790 if (ret_args != NULL) { 1791 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1792 "ret_args should be NULL when called in " 1793 "non-blocking mode"); 1794 return (IBT_INVALID_PARAM); 1795 } 1796 } else if (mode == IBT_BLOCKING) { 1797 if (ret_args == NULL) { 1798 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1799 "ret_args should be Non-NULL when called in " 1800 "blocking mode"); 1801 return (IBT_INVALID_PARAM); 1802 } 1803 } else { 1804 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1805 "invalid mode %x specified ", mode); 1806 return (IBT_INVALID_PARAM); 1807 } 1808 1809 if (attr->ud_sid == 0) { 1810 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1811 "ServiceID must be specified. "); 1812 return (IBT_INVALID_PARAM); 1813 } 1814 1815 if (attr->ud_addr == NULL) { 1816 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1817 "Address Info NULL"); 1818 return (IBT_INVALID_PARAM); 1819 } 1820 1821 /* Validate SGID */ 1822 if ((attr->ud_addr->av_sgid.gid_prefix == 0) || 1823 (attr->ud_addr->av_sgid.gid_guid == 0)) { 1824 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID"); 1825 return (IBT_INVALID_PARAM); 1826 } 1827 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>", 1828 attr->ud_addr->av_sgid.gid_prefix, 1829 attr->ud_addr->av_sgid.gid_guid); 1830 1831 /* Validate DGID */ 1832 if ((attr->ud_addr->av_dgid.gid_prefix == 0) || 1833 (attr->ud_addr->av_dgid.gid_guid == 0)) { 1834 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID"); 1835 return (IBT_INVALID_PARAM); 1836 } 1837 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>", 1838 attr->ud_addr->av_dgid.gid_prefix, 1839 attr->ud_addr->av_dgid.gid_guid); 1840 1841 return (IBT_SUCCESS); 1842 } 1843 1844 1845 /* Perform SIDR to retrieve DQPN and QKey. */ 1846 static ibt_status_t 1847 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1848 ibt_ud_returns_t *ret_args) 1849 { 1850 ibt_status_t retval; 1851 ib_pkey_t ud_pkey; 1852 ibmf_handle_t ibmf_hdl; 1853 ibmf_msg_t *ibmf_msg; 1854 ibcm_hca_info_t *hcap; 1855 ibcm_sidr_req_msg_t *sidr_req_msgp; 1856 ibcm_ud_state_data_t *ud_statep; 1857 ibtl_cm_hca_port_t port; 1858 ibcm_sidr_srch_t sidr_entry; 1859 ibcm_qp_list_t *cm_qp_entry; 1860 1861 /* Retrieve HCA GUID value from the available SGID info. */ 1862 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port); 1863 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) { 1864 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1865 "ibtl_cm_get_hca_port failed: %d", retval); 1866 return (retval); 1867 } 1868 1869 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: " 1870 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port); 1871 1872 /* Lookup the HCA info for this GUID */ 1873 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) { 1874 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL"); 1875 return (IBT_HCA_INVALID); 1876 } 1877 1878 /* Return failure if the HCA device or Port is not operational */ 1879 1880 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port, 1881 NULL, NULL)) != IBT_SUCCESS) { 1882 /* Device Port is not in good state, don't use it. */ 1883 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid " 1884 "port specified or port not active"); 1885 ibcm_dec_hca_acc_cnt(hcap); 1886 return (retval); 1887 } 1888 1889 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port, 1890 attr->ud_pkey_ix, &ud_pkey); 1891 if (retval != IBT_SUCCESS) { 1892 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1893 "Failed to convert index2pkey: %d", retval); 1894 ibcm_dec_hca_acc_cnt(hcap); 1895 return (retval); 1896 } 1897 1898 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(sidr_entry)) 1899 1900 /* Allocate a new request id */ 1901 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) { 1902 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1903 "no req id available"); 1904 ibcm_dec_hca_acc_cnt(hcap); 1905 return (IBT_INSUFF_KERNEL_RESOURCE); 1906 } 1907 1908 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) && 1909 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1)) 1910 != IBT_SUCCESS)) { 1911 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1912 "ibmf reg or callback setup failed during re-initialize"); 1913 return (retval); 1914 } 1915 1916 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl; 1917 1918 /* find the ibmf QP to post the SIDR REQ */ 1919 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) == 1920 NULL) { 1921 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation" 1922 " failed"); 1923 ibcm_dec_hca_acc_cnt(hcap); 1924 return (IBT_INSUFF_RESOURCE); 1925 } 1926 1927 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND)) 1928 != IBT_SUCCESS) { 1929 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation" 1930 " failed"); 1931 ibcm_release_qp(cm_qp_entry); 1932 ibcm_dec_hca_acc_cnt(hcap); 1933 return (retval); 1934 } 1935 1936 sidr_entry.srch_lid = port.hp_base_lid; 1937 sidr_entry.srch_gid = attr->ud_addr->av_sgid; 1938 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh; 1939 sidr_entry.srch_mode = IBCM_ACTIVE_MODE; 1940 1941 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(sidr_entry)) 1942 1943 /* do various allocations needed here */ 1944 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 1945 1946 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep, 1947 IBCM_FLAG_ADD); 1948 rw_exit(&hcap->hca_sidr_list_lock); 1949 1950 /* Increment hca's resource count */ 1951 ibcm_inc_hca_res_cnt(hcap); 1952 1953 /* After a resource created on hca, no need to hold the acc cnt */ 1954 ibcm_dec_hca_acc_cnt(hcap); 1955 1956 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 1957 1958 /* Initialize some ud_statep fields */ 1959 ud_statep->ud_stored_msg = ibmf_msg; 1960 ud_statep->ud_svc_id = attr->ud_sid; 1961 ud_statep->ud_pkt_life_time = 1962 ibt_ib2usec(attr->ud_pkt_lt); 1963 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry; 1964 1965 /* set remaining retry cnt */ 1966 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries; 1967 1968 /* 1969 * Get UD handler and corresponding args which is pass it back 1970 * as first argument for the handler. 1971 */ 1972 ud_statep->ud_state_cm_private = attr->ud_cm_private; 1973 1974 if (mode == IBT_BLOCKING) 1975 ud_statep->ud_return_data = ret_args; 1976 else 1977 ud_statep->ud_cm_handler = attr->ud_cm_handler; 1978 1979 /* Initialize the fields of ud_statep->ud_stored_reply_addr */ 1980 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh; 1981 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl; 1982 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit = 1983 attr->ud_addr->av_hop; 1984 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid = 1985 attr->ud_addr->av_sgid; 1986 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid = 1987 attr->ud_addr->av_dgid; 1988 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass = 1989 attr->ud_addr->av_tclass; 1990 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label = 1991 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK; 1992 1993 /* needs to be derived based on the base LID and path bits */ 1994 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid = 1995 port.hp_base_lid; 1996 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid = 1997 attr->ud_addr->av_dlid; 1998 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey; 1999 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY; 2000 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level = 2001 attr->ud_addr->av_srvl; 2002 2003 /* 2004 * This may be enchanced later, to use a remote qno based on past 2005 * redirect rej mad responses. This would be the place to specify 2006 * appropriate remote qno 2007 */ 2008 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1; 2009 2010 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2011 2012 /* Initialize the SIDR REQ message fields */ 2013 sidr_req_msgp = 2014 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg); 2015 2016 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id); 2017 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid); 2018 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey); 2019 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID = 2020 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID); 2021 2022 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) { 2023 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data, 2024 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ)); 2025 } 2026 2027 /* Send out the SIDR REQ message */ 2028 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT; 2029 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT; 2030 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */ 2031 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) + 2032 (ud_statep->ud_pkt_life_time * 2); 2033 2034 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID = 2035 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ, 2036 ud_statep->ud_req_id, 0)); 2037 2038 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x", 2039 ud_statep->ud_timer_value); 2040 2041 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 2042 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2043 2044 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg, 2045 ibcm_post_sidr_req_complete, ud_statep); 2046 2047 mutex_enter(&ud_statep->ud_state_mutex); 2048 2049 /* Wait for SIDR_REP */ 2050 if (mode == IBT_BLOCKING) { 2051 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking"); 2052 2053 while (ud_statep->ud_blocking_done != B_TRUE) { 2054 cv_wait(&ud_statep->ud_block_client_cv, 2055 &ud_statep->ud_state_mutex); 2056 } 2057 2058 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking"); 2059 2060 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) { 2061 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, " 2062 "status = %x, QKey = %x", ret_args->ud_dqpn, 2063 ret_args->ud_status, ret_args->ud_qkey); 2064 2065 } else { 2066 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>", 2067 ret_args->ud_status); 2068 retval = IBT_CM_FAILURE; 2069 } 2070 } 2071 2072 IBCM_UD_REF_CNT_DECR(ud_statep); 2073 mutex_exit(&ud_statep->ud_state_mutex); 2074 2075 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done"); 2076 2077 return (retval); 2078 } 2079 2080 2081 /* 2082 * Function: 2083 * ibt_request_ud_dest 2084 * Input: 2085 * ud_dest A previously allocated UD destination handle. 2086 * mode This function can execute in blocking or non blocking 2087 * modes. 2088 * attr UD destination attributes to be modified. 2089 * Output: 2090 * ud_ret_args If the function is called in blocking mode, ud_ret_args 2091 * should be a pointer to an ibt_ud_returns_t struct. 2092 * Returns: 2093 * IBT_SUCCESS 2094 * Description: 2095 * Modify a previously allocated UD destination handle based on the 2096 * results of doing the SIDR protocol. 2097 */ 2098 ibt_status_t 2099 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode, 2100 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args) 2101 { 2102 ibt_status_t retval; 2103 ibt_ud_dest_t *ud_destp; 2104 ibcm_local_handler_t *local_handler_priv = NULL; 2105 2106 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)", 2107 ud_dest, mode, attr, ud_ret_args); 2108 2109 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args); 2110 if (retval != IBT_SUCCESS) { 2111 return (retval); 2112 } 2113 2114 ud_destp = ud_dest; 2115 2116 /* Allocate an Address handle. */ 2117 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah, 2118 attr->ud_addr); 2119 if (retval != IBT_SUCCESS) { 2120 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2121 "Address Handle Modification failed: %d", retval); 2122 return (retval); 2123 } 2124 2125 if (mode == IBT_NONBLOCKING) { 2126 /* 2127 * In NON-BLOCKING mode, and we need to update the destination 2128 * handle with the DQPN and QKey that are obtained from 2129 * SIDR REP, hook-up our own handler, so that we can catch 2130 * the event, and we ourselves call the actual client's 2131 * ud_cm_handler, in our handler. 2132 */ 2133 2134 /* Allocate memory for local handler's private data. */ 2135 local_handler_priv = 2136 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP); 2137 2138 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2139 2140 local_handler_priv->actual_cm_handler = attr->ud_cm_handler; 2141 local_handler_priv->actual_cm_private = attr->ud_cm_private; 2142 local_handler_priv->dest_hdl = ud_destp; 2143 2144 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2145 2146 attr->ud_cm_handler = ibcm_local_cm_handler; 2147 attr->ud_cm_private = local_handler_priv; 2148 } 2149 2150 /* In order to get DQPN and Destination QKey, perform SIDR */ 2151 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args); 2152 if (retval != IBT_SUCCESS) { 2153 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2154 "Failed to get DQPN: %d", retval); 2155 2156 /* Free memory allocated for local handler's private data. */ 2157 if (local_handler_priv != NULL) 2158 kmem_free(local_handler_priv, 2159 sizeof (*local_handler_priv)); 2160 return (retval); 2161 } 2162 2163 /* 2164 * Fill in the dqpn and dqkey as obtained from ud_ret_args, 2165 * values will be valid only on BLOCKING mode. 2166 */ 2167 if (mode == IBT_BLOCKING) { 2168 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn; 2169 ud_destp->ud_qkey = ud_ret_args->ud_qkey; 2170 } 2171 2172 return (retval); 2173 } 2174 2175 /* 2176 * Function: 2177 * ibt_ud_get_dqpn 2178 * Input: 2179 * attr A pointer to an ibt_ud_dest_attr_t struct that are 2180 * required for SIDR REQ message. Not specified attributes 2181 * should be set to "NULL" or "0". 2182 * ud_sid, ud_addr and ud_pkt_lt must be specified. 2183 * mode This function can execute in blocking or non blocking 2184 * modes. 2185 * Output: 2186 * returns If the function is called in blocking mode, returns 2187 * should be a pointer to an ibt_ud_returns_t struct. 2188 * Return: 2189 * IBT_SUCCESS on success or respective failure on error. 2190 * Description: 2191 * Finds the destination QPN at the specified destination that the 2192 * specified service can be reached on. The IBTF CM initiates the 2193 * service ID resolution protocol (SIDR) to determine a destination QPN. 2194 * 2195 * NOTE: SIDR_REQ is initiated from active side. 2196 */ 2197 ibt_status_t 2198 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 2199 ibt_ud_returns_t *returns) 2200 { 2201 ibt_status_t retval; 2202 2203 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)", 2204 attr, mode, returns); 2205 2206 retval = ibcm_validate_dqpn_data(attr, mode, returns); 2207 if (retval != IBT_SUCCESS) { 2208 return (retval); 2209 } 2210 2211 return (ibcm_ud_get_dqpn(attr, mode, returns)); 2212 } 2213 2214 2215 /* 2216 * ibt_cm_delay: 2217 * A client CM handler function can call this function 2218 * to extend its response time to a CM event. 2219 * INPUTS: 2220 * flags Indicates what CM message processing is being delayed 2221 * by the CM handler, valid values are: 2222 * IBT_CM_DELAY_REQ 2223 * IBT_CM_DELAY_REP 2224 * IBT_CM_DELAY_LAP 2225 * cm_session_id The session ID that was passed to client srv_handler 2226 * by the CM 2227 * service_time The extended service time 2228 * priv_data Vendor specific data to be sent in the CM generated 2229 * MRA message. Should be NULL if not specified. 2230 * len The number of bytes of data specified by priv_data. 2231 * 2232 * RETURN VALUES: 2233 * IBT_SUCCESS on success (or respective failure on error) 2234 */ 2235 ibt_status_t 2236 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id, 2237 clock_t service_time, void *priv_data, ibt_priv_data_len_t len) 2238 { 2239 uint8_t msg_typ = 0; 2240 ibcm_mra_msg_t *mra_msgp; 2241 ibcm_state_data_t *statep; 2242 ibt_status_t status; 2243 2244 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)", 2245 flags, cm_session_id, service_time); 2246 2247 /* 2248 * Make sure channel is associated with a statep 2249 */ 2250 statep = (ibcm_state_data_t *)cm_session_id; 2251 2252 if (statep == NULL) { 2253 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL"); 2254 return (IBT_INVALID_PARAM); 2255 } 2256 2257 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep); 2258 2259 /* Allocate an ibmf msg for mra, if not allocated yet */ 2260 if (statep->mra_msg == NULL) { 2261 if ((status = ibcm_alloc_out_msg( 2262 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg, 2263 MAD_METHOD_SEND)) != IBT_SUCCESS) { 2264 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p" 2265 "IBMF MSG allocation failed", statep->channel); 2266 return (status); 2267 } 2268 } 2269 2270 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2271 2272 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg); 2273 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid); 2274 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid); 2275 2276 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */ 2277 if (flags == IBT_CM_DELAY_REQ) { 2278 msg_typ = IBT_CM_MRA_TYPE_REQ; 2279 } else if (flags == IBT_CM_DELAY_REP) { 2280 msg_typ = IBT_CM_MRA_TYPE_REP; 2281 } else if (flags == IBT_CM_DELAY_LAP) { 2282 msg_typ = IBT_CM_MRA_TYPE_LAP; 2283 } 2284 2285 mra_msgp->mra_message_type_plus = msg_typ << 6; 2286 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3; 2287 2288 len = min(len, IBT_MRA_PRIV_DATA_SZ); 2289 if (priv_data && (len > 0)) 2290 bcopy(priv_data, mra_msgp->mra_private_data, len); 2291 2292 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID = 2293 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID); 2294 2295 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2296 2297 mutex_enter(&statep->state_mutex); 2298 2299 if ((statep->mode == IBCM_ACTIVE_MODE) && 2300 (statep->state == IBCM_STATE_REP_RCVD)) { 2301 statep->state = IBCM_STATE_MRA_REP_SENT; 2302 } else if (statep->mode == IBCM_PASSIVE_MODE) { 2303 if (statep->state == IBCM_STATE_REQ_RCVD) { 2304 statep->state = IBCM_STATE_MRA_SENT; 2305 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) { 2306 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD; 2307 } else { 2308 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2309 "/ap_state/mode %x, %x, %x", statep->state, 2310 statep->ap_state, statep->mode); 2311 mutex_exit(&statep->state_mutex); 2312 return (IBT_CHAN_STATE_INVALID); 2313 } 2314 } else { 2315 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2316 "/ap_state/mode %x, %x, %x", statep->state, 2317 statep->ap_state, statep->mode); 2318 mutex_exit(&statep->state_mutex); 2319 2320 return (IBT_CHAN_STATE_INVALID); 2321 } 2322 /* service time is usecs, stale_clock is nsecs */ 2323 statep->stale_clock = gethrtime() + 2324 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 * 2325 statep->max_cm_retries); 2326 2327 statep->send_mad_flags |= IBCM_MRA_POST_BUSY; 2328 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */ 2329 mutex_exit(&statep->state_mutex); 2330 2331 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID = 2332 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID; 2333 2334 /* post the MRA mad in blocking mode, as no timers involved */ 2335 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete, 2336 statep); 2337 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_MRA); 2338 /* If this message isn't seen then ibt_cm_delay failed */ 2339 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay: done !!"); 2340 2341 return (IBT_SUCCESS); 2342 } 2343 2344 2345 /* 2346 * ibt_register_service() 2347 * Register a service with the IBCM 2348 * 2349 * INPUTS: 2350 * ibt_hdl The IBT client handle returned to the client 2351 * on an ibt_attach() call. 2352 * 2353 * srv The address of a ibt_srv_desc_t that describes 2354 * the service, containing the following: 2355 * 2356 * sd_ud_handler The Service CM UD event Handler. 2357 * sd_handler The Service CM RC/UC/RD event Handler. 2358 * sd_flags Service flags (peer-to-peer, or not). 2359 * 2360 * sid This tells CM if the service is local (sid is 0) or 2361 * wellknown (sid is the starting service id of the range). 2362 * 2363 * num_sids The number of contiguous service-ids to reserve. 2364 * 2365 * srv_hdl The address of a service identification handle, used 2366 * to deregister a service, and to bind GIDs to. 2367 * 2368 * ret_sid The address to store the Service ID return value. 2369 * If num_sids > 1, ret_sid is the first Service ID 2370 * in the range. 2371 * 2372 * ibt_register_service() returns: 2373 * IBT_SUCCESS - added a service successfully. 2374 * IBT_INVALID_PARAM - invalid input parameter. 2375 * IBT_CM_FAILURE - failed to add the service. 2376 * IBT_CM_SERVICE_EXISTS - service already exists. 2377 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should 2378 * never happen). 2379 */ 2380 ibt_status_t 2381 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv, 2382 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid) 2383 { 2384 ibcm_svc_info_t *svcinfop; 2385 2386 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p, %p, %llx, %d)", 2387 ibt_hdl, srv, (longlong_t)sid, num_sids); 2388 2389 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 2390 2391 *srv_hdl = NULL; 2392 2393 if (num_sids <= 0) { 2394 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2395 "Invalid number of service-ids specified (%d)", num_sids); 2396 return (IBT_INVALID_PARAM); 2397 } 2398 2399 if (sid == 0) { 2400 if (ret_sid == NULL) 2401 return (IBT_INVALID_PARAM); 2402 sid = ibcm_alloc_local_sids(num_sids); 2403 if (sid == 0) 2404 return (IBT_INSUFF_KERNEL_RESOURCE); 2405 2406 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */ 2407 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) { 2408 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2409 "Invalid non-LOCAL SID specified: 0x%llX", 2410 (longlong_t)sid); 2411 return (IBT_INVALID_PARAM); 2412 } 2413 2414 svcinfop = ibcm_create_svc_entry(sid, num_sids); 2415 2416 if (svcinfop == NULL) { 2417 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2418 "Service-ID 0x%llx already registered", (longlong_t)sid); 2419 return (IBT_CM_SERVICE_EXISTS); 2420 } 2421 2422 /* 2423 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry() 2424 */ 2425 svcinfop->svc_flags = srv->sd_flags; 2426 svcinfop->svc_rc_handler = srv->sd_handler; 2427 svcinfop->svc_ud_handler = srv->sd_ud_handler; 2428 2429 if (ret_sid != NULL) 2430 *ret_sid = sid; 2431 2432 *srv_hdl = svcinfop; 2433 2434 ibtl_cm_change_service_cnt(ibt_hdl, num_sids); 2435 2436 /* If this message isn't seen, then ibt_register_service failed. */ 2437 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)", 2438 svcinfop, sid); 2439 2440 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 2441 2442 return (IBT_SUCCESS); 2443 } 2444 2445 2446 static ibt_status_t 2447 ibcm_write_service_record(ibmf_saa_handle_t saa_handle, 2448 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type) 2449 { 2450 int rval; 2451 int retry; 2452 2453 ibcm_sa_access_enter(); 2454 for (retry = 0; retry < ibcm_max_sa_retries; retry++) { 2455 rval = ibmf_saa_update_service_record( 2456 saa_handle, srv_recp, saa_type, 0); 2457 if (rval != IBMF_TRANS_TIMEOUT) { 2458 break; 2459 } 2460 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2461 "ibmf_saa_update_service_record timed out" 2462 " SID = %llX, rval = %d, saa_type = %d", 2463 (longlong_t)srv_recp->ServiceID, rval, saa_type); 2464 delay(ibcm_sa_timeout_delay); 2465 } 2466 ibcm_sa_access_exit(); 2467 2468 if (rval != IBMF_SUCCESS) { 2469 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2470 "ibmf_saa_update_service_record() : Failed - %d", rval); 2471 return (ibcm_ibmf_analyze_error(rval)); 2472 } else 2473 return (IBT_SUCCESS); 2474 } 2475 2476 2477 static void 2478 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec) 2479 { 2480 ibt_status_t retval; 2481 uint_t num_found; 2482 size_t length; 2483 sa_service_record_t *srv_resp; 2484 void *results_p; 2485 uint_t i; 2486 uint64_t component_mask; 2487 ibmf_saa_access_args_t access_args; 2488 2489 component_mask = 2490 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID; 2491 2492 /* Call in SA Access retrieve routine to get Service Records. */ 2493 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 2494 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 2495 access_args.sq_component_mask = component_mask; 2496 access_args.sq_template = srec; 2497 access_args.sq_template_length = sizeof (sa_service_record_t); 2498 access_args.sq_callback = NULL; 2499 access_args.sq_callback_arg = NULL; 2500 2501 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 2502 &results_p); 2503 if (retval != IBT_SUCCESS) { 2504 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2505 "SA Access Failure"); 2506 return; 2507 } 2508 2509 num_found = length / sizeof (sa_service_record_t); 2510 2511 if (num_found) 2512 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2513 "Found %d matching Service Records.", num_found); 2514 2515 /* Validate the returned number of records. */ 2516 if ((results_p != NULL) && (num_found > 0)) { 2517 2518 /* Remove all the records. */ 2519 for (i = 0; i < num_found; i++) { 2520 2521 srv_resp = (sa_service_record_t *) 2522 ((uchar_t *)results_p + 2523 i * sizeof (sa_service_record_t)); 2524 2525 /* 2526 * Found some matching records, but check out whether 2527 * this Record is really stale or just happens to match 2528 * the current session records. If yes, don't remove it. 2529 */ 2530 mutex_enter(&ibcm_svc_info_lock); 2531 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) { 2532 /* This record is NOT STALE. */ 2533 mutex_exit(&ibcm_svc_info_lock); 2534 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2535 "This is not Stale, it's an active record"); 2536 continue; 2537 } 2538 mutex_exit(&ibcm_svc_info_lock); 2539 2540 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2541 "Removing Stale Rec: %s, %llX", 2542 srv_resp->ServiceName, srv_resp->ServiceID); 2543 2544 IBCM_DUMP_SERVICE_REC(srv_resp); 2545 2546 /* 2547 * Remove the Service Record Entry from SA. 2548 * 2549 * Get ServiceID info from Response Buf, other 2550 * attributes are already filled-in. 2551 */ 2552 2553 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2554 2555 srec->ServiceID = srv_resp->ServiceID; 2556 2557 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2558 2559 (void) ibcm_write_service_record(saa_handle, srec, 2560 IBMF_SAA_DELETE); 2561 } 2562 2563 /* Deallocate the memory for results_p. */ 2564 kmem_free(results_p, length); 2565 } 2566 } 2567 2568 2569 2570 /* 2571 * ibt_bind_service() 2572 * Register a service with the IBCM 2573 * 2574 * INPUTS: 2575 * srv_hdl The service id handle returned to the client 2576 * on an ibt_service_register() call. 2577 * 2578 * gid The GID to which to bind the service. 2579 * 2580 * srv_bind The address of a ibt_srv_bind_t that describes 2581 * the service record. This should be NULL if there 2582 * is to be no service record. This contains: 2583 * 2584 * sb_lease Lease period 2585 * sb_pkey Partition 2586 * sb_name pointer to ASCII string Service Name, 2587 * NULL terminated. 2588 * sb_key[] Key to secure the service record. 2589 * sb_data Service Data structure (64-byte) 2590 * 2591 * cm_private First argument of Service handler. 2592 * 2593 * sb_hdl_p The address of a service bind handle, used 2594 * to undo the service binding. 2595 * 2596 * ibt_bind_service() returns: 2597 * IBT_SUCCESS - added a service successfully. 2598 * IBT_INVALID_PARAM - invalid input parameter. 2599 * IBT_CM_FAILURE - failed to add the service. 2600 * IBT_CM_SERVICE_EXISTS - service already exists. 2601 */ 2602 ibt_status_t 2603 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind, 2604 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p) 2605 { 2606 ibt_status_t status; 2607 ibtl_cm_hca_port_t port; 2608 ibcm_svc_bind_t *sbindp, *sbp; 2609 ibcm_hca_info_t *hcap; 2610 ib_svc_id_t sid, start_sid, end_sid; 2611 ibmf_saa_handle_t saa_handle; 2612 sa_service_record_t srv_rec; 2613 uint16_t pkey_ix; 2614 2615 if (sb_hdl_p != NULL) 2616 *sb_hdl_p = NULL; /* return value for error cases */ 2617 2618 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)", 2619 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid); 2620 2621 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sbindp)) 2622 2623 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */ 2624 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) { 2625 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2626 "ibtl_cm_get_hca_port failed: %d", status); 2627 return (status); 2628 } 2629 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX", 2630 port.hp_port, port.hp_hca_guid); 2631 2632 hcap = ibcm_find_hca_entry(port.hp_hca_guid); 2633 if (hcap == NULL) { 2634 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found"); 2635 return (IBT_HCA_BUSY_DETACHING); 2636 } 2637 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap); 2638 2639 if (srv_bind != NULL) { 2640 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2641 if (saa_handle == NULL) { 2642 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2643 "saa_handle is NULL"); 2644 ibcm_dec_hca_acc_cnt(hcap); 2645 return (IBT_HCA_PORT_NOT_ACTIVE); 2646 } 2647 if (srv_bind->sb_pkey == 0) { 2648 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2649 "P_Key must not be 0"); 2650 ibcm_dec_hca_acc_cnt(hcap); 2651 return (IBT_INVALID_PARAM); 2652 } 2653 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) { 2654 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2655 "Service Name is too long"); 2656 ibcm_dec_hca_acc_cnt(hcap); 2657 return (IBT_INVALID_PARAM); 2658 } else 2659 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: " 2660 "Service Name='%s'", srv_bind->sb_name); 2661 status = ibt_pkey2index_byguid(port.hp_hca_guid, 2662 port.hp_port, srv_bind->sb_pkey, &pkey_ix); 2663 if (status != IBT_SUCCESS) { 2664 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2665 "P_Key 0x%x not found in P_Key_Table", 2666 srv_bind->sb_pkey); 2667 ibcm_dec_hca_acc_cnt(hcap); 2668 return (status); 2669 } 2670 } 2671 2672 /* assume success - allocate before locking */ 2673 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP); 2674 sbindp->sbind_cm_private = cm_private; 2675 sbindp->sbind_gid = gid; 2676 sbindp->sbind_hcaguid = port.hp_hca_guid; 2677 sbindp->sbind_port = port.hp_port; 2678 2679 mutex_enter(&ibcm_svc_info_lock); 2680 2681 sbp = srv_hdl->svc_bind_list; 2682 while (sbp != NULL) { 2683 if (sbp->sbind_gid.gid_guid == gid.gid_guid && 2684 sbp->sbind_gid.gid_prefix == gid.gid_prefix) { 2685 if (srv_bind == NULL || 2686 srv_bind->sb_pkey == sbp->sbind_pkey) { 2687 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2688 "failed: GID %llX:%llX and PKEY %x is " 2689 "already bound", gid.gid_guid, 2690 gid.gid_prefix, sbp->sbind_pkey); 2691 mutex_exit(&ibcm_svc_info_lock); 2692 ibcm_dec_hca_acc_cnt(hcap); 2693 kmem_free(sbindp, sizeof (*sbindp)); 2694 return (IBT_CM_SERVICE_EXISTS); 2695 } 2696 } 2697 sbp = sbp->sbind_link; 2698 } 2699 /* no entry found */ 2700 2701 sbindp->sbind_link = srv_hdl->svc_bind_list; 2702 srv_hdl->svc_bind_list = sbindp; 2703 2704 mutex_exit(&ibcm_svc_info_lock); 2705 2706 if (srv_bind != NULL) { 2707 bzero(&srv_rec, sizeof (srv_rec)); 2708 2709 srv_rec.ServiceLease = 2710 sbindp->sbind_lease = srv_bind->sb_lease; 2711 srv_rec.ServiceP_Key = 2712 sbindp->sbind_pkey = srv_bind->sb_pkey; 2713 srv_rec.ServiceKey_hi = 2714 sbindp->sbind_key[0] = srv_bind->sb_key[0]; 2715 srv_rec.ServiceKey_lo = 2716 sbindp->sbind_key[1] = srv_bind->sb_key[1]; 2717 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name); 2718 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name); 2719 srv_rec.ServiceGID = gid; 2720 2721 /* 2722 * Find out whether we have any stale Local Service records 2723 * matching the current attributes. If yes, we shall try to 2724 * remove them from SA using the current request's ServiceKey. 2725 * 2726 * We will perform this operation only for Local Services, as 2727 * it is handled by SA automatically for WellKnown Services. 2728 * 2729 * Ofcourse, clients can specify NOT to do this clean-up by 2730 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag). 2731 */ 2732 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) && 2733 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) { 2734 ibcm_rem_stale_srec(saa_handle, &srv_rec); 2735 } 2736 2737 /* Handle endianess for service data. */ 2738 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data); 2739 2740 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 2741 2742 /* insert srv record into the SA */ 2743 start_sid = srv_hdl->svc_id; 2744 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 2745 for (sid = start_sid; sid <= end_sid; sid++) { 2746 2747 srv_rec.ServiceID = sid; 2748 2749 IBCM_DUMP_SERVICE_REC(&srv_rec); 2750 2751 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: " 2752 "ibmf_saa_write_service_record, SvcId = %llX", 2753 (longlong_t)sid); 2754 2755 status = ibcm_write_service_record(saa_handle, &srv_rec, 2756 IBMF_SAA_UPDATE); 2757 if (status != IBT_SUCCESS) { 2758 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:" 2759 " ibcm_write_service_record fails %d, " 2760 "sid %llX", status, (longlong_t)sid); 2761 2762 if (sid != start_sid) { 2763 /* 2764 * Bind failed while bind SID other than 2765 * first in the sid_range. So we need 2766 * to unbind those, which are passed. 2767 * 2768 * Need to increment svc count to 2769 * compensate for ibt_unbind_service(). 2770 */ 2771 ibcm_inc_hca_svc_cnt(hcap); 2772 ibcm_dec_hca_acc_cnt(hcap); 2773 2774 (void) ibt_unbind_service(srv_hdl, 2775 sbindp); 2776 } else { 2777 ibcm_svc_bind_t **sbpp; 2778 2779 /* 2780 * Bind failed for the first SID or the 2781 * only SID in question, then no need 2782 * to unbind, just free memory and 2783 * return error. 2784 */ 2785 mutex_enter(&ibcm_svc_info_lock); 2786 2787 sbpp = &srv_hdl->svc_bind_list; 2788 sbp = *sbpp; 2789 while (sbp != NULL) { 2790 if (sbp == sbindp) { 2791 *sbpp = sbp->sbind_link; 2792 break; 2793 } 2794 sbpp = &sbp->sbind_link; 2795 sbp = *sbpp; 2796 } 2797 mutex_exit(&ibcm_svc_info_lock); 2798 ibcm_dec_hca_acc_cnt(hcap); 2799 2800 kmem_free(sbindp, sizeof (*sbindp)); 2801 } 2802 return (status); 2803 } 2804 } 2805 } 2806 ibcm_inc_hca_svc_cnt(hcap); 2807 ibcm_dec_hca_acc_cnt(hcap); 2808 2809 /* If this message isn't seen then ibt_bind_service failed */ 2810 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llX)", 2811 srv_hdl, gid.gid_prefix, gid.gid_guid); 2812 2813 if (sb_hdl_p != NULL) 2814 *sb_hdl_p = sbindp; 2815 2816 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sbindp)) 2817 2818 return (IBT_SUCCESS); 2819 } 2820 2821 ibt_status_t 2822 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp) 2823 { 2824 ib_svc_id_t sid, end_sid; 2825 ibt_status_t rval; 2826 ibcm_hca_info_t *hcap; 2827 ibcm_svc_bind_t *sbp, **sbpp; 2828 2829 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)", 2830 srv_hdl, sbindp); 2831 2832 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 2833 2834 /* If there is a service on hca, respective hcap cannot go away */ 2835 ASSERT(hcap != NULL); 2836 2837 mutex_enter(&ibcm_svc_info_lock); 2838 2839 sbpp = &srv_hdl->svc_bind_list; 2840 sbp = *sbpp; 2841 while (sbp != NULL) { 2842 if (sbp == sbindp) { 2843 *sbpp = sbp->sbind_link; 2844 break; 2845 } 2846 sbpp = &sbp->sbind_link; 2847 sbp = *sbpp; 2848 } 2849 sid = srv_hdl->svc_id; 2850 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1; 2851 if (sbp != NULL) 2852 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY) 2853 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 2854 mutex_exit(&ibcm_svc_info_lock); 2855 2856 if (sbp == NULL) { 2857 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2858 "service binding not found: srv_hdl %p, srv_bind %p", 2859 srv_hdl, sbindp); 2860 ibcm_dec_hca_acc_cnt(hcap); 2861 return (IBT_INVALID_PARAM); 2862 } 2863 2864 if (sbindp->sbind_pkey != 0) { /* Are there service records? */ 2865 ibtl_cm_hca_port_t port; 2866 sa_service_record_t srv_rec; 2867 ibmf_saa_handle_t saa_handle; 2868 ibt_status_t status; 2869 2870 /* get the default SGID of the port */ 2871 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port)) 2872 != IBT_SUCCESS) { 2873 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2874 "ibtl_cm_get_hca_port failed: %d", status); 2875 /* we're done, but there may be stale service records */ 2876 goto done; 2877 } 2878 2879 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2880 if (saa_handle == NULL) { 2881 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2882 "saa_handle is NULL"); 2883 /* we're done, but there may be stale service records */ 2884 goto done; 2885 } 2886 2887 /* Fill in fields of srv_rec */ 2888 bzero(&srv_rec, sizeof (srv_rec)); 2889 2890 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 2891 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 2892 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 2893 srv_rec.ServiceGID = sbindp->sbind_gid; 2894 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 2895 2896 while (sid <= end_sid) { 2897 2898 srv_rec.ServiceID = sid; 2899 IBCM_DUMP_SERVICE_REC(&srv_rec); 2900 2901 rval = ibcm_write_service_record(saa_handle, &srv_rec, 2902 IBMF_SAA_DELETE); 2903 2904 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: " 2905 "ibcm_write_service_record rval = %d, SID %llx", 2906 rval, sid); 2907 if (rval != IBT_SUCCESS) { 2908 /* this is not considered a reason to fail */ 2909 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2910 "ibcm_write_service_record fails %d, " 2911 "sid %llx", rval, sid); 2912 } 2913 sid++; 2914 } 2915 } 2916 done: 2917 ibcm_dec_hca_svc_cnt(hcap); 2918 ibcm_dec_hca_acc_cnt(hcap); 2919 kmem_free(sbindp, sizeof (*sbindp)); 2920 2921 /* If this message isn't seen then ibt_unbind_service failed */ 2922 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!"); 2923 2924 return (IBT_SUCCESS); 2925 } 2926 2927 /* 2928 * Simply pull off each binding from the list and unbind it. 2929 * If any of the unbind calls fail, we fail. 2930 */ 2931 ibt_status_t 2932 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl) 2933 { 2934 ibt_status_t status; 2935 ibcm_svc_bind_t *sbp; 2936 2937 mutex_enter(&ibcm_svc_info_lock); 2938 sbp = NULL; 2939 2940 /* this compare keeps the loop from being infinite */ 2941 while (sbp != srv_hdl->svc_bind_list) { 2942 sbp = srv_hdl->svc_bind_list; 2943 mutex_exit(&ibcm_svc_info_lock); 2944 status = ibt_unbind_service(srv_hdl, sbp); 2945 if (status != IBT_SUCCESS) 2946 return (status); 2947 mutex_enter(&ibcm_svc_info_lock); 2948 if (srv_hdl->svc_bind_list == NULL) 2949 break; 2950 } 2951 mutex_exit(&ibcm_svc_info_lock); 2952 return (IBT_SUCCESS); 2953 } 2954 2955 /* 2956 * ibt_deregister_service() 2957 * Deregister a service with the IBCM 2958 * 2959 * INPUTS: 2960 * ibt_hdl The IBT client handle returned to the client 2961 * on an ibt_attach() call. 2962 * 2963 * srv_hdl The address of a service identification handle, used 2964 * to de-register a service. 2965 * RETURN VALUES: 2966 * IBT_SUCCESS on success (or respective failure on error) 2967 */ 2968 ibt_status_t 2969 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl) 2970 { 2971 ibcm_svc_info_t *svcp; 2972 ibcm_svc_lookup_t svc; 2973 2974 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p, %p)", 2975 ibt_hdl, srv_hdl); 2976 2977 mutex_enter(&ibcm_svc_info_lock); 2978 2979 if (srv_hdl->svc_bind_list != NULL) { 2980 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:" 2981 " srv_hdl %p still has bindings", srv_hdl); 2982 mutex_exit(&ibcm_svc_info_lock); 2983 return (IBT_CM_SERVICE_BUSY); 2984 } 2985 svc.sid = srv_hdl->svc_id; 2986 svc.num_sids = 1; 2987 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d", 2988 srv_hdl->svc_id, srv_hdl->svc_num_sids); 2989 2990 #ifdef __lock_lint 2991 ibcm_svc_compare(NULL, NULL); 2992 #endif 2993 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 2994 if (svcp != srv_hdl) { 2995 mutex_exit(&ibcm_svc_info_lock); 2996 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): " 2997 "srv_hdl %p not found", srv_hdl); 2998 return (IBT_INVALID_PARAM); 2999 } 3000 avl_remove(&ibcm_svc_avl_tree, svcp); 3001 3002 /* wait for active REQ/SREQ handling to be done */ 3003 svcp->svc_to_delete = 1; 3004 while (svcp->svc_ref_cnt != 0) 3005 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3006 3007 mutex_exit(&ibcm_svc_info_lock); 3008 3009 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) 3010 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids); 3011 3012 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids); 3013 kmem_free(srv_hdl, sizeof (*srv_hdl)); 3014 3015 /* If this message isn't seen then ibt_deregister_service failed */ 3016 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!"); 3017 3018 return (IBT_SUCCESS); 3019 } 3020 3021 ibcm_status_t 3022 ibcm_ar_init(void) 3023 { 3024 ib_svc_id_t sid = IBCM_DAPL_ATS_SID; 3025 ibcm_svc_info_t *tmp_svcp; 3026 3027 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()"); 3028 3029 /* remove this special SID from the pool of available SIDs */ 3030 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) { 3031 IBTF_DPRINTF_L2(cmlog, "ibcm_ar_init: " 3032 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid); 3033 return (IBCM_FAILURE); 3034 } 3035 mutex_enter(&ibcm_svc_info_lock); 3036 ibcm_ar_svcinfop = tmp_svcp; 3037 ibcm_ar_list = NULL; /* no address records registered yet */ 3038 mutex_exit(&ibcm_svc_info_lock); 3039 return (IBCM_SUCCESS); 3040 } 3041 3042 ibcm_status_t 3043 ibcm_ar_fini(void) 3044 { 3045 ibcm_ar_t *ar_list; 3046 ibcm_svc_info_t *tmp_svcp; 3047 3048 mutex_enter(&ibcm_svc_info_lock); 3049 ar_list = ibcm_ar_list; 3050 3051 if (ar_list == NULL && 3052 avl_numnodes(&ibcm_svc_avl_tree) == 1 && 3053 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) { 3054 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop); 3055 tmp_svcp = ibcm_ar_svcinfop; 3056 mutex_exit(&ibcm_svc_info_lock); 3057 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop)); 3058 return (IBCM_SUCCESS); 3059 } 3060 mutex_exit(&ibcm_svc_info_lock); 3061 return (IBCM_FAILURE); 3062 } 3063 3064 3065 /* 3066 * Return to the caller: 3067 * IBT_SUCCESS Found a perfect match. 3068 * *arpp is set to the record. 3069 * IBT_INCONSISTENT_AR Found a record that's inconsistent. 3070 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and 3071 * found no record with same data. 3072 */ 3073 static ibt_status_t 3074 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp) 3075 { 3076 ibcm_ar_t *tmp; 3077 int i; 3078 3079 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3080 tmp = ibcm_ar_list; 3081 while (tmp != NULL) { 3082 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix && 3083 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid && 3084 tmp->ar.ar_pkey == arp->ar_pkey) { 3085 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3086 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3087 return (IBT_INCONSISTENT_AR); 3088 *arpp = tmp; 3089 return (IBT_SUCCESS); 3090 } else { 3091 /* if all the data bytes match, we have inconsistency */ 3092 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3093 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3094 break; 3095 if (i == IBCM_DAPL_ATS_NBYTES) 3096 return (IBT_INCONSISTENT_AR); 3097 /* try next address record */ 3098 } 3099 tmp = tmp->ar_link; 3100 } 3101 return (IBT_AR_NOT_REGISTERED); 3102 } 3103 3104 ibt_status_t 3105 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3106 { 3107 ibcm_ar_t *found; 3108 ibcm_ar_t *tmp; 3109 ibt_status_t status; 3110 ibt_status_t s1, s2; 3111 char *s; 3112 ibcm_ar_ref_t *hdlp; 3113 ibcm_ar_t *new; 3114 ibcm_ar_t **linkp; 3115 ibtl_cm_hca_port_t cm_port; 3116 uint16_t pkey_ix; 3117 ibcm_hca_info_t *hcap; 3118 ibmf_saa_handle_t saa_handle; 3119 sa_service_record_t *srv_recp; 3120 uint64_t gid_ored; 3121 3122 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX", 3123 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix, 3124 (longlong_t)arp->ar_gid.gid_guid); 3125 3126 /* 3127 * If P_Key is 0, but GID is not, this query is invalid. 3128 * If GID is 0, but P_Key is not, this query is invalid. 3129 */ 3130 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3131 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3132 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3133 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3134 "GID/P_Key is not valid"); 3135 return (IBT_INVALID_PARAM); 3136 } 3137 3138 /* assume success, so these might be needed */ 3139 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP); 3140 new = kmem_zalloc(sizeof (*new), KM_SLEEP); 3141 3142 mutex_enter(&ibcm_svc_info_lock); 3143 /* search for existing GID/pkey (there can be at most 1) */ 3144 status = ibcm_search_ar(arp, &found); 3145 if (status == IBT_INCONSISTENT_AR) { 3146 mutex_exit(&ibcm_svc_info_lock); 3147 kmem_free(new, sizeof (*new)); 3148 kmem_free(hdlp, sizeof (*hdlp)); 3149 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3150 "address record is inconsistent with a known one"); 3151 return (IBT_INCONSISTENT_AR); 3152 } else if (status == IBT_SUCCESS) { 3153 if (found->ar_flags == IBCM_AR_INITING) { 3154 found->ar_waiters++; 3155 cv_wait(&found->ar_cv, &ibcm_svc_info_lock); 3156 found->ar_waiters--; 3157 } 3158 if (found->ar_flags == IBCM_AR_FAILED) { 3159 if (found->ar_waiters == 0) { 3160 cv_destroy(&found->ar_cv); 3161 kmem_free(found, sizeof (*found)); 3162 } 3163 mutex_exit(&ibcm_svc_info_lock); 3164 kmem_free(new, sizeof (*new)); 3165 kmem_free(hdlp, sizeof (*hdlp)); 3166 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0)); 3167 } 3168 hdlp->ar_ibt_hdl = ibt_hdl; 3169 hdlp->ar_ref_link = found->ar_ibt_hdl_list; 3170 found->ar_ibt_hdl_list = hdlp; 3171 mutex_exit(&ibcm_svc_info_lock); 3172 kmem_free(new, sizeof (*new)); 3173 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3174 return (IBT_SUCCESS); 3175 } else { 3176 ASSERT(status == IBT_AR_NOT_REGISTERED); 3177 } 3178 hdlp->ar_ref_link = NULL; 3179 hdlp->ar_ibt_hdl = ibt_hdl; 3180 new->ar_ibt_hdl_list = hdlp; 3181 new->ar = *arp; 3182 new->ar_flags = IBCM_AR_INITING; 3183 new->ar_waiters = 0; 3184 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL); 3185 new->ar_link = ibcm_ar_list; 3186 ibcm_ar_list = new; 3187 3188 /* verify GID/pkey is valid for a local port, etc. */ 3189 hcap = NULL; 3190 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port)) 3191 != IBT_SUCCESS || 3192 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port, 3193 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS || 3194 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) { 3195 cv_destroy(&new->ar_cv); 3196 ibcm_ar_list = new->ar_link; 3197 mutex_exit(&ibcm_svc_info_lock); 3198 kmem_free(new, sizeof (*new)); 3199 kmem_free(hdlp, sizeof (*hdlp)); 3200 status = IBT_INVALID_PARAM; 3201 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) { 3202 s = "PORT DOWN"; 3203 status = IBT_HCA_PORT_NOT_ACTIVE; 3204 } else if (s1 != IBT_SUCCESS) 3205 s = "GID not found"; 3206 else if (s2 != IBT_SUCCESS) 3207 s = "PKEY not found"; 3208 else 3209 s = "CM could not find its HCA entry"; 3210 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d", 3211 s, status); 3212 return (status); 3213 } 3214 mutex_exit(&ibcm_svc_info_lock); 3215 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port); 3216 3217 /* create service record */ 3218 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP); 3219 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */ 3220 srv_recp->ServiceP_Key = arp->ar_pkey; 3221 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */ 3222 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */ 3223 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME); 3224 srv_recp->ServiceGID = arp->ar_gid; 3225 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES); 3226 srv_recp->ServiceID = IBCM_DAPL_ATS_SID; 3227 3228 /* insert service record into the SA */ 3229 3230 IBCM_DUMP_SERVICE_REC(srv_recp); 3231 3232 if (saa_handle != NULL) 3233 status = ibcm_write_service_record(saa_handle, srv_recp, 3234 IBMF_SAA_UPDATE); 3235 else 3236 status = IBT_HCA_PORT_NOT_ACTIVE; 3237 3238 if (status != IBT_SUCCESS) { 3239 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, " 3240 "sid %llX", status, (longlong_t)srv_recp->ServiceID); 3241 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid " 3242 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix, 3243 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3244 3245 kmem_free(srv_recp, sizeof (*srv_recp)); 3246 kmem_free(hdlp, sizeof (*hdlp)); 3247 3248 mutex_enter(&ibcm_svc_info_lock); 3249 linkp = &ibcm_ar_list; 3250 tmp = *linkp; 3251 while (tmp != NULL) { 3252 if (tmp == new) { 3253 *linkp = new->ar_link; 3254 break; 3255 } 3256 linkp = &tmp->ar_link; 3257 tmp = *linkp; 3258 } 3259 if (new->ar_waiters > 0) { 3260 new->ar_flags = IBCM_AR_FAILED; 3261 cv_broadcast(&new->ar_cv); 3262 mutex_exit(&ibcm_svc_info_lock); 3263 } else { 3264 cv_destroy(&new->ar_cv); 3265 mutex_exit(&ibcm_svc_info_lock); 3266 kmem_free(new, sizeof (*new)); 3267 } 3268 ibcm_dec_hca_acc_cnt(hcap); 3269 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3270 "IBMF_SAA failed to write address record"); 3271 } else { /* SUCCESS */ 3272 uint8_t *b; 3273 3274 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: SUCCESS for gid " 3275 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix, 3276 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3277 b = arp->ar_data; 3278 3279 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar:" 3280 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3281 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3282 b[10], b[11], b[12], b[13], b[14], b[15]); 3283 mutex_enter(&ibcm_svc_info_lock); 3284 new->ar_srv_recp = srv_recp; 3285 new->ar_saa_handle = saa_handle; 3286 new->ar_port = cm_port.hp_port; 3287 new->ar_hcap = hcap; 3288 new->ar_flags = IBCM_AR_SUCCESS; 3289 if (new->ar_waiters > 0) 3290 cv_broadcast(&new->ar_cv); 3291 mutex_exit(&ibcm_svc_info_lock); 3292 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3293 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */ 3294 } 3295 return (status); 3296 } 3297 3298 ibt_status_t 3299 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3300 { 3301 ibcm_ar_t *found; 3302 ibcm_ar_t *tmp; 3303 ibcm_ar_t **linkp; 3304 ibcm_ar_ref_t *hdlp; 3305 ibcm_ar_ref_t **hdlpp; 3306 ibt_status_t status; 3307 ibmf_saa_handle_t saa_handle; 3308 sa_service_record_t *srv_recp; 3309 uint64_t gid_ored; 3310 3311 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey); 3312 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: gid %llx:%llx", 3313 (longlong_t)arp->ar_gid.gid_prefix, 3314 (longlong_t)arp->ar_gid.gid_guid); 3315 3316 /* 3317 * If P_Key is 0, but GID is not, this query is invalid. 3318 * If GID is 0, but P_Key is not, this query is invalid. 3319 */ 3320 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3321 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3322 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3323 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3324 "GID/P_Key is not valid"); 3325 return (IBT_INVALID_PARAM); 3326 } 3327 3328 mutex_enter(&ibcm_svc_info_lock); 3329 /* search for existing GID/pkey (there can be at most 1) */ 3330 status = ibcm_search_ar(arp, &found); 3331 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) { 3332 mutex_exit(&ibcm_svc_info_lock); 3333 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3334 "address record not found"); 3335 return (IBT_AR_NOT_REGISTERED); 3336 } 3337 ASSERT(status == IBT_SUCCESS); 3338 3339 hdlpp = &found->ar_ibt_hdl_list; 3340 hdlp = *hdlpp; 3341 while (hdlp != NULL) { 3342 if (hdlp->ar_ibt_hdl == ibt_hdl) 3343 break; 3344 hdlpp = &hdlp->ar_ref_link; 3345 hdlp = *hdlpp; 3346 } 3347 if (hdlp == NULL) { /* could not find ibt_hdl on list */ 3348 mutex_exit(&ibcm_svc_info_lock); 3349 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3350 "address record found, but not for this client"); 3351 return (IBT_AR_NOT_REGISTERED); 3352 } 3353 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */ 3354 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) { 3355 /* last entry was removed */ 3356 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */ 3357 saa_handle = found->ar_saa_handle; 3358 srv_recp = found->ar_srv_recp; 3359 3360 /* wait if this service record is being rewritten */ 3361 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY) 3362 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3363 mutex_exit(&ibcm_svc_info_lock); 3364 3365 /* remove service record */ 3366 status = ibcm_write_service_record(saa_handle, srv_recp, 3367 IBMF_SAA_DELETE); 3368 if (status != IBT_SUCCESS) 3369 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3370 "IBMF_SAA failed to delete address record"); 3371 mutex_enter(&ibcm_svc_info_lock); 3372 if (found->ar_waiters == 0) { /* still no waiters */ 3373 linkp = &ibcm_ar_list; 3374 tmp = *linkp; 3375 while (tmp != found) { 3376 linkp = &tmp->ar_link; 3377 tmp = *linkp; 3378 } 3379 *linkp = tmp->ar_link; 3380 ibcm_dec_hca_acc_cnt(found->ar_hcap); 3381 kmem_free(srv_recp, sizeof (*srv_recp)); 3382 cv_destroy(&found->ar_cv); 3383 kmem_free(found, sizeof (*found)); 3384 } else { 3385 /* add service record back in for the waiters */ 3386 mutex_exit(&ibcm_svc_info_lock); 3387 status = ibcm_write_service_record(saa_handle, srv_recp, 3388 IBMF_SAA_UPDATE); 3389 mutex_enter(&ibcm_svc_info_lock); 3390 if (status == IBT_SUCCESS) 3391 found->ar_flags = IBCM_AR_SUCCESS; 3392 else { 3393 found->ar_flags = IBCM_AR_FAILED; 3394 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3395 "IBMF_SAA failed to write address record"); 3396 } 3397 cv_broadcast(&found->ar_cv); 3398 } 3399 } 3400 mutex_exit(&ibcm_svc_info_lock); 3401 kmem_free(hdlp, sizeof (*hdlp)); 3402 ibtl_cm_change_service_cnt(ibt_hdl, -1); 3403 return (status); 3404 } 3405 3406 ibt_status_t 3407 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp) 3408 { 3409 sa_service_record_t svcrec_req; 3410 sa_service_record_t *svcrec_resp; 3411 void *results_p; 3412 uint64_t component_mask = 0; 3413 uint64_t gid_ored; 3414 size_t length; 3415 int num_rec; 3416 int i; 3417 ibmf_saa_access_args_t access_args; 3418 ibt_status_t retval; 3419 ibtl_cm_hca_port_t cm_port; 3420 ibcm_hca_info_t *hcap; 3421 ibmf_saa_handle_t saa_handle; 3422 3423 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp); 3424 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx", 3425 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid); 3426 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey); 3427 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx", 3428 (longlong_t)queryp->ar_gid.gid_prefix, 3429 (longlong_t)queryp->ar_gid.gid_guid); 3430 3431 /* 3432 * If P_Key is 0, but GID is not, this query is invalid. 3433 * If GID is 0, but P_Key is not, this query is invalid. 3434 */ 3435 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix; 3436 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) || 3437 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) { 3438 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid"); 3439 return (IBT_INVALID_PARAM); 3440 } 3441 3442 hcap = NULL; 3443 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS || 3444 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL || 3445 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) { 3446 if (hcap != NULL) 3447 ibcm_dec_hca_acc_cnt(hcap); 3448 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid"); 3449 return (IBT_INVALID_PARAM); 3450 } 3451 3452 bzero(&svcrec_req, sizeof (svcrec_req)); 3453 3454 /* Is GID/P_Key Specified. */ 3455 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */ 3456 svcrec_req.ServiceP_Key = queryp->ar_pkey; 3457 component_mask |= SA_SR_COMPMASK_PKEY; 3458 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X", 3459 queryp->ar_pkey); 3460 svcrec_req.ServiceGID = queryp->ar_gid; 3461 component_mask |= SA_SR_COMPMASK_GID; 3462 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX", 3463 (longlong_t)queryp->ar_gid.gid_prefix, 3464 (longlong_t)queryp->ar_gid.gid_guid); 3465 } 3466 3467 /* Is ServiceData Specified. */ 3468 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) { 3469 if (queryp->ar_data[i] != 0) { 3470 bcopy(queryp->ar_data, svcrec_req.ServiceData, 3471 IBCM_DAPL_ATS_NBYTES); 3472 component_mask |= 0xFFFF << 7; /* all 16 Data8 */ 3473 /* components */ 3474 break; 3475 } 3476 } 3477 3478 /* Service Name */ 3479 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME); 3480 component_mask |= SA_SR_COMPMASK_NAME; 3481 3482 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID; 3483 component_mask |= SA_SR_COMPMASK_ID; 3484 3485 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3486 "Perform SA Access: Mask: 0x%X", component_mask); 3487 3488 /* 3489 * Call in SA Access retrieve routine to get Service Records. 3490 * 3491 * SA Access framework allocated memory for the "results_p". 3492 * Make sure to deallocate once we are done with the results_p. 3493 * The size of the buffer allocated will be as returned in 3494 * "length" field. 3495 */ 3496 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 3497 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 3498 access_args.sq_component_mask = component_mask; 3499 access_args.sq_template = &svcrec_req; 3500 access_args.sq_template_length = sizeof (sa_service_record_t); 3501 access_args.sq_callback = NULL; 3502 access_args.sq_callback_arg = NULL; 3503 3504 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 3505 &results_p); 3506 3507 ibcm_dec_hca_acc_cnt(hcap); 3508 if (retval != IBT_SUCCESS) { 3509 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed"); 3510 return (retval); 3511 } 3512 3513 num_rec = length / sizeof (sa_service_record_t); 3514 3515 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3516 "Found %d Service Records.", num_rec); 3517 3518 /* Validate the returned number of records. */ 3519 if ((results_p != NULL) && (num_rec > 0)) { 3520 uint8_t *b; 3521 3522 /* Just return info from the first service record. */ 3523 svcrec_resp = (sa_service_record_t *)results_p; 3524 3525 /* The Service GID and Service ID */ 3526 resultp->ar_gid = svcrec_resp->ServiceGID; 3527 resultp->ar_pkey = svcrec_resp->ServiceP_Key; 3528 bcopy(svcrec_resp->ServiceData, 3529 resultp->ar_data, IBCM_DAPL_ATS_NBYTES); 3530 3531 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3532 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey, 3533 (longlong_t)resultp->ar_gid.gid_prefix, 3534 (longlong_t)resultp->ar_gid.gid_guid); 3535 b = resultp->ar_data; 3536 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar:" 3537 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3538 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3539 b[10], b[11], b[12], b[13], b[14], b[15]); 3540 3541 /* Deallocate the memory for results_p. */ 3542 kmem_free(results_p, length); 3543 if (num_rec > 1) 3544 retval = IBT_MULTIPLE_AR; 3545 else 3546 retval = IBT_SUCCESS; 3547 } else { 3548 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3549 "ibmf_sa_access found 0 matching records"); 3550 retval = IBT_AR_NOT_REGISTERED; 3551 } 3552 return (retval); 3553 } 3554 3555 /* mark all ATS service records associated with the port */ 3556 static void 3557 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port) 3558 { 3559 ibcm_ar_t *tmp; 3560 3561 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3562 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3563 if (tmp->ar_hcap->hca_guid == hca_guid && 3564 tmp->ar_port == port) { 3565 /* even if it's busy, we mark it for rewrite */ 3566 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED; 3567 } 3568 } 3569 } 3570 3571 /* rewrite all ATS service records */ 3572 static int 3573 ibcm_rewrite_ar(void) 3574 { 3575 ibcm_ar_t *tmp; 3576 ibmf_saa_handle_t saa_handle; 3577 sa_service_record_t *srv_recp; 3578 ibt_status_t rval; 3579 int did_something = 0; 3580 3581 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3582 check_for_work: 3583 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3584 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) { 3585 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY; 3586 saa_handle = tmp->ar_saa_handle; 3587 srv_recp = tmp->ar_srv_recp; 3588 mutex_exit(&ibcm_svc_info_lock); 3589 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: " 3590 "rewriting ar @ %p", tmp); 3591 did_something = 1; 3592 rval = ibcm_write_service_record(saa_handle, srv_recp, 3593 IBMF_SAA_UPDATE); 3594 if (rval != IBT_SUCCESS) 3595 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: " 3596 "ibcm_write_service_record failed: " 3597 "status = %d", rval); 3598 mutex_enter(&ibcm_svc_info_lock); 3599 /* if it got marked again, then we want to rewrite */ 3600 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY) 3601 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE; 3602 /* in case there was a waiter... */ 3603 cv_broadcast(&ibcm_svc_info_cv); 3604 goto check_for_work; 3605 } 3606 } 3607 return (did_something); 3608 } 3609 3610 static void 3611 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp) 3612 { 3613 ibcm_hca_info_t *hcap; 3614 ib_svc_id_t sid, start_sid, end_sid; 3615 ibmf_saa_handle_t saa_handle; 3616 sa_service_record_t srv_rec; 3617 ibt_status_t rval; 3618 3619 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 3620 if (hcap == NULL) { 3621 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3622 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid); 3623 return; 3624 } 3625 3626 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port); 3627 if (saa_handle == NULL) { 3628 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3629 "saa_handle is NULL"); 3630 ibcm_dec_hca_acc_cnt(hcap); 3631 return; 3632 } 3633 3634 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: " 3635 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name, 3636 sbindp->sbind_gid.gid_guid); 3637 3638 bzero(&srv_rec, sizeof (srv_rec)); 3639 3640 srv_rec.ServiceLease = sbindp->sbind_lease; 3641 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 3642 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 3643 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 3644 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 3645 srv_rec.ServiceGID = sbindp->sbind_gid; 3646 3647 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 3648 3649 /* insert srv record into the SA */ 3650 start_sid = srv_hdl->svc_id; 3651 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 3652 for (sid = start_sid; sid <= end_sid; sid++) { 3653 srv_rec.ServiceID = sid; 3654 3655 rval = ibcm_write_service_record(saa_handle, &srv_rec, 3656 IBMF_SAA_UPDATE); 3657 3658 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: " 3659 "ibcm_write_service_record, SvcId = %llX, " 3660 "rval = %d", (longlong_t)sid, rval); 3661 if (rval != IBT_SUCCESS) { 3662 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:" 3663 " ibcm_write_service_record fails %d sid %llX", 3664 rval, (longlong_t)sid); 3665 } 3666 } 3667 ibcm_dec_hca_acc_cnt(hcap); 3668 } 3669 3670 /* 3671 * Task to mark all service records as needing to be rewritten to the SM/SA. 3672 * This task does not return until all of them have been rewritten. 3673 */ 3674 void 3675 ibcm_service_record_rewrite_task(void *arg) 3676 { 3677 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg; 3678 ib_guid_t hca_guid = pup->pup_hca_guid; 3679 uint8_t port = pup->pup_port; 3680 ibcm_svc_info_t *svcp; 3681 ibcm_svc_bind_t *sbp; 3682 avl_tree_t *avl_tree = &ibcm_svc_avl_tree; 3683 static int task_is_running = 0; 3684 3685 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task STARTED " 3686 "for hca_guid %llX, port %d", hca_guid, port); 3687 3688 mutex_enter(&ibcm_svc_info_lock); 3689 ibcm_mark_ar(hca_guid, port); 3690 for (svcp = avl_first(avl_tree); svcp != NULL; 3691 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3692 sbp = svcp->svc_bind_list; 3693 while (sbp != NULL) { 3694 if (sbp->sbind_pkey != 0 && 3695 sbp->sbind_port == port && 3696 sbp->sbind_hcaguid == hca_guid) { 3697 /* even if it's busy, we mark it for rewrite */ 3698 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED; 3699 } 3700 sbp = sbp->sbind_link; 3701 } 3702 } 3703 if (task_is_running) { 3704 /* let the other task thread finish the work */ 3705 mutex_exit(&ibcm_svc_info_lock); 3706 return; 3707 } 3708 task_is_running = 1; 3709 3710 (void) ibcm_rewrite_ar(); 3711 3712 check_for_work: 3713 for (svcp = avl_first(avl_tree); svcp != NULL; 3714 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3715 sbp = svcp->svc_bind_list; 3716 while (sbp != NULL) { 3717 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) { 3718 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY; 3719 mutex_exit(&ibcm_svc_info_lock); 3720 ibcm_rewrite_svc_record(svcp, sbp); 3721 mutex_enter(&ibcm_svc_info_lock); 3722 /* if it got marked again, we want to rewrite */ 3723 if (sbp->sbind_rewrite_state == 3724 IBCM_REWRITE_BUSY) 3725 sbp->sbind_rewrite_state = 3726 IBCM_REWRITE_IDLE; 3727 /* in case there was a waiter... */ 3728 cv_broadcast(&ibcm_svc_info_cv); 3729 goto check_for_work; 3730 } 3731 sbp = sbp->sbind_link; 3732 } 3733 } 3734 /* 3735 * If there were no service records to write, and we failed to 3736 * have to rewrite any more ATS service records, then we're done. 3737 */ 3738 if (ibcm_rewrite_ar() != 0) 3739 goto check_for_work; 3740 task_is_running = 0; 3741 mutex_exit(&ibcm_svc_info_lock); 3742 3743 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task DONE"); 3744 kmem_free(pup, sizeof (ibcm_port_up_t)); 3745 } 3746 3747 3748 /* 3749 * Function: 3750 * ibt_cm_proceed 3751 * 3752 * Verifies the arguments and dispatches the cm state machine processing 3753 * via taskq 3754 */ 3755 3756 ibt_status_t 3757 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id, 3758 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3759 void *priv_data, ibt_priv_data_len_t priv_data_len) 3760 { 3761 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3762 ibcm_proceed_targs_t *proceed_targs; 3763 ibcm_proceed_error_t proceed_error; 3764 3765 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x " 3766 "session_id %p", statep->channel, event, status, session_id); 3767 3768 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, " 3769 "priv_data %p priv_data_len %x", statep->channel, cm_event_data, 3770 priv_data, priv_data_len); 3771 3772 /* validate session_id and status */ 3773 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3774 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args"); 3775 return (IBT_INVALID_PARAM); 3776 } 3777 3778 /* If priv data len specified, then priv_data cannot be NULL */ 3779 if ((priv_data_len > 0) && (priv_data == NULL)) 3780 return (IBT_INVALID_PARAM); 3781 3782 proceed_error = IBCM_PROCEED_INVALID_NONE; 3783 3784 mutex_enter(&statep->state_mutex); 3785 if (event == IBT_CM_EVENT_REQ_RCV) { 3786 3787 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3788 (statep->state != IBCM_STATE_MRA_SENT)) 3789 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3790 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ) 3791 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3792 3793 } else if (event == IBT_CM_EVENT_REP_RCV) { 3794 if ((statep->state != IBCM_STATE_REP_RCVD) && 3795 (statep->state != IBCM_STATE_MRA_REP_SENT)) 3796 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3797 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ) 3798 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3799 } else if (event == IBT_CM_EVENT_LAP_RCV) { 3800 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) && 3801 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT)) 3802 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3803 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ) 3804 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3805 } else if (event == IBT_CM_EVENT_CONN_CLOSED) { 3806 if (statep->state != IBCM_STATE_DREQ_RCVD) 3807 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3808 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ) 3809 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3810 } else { 3811 proceed_error = IBCM_PROCEED_INVALID_EVENT; 3812 } 3813 3814 /* if there is an error, print an error message and return */ 3815 if (proceed_error != IBCM_PROCEED_INVALID_NONE) { 3816 mutex_exit(&statep->state_mutex); 3817 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) { 3818 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3819 "Invalid Event/State combination specified", 3820 statep->channel); 3821 return (IBT_INVALID_PARAM); 3822 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) { 3823 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3824 "Invalid Event/priv len combination specified", 3825 statep->channel); 3826 return (IBT_INVALID_PARAM); 3827 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) { 3828 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3829 "Invalid Event specified", statep->channel); 3830 return (IBT_INVALID_PARAM); 3831 } else { 3832 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP); 3833 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3834 "IBT_CM_EVENT_LAP_RCV not supported", 3835 statep->channel); 3836 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */ 3837 return (IBT_APM_NOT_SUPPORTED); 3838 } 3839 } 3840 3841 3842 /* wait until client's CM handler returns DEFER status back to CM */ 3843 3844 while (statep->clnt_proceed == IBCM_BLOCK) { 3845 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for " 3846 "return of client's cm handler", statep->channel); 3847 cv_wait(&statep->block_client_cv, &statep->state_mutex); 3848 } 3849 3850 if (statep->clnt_proceed == IBCM_FAIL) { 3851 mutex_exit(&statep->state_mutex); 3852 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as " 3853 "client returned non-DEFER status from cm handler", 3854 statep->channel); 3855 return (IBT_CHAN_STATE_INVALID); 3856 } 3857 3858 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK); 3859 statep->clnt_proceed = IBCM_FAIL; 3860 mutex_exit(&statep->state_mutex); 3861 3862 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3863 3864 /* the state machine processing is done in a separate thread */ 3865 3866 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 3867 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 3868 KM_SLEEP); 3869 3870 proceed_targs->event = event; 3871 proceed_targs->status = status; 3872 proceed_targs->priv_data_len = priv_data_len; 3873 3874 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 3875 3876 proceed_targs->tst.rc.statep = statep; 3877 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data, 3878 sizeof (ibt_cm_proceed_reply_t)); 3879 3880 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3881 3882 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq, 3883 proceed_targs, TQ_SLEEP); 3884 3885 return (IBT_SUCCESS); 3886 } 3887 3888 /* 3889 * Function: 3890 * ibcm_proceed_via_taskq 3891 * 3892 * Called from taskq, dispatched by ibt_cm_proceed 3893 * Completes the cm state processing for ibt_cm_proceed 3894 */ 3895 void 3896 ibcm_proceed_via_taskq(void *targs) 3897 { 3898 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 3899 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep; 3900 ibt_cm_reason_t reject_reason; 3901 uint8_t arej_len; 3902 ibcm_status_t response; 3903 ibcm_clnt_reply_info_t clnt_info; 3904 3905 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data; 3906 clnt_info.priv_data = proceed_targs->priv_data; 3907 clnt_info.priv_data_len = proceed_targs->priv_data_len; 3908 3909 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x", 3910 statep->channel, targs); 3911 3912 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) { 3913 response = 3914 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status, 3915 &clnt_info, &reject_reason, &arej_len, 3916 (ibcm_req_msg_t *)statep->defer_cm_msg); 3917 3918 ibcm_handle_cep_req_response(statep, response, reject_reason, 3919 arej_len); 3920 3921 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) { 3922 response = 3923 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status, 3924 &clnt_info, &reject_reason, &arej_len, 3925 (ibcm_rep_msg_t *)statep->defer_cm_msg); 3926 3927 ibcm_handle_cep_rep_response(statep, response, reject_reason, 3928 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg); 3929 3930 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) { 3931 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status, 3932 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg, 3933 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg)); 3934 3935 ibcm_post_apr_mad(statep); 3936 3937 } else { 3938 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED); 3939 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data, 3940 proceed_targs->priv_data_len); 3941 } 3942 3943 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 3944 } 3945 3946 /* 3947 * Function: 3948 * ibt_cm_ud_proceed 3949 * 3950 * Verifies the arguments and dispatches the cm state machine processing 3951 * via taskq 3952 */ 3953 ibt_status_t 3954 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel, 3955 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop, 3956 void *priv_data, ibt_priv_data_len_t priv_data_len) 3957 { 3958 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id; 3959 ibcm_proceed_targs_t *proceed_targs; 3960 ibt_qp_query_attr_t qp_attr; 3961 ibt_status_t retval; 3962 3963 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p " 3964 "ud_channel %p ", session_id, ud_channel); 3965 3966 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p " 3967 "priv_data_len %x", status, priv_data, priv_data_len); 3968 3969 /* validate session_id and status */ 3970 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) { 3971 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args"); 3972 return (IBT_INVALID_PARAM); 3973 } 3974 3975 /* If priv data len specified, then priv_data cannot be NULL */ 3976 if ((priv_data_len > 0) && (priv_data == NULL)) 3977 return (IBT_INVALID_PARAM); 3978 3979 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ) 3980 return (IBT_INVALID_PARAM); 3981 3982 /* retrieve qpn and qkey from ud channel */ 3983 3984 /* validate event and statep's state */ 3985 3986 if (status == IBT_CM_ACCEPT) { 3987 retval = ibt_query_qp(ud_channel, &qp_attr); 3988 if ((retval != IBT_SUCCESS) || 3989 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) { 3990 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: " 3991 "Failed to retrieve QPN from the channel: %d", 3992 retval); 3993 return (IBT_INVALID_PARAM); 3994 } 3995 } 3996 3997 3998 mutex_enter(&ud_statep->ud_state_mutex); 3999 4000 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) { 4001 mutex_exit(&ud_statep->ud_state_mutex); 4002 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State " 4003 "specified"); 4004 return (IBT_INVALID_PARAM); 4005 } 4006 4007 /* wait until client's CM handler returns DEFER status back to CM */ 4008 4009 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) { 4010 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return" 4011 " of client's ud cm handler"); 4012 cv_wait(&ud_statep->ud_block_client_cv, 4013 &ud_statep->ud_state_mutex); 4014 } 4015 4016 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) { 4017 mutex_exit(&ud_statep->ud_state_mutex); 4018 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client " 4019 "returned non-DEFER status from cm handler"); 4020 return (IBT_INVALID_PARAM); 4021 } 4022 4023 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK); 4024 ud_statep->ud_clnt_proceed = IBCM_FAIL; 4025 mutex_exit(&ud_statep->ud_state_mutex); 4026 4027 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4028 4029 /* the state machine processing is done in a separate thread */ 4030 4031 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4032 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 4033 KM_SLEEP); 4034 4035 proceed_targs->status = status; 4036 proceed_targs->priv_data_len = priv_data_len; 4037 4038 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4039 4040 if (status == IBT_CM_ACCEPT) { 4041 proceed_targs->tst.ud.ud_qkey = 4042 qp_attr.qp_info.qp_transport.ud.ud_qkey; 4043 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn; 4044 } 4045 4046 proceed_targs->tst.ud.ud_statep = ud_statep; 4047 4048 /* copy redirect info based on status */ 4049 if (status == IBT_CM_REDIRECT) 4050 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info, 4051 sizeof (ibt_redirect_info_t)); 4052 4053 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4054 4055 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq, 4056 proceed_targs, TQ_SLEEP); 4057 4058 return (IBT_SUCCESS); 4059 } 4060 4061 /* 4062 * Function: 4063 * ibcm_ud_proceed_via_taskq 4064 * 4065 * Called from taskq, dispatched by ibt_cm_ud_proceed 4066 * Completes the cm state processing for ibt_cm_ud_proceed 4067 */ 4068 void 4069 ibcm_ud_proceed_via_taskq(void *targs) 4070 { 4071 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4072 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep; 4073 ibcm_ud_clnt_reply_info_t ud_clnt_info; 4074 ibt_sidr_status_t sidr_status; 4075 4076 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs); 4077 4078 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn; 4079 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey; 4080 ud_clnt_info.priv_data = proceed_targs->priv_data; 4081 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len; 4082 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info; 4083 4084 /* validate event and statep's state */ 4085 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status, 4086 &ud_clnt_info, &sidr_status, 4087 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg)); 4088 4089 ibcm_post_sidr_rep_mad(ud_statep, sidr_status); 4090 4091 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */ 4092 mutex_enter(&ud_statep->ud_state_mutex); 4093 IBCM_UD_REF_CNT_DECR(ud_statep); 4094 mutex_exit(&ud_statep->ud_state_mutex); 4095 4096 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4097 } 4098 4099 /* 4100 * Function: 4101 * ibt_set_alt_path 4102 * Input: 4103 * channel Channel handle returned from ibt_alloc_rc_channel(9F). 4104 * 4105 * mode Execute in blocking or non blocking mode. 4106 * 4107 * alt_path A pointer to an ibt_alt_path_info_t as returned from an 4108 * ibt_get_alt_path(9F) call that specifies the new 4109 * alternate path. 4110 * 4111 * priv_data A pointer to a buffer specified by caller for the 4112 * private data in the outgoing CM Load Alternate Path 4113 * (LAP) message sent to the remote host. This can be NULL 4114 * if no private data is available to communicate to the 4115 * remote node. 4116 * 4117 * priv_data_len Length of valid data in priv_data, this should be less 4118 * than or equal to IBT_LAP_PRIV_DATA_SZ. 4119 * 4120 * Output: 4121 * ret_args If called in blocking mode, points to a return argument 4122 * structure of type ibt_ap_returns_t. 4123 * 4124 * Returns: 4125 * IBT_SUCCESS on Success else appropriate error. 4126 * Description: 4127 * Load the specified alternate path. Causes the CM to send an LAP message 4128 * to the remote node. 4129 * Can only be called on a previously opened RC channel. 4130 */ 4131 ibt_status_t 4132 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 4133 ibt_alt_path_info_t *alt_path, void *priv_data, 4134 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args) 4135 { 4136 ibmf_handle_t ibmf_hdl; 4137 ibt_status_t status = IBT_SUCCESS; 4138 ibcm_lap_msg_t *lap_msgp; 4139 ibcm_hca_info_t *hcap; 4140 ibcm_state_data_t *statep; 4141 uint8_t port_no; 4142 ib_lid_t alternate_slid; 4143 ibt_priv_data_len_t len; 4144 ib_lid_t base_lid; 4145 boolean_t alt_grh; 4146 4147 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)", 4148 channel, mode, alt_path, priv_data, priv_data_len, ret_args); 4149 4150 /* validate channel */ 4151 if (IBCM_INVALID_CHANNEL(channel)) { 4152 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel"); 4153 return (IBT_CHAN_HDL_INVALID); 4154 } 4155 4156 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 4157 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4158 "Invalid Channel type: Applicable only to RC Channel"); 4159 return (IBT_CHAN_SRV_TYPE_INVALID); 4160 } 4161 4162 if (mode == IBT_NONBLOCKING) { 4163 if (ret_args != NULL) { 4164 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4165 "ret_args should be NULL when called in " 4166 "non-blocking mode"); 4167 return (IBT_INVALID_PARAM); 4168 } 4169 } else if (mode == IBT_BLOCKING) { 4170 if (ret_args == NULL) { 4171 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4172 "ret_args should be Non-NULL when called in " 4173 "blocking mode"); 4174 return (IBT_INVALID_PARAM); 4175 } 4176 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) { 4177 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4178 "expected private data length is too large"); 4179 return (IBT_INVALID_PARAM); 4180 } 4181 if ((ret_args->ap_priv_data_len > 0) && 4182 (ret_args->ap_priv_data == NULL)) { 4183 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4184 "apr_priv_data_len > 0, but apr_priv_data NULL"); 4185 return (IBT_INVALID_PARAM); 4186 } 4187 } else { /* any other mode is not valid for ibt_set_alt_path */ 4188 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4189 "invalid mode %x specified", mode); 4190 return (IBT_INVALID_PARAM); 4191 } 4192 4193 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0) 4194 return (IBT_INVALID_PARAM); 4195 4196 /* get the statep */ 4197 IBCM_GET_CHAN_PRIVATE(channel, statep); 4198 if (statep == NULL) { 4199 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL"); 4200 return (IBT_CM_FAILURE); 4201 } 4202 4203 mutex_enter(&statep->state_mutex); 4204 IBCM_RELEASE_CHAN_PRIVATE(channel); 4205 IBCM_REF_CNT_INCR(statep); 4206 mutex_exit(&statep->state_mutex); 4207 4208 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep); 4209 4210 hcap = statep->hcap; 4211 4212 /* HCA must have been in active state. If not, it's a client bug */ 4213 if (!IBCM_ACCESS_HCA_OK(hcap)) 4214 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state"); 4215 4216 ASSERT(statep->cm_handler != NULL); 4217 4218 /* Check Alternate port */ 4219 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL, 4220 &base_lid); 4221 if (status != IBT_SUCCESS) { 4222 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4223 "ibt_get_port_state_byguid status %d ", status); 4224 mutex_enter(&statep->state_mutex); 4225 IBCM_REF_CNT_DECR(statep); 4226 mutex_exit(&statep->state_mutex); 4227 return (status); 4228 } 4229 4230 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 4231 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 4232 != IBT_SUCCESS)) { 4233 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4234 "ibmf reg or callback setup failed during re-initialize"); 4235 mutex_enter(&statep->state_mutex); 4236 IBCM_REF_CNT_DECR(statep); 4237 mutex_exit(&statep->state_mutex); 4238 return (status); 4239 } 4240 4241 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl; 4242 4243 alternate_slid = base_lid + 4244 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path; 4245 4246 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x", 4247 h2b16(alternate_slid)); 4248 4249 ibcm_lapr_enter(); /* limit how many run simultaneously */ 4250 4251 /* Allocate MAD for LAP */ 4252 if (statep->lapr_msg == NULL) 4253 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg, 4254 MAD_METHOD_SEND)) != IBT_SUCCESS) { 4255 ibcm_lapr_exit(); 4256 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4257 "chan 0x%p ibcm_alloc_out_msg failed", channel); 4258 mutex_enter(&statep->state_mutex); 4259 IBCM_REF_CNT_DECR(statep); 4260 mutex_exit(&statep->state_mutex); 4261 return (status); 4262 } 4263 4264 mutex_enter(&statep->state_mutex); 4265 4266 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is" 4267 " %x", statep->state); 4268 4269 /* Check state */ 4270 if ((statep->state != IBCM_STATE_ESTABLISHED) || 4271 (statep->ap_state != IBCM_AP_STATE_IDLE)) { 4272 IBCM_REF_CNT_DECR(statep); 4273 mutex_exit(&statep->state_mutex); 4274 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg); 4275 ibcm_lapr_exit(); 4276 return (IBT_CHAN_STATE_INVALID); 4277 } else { 4278 /* Set to LAP Sent state */ 4279 statep->ap_state = IBCM_AP_STATE_LAP_SENT; 4280 statep->ap_done = B_FALSE; 4281 statep->remaining_retry_cnt = statep->max_cm_retries; 4282 statep->timer_stored_state = statep->state; 4283 statep->timer_stored_ap_state = statep->ap_state; 4284 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */ 4285 } 4286 4287 mutex_exit(&statep->state_mutex); 4288 4289 /* No more failure returns below */ 4290 4291 /* Allocate MAD for LAP */ 4292 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:" 4293 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg)); 4294 4295 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4296 4297 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg); 4298 4299 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid); 4300 lap_msgp->lap_alt_r_port_lid = 4301 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid); 4302 4303 /* Fill in remote port gid */ 4304 lap_msgp->lap_alt_r_port_gid.gid_prefix = 4305 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix); 4306 lap_msgp->lap_alt_r_port_gid.gid_guid = 4307 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid); 4308 4309 /* Fill in local port gid */ 4310 lap_msgp->lap_alt_l_port_gid.gid_prefix = 4311 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix); 4312 lap_msgp->lap_alt_l_port_gid.gid_guid = 4313 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid); 4314 4315 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh; 4316 4317 /* alternate_flow_label, and alternate srate, alternate traffic class */ 4318 lap_msgp->lap_alt_srate_plus = 4319 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f; 4320 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ? 4321 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) | 4322 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass); 4323 4324 /* Alternate hop limit, service level */ 4325 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ? 4326 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 0xff; 4327 lap_msgp->lap_alt_sl_plus = 4328 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 | 4329 ((alt_grh == B_FALSE) ? 0x8 : 0); 4330 4331 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib( 4332 (2 * statep->rc_alt_pkt_lt) + 4333 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 4334 4335 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid); 4336 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid); 4337 4338 lap_msgp->lap_remote_qpn_eecn_plus = 4339 h2b32((statep->remote_qpn << 8) | 4340 ibt_usec2ib(ibcm_remote_response_time) << 3); 4341 4342 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ); 4343 if ((len > 0) && priv_data) { 4344 bcopy(priv_data, lap_msgp->lap_private_data, len); 4345 } 4346 4347 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */ 4348 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 4349 4350 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt); 4351 4352 /* return_data is filled up in the state machine code */ 4353 statep->ap_return_data = ret_args; 4354 4355 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 4356 4357 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID = 4358 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID); 4359 4360 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID = 4361 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid, 4362 0)); 4363 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx", 4364 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID); 4365 4366 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4367 4368 /* Send LAP */ 4369 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete, 4370 statep); 4371 4372 mutex_enter(&statep->state_mutex); 4373 4374 if (mode == IBT_BLOCKING) { 4375 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking"); 4376 4377 /* wait for APR */ 4378 while (statep->ap_done != B_TRUE) { 4379 cv_wait(&statep->block_client_cv, 4380 &statep->state_mutex); 4381 } 4382 4383 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking"); 4384 4385 /* 4386 * In the case that ibt_set_alt_path fails, 4387 * change retval to IBT_CM_FAILURE 4388 */ 4389 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED) 4390 status = IBT_CM_FAILURE; 4391 4392 } 4393 4394 /* decrement the ref-count before leaving here */ 4395 IBCM_REF_CNT_DECR(statep); 4396 4397 mutex_exit(&statep->state_mutex); 4398 4399 ibcm_lapr_exit(); 4400 4401 /* If this message isn't seen then ibt_set_alt_path failed */ 4402 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done"); 4403 4404 return (status); 4405 } 4406 4407 4408 #ifdef DEBUG 4409 4410 /* 4411 * ibcm_query_classport_info: 4412 * Query classportinfo 4413 * 4414 * INPUTS: 4415 * channel - Channel that is associated with a statep 4416 * 4417 * RETURN VALUE: NONE 4418 * This function is currently used to generate a valid get method classport 4419 * info, and test CM functionality. There is no ibtl client interface to 4420 * generate a classportinfo. It is possible that CM may use classportinfo 4421 * from other nodes in the future, and most of the code below could be re-used. 4422 */ 4423 void 4424 ibcm_query_classport_info(ibt_channel_hdl_t channel) 4425 { 4426 ibcm_state_data_t *statep; 4427 ibmf_msg_t *msgp; 4428 4429 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p)", channel); 4430 4431 /* validate channel, first */ 4432 if (IBCM_INVALID_CHANNEL(channel)) { 4433 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4434 "invalid channel (%p)", channel); 4435 return; 4436 } 4437 4438 /* get the statep */ 4439 IBCM_GET_CHAN_PRIVATE(channel, statep); 4440 4441 /* 4442 * This can happen, if the statep is already gone by a DREQ from 4443 * the remote side 4444 */ 4445 if (statep == NULL) { 4446 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4447 "statep NULL"); 4448 return; 4449 } 4450 4451 mutex_enter(&statep->state_mutex); 4452 IBCM_RELEASE_CHAN_PRIVATE(channel); 4453 IBCM_REF_CNT_INCR(statep); 4454 mutex_exit(&statep->state_mutex); 4455 4456 /* Debug/test code, so don't care about return status */ 4457 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp, 4458 MAD_METHOD_GET); 4459 4460 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid( 4461 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0)); 4462 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO); 4463 4464 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL); 4465 4466 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p) " 4467 "Get method MAD posted ", channel); 4468 4469 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp); 4470 4471 mutex_enter(&statep->state_mutex); 4472 IBCM_REF_CNT_DECR(statep); 4473 mutex_exit(&statep->state_mutex); 4474 } 4475 4476 static void 4477 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr) 4478 { 4479 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, " 4480 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid, 4481 cm_reply_addr->rcvd_addr.ia_remote_lid); 4482 4483 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, " 4484 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key, 4485 cm_reply_addr->rcvd_addr.ia_p_key, 4486 cm_reply_addr->rcvd_addr.ia_remote_qno, 4487 cm_reply_addr->rcvd_addr.ia_service_level); 4488 4489 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ", 4490 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix, 4491 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid); 4492 4493 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX", 4494 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix, 4495 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid); 4496 4497 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x", 4498 cm_reply_addr->grh_hdr.ig_flow_label, 4499 cm_reply_addr->grh_hdr.ig_tclass, 4500 cm_reply_addr->grh_hdr.ig_hop_limit); 4501 } 4502 4503 #endif 4504 4505 typedef struct ibcm_join_mcg_tqarg_s { 4506 ib_gid_t rgid; 4507 ibt_mcg_attr_t mcg_attr; 4508 ibt_mcg_info_t *mcg_infop; 4509 ibt_mcg_handler_t func; 4510 void *arg; 4511 } ibcm_join_mcg_tqarg_t; 4512 4513 _NOTE(READ_ONLY_DATA(ibcm_join_mcg_tqarg_s)) 4514 4515 /* 4516 * Function: 4517 * ibt_join_mcg 4518 * Input: 4519 * rgid The request GID that defines the HCA port from which a 4520 * contact to SA Access is performed to add the specified 4521 * endport GID ((mcg_attr->mc_pgid) to a multicast group. 4522 * If mcg_attr->mc_pgid is null, then this (rgid) will be 4523 * treated as endport GID that is to be added to the 4524 * multicast group. 4525 * 4526 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines 4527 * the attributes of the desired multicast group to be 4528 * created or joined. 4529 * 4530 * func NULL or a pointer to a function to call when 4531 * ibt_join_mcg() completes. If 'func' is not NULL then 4532 * ibt_join_mcg() will return as soon as possible after 4533 * initiating the multicast group join/create process. 4534 * 'func' is then called when the process completes. 4535 * 4536 * arg Argument to the 'func'. 4537 * 4538 * Output: 4539 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated 4540 * by the caller, where the attributes of the created or 4541 * joined multicast group are copied. 4542 * Returns: 4543 * IBT_SUCCESS 4544 * IBT_INVALID_PARAM 4545 * IBT_MCG_RECORDS_NOT_FOUND 4546 * IBT_INSUFF_RESOURCE 4547 * Description: 4548 * Join a multicast group. The first full member "join" causes the MCG 4549 * to be created. 4550 */ 4551 ibt_status_t 4552 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, 4553 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg) 4554 { 4555 ibcm_join_mcg_tqarg_t *mcg_tq; 4556 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP); 4557 4558 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix, 4559 rgid.gid_guid, mcg_attr); 4560 4561 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4562 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required"); 4563 return (IBT_INVALID_PARAM); 4564 } 4565 4566 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) || 4567 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) { 4568 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified"); 4569 return (IBT_INVALID_PARAM); 4570 } 4571 4572 if (mcg_attr->mc_join_state == 0) { 4573 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified"); 4574 return (IBT_INVALID_PARAM); 4575 } 4576 4577 if (mcg_info_p == NULL) { 4578 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL"); 4579 return (IBT_INVALID_PARAM); 4580 } 4581 4582 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4583 4584 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag); 4585 if (mcg_tq == NULL) { 4586 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: " 4587 "Unable to allocate memory for local usage."); 4588 return (IBT_INSUFF_KERNEL_RESOURCE); 4589 } 4590 4591 mcg_tq->rgid = rgid; 4592 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t)); 4593 mcg_tq->mcg_infop = mcg_info_p; 4594 mcg_tq->func = func; 4595 mcg_tq->arg = arg; 4596 4597 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4598 4599 if (func != NULL) { /* Non-Blocking */ 4600 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call"); 4601 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg, 4602 mcg_tq, TQ_NOSLEEP) == 0) { 4603 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to " 4604 "Dispatch the TaskQ"); 4605 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t)); 4606 return (IBT_INSUFF_KERNEL_RESOURCE); 4607 } else 4608 return (IBT_SUCCESS); 4609 } else { /* Blocking */ 4610 return (ibcm_process_join_mcg(mcg_tq)); 4611 } 4612 } 4613 4614 static void 4615 ibcm_process_async_join_mcg(void *tq_arg) 4616 { 4617 (void) ibcm_process_join_mcg(tq_arg); 4618 } 4619 4620 static ibt_status_t 4621 ibcm_process_join_mcg(void *taskq_arg) 4622 { 4623 sa_mcmember_record_t mcg_req; 4624 sa_mcmember_record_t *mcg_resp; 4625 ibmf_saa_access_args_t access_args; 4626 ibmf_saa_handle_t saa_handle; 4627 uint64_t component_mask = 0; 4628 ibt_status_t retval; 4629 ibtl_cm_hca_port_t hca_port; 4630 uint_t num_records; 4631 size_t length; 4632 ibcm_hca_info_t *hcap; 4633 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg; 4634 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop; 4635 4636 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg); 4637 4638 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port); 4639 if (retval != IBT_SUCCESS) { 4640 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get " 4641 "port info from specified RGID: status = %d", retval); 4642 goto ibcm_join_mcg_exit1; 4643 } 4644 4645 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4646 4647 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) || 4648 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) { 4649 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4650 "Request GID is Port GID"); 4651 mcg_req.PortGID = mcg_arg->rgid; 4652 } else { 4653 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid; 4654 } 4655 component_mask |= SA_MC_COMPMASK_PORTGID; 4656 4657 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey; 4658 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey; 4659 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state; 4660 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass; 4661 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow; 4662 mcg_req.SL = mcg_arg->mcg_attr.mc_sl; 4663 4664 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY | 4665 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS | 4666 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL; 4667 4668 /* If client has specified MGID, use it else SA will assign one. */ 4669 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4670 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid; 4671 component_mask |= SA_MC_COMPMASK_MGID; 4672 } 4673 4674 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: PGID=%llX:%llX, " 4675 "MGID=%llX:%llX", mcg_req.PortGID.gid_prefix, 4676 mcg_req.PortGID.gid_guid, mcg_req.MGID.gid_prefix, 4677 mcg_req.MGID.gid_guid); 4678 4679 /* Is MTU specified. */ 4680 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) { 4681 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu; 4682 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector; 4683 4684 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 4685 SA_MC_COMPMASK_MTU; 4686 } 4687 4688 /* Is RATE specified. */ 4689 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) { 4690 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate; 4691 mcg_req.RateSelector = 4692 mcg_arg->mcg_attr.mc_rate_req.r_selector; 4693 4694 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 4695 SA_MC_COMPMASK_RATE; 4696 } 4697 4698 /* Is Packet Life Time specified. */ 4699 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) { 4700 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt; 4701 mcg_req.RateSelector = 4702 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector; 4703 4704 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 4705 SA_MC_COMPMASK_PKTLT; 4706 } 4707 4708 if (mcg_arg->mcg_attr.mc_hop) { 4709 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop; 4710 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 4711 } 4712 4713 if (mcg_arg->mcg_attr.mc_scope) { 4714 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope; 4715 component_mask |= SA_MC_COMPMASK_SCOPE; 4716 } 4717 4718 if (mcg_arg->mcg_attr.mc_mlid) { 4719 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid; 4720 component_mask |= SA_MC_COMPMASK_MLID; 4721 } 4722 4723 /* Get SA Access Handle. */ 4724 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4725 if (hcap == NULL) { 4726 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found"); 4727 4728 retval = IBT_HCA_BUSY_DETACHING; 4729 goto ibcm_join_mcg_exit1; 4730 } 4731 4732 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4733 if (saa_handle == NULL) { 4734 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL"); 4735 4736 retval = IBT_HCA_PORT_NOT_ACTIVE; 4737 goto ibcm_join_mcg_exit; 4738 } 4739 4740 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) && 4741 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) { 4742 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0, 4743 &hca_port); 4744 if (retval != IBT_SUCCESS) { 4745 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed " 4746 "to get PortInfo of specified PGID: status = %d", 4747 retval); 4748 goto ibcm_join_mcg_exit1; 4749 } 4750 } 4751 4752 /* Contact SA Access */ 4753 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4754 access_args.sq_access_type = IBMF_SAA_UPDATE; 4755 access_args.sq_component_mask = component_mask; 4756 access_args.sq_template = &mcg_req; 4757 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4758 access_args.sq_callback = NULL; 4759 access_args.sq_callback_arg = NULL; 4760 4761 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 4762 (void **)&mcg_resp); 4763 if (retval != IBT_SUCCESS) { 4764 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: " 4765 "SA Access Failed"); 4766 goto ibcm_join_mcg_exit; 4767 } 4768 4769 num_records = length/sizeof (sa_mcmember_record_t); 4770 4771 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: " 4772 "Found %d MCMember Records", num_records); 4773 4774 /* Validate the returned number of records. */ 4775 if ((mcg_resp != NULL) && (num_records > 0)) { 4776 /* Update the return values. */ 4777 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID; 4778 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID; 4779 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate; 4780 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL; 4781 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel; 4782 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass; 4783 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit; 4784 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE; 4785 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID; 4786 mcg_info_p->mc_mtu = mcg_resp->MTU; 4787 mcg_info_p->mc_qkey = mcg_resp->Q_Key; 4788 4789 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid, 4790 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix); 4791 if (retval != IBT_SUCCESS) { 4792 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4793 "Pkey2Index Conversion failed<%d>", retval); 4794 mcg_info_p->mc_pkey_ix = 0; 4795 } 4796 4797 mcg_info_p->mc_scope = mcg_resp->Scope; 4798 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime; 4799 4800 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port; 4801 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix; 4802 mcg_info_p->mc_adds_vect.av_src_path = 0; 4803 4804 /* Deallocate the memory allocated by SA for mcg_resp. */ 4805 kmem_free(mcg_resp, length); 4806 retval = IBT_SUCCESS; 4807 } else { 4808 retval = IBT_MCG_RECORDS_NOT_FOUND; 4809 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4810 "MCG RECORDS NOT FOUND"); 4811 } 4812 4813 ibcm_join_mcg_exit: 4814 ibcm_dec_hca_acc_cnt(hcap); 4815 4816 ibcm_join_mcg_exit1: 4817 if (mcg_arg->func) 4818 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p); 4819 4820 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t)); 4821 4822 return (retval); 4823 } 4824 4825 4826 /* 4827 * Function: 4828 * ibt_leave_mcg 4829 * Input: 4830 * rgid The request GID that defines the HCA port upon which 4831 * to send the request to the Subnet Administrator, to 4832 * remove the specified port (port_gid) from the multicast 4833 * group. If 'port_gid' is the Reserved GID (i.e. 4834 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0), 4835 * then the end-port associated with 'rgid' is removed 4836 * from the multicast group. 4837 * 4838 * mc_gid A multicast group GID as returned from ibt_join_mcg() 4839 * call. This is optional, if not specified (i.e. 4840 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to 4841 * identify this as being a multicast GID), then the 4842 * port is removed from all the multicast groups of 4843 * which it is a member. 4844 * 4845 * port_gid This is optional, if not the Reserved GID (gid_prefix 4846 * and gid_guid not equal to 0), then this specifies the 4847 * endport GID of the multicast group member being deleted 4848 * from the group. If it is the Reserved GID (gid_prefix 4849 * and gid_guid equal to 0) then the member endport GID is 4850 * determined from 'rgid'. 4851 * 4852 * mc_join_state The Join State attribute used when the group was joined 4853 * using ibt_join_mcg(). This Join State component must 4854 * contains at least one bit set to 1 in the same position 4855 * as that used during ibt_join_mcg(). i.e. the logical 4856 * AND of the two JoinState components is not all zeros. 4857 * This Join State component must not have some bits set 4858 * which are not set using ibt_join_mcg(). 4859 * Output: 4860 * None. 4861 * Returns: 4862 * IBT_SUCCESS 4863 * IBT_INVALID_PARAM 4864 * IBT_MC_GROUP_INVALID 4865 * IBT_INSUFF_RESOURCE 4866 * Description: 4867 * The port associated with the port GID shall be removed from the 4868 * multicast group specified by MGID (mc_gid) or from all the multicast 4869 * groups of which it is a member if the MGID (mc_gid) is not specified. 4870 * 4871 * The last full member to leave causes the destruction of the Multicast 4872 * Group. 4873 */ 4874 ibt_status_t 4875 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid, 4876 uint8_t mc_join_state) 4877 { 4878 sa_mcmember_record_t mcg_req; 4879 ibmf_saa_access_args_t access_args; 4880 ibmf_saa_handle_t saa_handle; 4881 uint64_t component_mask = 0; 4882 int sa_retval; 4883 ibt_status_t retval; 4884 ibtl_cm_hca_port_t hca_port; 4885 size_t length; 4886 void *results_p; 4887 ibcm_hca_info_t *hcap; 4888 4889 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)", 4890 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid); 4891 4892 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)", 4893 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state); 4894 4895 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4896 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required"); 4897 return (IBT_INVALID_PARAM); 4898 } 4899 4900 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4901 4902 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX", 4903 mc_gid.gid_prefix, mc_gid.gid_guid); 4904 4905 /* Validate MGID */ 4906 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4907 mcg_req.MGID = mc_gid; 4908 component_mask |= SA_MC_COMPMASK_MGID; 4909 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) { 4910 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified"); 4911 return (IBT_MC_MGID_INVALID); 4912 } 4913 4914 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) { 4915 mcg_req.PortGID = rgid; 4916 } else { 4917 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave"); 4918 mcg_req.PortGID = port_gid; 4919 } 4920 component_mask |= SA_MC_COMPMASK_PORTGID; 4921 4922 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>", 4923 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 4924 4925 /* Join State */ 4926 mcg_req.JoinState = mc_join_state; 4927 component_mask |= SA_MC_COMPMASK_JOINSTATE; 4928 4929 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port); 4930 if (retval != IBT_SUCCESS) { 4931 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info " 4932 "from specified RGID : status = %d", retval); 4933 return (retval); 4934 } 4935 4936 /* Get SA Access Handle. */ 4937 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4938 if (hcap == NULL) { 4939 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: " 4940 "NO HCA found"); 4941 return (IBT_HCA_BUSY_DETACHING); 4942 } 4943 4944 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4945 if (saa_handle == NULL) { 4946 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL"); 4947 ibcm_dec_hca_acc_cnt(hcap); 4948 return (IBT_HCA_PORT_NOT_ACTIVE); 4949 } 4950 4951 /* Contact SA Access */ 4952 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4953 access_args.sq_access_type = IBMF_SAA_DELETE; 4954 access_args.sq_component_mask = component_mask; 4955 access_args.sq_template = &mcg_req; 4956 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4957 access_args.sq_callback = NULL; 4958 access_args.sq_callback_arg = NULL; 4959 4960 ibcm_sa_access_enter(); 4961 4962 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length, 4963 &results_p); 4964 if (sa_retval != IBMF_SUCCESS) { 4965 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d", 4966 sa_retval); 4967 (void) ibcm_ibmf_analyze_error(sa_retval); 4968 retval = IBT_MC_GROUP_INVALID; 4969 } 4970 4971 ibcm_sa_access_exit(); 4972 4973 ibcm_dec_hca_acc_cnt(hcap); 4974 4975 return (retval); 4976 } 4977 4978 4979 /* 4980 * Function: 4981 * ibt_query_mcg 4982 * Input: 4983 * rgid The request GID that defines the HCA port upon which 4984 * to send the request to the Subnet Administrator, to 4985 * retrieve Multicast Records matching attributes as 4986 * specified through 'mcg_attr' argument. 4987 * 4988 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that 4989 * specifies MCG attributes that are to be matched. 4990 * Attributes that are not required can be wild carded 4991 * by specifying as '0'. 4992 * 4993 * mcgs_max_num The maximum number of matching multicast groups to 4994 * return. If zero, then all available matching multicast 4995 * groups are returned. 4996 * Output: 4997 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where 4998 * multicast group information is returned. The actual 4999 * number of entries filled in the array is returned in 5000 * entries_p. 5001 * 5002 * entries_p The number of ibt_mcg_attr_t entries returned. 5003 * Returns: 5004 * IBT_SUCCESS 5005 * IBT_INVALID_PARAM 5006 * IBT_MCG_RECORDS_NOT_FOUND 5007 * Description: 5008 * Request information on multicast groups that match the parameters 5009 * specified in mcg_attr. Information on each multicast group is returned 5010 * to the caller in the form of an array of ibt_mcg_info_t. 5011 * ibt_query_mcg() allocates the memory for this array and returns a 5012 * pointer to the array (mcgs_p) and the number of entries in the array 5013 * (entries_p). This memory should be freed by the client using 5014 * ibt_free_mcg_info(). 5015 */ 5016 ibt_status_t 5017 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num, 5018 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p) 5019 { 5020 sa_mcmember_record_t mcg_req; 5021 sa_mcmember_record_t *mcg_resp; 5022 ibt_mcg_info_t *mcg_infop; 5023 ibmf_saa_access_args_t access_args; 5024 ibmf_saa_handle_t saa_handle; 5025 uint64_t component_mask = 0; 5026 ibt_status_t retval; 5027 ibtl_cm_hca_port_t hport; 5028 uint_t num_records; 5029 size_t length; 5030 void *results_p; 5031 ib_gid_t port_gid; 5032 ibcm_hca_info_t *hcap; 5033 5034 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num); 5035 5036 if ((entries_p == NULL) || (mcgs_info_p == NULL)) { 5037 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5038 "entries_p or mcgs_info_p is NULL"); 5039 return (IBT_INVALID_PARAM); 5040 } 5041 5042 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5043 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required"); 5044 return (IBT_INVALID_PARAM); 5045 } 5046 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>", 5047 rgid.gid_prefix, rgid.gid_guid); 5048 5049 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5050 port_gid.gid_prefix = port_gid.gid_guid = 0; 5051 5052 if (mcg_attr != NULL) { 5053 port_gid = mcg_attr->mc_pgid; 5054 5055 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5056 mcg_req.PortGID = mcg_attr->mc_pgid; 5057 component_mask |= SA_MC_COMPMASK_PORTGID; 5058 5059 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX", 5060 port_gid.gid_prefix, port_gid.gid_guid); 5061 } 5062 5063 /* Is Q_Key specified. */ 5064 if (mcg_attr->mc_qkey != 0) { 5065 mcg_req.Q_Key = mcg_attr->mc_qkey; 5066 component_mask |= SA_MC_COMPMASK_QKEY; 5067 } 5068 5069 /* Is P_Key specified. */ 5070 if (mcg_attr->mc_pkey != 0) { 5071 mcg_req.P_Key = mcg_attr->mc_pkey; 5072 component_mask |= SA_MC_COMPMASK_PKEY; 5073 } 5074 5075 /* Is MGID specified. */ 5076 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5077 mcg_req.MGID = mcg_attr->mc_mgid; 5078 component_mask |= SA_MC_COMPMASK_MGID; 5079 } 5080 5081 /* Is MTU specified. */ 5082 if (mcg_attr->mc_mtu_req.r_mtu) { 5083 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu; 5084 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector; 5085 5086 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 5087 SA_MC_COMPMASK_MTU; 5088 } 5089 5090 if (mcg_attr->mc_tclass) { 5091 mcg_req.TClass = mcg_attr->mc_tclass; 5092 component_mask |= SA_MC_COMPMASK_TCLASS; 5093 } 5094 5095 /* Is RATE specified. */ 5096 if (mcg_attr->mc_rate_req.r_srate) { 5097 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate; 5098 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector; 5099 5100 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 5101 SA_MC_COMPMASK_RATE; 5102 } 5103 5104 /* Is Packet Life Time specified. */ 5105 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) { 5106 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt; 5107 mcg_req.RateSelector = 5108 mcg_attr->mc_pkt_lt_req.p_selector; 5109 5110 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5111 SA_MC_COMPMASK_PKTLT; 5112 } 5113 5114 if (mcg_attr->mc_hop) { 5115 mcg_req.HopLimit = mcg_attr->mc_hop; 5116 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5117 } 5118 5119 if (mcg_attr->mc_flow) { 5120 mcg_req.FlowLabel = mcg_attr->mc_flow; 5121 component_mask |= SA_MC_COMPMASK_FLOWLABEL; 5122 } 5123 5124 if (mcg_attr->mc_sl) { 5125 mcg_req.SL = mcg_attr->mc_sl; 5126 component_mask |= SA_MC_COMPMASK_SL; 5127 } 5128 5129 if (mcg_attr->mc_scope) { 5130 mcg_req.Scope = mcg_attr->mc_scope; 5131 component_mask |= SA_MC_COMPMASK_SCOPE; 5132 } 5133 5134 if (mcg_attr->mc_join_state) { 5135 mcg_req.JoinState = mcg_attr->mc_join_state; 5136 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5137 } 5138 5139 if (mcg_attr->mc_mlid) { 5140 mcg_req.MLID = mcg_attr->mc_mlid; 5141 component_mask |= SA_MC_COMPMASK_MLID; 5142 } 5143 } 5144 5145 retval = ibtl_cm_get_hca_port(rgid, 0, &hport); 5146 if (retval != IBT_SUCCESS) { 5147 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info " 5148 "from specified RGID : status = %d", retval); 5149 return (retval); 5150 } 5151 5152 /* Get SA Access Handle. */ 5153 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5154 if (hcap == NULL) { 5155 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found"); 5156 return (IBT_HCA_BUSY_DETACHING); 5157 } 5158 5159 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port); 5160 if (saa_handle == NULL) { 5161 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL"); 5162 ibcm_dec_hca_acc_cnt(hcap); 5163 return (IBT_HCA_PORT_NOT_ACTIVE); 5164 } 5165 5166 /* Contact SA Access */ 5167 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5168 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 5169 access_args.sq_component_mask = component_mask; 5170 access_args.sq_template = &mcg_req; 5171 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5172 access_args.sq_callback = NULL; 5173 access_args.sq_callback_arg = NULL; 5174 5175 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5176 &results_p); 5177 if (retval != IBT_SUCCESS) { 5178 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed"); 5179 ibcm_dec_hca_acc_cnt(hcap); 5180 return (retval); 5181 } 5182 5183 num_records = length/sizeof (sa_mcmember_record_t); 5184 5185 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records", 5186 num_records); 5187 5188 /* Validate the returned number of records. */ 5189 if ((results_p != NULL) && (num_records > 0)) { 5190 uint_t i; 5191 5192 /* 5193 * If mcgs_max_num is zero, then return all records else 5194 * return only requested number of records 5195 */ 5196 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) { 5197 /* we are interested in only mcgs_max_num records */ 5198 num_records = mcgs_max_num; 5199 } 5200 5201 /* 5202 * The SGID returned in "mcg_info_p" buffer should be PortGID, 5203 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified, 5204 * else RequestGID (rgid) should be returned. 5205 */ 5206 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5207 5208 /* Get sgid_ix and port number of 'port_gid' */ 5209 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport); 5210 if (retval != IBT_SUCCESS) { 5211 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5212 "Failed to Get Portinfo for PortGID :" 5213 "status = %d", retval); 5214 return (retval); 5215 } 5216 } else { 5217 /* 5218 * The sgid_ix and port number related to RequestGID 5219 * are already obtained at the beginning. 5220 */ 5221 port_gid = rgid; 5222 } 5223 5224 /* 5225 * Allocate memory for return buffer, to be freed in 5226 * ibt_free_mcg_info(). 5227 */ 5228 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)), 5229 KM_SLEEP); 5230 5231 *mcgs_info_p = mcg_infop; 5232 *entries_p = num_records; 5233 5234 /* Update the return values. */ 5235 for (i = 0; i < num_records; i++) { 5236 5237 mcg_resp = (sa_mcmember_record_t *)((uchar_t *) 5238 results_p + i * sizeof (sa_mcmember_record_t)); 5239 5240 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID; 5241 mcg_infop[i].mc_adds_vect.av_sgid = port_gid; 5242 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate; 5243 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL; 5244 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5245 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass; 5246 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit; 5247 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port; 5248 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE; 5249 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID; 5250 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix; 5251 mcg_infop[i].mc_adds_vect.av_src_path = 0; 5252 mcg_infop[i].mc_mtu = mcg_resp->MTU; 5253 mcg_infop[i].mc_qkey = mcg_resp->Q_Key; 5254 mcg_infop[i].mc_scope = mcg_resp->Scope; 5255 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime; 5256 5257 if (ibt_pkey2index_byguid(hport.hp_hca_guid, 5258 hport.hp_port, mcg_resp->P_Key, 5259 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) { 5260 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: " 5261 "Pkey2Index Conversion failed"); 5262 mcg_infop[i].mc_pkey_ix = 0; 5263 } 5264 } 5265 5266 /* 5267 * Deallocate the memory allocated by SA for results_p. 5268 */ 5269 kmem_free(results_p, length); 5270 retval = IBT_SUCCESS; 5271 5272 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords", 5273 num_records); 5274 5275 } else { 5276 retval = IBT_MCG_RECORDS_NOT_FOUND; 5277 *entries_p = 0; 5278 5279 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND"); 5280 } 5281 5282 ibcm_dec_hca_acc_cnt(hcap); 5283 5284 return (retval); 5285 } 5286 5287 5288 /* 5289 * ibt_free_mcg_info() 5290 * Free the memory allocated by successful ibt_query_mcg() 5291 * 5292 * mcgs_info Pointer returned by ibt_query_mcg(). 5293 * 5294 * entries The number of ibt_mcg_info_t entries to free. 5295 */ 5296 void 5297 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries) 5298 { 5299 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: " 5300 "Free <%d> entries from 0x%p", entries, mcgs_info); 5301 5302 if ((mcgs_info != NULL) && (entries > 0)) 5303 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t)); 5304 else 5305 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: " 5306 "ERROR: NULL buf pointer or length specified."); 5307 } 5308 5309 5310 /* 5311 * Function: 5312 * ibt_gid_to_node_info() 5313 * Input: 5314 * gid Identifies the IB Node and port for which to obtain 5315 * Node information. 5316 * Output: 5317 * node_info_p A pointer to an ibt_node_info_t structure (allocated 5318 * by the caller) in which to return the node information. 5319 * Returns: 5320 * IBT_SUCCESS 5321 * IBT_INVALID_PARAM 5322 * IBT_NODE_RECORDS_NOT_FOUND 5323 * IBT_NO_HCAS_AVAILABLE 5324 * Description: 5325 * Retrieve Node Information for the specified GID. 5326 */ 5327 ibt_status_t 5328 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p) 5329 { 5330 sa_node_record_t nr_req, *nr_resp; 5331 ibmf_saa_handle_t saa_handle; 5332 ibt_status_t retval; 5333 ibcm_hca_info_t *hcap; 5334 ibtl_cm_hca_port_t hport; 5335 int i, j; 5336 uint_t num_rec; 5337 ib_guid_t *guid_array = NULL; 5338 sa_path_record_t *path; 5339 size_t len; 5340 uint8_t npaths; 5341 uint32_t num_hcas = 0; 5342 ib_lid_t node_lid; 5343 boolean_t local_node = B_FALSE; 5344 void *res_p; 5345 uint8_t num_ports = 0; 5346 5347 5348 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)", 5349 gid.gid_prefix, gid.gid_guid, node_info_p); 5350 5351 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) { 5352 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required"); 5353 return (IBT_INVALID_PARAM); 5354 } 5355 5356 if (node_info_p == NULL) { 5357 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5358 "Return Buf (node_info_p) is NULL."); 5359 return (IBT_INVALID_PARAM); 5360 } 5361 5362 /* 5363 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5364 * associated port) info via ibtl_cm_get_hca_port() call. 5365 */ 5366 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5367 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) { 5368 5369 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5370 if (hcap == NULL) { 5371 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5372 "HCA(%llX) info not found", hport.hp_hca_guid); 5373 return (IBT_NO_HCAS_AVAILABLE); 5374 } 5375 num_ports = 1; 5376 num_hcas = 1; 5377 node_lid = hport.hp_base_lid; 5378 local_node = B_TRUE; 5379 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: " 5380 "LID = 0x%X", node_lid); 5381 } else { 5382 /* Get the number of HCAs and their GUIDs */ 5383 num_hcas = ibt_get_hca_list(&guid_array); 5384 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list " 5385 "returned %d hcas", num_hcas); 5386 5387 if (num_hcas == 0) { 5388 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5389 "NO HCA's Found on this system"); 5390 return (IBT_NO_HCAS_AVAILABLE); 5391 } 5392 } 5393 5394 for (i = 0; i < num_hcas; i++) { 5395 if (local_node == B_FALSE) { 5396 hcap = ibcm_find_hca_entry(guid_array[i]); 5397 if (hcap == NULL) { 5398 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5399 "HCA(%llX) info not found", guid_array[i]); 5400 retval = IBT_NO_HCAS_AVAILABLE; 5401 continue; 5402 } 5403 num_ports = hcap->hca_num_ports; 5404 } 5405 5406 for (j = 0; j < num_ports; j++) { 5407 uint8_t port = 0; 5408 5409 if (local_node == B_TRUE) 5410 port = hport.hp_port; 5411 else 5412 port = j + 1; 5413 5414 /* Get SA Access Handle. */ 5415 saa_handle = ibcm_get_saa_handle(hcap, port); 5416 if (saa_handle == NULL) { 5417 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5418 "Port %d of HCA (%llX) is NOT ACTIVE", 5419 port, hport.hp_hca_guid); 5420 retval = IBT_NODE_RECORDS_NOT_FOUND; 5421 continue; 5422 } 5423 5424 if (local_node == B_FALSE) { 5425 ib_gid_t sgid; 5426 int sa_ret; 5427 5428 /* 5429 * Check whether 'gid' and this port has same 5430 * subnet prefix. If not, then there is no use 5431 * in searching from this port. 5432 */ 5433 sgid = hcap->hca_port_info[j].port_sgid0; 5434 if (gid.gid_prefix != sgid.gid_prefix) { 5435 IBTF_DPRINTF_L3(cmlog, 5436 "ibt_gid_to_node_info:Sn_Prefix of " 5437 "GID(%llX) and Port's(%llX) differ", 5438 gid.gid_prefix, sgid.gid_prefix); 5439 retval = IBT_NODE_RECORDS_NOT_FOUND; 5440 continue; 5441 } 5442 5443 /* 5444 * First Get Path Records for the specified DGID 5445 * from this port (SGID). From Path Records, 5446 * note down DLID, then use this DLID as Input 5447 * attribute to get NodeRecords from SA Access. 5448 */ 5449 npaths = 1; 5450 path = NULL; 5451 5452 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5453 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5454 &path); 5455 if (sa_ret != IBMF_SUCCESS) { 5456 IBTF_DPRINTF_L2(cmlog, 5457 "ibt_gid_to_node_info: " 5458 "ibmf_saa_gid_to_pathrecords() " 5459 "returned error: %d ", sa_ret); 5460 retval = 5461 ibcm_ibmf_analyze_error(sa_ret); 5462 continue; 5463 } else if ((npaths == 0) || (path == NULL)) { 5464 IBTF_DPRINTF_L3(cmlog, 5465 "ibt_gid_to_node_info: failed (%d) " 5466 "to get path records for the DGID " 5467 "0x%llX from SGID 0x%llX", sa_ret, 5468 gid.gid_guid, sgid.gid_guid); 5469 retval = IBT_NODE_RECORDS_NOT_FOUND; 5470 continue; 5471 } 5472 node_lid = path->DLID; /* LID */ 5473 5474 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5475 "Remote Node: LID = 0x%X", node_lid); 5476 5477 /* Free SA_Access memory for path record. */ 5478 kmem_free(path, len); 5479 } 5480 5481 /* Retrieve Node Records from SA Access. */ 5482 bzero(&nr_req, sizeof (sa_node_record_t)); 5483 5484 nr_req.LID = node_lid; /* LID */ 5485 5486 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5487 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5488 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5489 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5490 "failed (%d) to get Node records", retval); 5491 continue; 5492 } else if (retval != IBT_SUCCESS) { 5493 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5494 "failed (%d) to get Node records", retval); 5495 ibcm_dec_hca_acc_cnt(hcap); 5496 goto gid_to_ni_exit; 5497 } 5498 5499 num_rec = len/sizeof (sa_node_record_t); 5500 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5501 5502 /* Validate the returned number of records. */ 5503 if ((nr_resp != NULL) && (num_rec > 0)) { 5504 5505 IBCM_DUMP_NODE_REC(nr_resp); 5506 5507 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS( 5508 *node_info_p)) 5509 5510 node_info_p->n_sys_img_guid = 5511 nr_resp->NodeInfo.SystemImageGUID; 5512 node_info_p->n_node_guid = 5513 nr_resp->NodeInfo.NodeGUID; 5514 node_info_p->n_port_guid = 5515 nr_resp->NodeInfo.PortGUID; 5516 node_info_p->n_dev_id = 5517 nr_resp->NodeInfo.DeviceID; 5518 node_info_p->n_revision = 5519 nr_resp->NodeInfo.Revision; 5520 node_info_p->n_vendor_id = 5521 nr_resp->NodeInfo.VendorID; 5522 node_info_p->n_num_ports = 5523 nr_resp->NodeInfo.NumPorts; 5524 node_info_p->n_port_num = 5525 nr_resp->NodeInfo.LocalPortNum; 5526 node_info_p->n_node_type = 5527 nr_resp->NodeInfo.NodeType; 5528 (void) strncpy(node_info_p->n_description, 5529 (char *)&nr_resp->NodeDescription, 64); 5530 5531 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS( 5532 *node_info_p)) 5533 5534 /* 5535 * Deallocate the memory allocated by SA for 5536 * 'nr_resp'. 5537 */ 5538 ibcm_dec_hca_acc_cnt(hcap); 5539 kmem_free(nr_resp, len); 5540 retval = IBT_SUCCESS; 5541 5542 goto gid_to_ni_exit; 5543 } else { 5544 retval = IBT_NODE_RECORDS_NOT_FOUND; 5545 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5546 "Node Records NOT found - PortGUID %016llX", 5547 gid.gid_guid); 5548 } 5549 } 5550 ibcm_dec_hca_acc_cnt(hcap); 5551 5552 if (local_node == B_TRUE) 5553 break; 5554 } 5555 5556 gid_to_ni_exit: 5557 if (guid_array) 5558 ibt_free_hca_list(guid_array, num_hcas); 5559 5560 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval); 5561 5562 return (retval); 5563 } 5564 5565 5566 static ibt_status_t 5567 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req, 5568 uint64_t component_mask, void *result_p, size_t *len) 5569 { 5570 ibmf_saa_access_args_t args; 5571 size_t length; 5572 ibt_status_t retval; 5573 5574 args.sq_attr_id = SA_NODERECORD_ATTRID; 5575 args.sq_template = nr_req; 5576 args.sq_access_type = IBMF_SAA_RETRIEVE; 5577 args.sq_template_length = sizeof (sa_node_record_t); 5578 args.sq_component_mask = component_mask; 5579 args.sq_callback = NULL; 5580 args.sq_callback_arg = NULL; 5581 5582 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p); 5583 if (retval != IBT_SUCCESS) { 5584 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed"); 5585 return (retval); 5586 } 5587 5588 *len = length; 5589 5590 /* Validate the returned number of records. */ 5591 if ((result_p != NULL) && (length > 0)) { 5592 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND"); 5593 5594 /* Got it, done!. */ 5595 return (IBT_SUCCESS); 5596 } else { 5597 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found"); 5598 return (IBT_NODE_RECORDS_NOT_FOUND); 5599 } 5600 } 5601 5602 5603 /* 5604 * Function: 5605 * ibt_get_companion_port_gids() 5606 * Description: 5607 * Get list of GID's available on a companion port(s) of the specified 5608 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID. 5609 */ 5610 ibt_status_t 5611 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid, 5612 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p) 5613 { 5614 sa_node_record_t nr_req, *nr_resp; 5615 void *res_p; 5616 ibmf_saa_handle_t saa_handle; 5617 int sa_ret; 5618 ibt_status_t retval = IBT_SUCCESS; 5619 ibcm_hca_info_t *hcap; 5620 ibtl_cm_hca_port_t hport; 5621 int i, j; 5622 uint_t num_rec; 5623 ib_guid_t *guid_array = NULL; 5624 sa_path_record_t *path; 5625 size_t len; 5626 uint8_t npaths; 5627 uint32_t num_hcas = 0; 5628 boolean_t local_node = B_FALSE; 5629 boolean_t local_hca = B_FALSE; 5630 ib_guid_t h_guid = hca_guid; 5631 ib_gid_t *gidp = NULL, *t_gidp = NULL; 5632 int multi_hca_loop = 0; 5633 5634 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, " 5635 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid); 5636 5637 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) && 5638 (sysimg_guid == 0)) { 5639 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5640 "Null Input attribute specified."); 5641 return (IBT_INVALID_PARAM); 5642 } 5643 5644 if ((num_gids_p == NULL) || (gids_p == NULL)) { 5645 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5646 "num_gids_p or gids_p is NULL"); 5647 return (IBT_INVALID_PARAM); 5648 } 5649 5650 *num_gids_p = 0; 5651 5652 /* Get the number of HCAs and their GUIDs */ 5653 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) { 5654 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5655 "NO HCA's Found on this system"); 5656 return (IBT_NO_HCAS_AVAILABLE); 5657 } 5658 5659 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5660 "ibt_get_hca_list() returned %d hcas", num_hcas); 5661 5662 /* 5663 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5664 * associated port) info via ibtl_cm_get_hca_port() call. 5665 */ 5666 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5667 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) && 5668 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) { 5669 5670 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) { 5671 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5672 "Invalid GID<->HCAGUID combination specified."); 5673 retval = IBT_INVALID_PARAM; 5674 goto get_comp_pgid_exit; 5675 } 5676 h_guid = hport.hp_hca_guid; 5677 local_node = B_TRUE; 5678 5679 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5680 "Local Node: HCA (0x%llX)", h_guid); 5681 } else if (h_guid) { /* Is specified HCA GUID - local? */ 5682 for (i = 0; i < num_hcas; i++) { 5683 if (h_guid == guid_array[i]) { 5684 local_hca = B_TRUE; 5685 break; 5686 } 5687 } 5688 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */ 5689 for (i = 0; i < num_hcas; i++) { 5690 ibt_status_t ret; 5691 ibt_hca_attr_t hca_attr; 5692 5693 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr); 5694 if (ret != IBT_SUCCESS) { 5695 IBTF_DPRINTF_L2(cmlog, 5696 "ibt_get_companion_port_gids: HCA(%llX) " 5697 "info not found", guid_array[i]); 5698 retval = IBT_NO_HCAS_AVAILABLE; 5699 continue; 5700 } 5701 if (hca_attr.hca_si_guid == sysimg_guid) { 5702 if ((hca_guid != 0) && 5703 (hca_guid != hca_attr.hca_node_guid)) { 5704 IBTF_DPRINTF_L2(cmlog, 5705 "ibt_get_companion_port_gids: " 5706 "Invalid SysImg<->HCA GUID " 5707 "combination specified."); 5708 retval = IBT_INVALID_PARAM; 5709 goto get_comp_pgid_exit; 5710 } 5711 local_hca = B_TRUE; 5712 h_guid = hca_attr.hca_node_guid; 5713 break; 5714 } 5715 } 5716 } 5717 5718 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) { 5719 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p, 5720 num_gids_p); 5721 goto get_comp_pgid_exit; 5722 } 5723 5724 get_comp_for_multihca: 5725 /* We will be here, if request is for remote node */ 5726 for (i = 0; i < num_hcas; i++) { 5727 int multism; 5728 uint8_t count = 0; 5729 int multi_sm_loop = 0; 5730 uint_t k = 0, l; 5731 5732 hcap = ibcm_find_hca_entry(guid_array[i]); 5733 if (hcap == NULL) { 5734 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5735 "HCA(%llX) info not found", guid_array[i]); 5736 retval = IBT_NO_HCAS_AVAILABLE; 5737 continue; 5738 } 5739 5740 /* 1 - MultiSM, 0 - Single SM */ 5741 multism = ibtl_cm_is_multi_sm(guid_array[i]); 5742 5743 for (j = 0; j < hcap->hca_num_ports; j++) { 5744 ib_gid_t sgid; 5745 uint64_t c_mask = 0; 5746 ib_guid_t pg; 5747 uint_t port = j; 5748 5749 get_comp_for_multism: 5750 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5751 "Port %d, HCA %llX, MultiSM= %d, Loop=%d", 5752 port + 1, h_guid, multism, multi_sm_loop); 5753 5754 /* Get SA Access Handle. */ 5755 saa_handle = ibcm_get_saa_handle(hcap, port + 1); 5756 if (saa_handle == NULL) { 5757 IBTF_DPRINTF_L2(cmlog, 5758 "ibt_get_companion_port_gids: " 5759 "Port (%d) - NOT ACTIVE", port + 1); 5760 retval = IBT_GIDS_NOT_FOUND; 5761 continue; 5762 } 5763 5764 /* 5765 * Check whether 'gid' and this port has same subnet 5766 * prefix. If not, then there is no use in searching 5767 * from this port. 5768 */ 5769 sgid = hcap->hca_port_info[port].port_sgid0; 5770 if ((h_guid == 0) && (gid.gid_prefix != 0) && 5771 (multi_sm_loop == 0) && 5772 (gid.gid_prefix != sgid.gid_prefix)) { 5773 IBTF_DPRINTF_L2(cmlog, 5774 "ibt_get_companion_port_gids: SnPrefix of " 5775 "GID(%llX) and Port SN_Pfx(%llX) differ", 5776 gid.gid_prefix, sgid.gid_prefix); 5777 retval = IBT_GIDS_NOT_FOUND; 5778 continue; 5779 } 5780 5781 /* 5782 * If HCA GUID or System Image GUID is specified, then 5783 * we can achieve our goal sooner!. 5784 */ 5785 if ((h_guid == 0) && (sysimg_guid == 0)) { 5786 /* So only GID info is provided. */ 5787 5788 /* 5789 * First Get Path Records for the specified DGID 5790 * from this port (SGID). From Path Records, 5791 * note down DLID, then use this DLID as Input 5792 * attribute to get NodeRecords. 5793 */ 5794 npaths = 1; 5795 path = NULL; 5796 5797 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5798 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5799 &path); 5800 if (sa_ret != IBMF_SUCCESS) { 5801 IBTF_DPRINTF_L2(cmlog, 5802 "ibt_get_companion_port_gids: " 5803 "ibmf_saa_gid_to_pathrecords() " 5804 "returned error: %d ", sa_ret); 5805 retval = 5806 ibcm_ibmf_analyze_error(sa_ret); 5807 ibcm_dec_hca_acc_cnt(hcap); 5808 goto get_comp_pgid_exit; 5809 } else if ((npaths == 0) || (path == NULL)) { 5810 IBTF_DPRINTF_L2(cmlog, 5811 "ibt_get_companion_port_gids: " 5812 "failed (%d) to get path records " 5813 "for the DGID (0x%llX) from SGID " 5814 "(0x%llX)", sa_ret, gid.gid_guid, 5815 sgid.gid_guid); 5816 retval = IBT_GIDS_NOT_FOUND; 5817 continue; 5818 } 5819 5820 bzero(&nr_req, sizeof (sa_node_record_t)); 5821 nr_req.LID = path->DLID; /* LID */ 5822 5823 IBTF_DPRINTF_L3(cmlog, 5824 "ibt_get_companion_port_gids: " 5825 "Remote Node: LID = 0x%X", nr_req.LID); 5826 5827 /* Free SA_Access memory for path record. */ 5828 kmem_free(path, len); 5829 5830 IBTF_DPRINTF_L3(cmlog, 5831 "ibt_get_companion_port_gids: SAA Call: " 5832 "based on LID "); 5833 5834 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5835 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5836 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5837 IBTF_DPRINTF_L2(cmlog, 5838 "ibt_get_companion_port_gids: " 5839 "failed (%d) to get Node records", 5840 retval); 5841 continue; 5842 } else if (retval != IBT_SUCCESS) { 5843 IBTF_DPRINTF_L2(cmlog, 5844 "ibt_get_companion_port_gids: " 5845 "failed (%d) to get Node records", 5846 retval); 5847 ibcm_dec_hca_acc_cnt(hcap); 5848 goto get_comp_pgid_exit; 5849 } 5850 5851 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5852 /* Note down HCA GUID info. */ 5853 h_guid = nr_resp->NodeInfo.NodeGUID; 5854 5855 IBTF_DPRINTF_L3(cmlog, 5856 "ibt_get_companion_port_gids: " 5857 "Remote HCA GUID: 0x%llX", h_guid); 5858 5859 IBCM_DUMP_NODE_REC(nr_resp); 5860 5861 kmem_free(res_p, len); 5862 } 5863 5864 bzero(&nr_req, sizeof (sa_node_record_t)); 5865 if (h_guid != 0) { 5866 nr_req.NodeInfo.NodeGUID = h_guid; 5867 c_mask = SA_NODEINFO_COMPMASK_NODEGUID; 5868 } 5869 5870 if (sysimg_guid != 0) { 5871 nr_req.NodeInfo.SystemImageGUID = sysimg_guid; 5872 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID; 5873 } 5874 5875 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5876 "SAA Call: CMASK= 0x%llX", c_mask); 5877 5878 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask, 5879 &res_p, &len); 5880 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5881 IBTF_DPRINTF_L3(cmlog, 5882 "ibt_get_companion_port_gids: " 5883 "failed (%d) to get Node records", retval); 5884 continue; 5885 } else if (retval != IBT_SUCCESS) { 5886 IBTF_DPRINTF_L2(cmlog, 5887 "ibt_get_companion_port_gids: Error: (%d) " 5888 "while getting Node records", retval); 5889 ibcm_dec_hca_acc_cnt(hcap); 5890 goto get_comp_pgid_exit; 5891 } 5892 5893 num_rec = len/sizeof (sa_node_record_t); 5894 5895 /* We will be here, only if we found some NodeRec */ 5896 if (gid.gid_prefix && gid.gid_guid) { 5897 nr_resp = (sa_node_record_t *)res_p; 5898 for (l = 0; l < num_rec; l++, nr_resp++) { 5899 pg = nr_resp->NodeInfo.PortGUID; 5900 if (gid.gid_guid != pg) 5901 count++; 5902 } 5903 } else { 5904 count = num_rec; 5905 } 5906 5907 if (count != 0) { 5908 if (multi_sm_loop == 1) { 5909 count += k; 5910 t_gidp = kmem_zalloc(count * 5911 sizeof (ib_gid_t), KM_SLEEP); 5912 5913 if ((k != 0) && (gidp != NULL)) { 5914 bcopy(gidp, t_gidp, 5915 k * sizeof (ib_gid_t)); 5916 kmem_free(gidp, 5917 k * sizeof (ib_gid_t)); 5918 } 5919 gidp = t_gidp; 5920 } else { 5921 gidp = kmem_zalloc(count * 5922 sizeof (ib_gid_t), KM_SLEEP); 5923 } 5924 *num_gids_p = count; 5925 *gids_p = gidp; 5926 5927 nr_resp = (sa_node_record_t *)res_p; 5928 for (l = 0; l < num_rec; l++, nr_resp++) { 5929 IBCM_DUMP_NODE_REC(nr_resp); 5930 5931 pg = nr_resp->NodeInfo.PortGUID; 5932 IBTF_DPRINTF_L4(cmlog, 5933 "ibt_get_companion_port_gids: " 5934 "PortGID %llX", pg); 5935 5936 if (pg != gid.gid_guid) { 5937 gidp[k].gid_prefix = 5938 sgid.gid_prefix; 5939 gidp[k].gid_guid = pg; 5940 5941 IBTF_DPRINTF_L3(cmlog, 5942 "ibt_get_companion_pgids: " 5943 "GID[%d] = %llX:%llX", k, 5944 gidp[k].gid_prefix, 5945 gidp[k].gid_guid); 5946 5947 k++; 5948 if (k == count) 5949 break; 5950 } 5951 } 5952 retval = IBT_SUCCESS; /* done!. */ 5953 kmem_free(res_p, len); 5954 ibcm_dec_hca_acc_cnt(hcap); 5955 goto get_comp_pgid_exit; 5956 } else { 5957 IBTF_DPRINTF_L2(cmlog, 5958 "ibt_get_companion_port_gids: " 5959 "Companion PortGIDs not available"); 5960 retval = IBT_GIDS_NOT_FOUND; 5961 } 5962 /* Deallocate the memory for 'res_p'. */ 5963 kmem_free(res_p, len); 5964 5965 /* 5966 * If we are on MultiSM setup, then we need to lookout 5967 * from that subnet port too. 5968 */ 5969 if (multism) { 5970 /* break if already searched both the subnet */ 5971 if (multi_sm_loop == 1) 5972 break; 5973 5974 port = (j == 0) ? 1 : 0; 5975 multi_sm_loop = 1; 5976 goto get_comp_for_multism; 5977 } else { 5978 break; 5979 } 5980 } 5981 ibcm_dec_hca_acc_cnt(hcap); 5982 5983 /* 5984 * We may be on dual HCA with dual SM configured system. And 5985 * the input attr GID was visible from second HCA. So in order 5986 * to get the companion portgid we need to re-look from the 5987 * first HCA ports. 5988 */ 5989 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) && 5990 (multi_hca_loop != 1)) { 5991 multi_hca_loop = 1; 5992 goto get_comp_for_multihca; 5993 } 5994 } 5995 if (*num_gids_p == 0) 5996 retval = IBT_GIDS_NOT_FOUND; 5997 5998 get_comp_pgid_exit: 5999 if (guid_array) 6000 ibt_free_hca_list(guid_array, num_hcas); 6001 6002 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) { 6003 retval = IBT_SUCCESS; 6004 } 6005 6006 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, " 6007 "Found %d GIDs", retval, *num_gids_p); 6008 6009 return (retval); 6010 } 6011 6012 6013 /* Routines for warlock */ 6014 6015 /* ARGSUSED */ 6016 static void 6017 ibcm_dummy_mcg_handler(void *arg, ibt_status_t retval, ibt_mcg_info_t *minfo) 6018 { 6019 ibcm_join_mcg_tqarg_t dummy_mcg; 6020 6021 dummy_mcg.func = ibcm_dummy_mcg_handler; 6022 6023 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_mcg_handler: " 6024 "dummy_mcg.func %p", dummy_mcg.func); 6025 } 6026 6027 6028 /* ARGSUSED */ 6029 static void 6030 ibcm_dummy_recycle_rc_handler(ibt_status_t retval, void *arg) 6031 { 6032 ibcm_taskq_recycle_arg_t dummy_rc_recycle; 6033 6034 dummy_rc_recycle.func = ibcm_dummy_recycle_rc_handler; 6035 6036 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_recycle_rc_handler: " 6037 "dummy_rc_recycle.func %p", dummy_rc_recycle.func); 6038 } 6039 6040 6041 /* ARGSUSED */ 6042 static ibt_cm_status_t 6043 ibcm_dummy_ud_handler(void *priv, ibt_cm_ud_event_t *event, 6044 ibt_cm_ud_return_args_t *ret_args, 6045 void *priv_data, ibt_priv_data_len_t len) 6046 { 6047 /* 6048 * Let warlock see that ibcm_local_handler_s::actual_cm_handler 6049 * points to this routine. 6050 */ 6051 ibcm_local_handler_t p; 6052 ibcm_ud_state_data_t dummy_ud; 6053 6054 p.actual_cm_handler = ibcm_dummy_ud_handler; 6055 dummy_ud.ud_cm_handler = ibcm_dummy_ud_handler; 6056 6057 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_ud_handler: p.actual_cm_handler %p" 6058 "dummy_ud.ud_cm_handler %p", p.actual_cm_handler, 6059 dummy_ud.ud_cm_handler); 6060 /* 6061 * Call all routines that the client's callback routine could call. 6062 */ 6063 6064 return (IBT_CM_ACCEPT); 6065 } 6066 6067 /* ARGSUSED */ 6068 static ibt_cm_status_t 6069 ibcm_dummy_rc_handler(void *priv, ibt_cm_event_t *event, 6070 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 6071 { 6072 ibcm_state_data_t dummy_rc; 6073 6074 dummy_rc.cm_handler = ibcm_dummy_rc_handler; 6075 6076 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_rc_handler: " 6077 "dummy_ud.ud_cm_handler %p", dummy_rc.cm_handler); 6078 /* 6079 * Call all routines that the client's callback routine could call. 6080 */ 6081 6082 return (IBT_CM_ACCEPT); 6083 } 6084