1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 30 #include <sys/ib/ibtl/ibti.h> 31 32 /* 33 * ibcm_ti.c 34 * These routines implement the Communication Manager's interfaces to IBTL. 35 */ 36 37 /* CM rc recycle task args structure definition */ 38 typedef struct ibcm_taskq_recycle_arg_s { 39 ibt_channel_hdl_t rc_chan; 40 ibt_cep_flags_t control; 41 uint8_t hca_port_num; 42 ibt_recycle_handler_t func; 43 void *arg; 44 } ibcm_taskq_recycle_arg_t; 45 46 _NOTE(READ_ONLY_DATA(ibcm_taskq_recycle_arg_s)) 47 48 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap, 49 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args, 50 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid); 51 static void ibcm_process_abort_via_taskq(void *args); 52 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg); 53 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg); 54 static void ibcm_process_async_join_mcg(void *tq_arg); 55 56 static ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *, 57 uint64_t c_mask, void *, size_t *); 58 59 /* Address Record management definitions */ 60 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service" 61 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL 62 #define IBCM_DAPL_ATS_NBYTES 16 63 ibcm_svc_info_t *ibcm_ar_svcinfop; 64 ibcm_ar_t *ibcm_ar_list; 65 66 #ifdef DEBUG 67 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel, 68 ibcm_mad_addr_t *cm_reply_addr); 69 #endif 70 71 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_hdl})) 72 73 /* access is controlled between ibcm_sm.c and ibcm_ti.c by CVs */ 74 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by CV", {ibt_rc_returns_t 75 ibt_ud_returns_t ibt_ap_returns_t ibt_ar_t})) 76 77 /* 78 * Typically, clients initialize these args in one api call, and use in 79 * another api 80 */ 81 _NOTE(SCHEME_PROTECTS_DATA("Expected usage of ibtl api by client", 82 {ibt_path_info_s ibt_cep_path_s ibt_adds_vect_s ibt_mcg_info_s ib_gid_s 83 ibt_ud_dest_attr_s ibt_ud_dest_s ibt_srv_data_s ibt_redirect_info_s})) 84 85 /* 86 * ibt_open_rc_channel() 87 * ibt_open_rc_channel opens a communication channel on the specified 88 * channel to the specified service. For connection service type qp's 89 * the CM initiates the CEP to establish the connection and transitions 90 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's 91 * attributes as necessary. 92 * The implementation of this function assumes that alt path is different 93 * from primary path. It is assumed that the Path functions ensure that. 94 * 95 * RETURN VALUES: 96 * IBT_SUCCESS on success (or respective failure on error) 97 */ 98 ibt_status_t 99 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags, 100 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args, 101 ibt_rc_returns_t *ret_args) 102 { 103 /* all fields that are related to REQ MAD formation */ 104 105 ib_pkey_t prim_pkey; 106 ib_lid_t primary_slid, alternate_slid; 107 ib_qpn_t local_qpn = 0; 108 ib_guid_t hca_guid; 109 ib_qkey_t local_qkey = 0; 110 ib_eecn_t local_eecn = 0; 111 ib_eecn_t remote_eecn = 0; 112 boolean_t primary_grh; 113 boolean_t alternate_grh = B_FALSE; 114 ib_lid_t base_lid; 115 ib_com_id_t local_comid; 116 ibmf_msg_t *ibmf_msg; 117 ibcm_req_msg_t *req_msgp; 118 119 uint8_t rdma_in, rdma_out; 120 uint8_t cm_retries; 121 uint64_t local_cm_proc_time; /* In usec */ 122 uint8_t local_cm_resp_time; /* IB time */ 123 uint64_t remote_cm_resp_time; /* In usec */ 124 uint32_t starting_psn = 0; 125 126 /* CM path related fields */ 127 ibmf_handle_t ibmf_hdl; 128 ibcm_qp_list_t *cm_qp_entry; 129 ibcm_mad_addr_t cm_reply_addr; 130 131 uint8_t cm_pkt_lt; 132 133 /* Local args for ibtl/internal CM functions called within */ 134 ibt_status_t status; 135 ibcm_status_t lkup_status; 136 ibt_qp_query_attr_t qp_query_attr; 137 138 /* Other misc local args */ 139 ibt_priv_data_len_t len; 140 ibcm_hca_info_t *hcap; 141 ibcm_state_data_t *statep; 142 uint8_t port_no; 143 144 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)", 145 channel, flags, mode, chan_args, ret_args); 146 147 if (IBCM_INVALID_CHANNEL(channel)) { 148 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel"); 149 return (IBT_CHAN_HDL_INVALID); 150 } 151 152 /* cm handler should always be specified */ 153 if (chan_args->oc_cm_handler == NULL) { 154 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 155 "CM handler is not be specified", channel); 156 return (IBT_INVALID_PARAM); 157 } 158 159 if (mode == IBT_NONBLOCKING) { 160 if (ret_args != NULL) { 161 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 162 "ret_args should be NULL when called in " 163 "non-blocking mode", channel); 164 return (IBT_INVALID_PARAM); 165 } 166 } else if (mode == IBT_BLOCKING) { 167 if (ret_args == NULL) { 168 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 169 "ret_args should be Non-NULL when called in " 170 "blocking mode", channel); 171 return (IBT_INVALID_PARAM); 172 } 173 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) { 174 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 175 "private data length is too large", channel); 176 return (IBT_INVALID_PARAM); 177 } 178 if ((ret_args->rc_priv_data_len > 0) && 179 (ret_args->rc_priv_data == NULL)) { 180 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 181 "rc_priv_data_len > 0, but rc_priv_data NULL", 182 channel); 183 return (IBT_INVALID_PARAM); 184 } 185 } else { /* any other mode is not valid for ibt_open_rc_channel */ 186 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 187 "invalid mode %x specified", channel, mode); 188 return (IBT_INVALID_PARAM); 189 } 190 191 /* 192 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP 193 */ 194 if (flags & IBT_OCHAN_DUP) { 195 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 196 "Unsupported Flags specified: 0x%X", channel, flags); 197 return (IBT_INVALID_PARAM); 198 } 199 200 if ((flags & IBT_OCHAN_REDIRECTED) && 201 (flags & IBT_OCHAN_PORT_REDIRECTED)) { 202 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 203 "Illegal to specify IBT_OCHAN_REDIRECTED and " 204 "IBT_OCHAN_PORT_REDIRECTED flags together", channel); 205 return (IBT_INVALID_PARAM); 206 } 207 208 if (((flags & IBT_OCHAN_REDIRECTED) && 209 (chan_args->oc_cm_redirect_info == NULL)) || 210 ((flags & IBT_OCHAN_PORT_REDIRECTED) && 211 (chan_args->oc_cm_cep_path == NULL))) { 212 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 213 "Redirect flag specified, but respective arg is NULL", 214 channel); 215 return (IBT_INVALID_PARAM); 216 } 217 218 if ((flags & IBT_OCHAN_REDIRECTED) && 219 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) && 220 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) { 221 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 222 "Either rdi_dlid or rdi_gid must be specified for" 223 " IBT_OCHAN_REDIRECTED", channel); 224 return (IBT_INVALID_PARAM); 225 } 226 227 /* primary dlid and hca_port_num should never be zero */ 228 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 229 230 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) { 231 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 232 "Primary Path's information is not valid", channel); 233 return (IBT_INVALID_PARAM); 234 } 235 236 /* validate SID */ 237 if (chan_args->oc_path->pi_sid == 0) { 238 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 239 "ERROR: Service ID in path information is 0", channel); 240 return (IBT_INVALID_PARAM); 241 } 242 243 /* validate rnr_retry_cnt (enum has more than 3 bits) */ 244 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) { 245 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 246 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range", 247 channel, chan_args->oc_path_rnr_retry_cnt); 248 return (IBT_INVALID_PARAM); 249 } 250 251 /* 252 * Ensure that client is not re-using a QP that is still associated 253 * with a statep 254 */ 255 IBCM_GET_CHAN_PRIVATE(channel, statep); 256 if (statep != NULL) { 257 IBCM_RELEASE_CHAN_PRIVATE(channel); 258 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 259 "Channel being re-used on active side", channel); 260 return (IBT_CHAN_IN_USE); 261 } 262 263 /* Get GUID from Channel */ 264 hca_guid = ibt_channel_to_hca_guid(channel); 265 266 /* validate QP's hca guid with that from primary path */ 267 if (hca_guid != chan_args->oc_path->pi_hca_guid) { 268 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 269 "GUID from Channel and primary path don't match", channel); 270 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 271 "Channel GUID %llX primary path GUID %llX", channel, 272 hca_guid, chan_args->oc_path->pi_hca_guid); 273 return (IBT_CHAN_HDL_INVALID); 274 } 275 276 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 277 "Local HCA GUID %llX", channel, hca_guid); 278 279 ibcm_rc_flow_control_enter(); /* limit how many run simultaneously */ 280 281 status = ibt_query_qp(channel, &qp_query_attr); 282 if (status != IBT_SUCCESS) { 283 ibcm_rc_flow_control_exit(); 284 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 285 "ibt_query_qp failed %d", channel, status); 286 return (status); 287 } 288 289 /* If client specified "no port change on QP" */ 290 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 291 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) { 292 ibcm_rc_flow_control_exit(); 293 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 294 "chan port %d and path port %d does not match", channel, 295 qp_query_attr.qp_info.qp_transport.rc.rc_path. \ 296 cep_hca_port_num, port_no); 297 return (IBT_INVALID_PARAM); 298 } 299 300 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) { 301 ibcm_rc_flow_control_exit(); 302 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 303 "Invalid Channel type: Applicable only to RC Channel", 304 channel); 305 return (IBT_CHAN_SRV_TYPE_INVALID); 306 } 307 308 /* Check if QP is in INIT state or not */ 309 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) { 310 ibcm_rc_flow_control_exit(); 311 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 312 "QP is not in INIT state %x", channel, 313 qp_query_attr.qp_info.qp_state); 314 return (IBT_CHAN_STATE_INVALID); 315 } 316 317 local_qpn = qp_query_attr.qp_qpn; 318 319 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x", 320 channel, local_qpn); 321 322 #ifdef NO_EEC_SUPPORT_YET 323 324 if (flags & IBT_OCHAN_RDC_EXISTS) { 325 ibt_eec_query_attr_t eec_query_attr; 326 327 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey; 328 329 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD"); 330 331 status = ibt_query_eec(channel, &eec_query_attr); 332 if (status != IBT_SUCCESS) { 333 ibcm_rc_flow_control_exit(); 334 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 335 "ibt_query_eec failed %d", channel, status); 336 return (status); 337 } 338 local_eecn = eec_query_attr.eec_eecn; 339 } 340 341 #endif 342 343 /* If no HCA found return failure */ 344 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) { 345 ibcm_rc_flow_control_exit(); 346 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 347 "hcap is NULL. Probably hca is not in active state", 348 channel); 349 return (IBT_CHAN_HDL_INVALID); 350 } 351 352 rdma_out = chan_args->oc_rdma_ra_out; 353 rdma_in = chan_args->oc_rdma_ra_in; 354 355 if ((rdma_in > hcap->hca_max_rdma_in_qp) || 356 (rdma_out > hcap->hca_max_rdma_out_qp)) { 357 ibcm_rc_flow_control_exit(); 358 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 359 "rdma in %d/out %d values exceed hca limits", channel, 360 rdma_in, rdma_out); 361 ibcm_dec_hca_acc_cnt(hcap); 362 return (IBT_INVALID_PARAM); 363 } 364 365 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p" 366 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out); 367 368 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) { 369 ibcm_rc_flow_control_exit(); 370 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 371 "Huge Primary Pkt lt %d", channel, 372 chan_args->oc_path->pi_prim_pkt_lt); 373 ibcm_dec_hca_acc_cnt(hcap); 374 return (IBT_PATH_PKT_LT_TOO_HIGH); 375 } 376 377 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, 378 NULL, &base_lid); 379 if (status != IBT_SUCCESS) { 380 ibcm_rc_flow_control_exit(); 381 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 382 "primary port_num %d not active", channel, port_no); 383 ibcm_dec_hca_acc_cnt(hcap); 384 return (status); 385 } 386 387 /* Validate P_KEY Index */ 388 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no, 389 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey); 390 if (status != IBT_SUCCESS) { 391 ibcm_rc_flow_control_exit(); 392 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 393 "Invalid Primary PKeyIx %x", channel, 394 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix); 395 ibcm_dec_hca_acc_cnt(hcap); 396 return (status); 397 } 398 399 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p" 400 "primary_port_num %d primary_pkey 0x%x", channel, port_no, 401 prim_pkey); 402 403 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 404 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 405 != IBT_SUCCESS)) { 406 ibcm_rc_flow_control_exit(); 407 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 408 "ibmf reg or callback setup failed during re-initialize", 409 channel); 410 ibcm_dec_hca_acc_cnt(hcap); 411 return (status); 412 } 413 414 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl; 415 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 416 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl); 417 418 419 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 420 421 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p " 422 "primary SLID = %x", channel, primary_slid); 423 424 /* check first if alternate path exists or not as it is OPTIONAL */ 425 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) { 426 uint8_t alt_port_no; 427 428 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 429 430 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) { 431 ibcm_rc_flow_control_exit(); 432 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 433 "Huge Alt Pkt lt %d", channel, 434 chan_args->oc_path->pi_alt_pkt_lt); 435 ibcm_dec_hca_acc_cnt(hcap); 436 return (IBT_PATH_PKT_LT_TOO_HIGH); 437 } 438 439 if (port_no != alt_port_no) { 440 441 status = ibt_get_port_state_byguid(hcap->hca_guid, 442 alt_port_no, NULL, &base_lid); 443 if (status != IBT_SUCCESS) { 444 445 ibcm_rc_flow_control_exit(); 446 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 447 "chan 0x%p alt_port_num %d inactive %d", 448 channel, alt_port_no, status); 449 ibcm_dec_hca_acc_cnt(hcap); 450 return (status); 451 } 452 453 } 454 alternate_slid = 455 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 456 457 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan %0xp" 458 "alternate SLID = %x", channel, alternate_slid); 459 } 460 461 /* 462 * only pkey needs to be zero'ed, because all other fields are set in 463 * in ibcm_init_reply_addr. But, let's bzero the complete struct for 464 * any future modifications. 465 */ 466 bzero(&cm_reply_addr, sizeof (cm_reply_addr)); 467 468 /* Initialize the MAD destination address in stored_reply_addr */ 469 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args, 470 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) { 471 472 ibcm_rc_flow_control_exit(); 473 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 474 "ibcm_init_reply_addr failed status %d ", channel, status); 475 ibcm_dec_hca_acc_cnt(hcap); 476 return (status); 477 } 478 479 480 /* Initialize the pkey for CM MAD communication */ 481 if (cm_reply_addr.rcvd_addr.ia_p_key == 0) 482 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey; 483 484 #ifdef DEBUG 485 ibcm_print_reply_addr(channel, &cm_reply_addr); 486 #endif 487 488 /* Retrieve an ibmf qp for sending CM MADs */ 489 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no, 490 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) { 491 ibcm_rc_flow_control_exit(); 492 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 493 "unable to allocate ibmf qp for CM MADs", channel); 494 ibcm_dec_hca_acc_cnt(hcap); 495 return (IBT_INSUFF_RESOURCE); 496 } 497 498 499 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) { 500 ibcm_rc_flow_control_exit(); 501 ibcm_release_qp(cm_qp_entry); 502 ibcm_dec_hca_acc_cnt(hcap); 503 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 504 " Unable to allocate comid", channel); 505 return (IBT_INSUFF_KERNEL_RESOURCE); 506 } 507 508 /* allocate an IBMF mad buffer */ 509 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, 510 MAD_METHOD_SEND)) != IBT_SUCCESS) { 511 ibcm_rc_flow_control_exit(); 512 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 513 "chan 0x%p ibcm_alloc_out_msg failed", channel); 514 ibcm_release_qp(cm_qp_entry); 515 ibcm_free_comid(hcap, local_comid); 516 ibcm_dec_hca_acc_cnt(hcap); 517 return (status); 518 } 519 520 /* Init to Init, if QP's port does not match with path information */ 521 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 522 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) { 523 524 ibt_qp_info_t qp_info; 525 ibt_cep_modify_flags_t cep_flags; 526 527 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 528 "chan 0x%p chan port %d", channel, 529 qp_query_attr.qp_info.qp_transport.rc.rc_path.\ 530 cep_hca_port_num); 531 532 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 533 "chan 0x%p path port %d", channel, port_no); 534 535 bzero(&qp_info, sizeof (qp_info)); 536 /* For now, set it to RC type */ 537 538 qp_info.qp_trans = IBT_RC_SRV; 539 qp_info.qp_state = IBT_STATE_INIT; 540 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no; 541 542 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 543 544 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL); 545 546 if (status != IBT_SUCCESS) { 547 ibcm_rc_flow_control_exit(); 548 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 549 "chan 0x%p ibt_modify_qp() = %d", channel, status); 550 ibcm_release_qp(cm_qp_entry); 551 ibcm_free_comid(hcap, local_comid); 552 ibcm_dec_hca_acc_cnt(hcap); 553 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 554 return (status); 555 } else 556 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 557 "chan 0x%p ibt_modify_qp() = %d", channel, status); 558 } 559 560 /* allocate ibcm_state_data_t before grabbing the WRITER lock */ 561 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP); 562 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 563 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0, 564 hcap, &statep); 565 rw_exit(&hcap->hca_state_rwlock); 566 567 /* CM should be seeing this for the first time */ 568 ASSERT(lkup_status == IBCM_LOOKUP_NEW); 569 570 /* Increment the hca's resource count */ 571 ibcm_inc_hca_res_cnt(hcap); 572 573 /* Once a resource created on hca, no need to hold the acc cnt */ 574 ibcm_dec_hca_acc_cnt(hcap); 575 576 577 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 578 579 statep->timerid = 0; 580 statep->local_hca_guid = hca_guid; 581 statep->local_qpn = local_qpn; 582 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry; 583 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 584 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 585 586 587 /* Save "statep" as channel's CM private data. */ 588 statep->channel = channel; 589 IBCM_SET_CHAN_PRIVATE(statep->channel, statep); 590 591 statep->stored_msg = ibmf_msg; 592 593 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*req_msgp)) 594 595 /* Start filling in the REQ MAD */ 596 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg); 597 req_msgp->req_local_comm_id = h2b32(local_comid); 598 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid); 599 req_msgp->req_local_ca_guid = h2b64(hca_guid); 600 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */ 601 602 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */ 603 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in); 604 605 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */ 606 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out); 607 608 if (flags & IBT_OCHAN_REMOTE_CM_TM) 609 remote_cm_resp_time = chan_args->oc_remote_cm_time; 610 else 611 remote_cm_resp_time = ibcm_remote_response_time; 612 613 /* 614 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type, 615 * IBT_CM_FLOW_CONTROL is always set by default. 616 */ 617 req_msgp->req_remote_eecn_plus = h2b32( 618 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 | 619 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL); 620 621 if (flags & IBT_OCHAN_LOCAL_CM_TM) 622 local_cm_proc_time = chan_args->oc_local_cm_time; 623 else 624 local_cm_proc_time = ibcm_local_processing_time; 625 626 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time + 627 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) + 628 ibcm_sw_delay); 629 630 /* save retry count */ 631 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt; 632 633 if (flags & IBT_OCHAN_STARTING_PSN) 634 starting_psn = chan_args->oc_starting_psn; 635 636 if (local_cm_resp_time > 0x1f) 637 local_cm_resp_time = 0x1f; 638 639 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */ 640 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 | 641 local_cm_resp_time << 3 | statep->cep_retry_cnt); 642 643 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 644 "Prim Pkt lt (IB time) 0x%x", channel, 645 chan_args->oc_path->pi_prim_pkt_lt); 646 647 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 648 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time); 649 650 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 651 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time); 652 653 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 654 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time); 655 656 statep->starting_psn = starting_psn; 657 658 /* Pkey - bytes 48-49 */ 659 req_msgp->req_part_key = h2b16(prim_pkey); 660 661 if (flags & IBT_OCHAN_CM_RETRY) 662 cm_retries = chan_args->oc_cm_retry_cnt; 663 else 664 cm_retries = ibcm_max_retries; 665 666 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries; 667 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4; 668 669 /* 670 * Check whether SRQ is associated with this Channel, if yes, then 671 * set the SRQ Exists bit in the REQ. 672 */ 673 if (qp_query_attr.qp_srq != NULL) { 674 req_msgp->req_max_cm_retries_plus |= (1 << 3); 675 } 676 677 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 | 678 chan_args->oc_path_rnr_retry_cnt; 679 680 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d" 681 "staring PSN %x", channel, cm_retries, starting_psn); 682 683 684 #ifdef NO_EEC_SUPPORT_YET 685 if (flags & IBT_OCHAN_RDC_EXISTS) 686 req_msgp->req_mtu_plus |= 8; 687 #endif 688 689 /* Initialize the "primary" port stuff next - bytes 52-95 */ 690 req_msgp->req_primary_l_port_lid = h2b16(primary_slid); 691 req_msgp->req_primary_r_port_lid = 692 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 693 req_msgp->req_primary_l_port_gid.gid_prefix = 694 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix); 695 req_msgp->req_primary_l_port_gid.gid_guid = 696 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid); 697 req_msgp->req_primary_r_port_gid.gid_prefix = 698 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 699 req_msgp->req_primary_r_port_gid.gid_guid = 700 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 701 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh; 702 703 /* Bytes 88-91 - primary_flowlbl, and primary_srate */ 704 req_msgp->req_primary_flow_label_plus = 705 h2b32(((primary_grh == B_TRUE) ? 706 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) | 707 IBCM_PRIM_ADDS_VECT(chan_args).av_srate); 708 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ? 709 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0; 710 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ? 711 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 0xff; 712 req_msgp->req_primary_sl_plus = 713 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 | 714 ((primary_grh == B_TRUE) ? 0 : 8); 715 716 req_msgp->req_primary_localtime_plus = 717 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) + 718 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 719 720 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p", 721 channel, statep); 722 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 723 "active hca_ack_delay (usec) %d", channel, 724 req_msgp->req_primary_localtime_plus); 725 726 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 727 "Sent primary cep timeout (IB Time) %d", channel, 728 hcap->hca_ack_delay); 729 730 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ", 731 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 732 733 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 734 "prim gid prefix %llX", channel, 735 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 736 737 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 738 "prim gid guid %llX ", channel, 739 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 740 741 /* Initialize the "alternate" port stuff - optional */ 742 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) { 743 ib_gid_t tmp_gid; 744 745 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid); 746 req_msgp->req_alt_r_port_lid = 747 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 748 /* 749 * doing all this as req_alt_r/l_port_gid is at offset 750 * 100, 116 which is not divisible by 8 751 */ 752 753 tmp_gid.gid_prefix = 754 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 755 tmp_gid.gid_guid = 756 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 757 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0], 758 sizeof (ib_gid_t)); 759 tmp_gid.gid_prefix = 760 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix); 761 tmp_gid.gid_guid = 762 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid); 763 764 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0], 765 sizeof (ib_gid_t)); 766 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh; 767 768 /* Bytes 132-135 - alternate_flow_label, and alternate srate */ 769 req_msgp->req_alt_flow_label_plus = h2b32( 770 (((alternate_grh == B_TRUE) ? 771 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) | 772 IBCM_ALT_ADDS_VECT(chan_args).av_srate)); 773 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ? 774 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0; 775 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ? 776 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 0xff; 777 req_msgp->req_alt_sl_plus = 778 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 | 779 ((alternate_grh == B_TRUE) ? 0 : 8); 780 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 * 781 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) + 782 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 783 784 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 785 "alt_dlid %x ", channel, 786 IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 787 788 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 789 "alt gid prefix %llX", channel, 790 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 791 792 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 793 "alt gid guid %llX ", channel, 794 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 795 } 796 797 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ); 798 if ((len > 0) && chan_args->oc_priv_data) 799 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len); 800 801 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*req_msgp)) 802 803 /* return_data is filled up in the state machine code */ 804 if (ret_args != NULL) { 805 statep->open_return_data = ret_args; 806 } 807 808 /* initialize some statep fields here */ 809 statep->mode = IBCM_ACTIVE_MODE; 810 statep->hcap = hcap; 811 812 statep->cm_handler = chan_args->oc_cm_handler; 813 statep->state_cm_private = chan_args->oc_cm_clnt_private; 814 815 statep->pkt_life_time = 816 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt); 817 818 statep->timer_value = 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time; 819 820 /* Initialize statep->stored_reply_addr */ 821 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl; 822 823 /* Initialize stored reply addr fields */ 824 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr; 825 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr; 826 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists; 827 statep->stored_reply_addr.port_num = cm_reply_addr.port_num; 828 829 /* 830 * The IPD on local/active side is calculated by path functions, 831 * hence available in the args of ibt_open_rc_channel 832 */ 833 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate; 834 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate; 835 836 /* Store the source path bits for primary and alt paths */ 837 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 838 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 839 840 statep->open_done = B_FALSE; 841 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT; 842 IBCM_REF_CNT_INCR(statep); /* For non-blocking REQ post */ 843 statep->send_mad_flags |= IBCM_REQ_POST_BUSY; 844 845 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID = 846 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID); 847 848 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID = 849 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid, 850 0)); 851 852 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 853 854 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_REQ); 855 856 /* Send REQ */ 857 ibcm_post_rc_mad(statep, statep->stored_msg, ibcm_post_req_complete, 858 statep); 859 860 mutex_enter(&statep->state_mutex); 861 862 if (mode == IBT_BLOCKING) { 863 864 /* wait for REQ/REP/RTU */ 865 while (statep->open_done != B_TRUE) { 866 cv_wait(&statep->block_client_cv, &statep->state_mutex); 867 } 868 869 /* 870 * In the case that open_channel() fails because of a 871 * REJ or timeout, change retval to IBT_CM_FAILURE 872 */ 873 if (statep->open_return_data->rc_status != IBT_CM_ACCEPT) 874 status = IBT_CM_FAILURE; 875 876 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p " 877 "ret status %d cm status %d", channel, status, 878 statep->open_return_data->rc_status); 879 } 880 881 882 /* decrement the ref-count before leaving here */ 883 IBCM_REF_CNT_DECR(statep); 884 885 mutex_exit(&statep->state_mutex); 886 887 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel); 888 return (status); 889 } 890 891 /* 892 * ibcm_init_reply_addr: 893 * 894 * The brief description of functionality below. 895 * 896 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case): 897 * Build CM path from chan_args->oc_cm_cep_path 898 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt 899 * 900 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case): 901 * If Redirect LID is specified, 902 * If Redirect GID is not specified or specified to be on the same 903 * subnet, then 904 * Build CM path from chan_args->oc_cm_redirect_info 905 * Set CM pkt lt to subnet timeout 906 * Else (ie., GID specified, but on a different subnet) 907 * Do a path lookup to build CM Path and set CM pkt lt 908 * 909 */ 910 static ibt_status_t 911 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr, 912 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags, 913 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid) 914 { 915 ibt_adds_vect_t *cm_adds; 916 ibt_path_info_t path; 917 boolean_t cm_grh; 918 ibt_status_t status; 919 920 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:"); 921 922 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*reply_addr)) 923 924 /* 925 * sending side CM lid/gid/port num are not based on any redirect 926 * params. These values are set to primary RC path lid/gid/port num. 927 * In the future, these values can be set based on framework policy 928 * decisions ensuring reachability. 929 */ 930 reply_addr->grh_hdr.ig_sender_gid = 931 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid; 932 reply_addr->rcvd_addr.ia_local_lid = prim_slid; 933 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 934 935 if (flags & IBT_OCHAN_PORT_REDIRECTED) { 936 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 937 "IBT_OCHAN_PORT_REDIRECTED specified"); 938 939 status = ibt_index2pkey_byguid(hcap->hca_guid, 940 chan_args->oc_cm_cep_path->cep_hca_port_num, 941 chan_args->oc_cm_cep_path->cep_pkey_ix, 942 &reply_addr->rcvd_addr.ia_p_key); 943 944 if (status != IBT_SUCCESS) { 945 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid " 946 "CM PKeyIx %x port_num %x", 947 chan_args->oc_cm_cep_path->cep_pkey_ix, 948 chan_args->oc_cm_cep_path->cep_hca_port_num); 949 return (status); 950 } 951 952 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect); 953 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x", 954 cm_adds->av_dlid); 955 956 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 957 reply_addr->rcvd_addr.ia_remote_qno = 1; 958 *cm_pkt_lt = chan_args->oc_cm_pkt_lt; 959 960 } else if (flags & IBT_OCHAN_REDIRECTED) { 961 ibt_redirect_info_t *redirect_info; 962 963 ibt_hca_portinfo_t *port_infop; 964 uint_t psize, nports; 965 966 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 967 "IBT_OCHAN_REDIRECTED specified"); 968 969 redirect_info = chan_args->oc_cm_redirect_info; 970 971 /* As per spec definition 1.1, it's always IB_GSI_QKEY */ 972 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey; 973 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn; 974 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey; 975 976 /* 977 * if LID is non-zero in classportinfo and DGID is in the same 978 * subnet as SGID, use classportinfo fields to form 979 * CM MAD destination address. 980 */ 981 982 if ((redirect_info->rdi_dlid != 0) && 983 ((redirect_info->rdi_gid.gid_guid == 0) || 984 (redirect_info->rdi_gid.gid_guid == 985 reply_addr->grh_hdr.ig_sender_gid.gid_guid))) { 986 987 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid, 988 reply_addr->port_num, &port_infop, &nports, &psize); 989 990 if ((status != IBT_SUCCESS) || (nports == 0)) { 991 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 992 "Query Ports Failed: %d", status); 993 return (status); 994 } else if (port_infop->p_subnet_timeout > 995 IBCM_MAX_IB_PKT_LT) { 996 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 997 "large subnet timeout %x port_no %x", 998 port_infop->p_subnet_timeout, 999 reply_addr->port_num); 1000 ibt_free_portinfo(port_infop, psize); 1001 return (IBT_PATH_PKT_LT_TOO_HIGH); 1002 } else { 1003 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: " 1004 "subnet timeout %x port_no %x", 1005 port_infop->p_subnet_timeout, 1006 reply_addr->port_num); 1007 1008 *cm_pkt_lt = 1009 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt, 1010 port_infop->p_subnet_timeout)); 1011 1012 ibt_free_portinfo(port_infop, psize); 1013 } 1014 1015 reply_addr->rcvd_addr.ia_remote_lid = 1016 redirect_info->rdi_dlid; 1017 reply_addr->rcvd_addr.ia_service_level = 1018 redirect_info->rdi_sl; 1019 1020 /* If GID is specified, copy GRH fields */ 1021 if (redirect_info->rdi_gid.gid_guid != 0) { 1022 reply_addr->grh_exists = B_TRUE; 1023 reply_addr->grh_hdr.ig_recver_gid = 1024 redirect_info->rdi_gid; 1025 reply_addr->grh_hdr.ig_tclass = 1026 redirect_info->rdi_tclass; 1027 reply_addr->grh_hdr.ig_flow_label = 1028 redirect_info->rdi_flow; 1029 1030 /* Classportinfo doesn't have hoplimit field */ 1031 reply_addr->grh_hdr.ig_hop_limit = 0xff; 1032 } 1033 return (IBT_SUCCESS); 1034 1035 } else { /* redirect GID on a different subnet */ 1036 ibt_path_attr_t path_attr; 1037 ib_gid_t path_dgid[1]; 1038 1039 /* 1040 * If GID is specified, and LID is zero in classportinfo 1041 * do a path lookup using specified GID, Pkey, FL, TC 1042 * in classportinfo 1043 */ 1044 1045 ASSERT(redirect_info->rdi_gid.gid_guid != 0); 1046 1047 bzero(&path_attr, sizeof (path_attr)); 1048 1049 path_attr.pa_dgids = &path_dgid[0]; 1050 path_attr.pa_dgids[0] = redirect_info->rdi_gid; 1051 1052 /* 1053 * use reply_addr below, as sender_gid in reply_addr 1054 * may have been set above based on some policy decision 1055 * for originating end point for CM MADs above 1056 */ 1057 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid; 1058 1059 path_attr.pa_num_dgids = 1; 1060 path_attr.pa_flow = redirect_info->rdi_flow; 1061 path_attr.pa_tclass = redirect_info->rdi_tclass; 1062 path_attr.pa_pkey = redirect_info->rdi_pkey; 1063 1064 if ((status = ibt_get_paths(ibcm_ibt_handle, 1065 IBT_PATH_NO_FLAGS, &path_attr, 1, &path, NULL)) != 1066 IBT_SUCCESS) 1067 return (status); 1068 1069 /* Initialize cm_adds */ 1070 cm_adds = &path.pi_prim_cep_path.cep_adds_vect; 1071 *cm_pkt_lt = path.pi_prim_pkt_lt; 1072 } 1073 1074 } else { /* cm_pkey initialized in ibt_open_rc_channel */ 1075 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 1076 reply_addr->rcvd_addr.ia_remote_qno = 1; 1077 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt; 1078 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args)); 1079 } 1080 1081 1082 cm_grh = cm_adds->av_send_grh; 1083 reply_addr->grh_exists = cm_grh; 1084 1085 reply_addr->rcvd_addr.ia_remote_lid = 1086 cm_adds->av_dlid; 1087 1088 reply_addr->grh_hdr.ig_recver_gid = 1089 cm_adds->av_dgid; 1090 reply_addr->grh_hdr.ig_flow_label = 1091 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK; 1092 reply_addr->grh_hdr.ig_tclass = 1093 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0; 1094 reply_addr->grh_hdr.ig_hop_limit = 1095 (cm_grh == B_TRUE) ? cm_adds->av_hop : 0xff; 1096 reply_addr->rcvd_addr.ia_service_level = 1097 cm_adds->av_srvl; 1098 1099 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*reply_addr)) 1100 1101 return (IBT_SUCCESS); 1102 } 1103 1104 1105 /* 1106 * ibt_prime_close_rc_channel() 1107 * It allocates resources required for close channel operation, so 1108 * ibt_close_rc_channel can be called from interrupt routine. 1109 * 1110 * INPUTS: 1111 * channel The address of an ibt_channel_t struct that 1112 * specifies the channel to open. 1113 * 1114 * RETURN VALUES: 1115 * IBT_SUCCESS on success(or respective failure on error) 1116 * 1117 * Clients are typically expected to call this function in established state 1118 */ 1119 ibt_status_t 1120 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel) 1121 { 1122 ibcm_state_data_t *statep; 1123 ibt_status_t status = IBT_SUCCESS; 1124 1125 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel); 1126 1127 /* validate channel, first */ 1128 if (IBCM_INVALID_CHANNEL(channel)) { 1129 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p" 1130 "invalid channel", channel); 1131 return (IBT_CHAN_HDL_INVALID); 1132 } 1133 1134 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1135 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p" 1136 "Invalid Channel type: Applicable only to RC Channel", 1137 channel); 1138 return (IBT_CHAN_SRV_TYPE_INVALID); 1139 } 1140 1141 /* get the statep */ 1142 IBCM_GET_CHAN_PRIVATE(channel, statep); 1143 1144 /* 1145 * This can happen, if the statep is already gone by a DREQ from 1146 * the remote side 1147 */ 1148 1149 if (statep == NULL) { 1150 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p" 1151 "statep NULL", channel); 1152 return (IBT_SUCCESS); 1153 } 1154 1155 mutex_enter(&statep->state_mutex); 1156 IBCM_RELEASE_CHAN_PRIVATE(channel); 1157 if (statep->state != IBCM_STATE_ESTABLISHED) { 1158 mutex_exit(&statep->state_mutex); 1159 return (IBT_CHAN_STATE_INVALID); 1160 } 1161 IBCM_REF_CNT_INCR(statep); 1162 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p" 1163 "state %x", channel, statep, statep->state); 1164 mutex_exit(&statep->state_mutex); 1165 1166 /* clients could pre-allocate dreq mad, even before connection est */ 1167 if (statep->dreq_msg == NULL) 1168 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, 1169 &statep->dreq_msg, MAD_METHOD_SEND); 1170 1171 mutex_enter(&statep->state_mutex); 1172 IBCM_REF_CNT_DECR(statep); 1173 mutex_exit(&statep->state_mutex); 1174 1175 if (status != IBT_SUCCESS) { 1176 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1177 "ibcm_alloc_out_msg failed ", channel); 1178 return (status); 1179 } 1180 1181 /* If this message isn't seen then ibt_prime_close_rc_channel failed */ 1182 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done", 1183 channel); 1184 1185 return (IBT_SUCCESS); 1186 } 1187 1188 /* 1189 * ibt_close_rc_channel() 1190 * It closes an established channel. 1191 * 1192 * RETURN VALUES: 1193 * IBT_SUCCESS on success(or respective failure on error) 1194 */ 1195 ibt_status_t 1196 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 1197 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status, 1198 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p) 1199 { 1200 ibcm_hca_info_t *hcap; 1201 ibcm_state_data_t *statep; 1202 ibt_status_t status; 1203 1204 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)", 1205 channel, mode, priv_data, priv_data_len, 1206 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p); 1207 1208 /* validate channel, first */ 1209 if (IBCM_INVALID_CHANNEL(channel)) { 1210 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1211 "invalid channel", channel); 1212 return (IBT_CHAN_HDL_INVALID); 1213 } 1214 1215 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1216 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1217 "Invalid Channel type: Applicable only to RC Channel", 1218 channel); 1219 return (IBT_CHAN_SRV_TYPE_INVALID); 1220 } 1221 1222 if (mode == IBT_BLOCKING) { 1223 /* valid only for BLOCKING MODE */ 1224 if ((ret_priv_data_len_p != NULL) && 1225 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) { 1226 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1227 "private data len %d is too large", channel, 1228 *ret_priv_data_len_p); 1229 return (IBT_INVALID_PARAM); 1230 } 1231 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) { 1232 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1233 "invalid mode %x specified", channel, mode); 1234 return (IBT_INVALID_PARAM); 1235 } 1236 1237 /* get the statep */ 1238 IBCM_GET_CHAN_PRIVATE(channel, statep); 1239 1240 if (statep == NULL) { 1241 1242 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1243 "statep NULL", channel); 1244 if (ibtl_cm_is_chan_closing(channel) || 1245 ibtl_cm_is_chan_closed(channel)) { 1246 if (ret_status) 1247 *ret_status = IBT_CM_CLOSED_ALREADY; 1248 1249 /* No private data to return to the client */ 1250 if (ret_priv_data_len_p != NULL) 1251 *ret_priv_data_len_p = 0; 1252 1253 return (IBT_SUCCESS); 1254 } 1255 return (IBT_CHAN_STATE_INVALID); 1256 } 1257 1258 mutex_enter(&statep->state_mutex); 1259 IBCM_RELEASE_CHAN_PRIVATE(channel); 1260 IBCM_REF_CNT_INCR(statep); 1261 mutex_exit(&statep->state_mutex); 1262 1263 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p statep %p", 1264 channel, statep); 1265 1266 mutex_enter(&statep->state_mutex); 1267 hcap = statep->hcap; 1268 1269 /* HCA must have been in active state. If not, it's a client bug */ 1270 if (!IBCM_ACCESS_HCA_OK(hcap)) { 1271 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1272 "hcap 0x%p not active", channel, hcap); 1273 IBCM_REF_CNT_DECR(statep); 1274 mutex_exit(&statep->state_mutex); 1275 return (IBT_CHAN_HDL_INVALID); 1276 } 1277 1278 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) { 1279 while (statep->cep_in_rts == IBCM_BLOCK) 1280 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1281 } 1282 1283 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */ 1284 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT) 1285 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1286 1287 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1288 "connection state is %x", channel, statep->state); 1289 1290 statep->close_ret_status = ret_status; 1291 1292 /* If state is in pre-established states, abort the connection est */ 1293 if (statep->state != IBCM_STATE_ESTABLISHED) { 1294 1295 /* No DREP private data possible */ 1296 if (ret_priv_data_len_p != NULL) 1297 *ret_priv_data_len_p = 0; 1298 1299 /* 1300 * If waiting for a response mad, then cancel the timer, 1301 * and delete the connection 1302 */ 1303 if (statep->state == IBCM_STATE_REQ_SENT || 1304 statep->state == IBCM_STATE_REP_SENT || 1305 statep->state == IBCM_STATE_REP_WAIT || 1306 statep->state == IBCM_STATE_MRA_REP_RCVD) { 1307 timeout_id_t timer_val = statep->timerid; 1308 ibcm_conn_state_t old_state; 1309 1310 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1311 "chan 0x%p connection aborted in state %x", channel, 1312 statep->state); 1313 1314 old_state = statep->state; 1315 statep->state = IBCM_STATE_DELETE; 1316 1317 if (mode == IBT_NONBLOCKING) { 1318 if (taskq_dispatch(ibcm_taskq, 1319 ibcm_process_abort_via_taskq, statep, 1320 TQ_NOSLEEP) == 0) { 1321 1322 IBCM_REF_CNT_DECR(statep); 1323 statep->state = old_state; 1324 mutex_exit(&statep->state_mutex); 1325 return (IBT_INSUFF_KERNEL_RESOURCE); 1326 } /* if taskq_dispatch succeeds */ 1327 /* Cancel the timer */ 1328 statep->timerid = 0; 1329 mutex_exit(&statep->state_mutex); 1330 } else { 1331 /* Cancel the timer */ 1332 statep->timerid = 0; 1333 mutex_exit(&statep->state_mutex); 1334 (void) taskq_dispatch(ibcm_taskq, 1335 ibcm_process_abort_via_taskq, statep, 1336 TQ_SLEEP); 1337 } 1338 1339 /* cancel the currently running timer */ 1340 if (timer_val != 0) 1341 (void) untimeout(timer_val); 1342 1343 /* wait until cm handler returns for BLOCKING cases */ 1344 if ((mode == IBT_BLOCKING) || 1345 (mode == IBT_NOCALLBACKS)) { 1346 mutex_enter(&statep->state_mutex); 1347 while (statep->close_done != B_TRUE) 1348 cv_wait(&statep->block_client_cv, 1349 &statep->state_mutex); 1350 mutex_exit(&statep->state_mutex); 1351 } 1352 1353 if (ret_status) 1354 *ret_status = IBT_CM_CLOSED_ABORT; 1355 1356 /* 1357 * It would ideal to post a REJ MAD, but that would 1358 * be non-conformance to spec. Hence, delete the state 1359 * data. Assuming that happens quickly, any retransmits 1360 * from the remote are replied by CM with reject 1361 * reason " no valid com id". That would stop remote 1362 * sending any more MADs. 1363 */ 1364 ibcm_delete_state_data(statep); 1365 return (IBT_SUCCESS); 1366 1367 /* if CM busy in cm handler, wait until cm handler returns */ 1368 } else if (statep->state == IBCM_STATE_REQ_RCVD || 1369 statep->state == IBCM_STATE_REP_RCVD || 1370 statep->state == IBCM_STATE_MRA_SENT || 1371 statep->state == IBCM_STATE_MRA_REP_SENT) { 1372 1373 /* take control of statep */ 1374 statep->abort_flag |= IBCM_ABORT_CLIENT; 1375 1376 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1377 "chan 0x%p connection aborted in state = %x", 1378 channel, statep->state); 1379 1380 /* 1381 * wait until state machine modifies qp state to error, 1382 * including disassociating statep and QP 1383 */ 1384 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS)) 1385 while (statep->close_done != B_TRUE) 1386 cv_wait(&statep->block_client_cv, 1387 &statep->state_mutex); 1388 1389 /* a sanity setting */ 1390 if (mode == IBT_NOCALLBACKS) 1391 statep->cm_handler = NULL; 1392 IBCM_REF_CNT_DECR(statep); 1393 mutex_exit(&statep->state_mutex); 1394 1395 /* 1396 * In rare situations, connection attempt could be 1397 * terminated for some other reason, before abort is 1398 * processed, but CM still returns ret_status as abort 1399 */ 1400 if (ret_status) 1401 *ret_status = IBT_CM_CLOSED_ABORT; 1402 1403 /* 1404 * REJ MAD is posted by the CM state machine for this 1405 * case, hence state structure is deleted in the 1406 * state machine processing. 1407 */ 1408 return (IBT_SUCCESS); 1409 1410 } else if ((statep->state == IBCM_STATE_TIMEWAIT) || 1411 (statep->state == IBCM_STATE_DELETE)) { 1412 1413 /* State already in timewait, so no return priv data */ 1414 IBCM_REF_CNT_DECR(statep); 1415 mutex_exit(&statep->state_mutex); 1416 1417 /* The teardown has already been done */ 1418 if (ret_status) 1419 *ret_status = IBT_CM_CLOSED_ALREADY; 1420 1421 return (IBT_SUCCESS); 1422 1423 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) || 1424 (statep->state == IBCM_STATE_DREQ_SENT) || 1425 (statep->state == IBCM_STATE_DREP_RCVD) || 1426 ((statep->state == IBCM_STATE_TIMED_OUT) && 1427 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) { 1428 1429 /* 1430 * Either the remote or local client has already 1431 * initiated the teardown. IBCM_STATE_DREP_RCVD is 1432 * possible, if CM initiated teardown without client's 1433 * knowledge, for stale handling, etc., 1434 */ 1435 if (mode == IBT_NOCALLBACKS) { 1436 if (statep->close_nocb_state == IBCM_UNBLOCK) { 1437 statep->close_nocb_state = IBCM_FAIL; 1438 /* enable free qp after return */ 1439 ibtl_cm_chan_is_closing( 1440 statep->channel); 1441 } else while (statep->close_nocb_state == 1442 IBCM_BLOCK) 1443 cv_wait(&statep->block_client_cv, 1444 &statep->state_mutex); 1445 statep->cm_handler = NULL; /* sanity setting */ 1446 if (ret_status) 1447 *ret_status = IBT_CM_CLOSED_ALREADY; 1448 } else if (mode == IBT_BLOCKING) { 1449 /* wait until state is moved to timewait */ 1450 while (statep->close_done != B_TRUE) 1451 cv_wait(&statep->block_client_cv, 1452 &statep->state_mutex); 1453 } 1454 1455 IBCM_REF_CNT_DECR(statep); 1456 mutex_exit(&statep->state_mutex); 1457 1458 /* ret_status is set in state machine code */ 1459 return (IBT_SUCCESS); 1460 1461 } else if (statep->state == IBCM_STATE_TIMED_OUT) { 1462 1463 if ((mode == IBT_BLOCKING) || 1464 (mode == IBT_NOCALLBACKS)) { 1465 1466 /* 1467 * wait until cm handler invocation and 1468 * disassociation between statep and channel 1469 * is complete 1470 */ 1471 while (statep->close_done != B_TRUE) 1472 cv_wait(&statep->block_client_cv, 1473 &statep->state_mutex); 1474 } 1475 1476 if (ret_status) 1477 *ret_status = IBT_CM_CLOSED_ABORT; 1478 IBCM_REF_CNT_DECR(statep); 1479 mutex_exit(&statep->state_mutex); 1480 1481 return (IBT_SUCCESS); 1482 } else { 1483 IBCM_REF_CNT_DECR(statep); 1484 mutex_exit(&statep->state_mutex); 1485 1486 return (IBT_CM_FAILURE); 1487 } 1488 } 1489 1490 ASSERT(statep->close_nocb_state != IBCM_BLOCK); 1491 1492 if (mode == IBT_NOCALLBACKS) { 1493 statep->close_nocb_state = IBCM_FAIL; 1494 statep->cm_handler = NULL; 1495 ibtl_cm_chan_is_closing(statep->channel); 1496 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: " 1497 "NOCALLBACKS on in statep = %p", statep); 1498 } 1499 1500 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT; 1501 statep->timerid = 0; 1502 statep->close_done = B_FALSE; 1503 statep->close_flow = 1; 1504 1505 /* 1506 * Cancel/wait for any pending ibt_set_alt_path, and 1507 * release state mutex 1508 */ 1509 ibcm_sync_lapr_idle(statep); 1510 1511 ibcm_close_flow_control_enter(); 1512 if (statep->dreq_msg == NULL) { 1513 if ((status = ibcm_alloc_out_msg( 1514 statep->stored_reply_addr.ibmf_hdl, &statep->dreq_msg, 1515 MAD_METHOD_SEND)) != IBT_SUCCESS) { 1516 1517 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: " 1518 "chan 0x%p ibcm_alloc_out_msg failed ", channel); 1519 mutex_enter(&statep->state_mutex); 1520 statep->state = IBCM_STATE_ESTABLISHED; 1521 IBCM_REF_CNT_DECR(statep); 1522 cv_broadcast(&statep->block_mad_cv); 1523 statep->close_flow = 0; 1524 ibcm_close_flow_control_exit(); 1525 mutex_exit(&statep->state_mutex); 1526 return (status); 1527 } 1528 } else 1529 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel:" 1530 "DREQ MAD already allocated in statep %p", statep); 1531 1532 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) { 1533 statep->close_priv_data = NULL; 1534 statep->close_priv_data_len = NULL; 1535 } else { 1536 statep->close_priv_data = ret_priv_data; 1537 statep->close_priv_data_len = ret_priv_data_len_p; 1538 } 1539 1540 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ); 1541 if ((priv_data != NULL) && (priv_data_len > 0)) 1542 bcopy(priv_data, ((ibcm_dreq_msg_t *) 1543 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data, 1544 priv_data_len); 1545 1546 /* statep mutex not held while posting the MAD */ 1547 if (mode == IBT_NONBLOCKING) { 1548 if (taskq_dispatch(ibcm_taskq, ibcm_post_dreq_mad, statep, 1549 TQ_NOSLEEP) == 0) { 1550 mutex_enter(&statep->state_mutex); 1551 statep->state = IBCM_STATE_ESTABLISHED; 1552 IBCM_REF_CNT_DECR(statep); 1553 cv_broadcast(&statep->block_mad_cv); 1554 statep->close_flow = 0; 1555 ibcm_close_flow_control_exit(); 1556 mutex_exit(&statep->state_mutex); 1557 return (IBT_INSUFF_KERNEL_RESOURCE); 1558 } 1559 } else { 1560 ibcm_post_dreq_mad(statep); 1561 } 1562 1563 mutex_enter(&statep->state_mutex); 1564 1565 if (mode == IBT_BLOCKING) { 1566 1567 /* wait for DREP */ 1568 while (statep->close_done != B_TRUE) 1569 cv_wait(&statep->block_client_cv, 1570 &statep->state_mutex); 1571 1572 IBTF_DPRINTF_L4(cmlog, "ibt_close_rc_channel: chan 0x%p " 1573 "done blocking", channel); 1574 } 1575 1576 IBCM_REF_CNT_DECR(statep); 1577 mutex_exit(&statep->state_mutex); 1578 1579 /* If this message isn't seen then ibt_close_rc_channel failed */ 1580 IBTF_DPRINTF_L5(cmlog, "ibt_close_rc_channel: chan 0x%p done", 1581 channel); 1582 1583 return (IBT_SUCCESS); 1584 } 1585 1586 ibt_status_t 1587 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control, 1588 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg) 1589 { 1590 ibcm_state_data_t *statep; 1591 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg; 1592 ibt_qp_query_attr_t qp_attr; 1593 ibt_status_t retval; 1594 1595 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan, 1596 control, hca_port_num, func, arg); 1597 1598 if (IBCM_INVALID_CHANNEL(rc_chan)) { 1599 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel"); 1600 return (IBT_CHAN_HDL_INVALID); 1601 } 1602 1603 /* check qp state */ 1604 retval = ibt_query_qp(rc_chan, &qp_attr); 1605 1606 if (retval != IBT_SUCCESS) 1607 return (retval); 1608 1609 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV) 1610 return (IBT_CHAN_SRV_TYPE_INVALID); 1611 1612 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR) 1613 return (IBT_CHAN_STATE_INVALID); 1614 1615 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1616 1617 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t), 1618 KM_SLEEP); 1619 1620 ibcm_tq_recycle_arg->rc_chan = rc_chan; 1621 ibcm_tq_recycle_arg->control = control; 1622 ibcm_tq_recycle_arg->hca_port_num = hca_port_num; 1623 ibcm_tq_recycle_arg->func = func; 1624 ibcm_tq_recycle_arg->arg = arg; 1625 1626 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1627 1628 IBCM_GET_CHAN_PRIVATE(rc_chan, statep); 1629 1630 /* 1631 * If non-blocking ie., func specified and channel has not yet completed 1632 * the timewait, then schedule the work for later 1633 */ 1634 if ((func != NULL) && (statep != NULL)) { 1635 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1636 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1637 statep->recycle_arg = ibcm_tq_recycle_arg; 1638 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1639 return (IBT_SUCCESS); 1640 } 1641 1642 /* 1643 * if blocking ie., func specified, and channel has not yet completed 1644 * the timewait, then block until the channel completes the timewait 1645 */ 1646 if (statep != NULL) 1647 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1648 IBCM_WAIT_CHAN_PRIVATE(rc_chan); 1649 1650 if (func) { /* NON BLOCKING case. Taskq for QP state change */ 1651 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle, 1652 ibcm_tq_recycle_arg, TQ_SLEEP); 1653 return (IBT_SUCCESS); 1654 } else /* BLOCKING case */ 1655 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg)); 1656 } 1657 1658 void 1659 ibcm_process_rc_recycle(void *recycle_arg) 1660 { 1661 (void) ibcm_process_rc_recycle_ret(recycle_arg); 1662 } 1663 1664 static ibt_status_t 1665 ibcm_process_rc_recycle_ret(void *recycle_arg) 1666 { 1667 ibt_qp_info_t qp_info; 1668 ibt_status_t ibt_status = IBT_SUCCESS; 1669 ibt_cep_modify_flags_t cep_flags; 1670 ibt_qp_query_attr_t qp_attr; 1671 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg = 1672 (ibcm_taskq_recycle_arg_t *)recycle_arg; 1673 1674 /* QP must have been in error state */ 1675 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr); 1676 if (ibt_status != IBT_SUCCESS) 1677 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1678 "chanp %p ibt_query_qp() = %d", 1679 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1680 else { 1681 /* perform the QP state change from ERROR to RESET */ 1682 bzero(&qp_info, sizeof (qp_info)); 1683 1684 qp_info.qp_trans = IBT_RC_SRV; 1685 qp_info.qp_state = IBT_STATE_RESET; 1686 1687 /* Call modify_qp to move to RESET state */ 1688 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1689 IBT_CEP_SET_STATE, &qp_info, NULL); 1690 1691 if (ibt_status != IBT_SUCCESS) 1692 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1693 "chanp %p ibt_modify_qp() = %d for ERROR to RESET", 1694 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1695 } 1696 1697 if (ibt_status == IBT_SUCCESS) { 1698 1699 qp_info.qp_state = IBT_STATE_INIT; 1700 1701 /* set flags for all mandatory args from RESET to INIT */ 1702 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 1703 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W; 1704 cep_flags |= IBT_CEP_SET_ATOMIC; 1705 1706 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 1707 ibcm_tq_recycle_arg->hca_port_num; 1708 qp_info.qp_flags |= 1709 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD; 1710 qp_info.qp_flags |= 1711 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR; 1712 qp_info.qp_flags |= 1713 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC; 1714 1715 /* Always use the existing pkey */ 1716 qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 1717 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix; 1718 1719 /* Call modify_qp to move to INIT state */ 1720 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1721 cep_flags, &qp_info, NULL); 1722 1723 if (ibt_status != IBT_SUCCESS) 1724 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1725 "chanp %p ibt_modify_qp() = %d for RESET to INIT", 1726 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1727 } 1728 1729 /* Change the QP CM state to indicate QP being re-used */ 1730 if (ibt_status == IBT_SUCCESS) 1731 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan); 1732 1733 /* Call func, if defined */ 1734 if (ibcm_tq_recycle_arg->func) 1735 (*(ibcm_tq_recycle_arg->func))(ibt_status, 1736 ibcm_tq_recycle_arg->arg); 1737 1738 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t)); 1739 1740 return (ibt_status); 1741 } 1742 1743 static void 1744 ibcm_process_abort_via_taskq(void *args) 1745 { 1746 ibcm_state_data_t *statep = (ibcm_state_data_t *)args; 1747 1748 ibcm_process_abort(statep); 1749 mutex_enter(&statep->state_mutex); 1750 IBCM_REF_CNT_DECR(statep); 1751 mutex_exit(&statep->state_mutex); 1752 } 1753 1754 /* 1755 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in 1756 * Non-Blocking mode operations. 1757 */ 1758 typedef struct ibcm_local_handler_s { 1759 ibt_cm_ud_handler_t actual_cm_handler; 1760 void *actual_cm_private; 1761 ibt_ud_dest_t *dest_hdl; 1762 } ibcm_local_handler_t; 1763 1764 _NOTE(READ_ONLY_DATA(ibcm_local_handler_s)) 1765 1766 /* 1767 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in 1768 * NON-Blocking mode. 1769 * 1770 * Out here, we update the UD Destination handle with 1771 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client 1772 * handler that was specified by the client. 1773 */ 1774 static ibt_cm_status_t 1775 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event, 1776 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 1777 { 1778 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv; 1779 1780 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d", 1781 event->cm_type); 1782 1783 ASSERT(handler_priv != NULL); 1784 1785 switch (event->cm_type) { 1786 case IBT_CM_UD_EVENT_SIDR_REP: 1787 /* Update QPN & QKey from event into destination handle. */ 1788 if (handler_priv->dest_hdl != NULL) { 1789 handler_priv->dest_hdl->ud_dst_qpn = 1790 event->cm_event.sidr_rep.srep_remote_qpn; 1791 handler_priv->dest_hdl->ud_qkey = 1792 event->cm_event.sidr_rep.srep_remote_qkey; 1793 } 1794 1795 /* Invoke the client handler - inform only, so ignore retval */ 1796 (void) handler_priv->actual_cm_handler( 1797 handler_priv->actual_cm_private, event, ret_args, priv_data, 1798 len); 1799 1800 /* Free memory allocated for local handler's private data. */ 1801 if (handler_priv != NULL) 1802 kmem_free(handler_priv, sizeof (*handler_priv)); 1803 1804 break; 1805 default: 1806 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR"); 1807 break; 1808 } 1809 1810 return (IBT_CM_ACCEPT); 1811 } 1812 1813 1814 /* Validate the input UD destination attributes. */ 1815 static ibt_status_t 1816 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1817 ibt_ud_returns_t *ret_args) 1818 { 1819 /* cm handler must always be specified */ 1820 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) { 1821 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1822 "CM handler is not specified "); 1823 return (IBT_INVALID_PARAM); 1824 } 1825 1826 if (mode == IBT_NONBLOCKING) { 1827 if (ret_args != NULL) { 1828 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1829 "ret_args should be NULL when called in " 1830 "non-blocking mode"); 1831 return (IBT_INVALID_PARAM); 1832 } 1833 } else if (mode == IBT_BLOCKING) { 1834 if (ret_args == NULL) { 1835 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1836 "ret_args should be Non-NULL when called in " 1837 "blocking mode"); 1838 return (IBT_INVALID_PARAM); 1839 } 1840 } else { 1841 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1842 "invalid mode %x specified ", mode); 1843 return (IBT_INVALID_PARAM); 1844 } 1845 1846 if (attr->ud_sid == 0) { 1847 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1848 "ServiceID must be specified. "); 1849 return (IBT_INVALID_PARAM); 1850 } 1851 1852 if (attr->ud_addr == NULL) { 1853 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1854 "Address Info NULL"); 1855 return (IBT_INVALID_PARAM); 1856 } 1857 1858 /* Validate SGID */ 1859 if ((attr->ud_addr->av_sgid.gid_prefix == 0) || 1860 (attr->ud_addr->av_sgid.gid_guid == 0)) { 1861 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID"); 1862 return (IBT_INVALID_PARAM); 1863 } 1864 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>", 1865 attr->ud_addr->av_sgid.gid_prefix, 1866 attr->ud_addr->av_sgid.gid_guid); 1867 1868 /* Validate DGID */ 1869 if ((attr->ud_addr->av_dgid.gid_prefix == 0) || 1870 (attr->ud_addr->av_dgid.gid_guid == 0)) { 1871 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID"); 1872 return (IBT_INVALID_PARAM); 1873 } 1874 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>", 1875 attr->ud_addr->av_dgid.gid_prefix, 1876 attr->ud_addr->av_dgid.gid_guid); 1877 1878 return (IBT_SUCCESS); 1879 } 1880 1881 1882 /* Perform SIDR to retrieve DQPN and QKey. */ 1883 static ibt_status_t 1884 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1885 ibt_ud_returns_t *ret_args) 1886 { 1887 ibt_status_t retval; 1888 ib_pkey_t ud_pkey; 1889 ibmf_handle_t ibmf_hdl; 1890 ibmf_msg_t *ibmf_msg; 1891 ibcm_hca_info_t *hcap; 1892 ibcm_sidr_req_msg_t *sidr_req_msgp; 1893 ibcm_ud_state_data_t *ud_statep; 1894 ibtl_cm_hca_port_t port; 1895 ibcm_sidr_srch_t sidr_entry; 1896 ibcm_qp_list_t *cm_qp_entry; 1897 1898 /* Retrieve HCA GUID value from the available SGID info. */ 1899 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port); 1900 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) { 1901 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1902 "ibtl_cm_get_hca_port failed: %d", retval); 1903 return (retval); 1904 } 1905 1906 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: " 1907 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port); 1908 1909 /* Lookup the HCA info for this GUID */ 1910 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) { 1911 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL"); 1912 return (IBT_HCA_INVALID); 1913 } 1914 1915 /* Return failure if the HCA device or Port is not operational */ 1916 1917 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port, 1918 NULL, NULL)) != IBT_SUCCESS) { 1919 /* Device Port is not in good state, don't use it. */ 1920 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid " 1921 "port specified or port not active"); 1922 ibcm_dec_hca_acc_cnt(hcap); 1923 return (retval); 1924 } 1925 1926 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port, 1927 attr->ud_pkey_ix, &ud_pkey); 1928 if (retval != IBT_SUCCESS) { 1929 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1930 "Failed to convert index2pkey: %d", retval); 1931 ibcm_dec_hca_acc_cnt(hcap); 1932 return (retval); 1933 } 1934 1935 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(sidr_entry)) 1936 1937 /* Allocate a new request id */ 1938 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) { 1939 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1940 "no req id available"); 1941 ibcm_dec_hca_acc_cnt(hcap); 1942 return (IBT_INSUFF_KERNEL_RESOURCE); 1943 } 1944 1945 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) && 1946 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1)) 1947 != IBT_SUCCESS)) { 1948 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1949 "ibmf reg or callback setup failed during re-initialize"); 1950 return (retval); 1951 } 1952 1953 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl; 1954 1955 /* find the ibmf QP to post the SIDR REQ */ 1956 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) == 1957 NULL) { 1958 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation" 1959 " failed"); 1960 ibcm_dec_hca_acc_cnt(hcap); 1961 return (IBT_INSUFF_RESOURCE); 1962 } 1963 1964 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND)) 1965 != IBT_SUCCESS) { 1966 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation" 1967 " failed"); 1968 ibcm_release_qp(cm_qp_entry); 1969 ibcm_dec_hca_acc_cnt(hcap); 1970 return (retval); 1971 } 1972 1973 sidr_entry.srch_lid = port.hp_base_lid; 1974 sidr_entry.srch_gid = attr->ud_addr->av_sgid; 1975 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh; 1976 sidr_entry.srch_mode = IBCM_ACTIVE_MODE; 1977 1978 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(sidr_entry)) 1979 1980 /* do various allocations needed here */ 1981 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 1982 1983 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep, 1984 IBCM_FLAG_ADD); 1985 rw_exit(&hcap->hca_sidr_list_lock); 1986 1987 /* Increment hca's resource count */ 1988 ibcm_inc_hca_res_cnt(hcap); 1989 1990 /* After a resource created on hca, no need to hold the acc cnt */ 1991 ibcm_dec_hca_acc_cnt(hcap); 1992 1993 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 1994 1995 /* Initialize some ud_statep fields */ 1996 ud_statep->ud_stored_msg = ibmf_msg; 1997 ud_statep->ud_svc_id = attr->ud_sid; 1998 ud_statep->ud_pkt_life_time = 1999 ibt_ib2usec(attr->ud_pkt_lt); 2000 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry; 2001 2002 /* set remaining retry cnt */ 2003 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries; 2004 2005 /* 2006 * Get UD handler and corresponding args which is pass it back 2007 * as first argument for the handler. 2008 */ 2009 ud_statep->ud_state_cm_private = attr->ud_cm_private; 2010 2011 if (mode == IBT_BLOCKING) 2012 ud_statep->ud_return_data = ret_args; 2013 else 2014 ud_statep->ud_cm_handler = attr->ud_cm_handler; 2015 2016 /* Initialize the fields of ud_statep->ud_stored_reply_addr */ 2017 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh; 2018 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl; 2019 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit = 2020 attr->ud_addr->av_hop; 2021 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid = 2022 attr->ud_addr->av_sgid; 2023 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid = 2024 attr->ud_addr->av_dgid; 2025 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass = 2026 attr->ud_addr->av_tclass; 2027 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label = 2028 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK; 2029 2030 /* needs to be derived based on the base LID and path bits */ 2031 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid = 2032 port.hp_base_lid; 2033 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid = 2034 attr->ud_addr->av_dlid; 2035 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey; 2036 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY; 2037 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level = 2038 attr->ud_addr->av_srvl; 2039 2040 /* 2041 * This may be enchanced later, to use a remote qno based on past 2042 * redirect rej mad responses. This would be the place to specify 2043 * appropriate remote qno 2044 */ 2045 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1; 2046 2047 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2048 2049 /* Initialize the SIDR REQ message fields */ 2050 sidr_req_msgp = 2051 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg); 2052 2053 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id); 2054 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid); 2055 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey); 2056 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID = 2057 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID); 2058 2059 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) { 2060 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data, 2061 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ)); 2062 } 2063 2064 /* Send out the SIDR REQ message */ 2065 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT; 2066 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT; 2067 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */ 2068 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) + 2069 (ud_statep->ud_pkt_life_time * 2); 2070 2071 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID = 2072 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ, 2073 ud_statep->ud_req_id, 0)); 2074 2075 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x", 2076 ud_statep->ud_timer_value); 2077 2078 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 2079 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2080 2081 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg, 2082 ibcm_post_sidr_req_complete, ud_statep); 2083 2084 mutex_enter(&ud_statep->ud_state_mutex); 2085 2086 /* Wait for SIDR_REP */ 2087 if (mode == IBT_BLOCKING) { 2088 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking"); 2089 2090 while (ud_statep->ud_blocking_done != B_TRUE) { 2091 cv_wait(&ud_statep->ud_block_client_cv, 2092 &ud_statep->ud_state_mutex); 2093 } 2094 2095 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking"); 2096 2097 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) { 2098 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, " 2099 "status = %x, QKey = %x", ret_args->ud_dqpn, 2100 ret_args->ud_status, ret_args->ud_qkey); 2101 2102 } else { 2103 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>", 2104 ret_args->ud_status); 2105 retval = IBT_CM_FAILURE; 2106 } 2107 } 2108 2109 IBCM_UD_REF_CNT_DECR(ud_statep); 2110 mutex_exit(&ud_statep->ud_state_mutex); 2111 2112 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done"); 2113 2114 return (retval); 2115 } 2116 2117 2118 /* 2119 * Function: 2120 * ibt_request_ud_dest 2121 * Input: 2122 * ud_dest A previously allocated UD destination handle. 2123 * mode This function can execute in blocking or non blocking 2124 * modes. 2125 * attr UD destination attributes to be modified. 2126 * Output: 2127 * ud_ret_args If the function is called in blocking mode, ud_ret_args 2128 * should be a pointer to an ibt_ud_returns_t struct. 2129 * Returns: 2130 * IBT_SUCCESS 2131 * Description: 2132 * Modify a previously allocated UD destination handle based on the 2133 * results of doing the SIDR protocol. 2134 */ 2135 ibt_status_t 2136 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode, 2137 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args) 2138 { 2139 ibt_status_t retval; 2140 ibt_ud_dest_t *ud_destp; 2141 ibcm_local_handler_t *local_handler_priv = NULL; 2142 2143 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)", 2144 ud_dest, mode, attr, ud_ret_args); 2145 2146 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args); 2147 if (retval != IBT_SUCCESS) { 2148 return (retval); 2149 } 2150 2151 ud_destp = ud_dest; 2152 2153 /* Allocate an Address handle. */ 2154 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah, 2155 attr->ud_addr); 2156 if (retval != IBT_SUCCESS) { 2157 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2158 "Address Handle Modification failed: %d", retval); 2159 return (retval); 2160 } 2161 2162 if (mode == IBT_NONBLOCKING) { 2163 /* 2164 * In NON-BLOCKING mode, and we need to update the destination 2165 * handle with the DQPN and QKey that are obtained from 2166 * SIDR REP, hook-up our own handler, so that we can catch 2167 * the event, and we ourselves call the actual client's 2168 * ud_cm_handler, in our handler. 2169 */ 2170 2171 /* Allocate memory for local handler's private data. */ 2172 local_handler_priv = 2173 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP); 2174 2175 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2176 2177 local_handler_priv->actual_cm_handler = attr->ud_cm_handler; 2178 local_handler_priv->actual_cm_private = attr->ud_cm_private; 2179 local_handler_priv->dest_hdl = ud_destp; 2180 2181 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2182 2183 attr->ud_cm_handler = ibcm_local_cm_handler; 2184 attr->ud_cm_private = local_handler_priv; 2185 } 2186 2187 /* In order to get DQPN and Destination QKey, perform SIDR */ 2188 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args); 2189 if (retval != IBT_SUCCESS) { 2190 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2191 "Failed to get DQPN: %d", retval); 2192 2193 /* Free memory allocated for local handler's private data. */ 2194 if (local_handler_priv != NULL) 2195 kmem_free(local_handler_priv, 2196 sizeof (*local_handler_priv)); 2197 return (retval); 2198 } 2199 2200 /* 2201 * Fill in the dqpn and dqkey as obtained from ud_ret_args, 2202 * values will be valid only on BLOCKING mode. 2203 */ 2204 if (mode == IBT_BLOCKING) { 2205 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn; 2206 ud_destp->ud_qkey = ud_ret_args->ud_qkey; 2207 } 2208 2209 return (retval); 2210 } 2211 2212 /* 2213 * Function: 2214 * ibt_ud_get_dqpn 2215 * Input: 2216 * attr A pointer to an ibt_ud_dest_attr_t struct that are 2217 * required for SIDR REQ message. Not specified attributes 2218 * should be set to "NULL" or "0". 2219 * ud_sid, ud_addr and ud_pkt_lt must be specified. 2220 * mode This function can execute in blocking or non blocking 2221 * modes. 2222 * Output: 2223 * returns If the function is called in blocking mode, returns 2224 * should be a pointer to an ibt_ud_returns_t struct. 2225 * Return: 2226 * IBT_SUCCESS on success or respective failure on error. 2227 * Description: 2228 * Finds the destination QPN at the specified destination that the 2229 * specified service can be reached on. The IBTF CM initiates the 2230 * service ID resolution protocol (SIDR) to determine a destination QPN. 2231 * 2232 * NOTE: SIDR_REQ is initiated from active side. 2233 */ 2234 ibt_status_t 2235 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 2236 ibt_ud_returns_t *returns) 2237 { 2238 ibt_status_t retval; 2239 2240 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)", 2241 attr, mode, returns); 2242 2243 retval = ibcm_validate_dqpn_data(attr, mode, returns); 2244 if (retval != IBT_SUCCESS) { 2245 return (retval); 2246 } 2247 2248 return (ibcm_ud_get_dqpn(attr, mode, returns)); 2249 } 2250 2251 2252 /* 2253 * ibt_cm_delay: 2254 * A client CM handler function can call this function 2255 * to extend its response time to a CM event. 2256 * INPUTS: 2257 * flags Indicates what CM message processing is being delayed 2258 * by the CM handler, valid values are: 2259 * IBT_CM_DELAY_REQ 2260 * IBT_CM_DELAY_REP 2261 * IBT_CM_DELAY_LAP 2262 * cm_session_id The session ID that was passed to client srv_handler 2263 * by the CM 2264 * service_time The extended service time 2265 * priv_data Vendor specific data to be sent in the CM generated 2266 * MRA message. Should be NULL if not specified. 2267 * len The number of bytes of data specified by priv_data. 2268 * 2269 * RETURN VALUES: 2270 * IBT_SUCCESS on success (or respective failure on error) 2271 */ 2272 ibt_status_t 2273 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id, 2274 clock_t service_time, void *priv_data, ibt_priv_data_len_t len) 2275 { 2276 uint8_t msg_typ = 0; 2277 ibcm_mra_msg_t *mra_msgp; 2278 ibcm_state_data_t *statep; 2279 ibt_status_t status; 2280 2281 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)", 2282 flags, cm_session_id, service_time); 2283 2284 /* 2285 * Make sure channel is associated with a statep 2286 */ 2287 statep = (ibcm_state_data_t *)cm_session_id; 2288 2289 if (statep == NULL) { 2290 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL"); 2291 return (IBT_INVALID_PARAM); 2292 } 2293 2294 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep); 2295 2296 /* Allocate an ibmf msg for mra, if not allocated yet */ 2297 if (statep->mra_msg == NULL) { 2298 if ((status = ibcm_alloc_out_msg( 2299 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg, 2300 MAD_METHOD_SEND)) != IBT_SUCCESS) { 2301 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p" 2302 "IBMF MSG allocation failed", statep->channel); 2303 return (status); 2304 } 2305 } 2306 2307 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2308 2309 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg); 2310 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid); 2311 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid); 2312 2313 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */ 2314 if (flags == IBT_CM_DELAY_REQ) { 2315 msg_typ = IBT_CM_MRA_TYPE_REQ; 2316 } else if (flags == IBT_CM_DELAY_REP) { 2317 msg_typ = IBT_CM_MRA_TYPE_REP; 2318 } else if (flags == IBT_CM_DELAY_LAP) { 2319 msg_typ = IBT_CM_MRA_TYPE_LAP; 2320 } 2321 2322 mra_msgp->mra_message_type_plus = msg_typ << 6; 2323 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3; 2324 2325 len = min(len, IBT_MRA_PRIV_DATA_SZ); 2326 if (priv_data && (len > 0)) 2327 bcopy(priv_data, mra_msgp->mra_private_data, len); 2328 2329 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID = 2330 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID); 2331 2332 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2333 2334 mutex_enter(&statep->state_mutex); 2335 2336 if ((statep->mode == IBCM_ACTIVE_MODE) && 2337 (statep->state == IBCM_STATE_REP_RCVD)) { 2338 statep->state = IBCM_STATE_MRA_REP_SENT; 2339 } else if (statep->mode == IBCM_PASSIVE_MODE) { 2340 if (statep->state == IBCM_STATE_REQ_RCVD) { 2341 statep->state = IBCM_STATE_MRA_SENT; 2342 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) { 2343 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD; 2344 } else { 2345 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state" 2346 "/ap_state/mode %x, %x, %x", statep->state, 2347 statep->ap_state, statep->mode); 2348 mutex_exit(&statep->state_mutex); 2349 return (IBT_CHAN_STATE_INVALID); 2350 } 2351 } else { 2352 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state" 2353 "/ap_state/mode %x, %x, %x", statep->state, 2354 statep->ap_state, statep->mode); 2355 mutex_exit(&statep->state_mutex); 2356 2357 return (IBT_CHAN_STATE_INVALID); 2358 } 2359 /* service time is usecs, stale_clock is nsecs */ 2360 statep->stale_clock = gethrtime() + 2361 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 * 2362 statep->max_cm_retries); 2363 2364 statep->send_mad_flags |= IBCM_MRA_POST_BUSY; 2365 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */ 2366 mutex_exit(&statep->state_mutex); 2367 2368 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID = 2369 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID; 2370 2371 /* post the MRA mad in blocking mode, as no timers involved */ 2372 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete, 2373 statep); 2374 2375 /* If this message isn't seen then ibt_cm_delay failed */ 2376 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: done !!"); 2377 2378 return (IBT_SUCCESS); 2379 } 2380 2381 2382 /* 2383 * ibt_register_service() 2384 * Register a service with the IBCM 2385 * 2386 * INPUTS: 2387 * ibt_hdl The IBT client handle returned to the client 2388 * on an ibt_attach() call. 2389 * 2390 * srv The address of a ibt_srv_desc_t that describes 2391 * the service, containing the following: 2392 * 2393 * sd_ud_handler The Service CM UD event Handler. 2394 * sd_handler The Service CM RC/UC/RD event Handler. 2395 * sd_flags Service flags (peer-to-peer, or not). 2396 * 2397 * sid This tells CM if the service is local (sid is 0) or 2398 * wellknown (sid is the starting service id of the range). 2399 * 2400 * num_sids The number of contiguous service-ids to reserve. 2401 * 2402 * srv_hdl The address of a service identification handle, used 2403 * to deregister a service, and to bind GIDs to. 2404 * 2405 * ret_sid The address to store the Service ID return value. 2406 * If num_sids > 1, ret_sid is the first Service ID 2407 * in the range. 2408 * 2409 * ibt_register_service() returns: 2410 * IBT_SUCCESS - added a service successfully. 2411 * IBT_INVALID_PARAM - invalid input parameter. 2412 * IBT_CM_FAILURE - failed to add the service. 2413 * IBT_CM_SERVICE_EXISTS - service already exists. 2414 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should 2415 * never happen). 2416 */ 2417 ibt_status_t 2418 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv, 2419 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid) 2420 { 2421 ibcm_svc_info_t *svcinfop; 2422 2423 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p, %p, %llx, %d)", 2424 ibt_hdl, srv, (longlong_t)sid, num_sids); 2425 2426 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 2427 2428 *srv_hdl = NULL; 2429 2430 if (num_sids <= 0) { 2431 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2432 "Invalid number of service-ids specified (%d)", num_sids); 2433 return (IBT_INVALID_PARAM); 2434 } 2435 2436 if (sid == 0) { 2437 if (ret_sid == NULL) 2438 return (IBT_INVALID_PARAM); 2439 sid = ibcm_alloc_local_sids(num_sids); 2440 if (sid == 0) 2441 return (IBT_INSUFF_KERNEL_RESOURCE); 2442 2443 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */ 2444 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) { 2445 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2446 "Invalid non-LOCAL SID specified: 0x%llX", 2447 (longlong_t)sid); 2448 return (IBT_INVALID_PARAM); 2449 } 2450 2451 svcinfop = ibcm_create_svc_entry(sid, num_sids); 2452 2453 if (svcinfop == NULL) { 2454 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2455 "Service-ID 0x%llx already registered", (longlong_t)sid); 2456 return (IBT_CM_SERVICE_EXISTS); 2457 } 2458 2459 /* 2460 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry() 2461 */ 2462 svcinfop->svc_flags = srv->sd_flags; 2463 svcinfop->svc_rc_handler = srv->sd_handler; 2464 svcinfop->svc_ud_handler = srv->sd_ud_handler; 2465 2466 if (ret_sid != NULL) 2467 *ret_sid = sid; 2468 2469 *srv_hdl = svcinfop; 2470 2471 ibtl_cm_change_service_cnt(ibt_hdl, num_sids); 2472 2473 /* If this message isn't seen, then ibt_register_service failed. */ 2474 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)", 2475 svcinfop, sid); 2476 2477 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 2478 2479 return (IBT_SUCCESS); 2480 } 2481 2482 2483 static ibt_status_t 2484 ibcm_write_service_record(ibmf_saa_handle_t saa_handle, 2485 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type) 2486 { 2487 int rval; 2488 int retry; 2489 2490 ibcm_sa_access_enter(); 2491 for (retry = 0; retry < ibcm_max_sa_retries; retry++) { 2492 rval = ibmf_saa_update_service_record( 2493 saa_handle, srv_recp, saa_type, 0); 2494 if (rval != IBMF_TRANS_TIMEOUT) { 2495 break; 2496 } 2497 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2498 "ibmf_saa_update_service_record timed out" 2499 " SID = %llX, rval = %d, saa_type = %d", 2500 (longlong_t)srv_recp->ServiceID, rval, saa_type); 2501 delay(ibcm_sa_timeout_delay); 2502 } 2503 ibcm_sa_access_exit(); 2504 2505 if (rval != IBMF_SUCCESS) { 2506 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2507 "ibmf_saa_update_service_record() : Failed - %d", rval); 2508 return (ibcm_ibmf_analyze_error(rval)); 2509 } else 2510 return (IBT_SUCCESS); 2511 } 2512 2513 2514 static void 2515 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec) 2516 { 2517 ibt_status_t retval; 2518 uint_t num_found; 2519 size_t length; 2520 sa_service_record_t *srv_resp; 2521 void *results_p; 2522 uint_t i; 2523 uint64_t component_mask; 2524 ibmf_saa_access_args_t access_args; 2525 2526 component_mask = 2527 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID; 2528 2529 /* Call in SA Access retrieve routine to get Service Records. */ 2530 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 2531 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 2532 access_args.sq_component_mask = component_mask; 2533 access_args.sq_template = srec; 2534 access_args.sq_template_length = sizeof (sa_service_record_t); 2535 access_args.sq_callback = NULL; 2536 access_args.sq_callback_arg = NULL; 2537 2538 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 2539 &results_p); 2540 if (retval != IBT_SUCCESS) { 2541 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2542 "SA Access Failure"); 2543 return; 2544 } 2545 2546 num_found = length / sizeof (sa_service_record_t); 2547 2548 if (num_found) 2549 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2550 "Found %d matching Service Records.", num_found); 2551 2552 /* Validate the returned number of records. */ 2553 if ((results_p != NULL) && (num_found > 0)) { 2554 2555 /* Remove all the records. */ 2556 for (i = 0; i < num_found; i++) { 2557 2558 srv_resp = (sa_service_record_t *) 2559 ((uchar_t *)results_p + 2560 i * sizeof (sa_service_record_t)); 2561 2562 /* 2563 * Found some matching records, but check out whether 2564 * this Record is really stale or just happens to match 2565 * the current session records. If yes, don't remove it. 2566 */ 2567 mutex_enter(&ibcm_svc_info_lock); 2568 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) { 2569 /* This record is NOT STALE. */ 2570 mutex_exit(&ibcm_svc_info_lock); 2571 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2572 "This is not Stale, it's an active record"); 2573 continue; 2574 } 2575 mutex_exit(&ibcm_svc_info_lock); 2576 2577 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2578 "Removing Stale Rec: %s, %llX", 2579 srv_resp->ServiceName, srv_resp->ServiceID); 2580 2581 IBCM_DUMP_SERVICE_REC(srv_resp); 2582 2583 /* 2584 * Remove the Service Record Entry from SA. 2585 * 2586 * Get ServiceID info from Response Buf, other 2587 * attributes are already filled-in. 2588 */ 2589 2590 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2591 2592 srec->ServiceID = srv_resp->ServiceID; 2593 2594 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2595 2596 (void) ibcm_write_service_record(saa_handle, srec, 2597 IBMF_SAA_DELETE); 2598 } 2599 2600 /* Deallocate the memory for results_p. */ 2601 kmem_free(results_p, length); 2602 } 2603 } 2604 2605 2606 2607 /* 2608 * ibt_bind_service() 2609 * Register a service with the IBCM 2610 * 2611 * INPUTS: 2612 * srv_hdl The service id handle returned to the client 2613 * on an ibt_service_register() call. 2614 * 2615 * gid The GID to which to bind the service. 2616 * 2617 * srv_bind The address of a ibt_srv_bind_t that describes 2618 * the service record. This should be NULL if there 2619 * is to be no service record. This contains: 2620 * 2621 * sb_lease Lease period 2622 * sb_pkey Partition 2623 * sb_name pointer to ASCII string Service Name, 2624 * NULL terminated. 2625 * sb_key[] Key to secure the service record. 2626 * sb_data Service Data structure (64-byte) 2627 * 2628 * cm_private First argument of Service handler. 2629 * 2630 * sb_hdl_p The address of a service bind handle, used 2631 * to undo the service binding. 2632 * 2633 * ibt_bind_service() returns: 2634 * IBT_SUCCESS - added a service successfully. 2635 * IBT_INVALID_PARAM - invalid input parameter. 2636 * IBT_CM_FAILURE - failed to add the service. 2637 * IBT_CM_SERVICE_EXISTS - service already exists. 2638 */ 2639 ibt_status_t 2640 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind, 2641 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p) 2642 { 2643 ibt_status_t status; 2644 ibtl_cm_hca_port_t port; 2645 ibcm_svc_bind_t *sbindp, *sbp; 2646 ibcm_hca_info_t *hcap; 2647 ib_svc_id_t sid, start_sid, end_sid; 2648 ibmf_saa_handle_t saa_handle; 2649 sa_service_record_t srv_rec; 2650 uint16_t pkey_ix; 2651 2652 if (sb_hdl_p != NULL) 2653 *sb_hdl_p = NULL; /* return value for error cases */ 2654 2655 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)", 2656 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid); 2657 2658 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sbindp)) 2659 2660 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */ 2661 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) { 2662 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2663 "ibtl_cm_get_hca_port failed: %d", status); 2664 return (status); 2665 } 2666 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX", 2667 port.hp_port, port.hp_hca_guid); 2668 2669 hcap = ibcm_find_hca_entry(port.hp_hca_guid); 2670 if (hcap == NULL) { 2671 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found"); 2672 return (IBT_HCA_BUSY_DETACHING); 2673 } 2674 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap); 2675 2676 if (srv_bind != NULL) { 2677 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2678 if (saa_handle == NULL) { 2679 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2680 "saa_handle is NULL"); 2681 ibcm_dec_hca_acc_cnt(hcap); 2682 return (IBT_HCA_PORT_NOT_ACTIVE); 2683 } 2684 if (srv_bind->sb_pkey == 0) { 2685 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2686 "P_Key must not be 0"); 2687 ibcm_dec_hca_acc_cnt(hcap); 2688 return (IBT_INVALID_PARAM); 2689 } 2690 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) { 2691 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2692 "Service Name is too long"); 2693 ibcm_dec_hca_acc_cnt(hcap); 2694 return (IBT_INVALID_PARAM); 2695 } else 2696 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: " 2697 "Service Name='%s'", srv_bind->sb_name); 2698 status = ibt_pkey2index_byguid(port.hp_hca_guid, 2699 port.hp_port, srv_bind->sb_pkey, &pkey_ix); 2700 if (status != IBT_SUCCESS) { 2701 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2702 "P_Key 0x%x not found in P_Key_Table", 2703 srv_bind->sb_pkey); 2704 ibcm_dec_hca_acc_cnt(hcap); 2705 return (status); 2706 } 2707 } 2708 2709 /* assume success - allocate before locking */ 2710 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP); 2711 sbindp->sbind_cm_private = cm_private; 2712 sbindp->sbind_gid = gid; 2713 sbindp->sbind_hcaguid = port.hp_hca_guid; 2714 sbindp->sbind_port = port.hp_port; 2715 2716 mutex_enter(&ibcm_svc_info_lock); 2717 2718 sbp = srv_hdl->svc_bind_list; 2719 while (sbp != NULL) { 2720 if (sbp->sbind_gid.gid_guid == gid.gid_guid && 2721 sbp->sbind_gid.gid_prefix == gid.gid_prefix) { 2722 if (srv_bind == NULL || 2723 srv_bind->sb_pkey == sbp->sbind_pkey) { 2724 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2725 "failed: GID %llx:%llx and PKEY %x is " 2726 "already bound", gid.gid_guid, 2727 gid.gid_prefix, srv_bind->sb_pkey); 2728 mutex_exit(&ibcm_svc_info_lock); 2729 ibcm_dec_hca_acc_cnt(hcap); 2730 kmem_free(sbindp, sizeof (*sbindp)); 2731 return (IBT_CM_SERVICE_EXISTS); 2732 } 2733 } 2734 sbp = sbp->sbind_link; 2735 } 2736 /* no entry found */ 2737 2738 sbindp->sbind_link = srv_hdl->svc_bind_list; 2739 srv_hdl->svc_bind_list = sbindp; 2740 2741 mutex_exit(&ibcm_svc_info_lock); 2742 2743 if (srv_bind != NULL) { 2744 bzero(&srv_rec, sizeof (srv_rec)); 2745 2746 srv_rec.ServiceLease = 2747 sbindp->sbind_lease = srv_bind->sb_lease; 2748 srv_rec.ServiceP_Key = 2749 sbindp->sbind_pkey = srv_bind->sb_pkey; 2750 srv_rec.ServiceKey_hi = 2751 sbindp->sbind_key[0] = srv_bind->sb_key[0]; 2752 srv_rec.ServiceKey_lo = 2753 sbindp->sbind_key[1] = srv_bind->sb_key[1]; 2754 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name); 2755 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name); 2756 srv_rec.ServiceGID = gid; 2757 2758 /* 2759 * Find out whether we have any stale Local Service records 2760 * matching the current attributes. If yes, we shall try to 2761 * remove them from SA using the current request's ServiceKey. 2762 * 2763 * We will perform this operation only for Local Services, as 2764 * it is handled by SA automatically for WellKnown Services. 2765 * 2766 * Ofcourse, clients can specify NOT to do this clean-up by 2767 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag). 2768 */ 2769 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) && 2770 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) { 2771 ibcm_rem_stale_srec(saa_handle, &srv_rec); 2772 } 2773 2774 /* Handle endianess for service data. */ 2775 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data); 2776 2777 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 2778 2779 /* insert srv record into the SA */ 2780 start_sid = srv_hdl->svc_id; 2781 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 2782 for (sid = start_sid; sid <= end_sid; sid++) { 2783 2784 srv_rec.ServiceID = sid; 2785 2786 IBCM_DUMP_SERVICE_REC(&srv_rec); 2787 2788 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: " 2789 "ibmf_saa_write_service_record, SvcId = %llX", 2790 (longlong_t)sid); 2791 2792 status = ibcm_write_service_record(saa_handle, &srv_rec, 2793 IBMF_SAA_UPDATE); 2794 if (status != IBT_SUCCESS) { 2795 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:" 2796 " ibcm_write_service_record fails %d, " 2797 "sid %llX", status, (longlong_t)sid); 2798 2799 if (sid != start_sid) { 2800 /* 2801 * Bind failed while bind SID other than 2802 * first in the sid_range. So we need 2803 * to unbind those, which are passed. 2804 * 2805 * Need to increment svc count to 2806 * compensate for ibt_unbind_service(). 2807 */ 2808 ibcm_inc_hca_svc_cnt(hcap); 2809 ibcm_dec_hca_acc_cnt(hcap); 2810 2811 (void) ibt_unbind_service(srv_hdl, 2812 sbindp); 2813 } else { 2814 ibcm_svc_bind_t **sbpp; 2815 2816 /* 2817 * Bind failed for the first SID or the 2818 * only SID in question, then no need 2819 * to unbind, just free memory and 2820 * return error. 2821 */ 2822 mutex_enter(&ibcm_svc_info_lock); 2823 2824 sbpp = &srv_hdl->svc_bind_list; 2825 sbp = *sbpp; 2826 while (sbp != NULL) { 2827 if (sbp == sbindp) { 2828 *sbpp = sbp->sbind_link; 2829 break; 2830 } 2831 sbpp = &sbp->sbind_link; 2832 sbp = *sbpp; 2833 } 2834 mutex_exit(&ibcm_svc_info_lock); 2835 ibcm_dec_hca_acc_cnt(hcap); 2836 2837 kmem_free(sbindp, sizeof (*sbindp)); 2838 } 2839 return (status); 2840 } 2841 } 2842 } 2843 ibcm_inc_hca_svc_cnt(hcap); 2844 ibcm_dec_hca_acc_cnt(hcap); 2845 2846 /* If this message isn't seen then ibt_bind_service failed */ 2847 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llx)", 2848 srv_hdl, gid.gid_prefix, gid.gid_guid); 2849 2850 if (sb_hdl_p != NULL) 2851 *sb_hdl_p = sbindp; 2852 2853 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sbindp)) 2854 2855 return (IBT_SUCCESS); 2856 } 2857 2858 ibt_status_t 2859 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp) 2860 { 2861 ib_svc_id_t sid, end_sid; 2862 ibt_status_t rval; 2863 ibcm_hca_info_t *hcap; 2864 ibcm_svc_bind_t *sbp, **sbpp; 2865 2866 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)", 2867 srv_hdl, sbindp); 2868 2869 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 2870 2871 /* If there is a service on hca, respective hcap cannot go away */ 2872 ASSERT(hcap != NULL); 2873 2874 mutex_enter(&ibcm_svc_info_lock); 2875 2876 sbpp = &srv_hdl->svc_bind_list; 2877 sbp = *sbpp; 2878 while (sbp != NULL) { 2879 if (sbp == sbindp) { 2880 *sbpp = sbp->sbind_link; 2881 break; 2882 } 2883 sbpp = &sbp->sbind_link; 2884 sbp = *sbpp; 2885 } 2886 sid = srv_hdl->svc_id; 2887 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1; 2888 if (sbp != NULL) 2889 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY) 2890 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 2891 mutex_exit(&ibcm_svc_info_lock); 2892 2893 if (sbp == NULL) { 2894 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2895 "service binding not found: srv_hdl %p, srv_bind %p", 2896 srv_hdl, sbindp); 2897 ibcm_dec_hca_acc_cnt(hcap); 2898 return (IBT_INVALID_PARAM); 2899 } 2900 2901 if (sbindp->sbind_pkey != 0) { /* Are there service records? */ 2902 ibtl_cm_hca_port_t port; 2903 sa_service_record_t srv_rec; 2904 ibmf_saa_handle_t saa_handle; 2905 ibt_status_t status; 2906 2907 /* get the default SGID of the port */ 2908 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port)) 2909 != IBT_SUCCESS) { 2910 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2911 "ibtl_cm_get_hca_port failed: %d", status); 2912 /* we're done, but there may be stale service records */ 2913 goto done; 2914 } 2915 2916 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2917 if (saa_handle == NULL) { 2918 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2919 "saa_handle is NULL"); 2920 /* we're done, but there may be stale service records */ 2921 goto done; 2922 } 2923 2924 /* Fill in fields of srv_rec */ 2925 bzero(&srv_rec, sizeof (srv_rec)); 2926 2927 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 2928 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 2929 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 2930 srv_rec.ServiceGID = sbindp->sbind_gid; 2931 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 2932 2933 while (sid <= end_sid) { 2934 2935 srv_rec.ServiceID = sid; 2936 IBCM_DUMP_SERVICE_REC(&srv_rec); 2937 2938 rval = ibcm_write_service_record(saa_handle, &srv_rec, 2939 IBMF_SAA_DELETE); 2940 2941 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: " 2942 "ibcm_write_service_record rval = %d, SID %llx", 2943 rval, sid); 2944 if (rval != IBT_SUCCESS) { 2945 /* this is not considered a reason to fail */ 2946 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2947 "ibcm_write_service_record fails %d, " 2948 "sid %llx", rval, sid); 2949 } 2950 sid++; 2951 } 2952 } 2953 done: 2954 ibcm_dec_hca_svc_cnt(hcap); 2955 ibcm_dec_hca_acc_cnt(hcap); 2956 kmem_free(sbindp, sizeof (*sbindp)); 2957 2958 /* If this message isn't seen then ibt_unbind_service failed */ 2959 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!"); 2960 2961 return (IBT_SUCCESS); 2962 } 2963 2964 /* 2965 * Simply pull off each binding from the list and unbind it. 2966 * If any of the unbind calls fail, we fail. 2967 */ 2968 ibt_status_t 2969 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl) 2970 { 2971 ibt_status_t status; 2972 ibcm_svc_bind_t *sbp; 2973 2974 mutex_enter(&ibcm_svc_info_lock); 2975 sbp = NULL; 2976 2977 /* this compare keeps the loop from being infinite */ 2978 while (sbp != srv_hdl->svc_bind_list) { 2979 sbp = srv_hdl->svc_bind_list; 2980 mutex_exit(&ibcm_svc_info_lock); 2981 status = ibt_unbind_service(srv_hdl, sbp); 2982 if (status != IBT_SUCCESS) 2983 return (status); 2984 mutex_enter(&ibcm_svc_info_lock); 2985 if (srv_hdl->svc_bind_list == NULL) 2986 break; 2987 } 2988 mutex_exit(&ibcm_svc_info_lock); 2989 return (IBT_SUCCESS); 2990 } 2991 2992 /* 2993 * ibt_deregister_service() 2994 * Deregister a service with the IBCM 2995 * 2996 * INPUTS: 2997 * ibt_hdl The IBT client handle returned to the client 2998 * on an ibt_attach() call. 2999 * 3000 * srv_hdl The address of a service identification handle, used 3001 * to de-register a service. 3002 * RETURN VALUES: 3003 * IBT_SUCCESS on success (or respective failure on error) 3004 */ 3005 ibt_status_t 3006 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl) 3007 { 3008 ibcm_svc_info_t *svcp; 3009 ibcm_svc_lookup_t svc; 3010 3011 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p, %p)", 3012 ibt_hdl, srv_hdl); 3013 3014 mutex_enter(&ibcm_svc_info_lock); 3015 3016 if (srv_hdl->svc_bind_list != NULL) { 3017 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:" 3018 " srv_hdl %p still has bindings", srv_hdl); 3019 mutex_exit(&ibcm_svc_info_lock); 3020 return (IBT_CM_SERVICE_BUSY); 3021 } 3022 svc.sid = srv_hdl->svc_id; 3023 svc.num_sids = 1; 3024 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d", 3025 srv_hdl->svc_id, srv_hdl->svc_num_sids); 3026 3027 #ifdef __lock_lint 3028 ibcm_svc_compare(NULL, NULL); 3029 #endif 3030 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 3031 if (svcp != srv_hdl) { 3032 mutex_exit(&ibcm_svc_info_lock); 3033 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): " 3034 "srv_hdl %p not found", srv_hdl); 3035 return (IBT_INVALID_PARAM); 3036 } 3037 avl_remove(&ibcm_svc_avl_tree, svcp); 3038 3039 /* wait for active REQ/SREQ handling to be done */ 3040 svcp->svc_to_delete = 1; 3041 while (svcp->svc_ref_cnt != 0) 3042 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3043 3044 mutex_exit(&ibcm_svc_info_lock); 3045 3046 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) 3047 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids); 3048 3049 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids); 3050 kmem_free(srv_hdl, sizeof (*srv_hdl)); 3051 3052 /* If this message isn't seen then ibt_deregister_service failed */ 3053 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!"); 3054 3055 return (IBT_SUCCESS); 3056 } 3057 3058 ibcm_status_t 3059 ibcm_ar_init(void) 3060 { 3061 ib_svc_id_t sid = IBCM_DAPL_ATS_SID; 3062 ibcm_svc_info_t *tmp_svcp; 3063 3064 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()"); 3065 3066 /* remove this special SID from the pool of available SIDs */ 3067 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) { 3068 IBTF_DPRINTF_L2(cmlog, "ibcm_ar_init: " 3069 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid); 3070 return (IBCM_FAILURE); 3071 } 3072 mutex_enter(&ibcm_svc_info_lock); 3073 ibcm_ar_svcinfop = tmp_svcp; 3074 ibcm_ar_list = NULL; /* no address records registered yet */ 3075 mutex_exit(&ibcm_svc_info_lock); 3076 return (IBCM_SUCCESS); 3077 } 3078 3079 ibcm_status_t 3080 ibcm_ar_fini(void) 3081 { 3082 ibcm_ar_t *ar_list; 3083 ibcm_svc_info_t *tmp_svcp; 3084 3085 mutex_enter(&ibcm_svc_info_lock); 3086 ar_list = ibcm_ar_list; 3087 3088 if (ar_list == NULL && 3089 avl_numnodes(&ibcm_svc_avl_tree) == 1 && 3090 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) { 3091 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop); 3092 tmp_svcp = ibcm_ar_svcinfop; 3093 mutex_exit(&ibcm_svc_info_lock); 3094 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop)); 3095 return (IBCM_SUCCESS); 3096 } 3097 mutex_exit(&ibcm_svc_info_lock); 3098 return (IBCM_FAILURE); 3099 } 3100 3101 3102 /* 3103 * Return to the caller: 3104 * IBT_SUCCESS Found a perfect match. 3105 * *arpp is set to the record. 3106 * IBT_INCONSISTENT_AR Found a record that's inconsistent. 3107 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and 3108 * found no record with same data. 3109 */ 3110 static ibt_status_t 3111 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp) 3112 { 3113 ibcm_ar_t *tmp; 3114 int i; 3115 3116 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3117 tmp = ibcm_ar_list; 3118 while (tmp != NULL) { 3119 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix && 3120 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid && 3121 tmp->ar.ar_pkey == arp->ar_pkey) { 3122 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3123 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3124 return (IBT_INCONSISTENT_AR); 3125 *arpp = tmp; 3126 return (IBT_SUCCESS); 3127 } else { 3128 /* if all the data bytes match, we have inconsistency */ 3129 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3130 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3131 break; 3132 if (i == IBCM_DAPL_ATS_NBYTES) 3133 return (IBT_INCONSISTENT_AR); 3134 /* try next address record */ 3135 } 3136 tmp = tmp->ar_link; 3137 } 3138 return (IBT_AR_NOT_REGISTERED); 3139 } 3140 3141 ibt_status_t 3142 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3143 { 3144 ibcm_ar_t *found; 3145 ibcm_ar_t *tmp; 3146 ibt_status_t status; 3147 ibt_status_t s1, s2; 3148 char *s; 3149 ibcm_ar_ref_t *hdlp; 3150 ibcm_ar_t *new; 3151 ibcm_ar_t **linkp; 3152 ibtl_cm_hca_port_t cm_port; 3153 uint16_t pkey_ix; 3154 ibcm_hca_info_t *hcap; 3155 ibmf_saa_handle_t saa_handle; 3156 sa_service_record_t *srv_recp; 3157 uint64_t gid_ored; 3158 3159 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX", 3160 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix, 3161 (longlong_t)arp->ar_gid.gid_guid); 3162 3163 /* 3164 * If P_Key is 0, but GID is not, this query is invalid. 3165 * If GID is 0, but P_Key is not, this query is invalid. 3166 */ 3167 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3168 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3169 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3170 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3171 "GID/P_Key is not valid"); 3172 return (IBT_INVALID_PARAM); 3173 } 3174 3175 /* assume success, so these might be needed */ 3176 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP); 3177 new = kmem_alloc(sizeof (*new), KM_SLEEP); 3178 3179 mutex_enter(&ibcm_svc_info_lock); 3180 /* search for existing GID/pkey (there can be at most 1) */ 3181 status = ibcm_search_ar(arp, &found); 3182 if (status == IBT_INCONSISTENT_AR) { 3183 mutex_exit(&ibcm_svc_info_lock); 3184 kmem_free(new, sizeof (*new)); 3185 kmem_free(hdlp, sizeof (*hdlp)); 3186 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3187 "address record is inconsistent with a known one"); 3188 return (IBT_INCONSISTENT_AR); 3189 } else if (status == IBT_SUCCESS) { 3190 if (found->ar_flags == IBCM_AR_INITING) { 3191 found->ar_waiters++; 3192 cv_wait(&found->ar_cv, &ibcm_svc_info_lock); 3193 found->ar_waiters--; 3194 } 3195 if (found->ar_flags == IBCM_AR_FAILED) { 3196 if (found->ar_waiters == 0) { 3197 cv_destroy(&found->ar_cv); 3198 kmem_free(found, sizeof (*found)); 3199 } 3200 mutex_exit(&ibcm_svc_info_lock); 3201 kmem_free(new, sizeof (*new)); 3202 kmem_free(hdlp, sizeof (*hdlp)); 3203 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0)); 3204 } 3205 hdlp->ar_ibt_hdl = ibt_hdl; 3206 hdlp->ar_ref_link = found->ar_ibt_hdl_list; 3207 found->ar_ibt_hdl_list = hdlp; 3208 mutex_exit(&ibcm_svc_info_lock); 3209 kmem_free(new, sizeof (*new)); 3210 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3211 return (IBT_SUCCESS); 3212 } else { 3213 ASSERT(status == IBT_AR_NOT_REGISTERED); 3214 } 3215 hdlp->ar_ref_link = NULL; 3216 hdlp->ar_ibt_hdl = ibt_hdl; 3217 new->ar_ibt_hdl_list = hdlp; 3218 new->ar = *arp; 3219 new->ar_flags = IBCM_AR_INITING; 3220 new->ar_waiters = 0; 3221 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL); 3222 new->ar_link = ibcm_ar_list; 3223 ibcm_ar_list = new; 3224 3225 /* verify GID/pkey is valid for a local port, etc. */ 3226 hcap = NULL; 3227 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port)) 3228 != IBT_SUCCESS || 3229 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port, 3230 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS || 3231 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) { 3232 cv_destroy(&new->ar_cv); 3233 ibcm_ar_list = new->ar_link; 3234 mutex_exit(&ibcm_svc_info_lock); 3235 kmem_free(new, sizeof (*new)); 3236 kmem_free(hdlp, sizeof (*hdlp)); 3237 status = IBT_INVALID_PARAM; 3238 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) { 3239 s = "PORT DOWN"; 3240 status = IBT_HCA_PORT_NOT_ACTIVE; 3241 } else if (s1 != IBT_SUCCESS) 3242 s = "GID not found"; 3243 else if (s2 != IBT_SUCCESS) 3244 s = "PKEY not found"; 3245 else 3246 s = "CM could not find its HCA entry"; 3247 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d", 3248 s, status); 3249 return (status); 3250 } 3251 mutex_exit(&ibcm_svc_info_lock); 3252 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port); 3253 3254 /* create service record */ 3255 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP); 3256 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */ 3257 srv_recp->ServiceP_Key = arp->ar_pkey; 3258 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */ 3259 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */ 3260 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME); 3261 srv_recp->ServiceGID = arp->ar_gid; 3262 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES); 3263 srv_recp->ServiceID = IBCM_DAPL_ATS_SID; 3264 3265 /* insert service record into the SA */ 3266 3267 IBCM_DUMP_SERVICE_REC(srv_recp); 3268 3269 if (saa_handle != NULL) 3270 status = ibcm_write_service_record(saa_handle, srv_recp, 3271 IBMF_SAA_UPDATE); 3272 else 3273 status = IBT_HCA_PORT_NOT_ACTIVE; 3274 3275 if (status != IBT_SUCCESS) { 3276 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, " 3277 "sid %llX", status, (longlong_t)srv_recp->ServiceID); 3278 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid " 3279 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix, 3280 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3281 3282 kmem_free(srv_recp, sizeof (*srv_recp)); 3283 kmem_free(hdlp, sizeof (*hdlp)); 3284 3285 mutex_enter(&ibcm_svc_info_lock); 3286 linkp = &ibcm_ar_list; 3287 tmp = *linkp; 3288 while (tmp != NULL) { 3289 if (tmp == new) { 3290 *linkp = new->ar_link; 3291 break; 3292 } 3293 linkp = &tmp->ar_link; 3294 tmp = *linkp; 3295 } 3296 if (new->ar_waiters > 0) { 3297 new->ar_flags = IBCM_AR_FAILED; 3298 cv_broadcast(&new->ar_cv); 3299 mutex_exit(&ibcm_svc_info_lock); 3300 } else { 3301 cv_destroy(&new->ar_cv); 3302 mutex_exit(&ibcm_svc_info_lock); 3303 kmem_free(new, sizeof (*new)); 3304 } 3305 ibcm_dec_hca_acc_cnt(hcap); 3306 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3307 "IBMF_SAA failed to write address record"); 3308 } else { /* SUCCESS */ 3309 uint8_t *b; 3310 3311 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: SUCCESS for gid " 3312 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix, 3313 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3314 b = arp->ar_data; 3315 3316 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar:" 3317 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3318 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3319 b[10], b[11], b[12], b[13], b[14], b[15]); 3320 mutex_enter(&ibcm_svc_info_lock); 3321 new->ar_srv_recp = srv_recp; 3322 new->ar_saa_handle = saa_handle; 3323 new->ar_port = cm_port.hp_port; 3324 new->ar_hcap = hcap; 3325 new->ar_flags = IBCM_AR_SUCCESS; 3326 if (new->ar_waiters > 0) 3327 cv_broadcast(&new->ar_cv); 3328 mutex_exit(&ibcm_svc_info_lock); 3329 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3330 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */ 3331 } 3332 return (status); 3333 } 3334 3335 ibt_status_t 3336 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3337 { 3338 ibcm_ar_t *found; 3339 ibcm_ar_t *tmp; 3340 ibcm_ar_t **linkp; 3341 ibcm_ar_ref_t *hdlp; 3342 ibcm_ar_ref_t **hdlpp; 3343 ibt_status_t status; 3344 ibmf_saa_handle_t saa_handle; 3345 sa_service_record_t *srv_recp; 3346 uint64_t gid_ored; 3347 3348 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey); 3349 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: gid %llx:%llx", 3350 (longlong_t)arp->ar_gid.gid_prefix, 3351 (longlong_t)arp->ar_gid.gid_guid); 3352 3353 /* 3354 * If P_Key is 0, but GID is not, this query is invalid. 3355 * If GID is 0, but P_Key is not, this query is invalid. 3356 */ 3357 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3358 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3359 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3360 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3361 "GID/P_Key is not valid"); 3362 return (IBT_INVALID_PARAM); 3363 } 3364 3365 mutex_enter(&ibcm_svc_info_lock); 3366 /* search for existing GID/pkey (there can be at most 1) */ 3367 status = ibcm_search_ar(arp, &found); 3368 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) { 3369 mutex_exit(&ibcm_svc_info_lock); 3370 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3371 "address record not found"); 3372 return (IBT_AR_NOT_REGISTERED); 3373 } 3374 ASSERT(status == IBT_SUCCESS); 3375 3376 hdlpp = &found->ar_ibt_hdl_list; 3377 hdlp = *hdlpp; 3378 while (hdlp != NULL) { 3379 if (hdlp->ar_ibt_hdl == ibt_hdl) 3380 break; 3381 hdlpp = &hdlp->ar_ref_link; 3382 hdlp = *hdlpp; 3383 } 3384 if (hdlp == NULL) { /* could not find ibt_hdl on list */ 3385 mutex_exit(&ibcm_svc_info_lock); 3386 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3387 "address record found, but not for this client"); 3388 return (IBT_AR_NOT_REGISTERED); 3389 } 3390 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */ 3391 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) { 3392 /* last entry was removed */ 3393 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */ 3394 saa_handle = found->ar_saa_handle; 3395 srv_recp = found->ar_srv_recp; 3396 3397 /* wait if this service record is being rewritten */ 3398 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY) 3399 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3400 mutex_exit(&ibcm_svc_info_lock); 3401 3402 /* remove service record */ 3403 status = ibcm_write_service_record(saa_handle, srv_recp, 3404 IBMF_SAA_DELETE); 3405 if (status != IBT_SUCCESS) 3406 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3407 "IBMF_SAA failed to delete address record"); 3408 mutex_enter(&ibcm_svc_info_lock); 3409 if (found->ar_waiters == 0) { /* still no waiters */ 3410 linkp = &ibcm_ar_list; 3411 tmp = *linkp; 3412 while (tmp != found) { 3413 linkp = &tmp->ar_link; 3414 tmp = *linkp; 3415 } 3416 *linkp = tmp->ar_link; 3417 ibcm_dec_hca_acc_cnt(found->ar_hcap); 3418 kmem_free(srv_recp, sizeof (*srv_recp)); 3419 cv_destroy(&found->ar_cv); 3420 kmem_free(found, sizeof (*found)); 3421 } else { 3422 /* add service record back in for the waiters */ 3423 mutex_exit(&ibcm_svc_info_lock); 3424 status = ibcm_write_service_record(saa_handle, srv_recp, 3425 IBMF_SAA_UPDATE); 3426 mutex_enter(&ibcm_svc_info_lock); 3427 if (status == IBT_SUCCESS) 3428 found->ar_flags = IBCM_AR_SUCCESS; 3429 else { 3430 found->ar_flags = IBCM_AR_FAILED; 3431 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3432 "IBMF_SAA failed to write address record"); 3433 } 3434 cv_broadcast(&found->ar_cv); 3435 } 3436 } 3437 mutex_exit(&ibcm_svc_info_lock); 3438 kmem_free(hdlp, sizeof (*hdlp)); 3439 ibtl_cm_change_service_cnt(ibt_hdl, -1); 3440 return (status); 3441 } 3442 3443 ibt_status_t 3444 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp) 3445 { 3446 sa_service_record_t svcrec_req; 3447 sa_service_record_t *svcrec_resp; 3448 void *results_p; 3449 uint64_t component_mask = 0; 3450 uint64_t gid_ored; 3451 size_t length; 3452 int num_rec; 3453 int i; 3454 ibmf_saa_access_args_t access_args; 3455 ibt_status_t retval; 3456 ibtl_cm_hca_port_t cm_port; 3457 ibcm_hca_info_t *hcap; 3458 ibmf_saa_handle_t saa_handle; 3459 3460 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp); 3461 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx", 3462 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid); 3463 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey); 3464 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx", 3465 (longlong_t)queryp->ar_gid.gid_prefix, 3466 (longlong_t)queryp->ar_gid.gid_guid); 3467 3468 /* 3469 * If P_Key is 0, but GID is not, this query is invalid. 3470 * If GID is 0, but P_Key is not, this query is invalid. 3471 */ 3472 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix; 3473 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) || 3474 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) { 3475 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid"); 3476 return (IBT_INVALID_PARAM); 3477 } 3478 3479 hcap = NULL; 3480 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS || 3481 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL || 3482 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) { 3483 if (hcap != NULL) 3484 ibcm_dec_hca_acc_cnt(hcap); 3485 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid"); 3486 return (IBT_INVALID_PARAM); 3487 } 3488 3489 bzero(&svcrec_req, sizeof (svcrec_req)); 3490 3491 /* Is GID/P_Key Specified. */ 3492 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */ 3493 svcrec_req.ServiceP_Key = queryp->ar_pkey; 3494 component_mask |= SA_SR_COMPMASK_PKEY; 3495 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X", 3496 queryp->ar_pkey); 3497 svcrec_req.ServiceGID = queryp->ar_gid; 3498 component_mask |= SA_SR_COMPMASK_GID; 3499 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX", 3500 (longlong_t)queryp->ar_gid.gid_prefix, 3501 (longlong_t)queryp->ar_gid.gid_guid); 3502 } 3503 3504 /* Is ServiceData Specified. */ 3505 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) { 3506 if (queryp->ar_data[i] != 0) { 3507 bcopy(queryp->ar_data, svcrec_req.ServiceData, 3508 IBCM_DAPL_ATS_NBYTES); 3509 component_mask |= 0xFFFF << 7; /* all 16 Data8 */ 3510 /* components */ 3511 break; 3512 } 3513 } 3514 3515 /* Service Name */ 3516 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME); 3517 component_mask |= SA_SR_COMPMASK_NAME; 3518 3519 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID; 3520 component_mask |= SA_SR_COMPMASK_ID; 3521 3522 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3523 "Perform SA Access: Mask: 0x%X", component_mask); 3524 3525 /* 3526 * Call in SA Access retrieve routine to get Service Records. 3527 * 3528 * SA Access framework allocated memory for the "results_p". 3529 * Make sure to deallocate once we are done with the results_p. 3530 * The size of the buffer allocated will be as returned in 3531 * "length" field. 3532 */ 3533 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 3534 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 3535 access_args.sq_component_mask = component_mask; 3536 access_args.sq_template = &svcrec_req; 3537 access_args.sq_template_length = sizeof (sa_service_record_t); 3538 access_args.sq_callback = NULL; 3539 access_args.sq_callback_arg = NULL; 3540 3541 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 3542 &results_p); 3543 3544 ibcm_dec_hca_acc_cnt(hcap); 3545 if (retval != IBT_SUCCESS) { 3546 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed"); 3547 return (retval); 3548 } 3549 3550 num_rec = length / sizeof (sa_service_record_t); 3551 3552 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3553 "Found %d Service Records.", num_rec); 3554 3555 /* Validate the returned number of records. */ 3556 if ((results_p != NULL) && (num_rec > 0)) { 3557 uint8_t *b; 3558 3559 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3560 "Found %d Service Record(s).", num_rec); 3561 3562 /* Just return info from the first service record. */ 3563 svcrec_resp = (sa_service_record_t *)results_p; 3564 3565 /* The Service GID and Service ID */ 3566 resultp->ar_gid = svcrec_resp->ServiceGID; 3567 resultp->ar_pkey = svcrec_resp->ServiceP_Key; 3568 bcopy(svcrec_resp->ServiceData, 3569 resultp->ar_data, IBCM_DAPL_ATS_NBYTES); 3570 3571 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3572 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey, 3573 (longlong_t)resultp->ar_gid.gid_prefix, 3574 (longlong_t)resultp->ar_gid.gid_guid); 3575 b = resultp->ar_data; 3576 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar:" 3577 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3578 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3579 b[10], b[11], b[12], b[13], b[14], b[15]); 3580 3581 /* Deallocate the memory for results_p. */ 3582 kmem_free(results_p, length); 3583 if (num_rec > 1) 3584 retval = IBT_MULTIPLE_AR; 3585 else 3586 retval = IBT_SUCCESS; 3587 } else { 3588 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3589 "ibmf_sa_access found 0 matching records"); 3590 retval = IBT_AR_NOT_REGISTERED; 3591 } 3592 return (retval); 3593 } 3594 3595 /* mark all ATS service records associated with the port */ 3596 static void 3597 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port) 3598 { 3599 ibcm_ar_t *tmp; 3600 3601 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3602 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3603 if (tmp->ar_hcap->hca_guid == hca_guid && 3604 tmp->ar_port == port) { 3605 /* even if it's busy, we mark it for rewrite */ 3606 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED; 3607 } 3608 } 3609 } 3610 3611 /* rewrite all ATS service records */ 3612 static int 3613 ibcm_rewrite_ar(void) 3614 { 3615 ibcm_ar_t *tmp; 3616 ibmf_saa_handle_t saa_handle; 3617 sa_service_record_t *srv_recp; 3618 ibt_status_t rval; 3619 int did_something = 0; 3620 3621 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3622 check_for_work: 3623 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3624 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) { 3625 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY; 3626 saa_handle = tmp->ar_saa_handle; 3627 srv_recp = tmp->ar_srv_recp; 3628 mutex_exit(&ibcm_svc_info_lock); 3629 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: " 3630 "rewriting ar @ %p", tmp); 3631 did_something = 1; 3632 rval = ibcm_write_service_record(saa_handle, srv_recp, 3633 IBMF_SAA_UPDATE); 3634 if (rval != IBT_SUCCESS) 3635 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: " 3636 "ibcm_write_service_record failed: " 3637 "status = %d", rval); 3638 mutex_enter(&ibcm_svc_info_lock); 3639 /* if it got marked again, then we want to rewrite */ 3640 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY) 3641 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE; 3642 /* in case there was a waiter... */ 3643 cv_broadcast(&ibcm_svc_info_cv); 3644 goto check_for_work; 3645 } 3646 } 3647 return (did_something); 3648 } 3649 3650 static void 3651 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp) 3652 { 3653 ibcm_hca_info_t *hcap; 3654 ib_svc_id_t sid, start_sid, end_sid; 3655 ibmf_saa_handle_t saa_handle; 3656 sa_service_record_t srv_rec; 3657 ibt_status_t rval; 3658 3659 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 3660 if (hcap == NULL) { 3661 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3662 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid); 3663 return; 3664 } 3665 3666 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port); 3667 if (saa_handle == NULL) { 3668 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3669 "saa_handle is NULL"); 3670 ibcm_dec_hca_acc_cnt(hcap); 3671 return; 3672 } 3673 3674 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: " 3675 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name, 3676 sbindp->sbind_gid.gid_guid); 3677 3678 bzero(&srv_rec, sizeof (srv_rec)); 3679 3680 srv_rec.ServiceLease = sbindp->sbind_lease; 3681 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 3682 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 3683 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 3684 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 3685 srv_rec.ServiceGID = sbindp->sbind_gid; 3686 3687 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 3688 3689 /* insert srv record into the SA */ 3690 start_sid = srv_hdl->svc_id; 3691 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 3692 for (sid = start_sid; sid <= end_sid; sid++) { 3693 srv_rec.ServiceID = sid; 3694 3695 rval = ibcm_write_service_record(saa_handle, &srv_rec, 3696 IBMF_SAA_UPDATE); 3697 3698 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: " 3699 "ibcm_write_service_record, SvcId = %llX, " 3700 "rval = %d", (longlong_t)sid, rval); 3701 if (rval != IBT_SUCCESS) { 3702 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:" 3703 " ibcm_write_service_record fails %d sid %llX", 3704 rval, (longlong_t)sid); 3705 } 3706 } 3707 ibcm_dec_hca_acc_cnt(hcap); 3708 } 3709 3710 /* 3711 * Task to mark all service records as needing to be rewritten to the SM/SA. 3712 * This task does not return until all of them have been rewritten. 3713 */ 3714 void 3715 ibcm_service_record_rewrite_task(void *arg) 3716 { 3717 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg; 3718 ib_guid_t hca_guid = pup->pup_hca_guid; 3719 uint8_t port = pup->pup_port; 3720 ibcm_svc_info_t *svcp; 3721 ibcm_svc_bind_t *sbp; 3722 avl_tree_t *avl_tree = &ibcm_svc_avl_tree; 3723 static int task_is_running = 0; 3724 3725 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task STARTED " 3726 "for hca_guid %llX, port %d", hca_guid, port); 3727 3728 mutex_enter(&ibcm_svc_info_lock); 3729 ibcm_mark_ar(hca_guid, port); 3730 for (svcp = avl_first(avl_tree); svcp != NULL; 3731 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3732 sbp = svcp->svc_bind_list; 3733 while (sbp != NULL) { 3734 if (sbp->sbind_pkey != 0 && 3735 sbp->sbind_port == port && 3736 sbp->sbind_hcaguid == hca_guid) { 3737 /* even if it's busy, we mark it for rewrite */ 3738 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED; 3739 } 3740 sbp = sbp->sbind_link; 3741 } 3742 } 3743 if (task_is_running) { 3744 /* let the other task thread finish the work */ 3745 mutex_exit(&ibcm_svc_info_lock); 3746 return; 3747 } 3748 task_is_running = 1; 3749 3750 (void) ibcm_rewrite_ar(); 3751 3752 check_for_work: 3753 for (svcp = avl_first(avl_tree); svcp != NULL; 3754 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3755 sbp = svcp->svc_bind_list; 3756 while (sbp != NULL) { 3757 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) { 3758 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY; 3759 mutex_exit(&ibcm_svc_info_lock); 3760 ibcm_rewrite_svc_record(svcp, sbp); 3761 mutex_enter(&ibcm_svc_info_lock); 3762 /* if it got marked again, we want to rewrite */ 3763 if (sbp->sbind_rewrite_state == 3764 IBCM_REWRITE_BUSY) 3765 sbp->sbind_rewrite_state = 3766 IBCM_REWRITE_IDLE; 3767 /* in case there was a waiter... */ 3768 cv_broadcast(&ibcm_svc_info_cv); 3769 goto check_for_work; 3770 } 3771 sbp = sbp->sbind_link; 3772 } 3773 } 3774 /* 3775 * If there were no service records to write, and we failed to 3776 * have to rewrite any more ATS service records, then we're done. 3777 */ 3778 if (ibcm_rewrite_ar() != 0) 3779 goto check_for_work; 3780 task_is_running = 0; 3781 mutex_exit(&ibcm_svc_info_lock); 3782 3783 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task DONE"); 3784 kmem_free(pup, sizeof (ibcm_port_up_t)); 3785 } 3786 3787 3788 /* 3789 * Function: 3790 * ibt_cm_proceed 3791 * 3792 * Verifies the arguments and dispatches the cm state machine processing 3793 * via taskq 3794 */ 3795 3796 ibt_status_t 3797 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id, 3798 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3799 void *priv_data, ibt_priv_data_len_t priv_data_len) 3800 { 3801 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3802 ibcm_proceed_targs_t *proceed_targs; 3803 ibcm_proceed_error_t proceed_error; 3804 3805 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x " 3806 "session_id %p", statep->channel, event, status, session_id); 3807 3808 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, " 3809 "priv_data %p priv_data_len %x", statep->channel, cm_event_data, 3810 priv_data, priv_data_len); 3811 3812 /* validate session_id and status */ 3813 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3814 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args"); 3815 return (IBT_INVALID_PARAM); 3816 } 3817 3818 /* If priv data len specified, then priv_data cannot be NULL */ 3819 if ((priv_data_len > 0) && (priv_data == NULL)) 3820 return (IBT_INVALID_PARAM); 3821 3822 proceed_error = IBCM_PROCEED_INVALID_NONE; 3823 3824 mutex_enter(&statep->state_mutex); 3825 if (event == IBT_CM_EVENT_REQ_RCV) { 3826 3827 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3828 (statep->state != IBCM_STATE_MRA_SENT)) 3829 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3830 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ) 3831 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3832 3833 } else if (event == IBT_CM_EVENT_REP_RCV) { 3834 if ((statep->state != IBCM_STATE_REP_RCVD) && 3835 (statep->state != IBCM_STATE_MRA_REP_SENT)) 3836 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3837 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ) 3838 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3839 } else if (event == IBT_CM_EVENT_LAP_RCV) { 3840 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) && 3841 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT)) 3842 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3843 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ) 3844 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3845 } else if (event == IBT_CM_EVENT_CONN_CLOSED) { 3846 if (statep->state != IBCM_STATE_DREQ_RCVD) 3847 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3848 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ) 3849 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3850 } else { 3851 proceed_error = IBCM_PROCEED_INVALID_EVENT; 3852 } 3853 3854 /* if there is an error, print an error message and return */ 3855 if (proceed_error != IBCM_PROCEED_INVALID_NONE) { 3856 mutex_exit(&statep->state_mutex); 3857 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) { 3858 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3859 "Invalid Event/State combination specified", 3860 statep->channel); 3861 return (IBT_INVALID_PARAM); 3862 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) { 3863 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3864 "Invalid Event/priv len combination specified", 3865 statep->channel); 3866 return (IBT_INVALID_PARAM); 3867 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) { 3868 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3869 "Invalid Event specified", statep->channel); 3870 return (IBT_INVALID_PARAM); 3871 } else { 3872 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP); 3873 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3874 "IBT_CM_EVENT_LAP_RCV not supported", 3875 statep->channel); 3876 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */ 3877 return (IBT_APM_NOT_SUPPORTED); 3878 } 3879 } 3880 3881 3882 /* wait until client's CM handler returns DEFER status back to CM */ 3883 3884 while (statep->clnt_proceed == IBCM_BLOCK) { 3885 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for " 3886 "return of client's cm handler", statep->channel); 3887 cv_wait(&statep->block_client_cv, &statep->state_mutex); 3888 } 3889 3890 if (statep->clnt_proceed == IBCM_FAIL) { 3891 mutex_exit(&statep->state_mutex); 3892 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as " 3893 "client returned non-DEFER status from cm handler", 3894 statep->channel); 3895 return (IBT_CHAN_STATE_INVALID); 3896 } 3897 3898 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK); 3899 statep->clnt_proceed = IBCM_FAIL; 3900 mutex_exit(&statep->state_mutex); 3901 3902 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3903 3904 /* the state machine processing is done in a separate thread */ 3905 3906 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 3907 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 3908 KM_SLEEP); 3909 3910 proceed_targs->event = event; 3911 proceed_targs->status = status; 3912 proceed_targs->priv_data_len = priv_data_len; 3913 3914 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 3915 3916 proceed_targs->tst.rc.statep = statep; 3917 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data, 3918 sizeof (ibt_cm_proceed_reply_t)); 3919 3920 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3921 3922 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq, 3923 proceed_targs, TQ_SLEEP); 3924 3925 return (IBT_SUCCESS); 3926 } 3927 3928 /* 3929 * Function: 3930 * ibcm_proceed_via_taskq 3931 * 3932 * Called from taskq, dispatched by ibt_cm_proceed 3933 * Completes the cm state processing for ibt_cm_proceed 3934 */ 3935 void 3936 ibcm_proceed_via_taskq(void *targs) 3937 { 3938 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 3939 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep; 3940 ibt_cm_reason_t reject_reason; 3941 uint8_t arej_len; 3942 ibcm_status_t response; 3943 ibcm_clnt_reply_info_t clnt_info; 3944 3945 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data; 3946 clnt_info.priv_data = proceed_targs->priv_data; 3947 clnt_info.priv_data_len = proceed_targs->priv_data_len; 3948 3949 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x", 3950 statep->channel, targs); 3951 3952 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) { 3953 response = 3954 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status, 3955 &clnt_info, &reject_reason, &arej_len, 3956 (ibcm_req_msg_t *)statep->defer_cm_msg); 3957 3958 ibcm_handle_cep_req_response(statep, response, reject_reason, 3959 arej_len); 3960 3961 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) { 3962 response = 3963 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status, 3964 &clnt_info, &reject_reason, &arej_len, 3965 (ibcm_rep_msg_t *)statep->defer_cm_msg); 3966 3967 ibcm_handle_cep_rep_response(statep, response, reject_reason, 3968 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg); 3969 3970 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) { 3971 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status, 3972 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg, 3973 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg)); 3974 3975 ibcm_post_apr_mad(statep); 3976 3977 } else { 3978 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED); 3979 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data, 3980 proceed_targs->priv_data_len); 3981 } 3982 3983 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 3984 } 3985 3986 /* 3987 * Function: 3988 * ibt_cm_ud_proceed 3989 * 3990 * Verifies the arguments and dispatches the cm state machine processing 3991 * via taskq 3992 */ 3993 ibt_status_t 3994 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel, 3995 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop, 3996 void *priv_data, ibt_priv_data_len_t priv_data_len) 3997 { 3998 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id; 3999 ibcm_proceed_targs_t *proceed_targs; 4000 ibt_qp_query_attr_t qp_attr; 4001 ibt_status_t retval; 4002 4003 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p " 4004 "ud_channel %p ", session_id, ud_channel); 4005 4006 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p " 4007 "priv_data_len %x", status, priv_data, priv_data_len); 4008 4009 /* validate session_id and status */ 4010 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) { 4011 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args"); 4012 return (IBT_INVALID_PARAM); 4013 } 4014 4015 /* If priv data len specified, then priv_data cannot be NULL */ 4016 if ((priv_data_len > 0) && (priv_data == NULL)) 4017 return (IBT_INVALID_PARAM); 4018 4019 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ) 4020 return (IBT_INVALID_PARAM); 4021 4022 /* retrieve qpn and qkey from ud channel */ 4023 4024 /* validate event and statep's state */ 4025 4026 if (status == IBT_CM_ACCEPT) { 4027 retval = ibt_query_qp(ud_channel, &qp_attr); 4028 if ((retval != IBT_SUCCESS) || 4029 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) { 4030 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: " 4031 "Failed to retrieve QPN from the channel: %d", 4032 retval); 4033 return (IBT_INVALID_PARAM); 4034 } 4035 } 4036 4037 4038 mutex_enter(&ud_statep->ud_state_mutex); 4039 4040 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) { 4041 mutex_exit(&ud_statep->ud_state_mutex); 4042 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State " 4043 "specified"); 4044 return (IBT_INVALID_PARAM); 4045 } 4046 4047 /* wait until client's CM handler returns DEFER status back to CM */ 4048 4049 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) { 4050 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return" 4051 " of client's ud cm handler"); 4052 cv_wait(&ud_statep->ud_block_client_cv, 4053 &ud_statep->ud_state_mutex); 4054 } 4055 4056 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) { 4057 mutex_exit(&ud_statep->ud_state_mutex); 4058 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client " 4059 "returned non-DEFER status from cm handler"); 4060 return (IBT_INVALID_PARAM); 4061 } 4062 4063 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK); 4064 ud_statep->ud_clnt_proceed = IBCM_FAIL; 4065 mutex_exit(&ud_statep->ud_state_mutex); 4066 4067 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4068 4069 /* the state machine processing is done in a separate thread */ 4070 4071 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4072 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 4073 KM_SLEEP); 4074 4075 proceed_targs->status = status; 4076 proceed_targs->priv_data_len = priv_data_len; 4077 4078 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4079 4080 if (status == IBT_CM_ACCEPT) { 4081 proceed_targs->tst.ud.ud_qkey = 4082 qp_attr.qp_info.qp_transport.ud.ud_qkey; 4083 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn; 4084 } 4085 4086 proceed_targs->tst.ud.ud_statep = ud_statep; 4087 4088 /* copy redirect info based on status */ 4089 if (status == IBT_CM_REDIRECT) 4090 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info, 4091 sizeof (ibt_redirect_info_t)); 4092 4093 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4094 4095 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq, 4096 proceed_targs, TQ_SLEEP); 4097 4098 return (IBT_SUCCESS); 4099 } 4100 4101 /* 4102 * Function: 4103 * ibcm_ud_proceed_via_taskq 4104 * 4105 * Called from taskq, dispatched by ibt_cm_ud_proceed 4106 * Completes the cm state processing for ibt_cm_ud_proceed 4107 */ 4108 void 4109 ibcm_ud_proceed_via_taskq(void *targs) 4110 { 4111 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4112 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep; 4113 ibcm_ud_clnt_reply_info_t ud_clnt_info; 4114 ibt_sidr_status_t sidr_status; 4115 4116 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs); 4117 4118 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn; 4119 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey; 4120 ud_clnt_info.priv_data = proceed_targs->priv_data; 4121 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len; 4122 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info; 4123 4124 /* validate event and statep's state */ 4125 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status, 4126 &ud_clnt_info, &sidr_status, 4127 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg)); 4128 4129 ibcm_post_sidr_rep_mad(ud_statep, sidr_status); 4130 4131 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */ 4132 mutex_enter(&ud_statep->ud_state_mutex); 4133 IBCM_UD_REF_CNT_DECR(ud_statep); 4134 mutex_exit(&ud_statep->ud_state_mutex); 4135 4136 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4137 } 4138 4139 /* 4140 * Function: 4141 * ibt_set_alt_path 4142 * Input: 4143 * channel Channel handle returned from ibt_alloc_rc_channel(9F). 4144 * 4145 * mode Execute in blocking or non blocking mode. 4146 * 4147 * alt_path A pointer to an ibt_alt_path_info_t as returned from an 4148 * ibt_get_alt_path(9F) call that specifies the new 4149 * alternate path. 4150 * 4151 * priv_data A pointer to a buffer specified by caller for the 4152 * private data in the outgoing CM Load Alternate Path 4153 * (LAP) message sent to the remote host. This can be NULL 4154 * if no private data is available to communicate to the 4155 * remote node. 4156 * 4157 * priv_data_len Length of valid data in priv_data, this should be less 4158 * than or equal to IBT_LAP_PRIV_DATA_SZ. 4159 * 4160 * Output: 4161 * ret_args If called in blocking mode, points to a return argument 4162 * structure of type ibt_ap_returns_t. 4163 * 4164 * Returns: 4165 * IBT_SUCCESS on Success else appropriate error. 4166 * Description: 4167 * Load the specified alternate path. Causes the CM to send an LAP message 4168 * to the remote node. 4169 * Can only be called on a previously opened RC channel. 4170 */ 4171 ibt_status_t 4172 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 4173 ibt_alt_path_info_t *alt_path, void *priv_data, 4174 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args) 4175 { 4176 ibmf_handle_t ibmf_hdl; 4177 ibt_status_t status = IBT_SUCCESS; 4178 ibcm_lap_msg_t *lap_msgp; 4179 ibcm_hca_info_t *hcap; 4180 ibcm_state_data_t *statep; 4181 uint8_t port_no; 4182 ib_lid_t alternate_slid; 4183 ibt_priv_data_len_t len; 4184 ib_lid_t base_lid; 4185 boolean_t alt_grh; 4186 4187 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)", 4188 channel, mode, alt_path, priv_data, priv_data_len, ret_args); 4189 4190 /* validate channel */ 4191 if (IBCM_INVALID_CHANNEL(channel)) { 4192 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel"); 4193 return (IBT_CHAN_HDL_INVALID); 4194 } 4195 4196 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 4197 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4198 "Invalid Channel type: Applicable only to RC Channel"); 4199 return (IBT_CHAN_SRV_TYPE_INVALID); 4200 } 4201 4202 if (mode == IBT_NONBLOCKING) { 4203 if (ret_args != NULL) { 4204 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4205 "ret_args should be NULL when called in " 4206 "non-blocking mode"); 4207 return (IBT_INVALID_PARAM); 4208 } 4209 } else if (mode == IBT_BLOCKING) { 4210 if (ret_args == NULL) { 4211 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4212 "ret_args should be Non-NULL when called in " 4213 "blocking mode"); 4214 return (IBT_INVALID_PARAM); 4215 } 4216 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) { 4217 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4218 "expected private data length is too large"); 4219 return (IBT_INVALID_PARAM); 4220 } 4221 if ((ret_args->ap_priv_data_len > 0) && 4222 (ret_args->ap_priv_data == NULL)) { 4223 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4224 "apr_priv_data_len > 0, but apr_priv_data NULL"); 4225 return (IBT_INVALID_PARAM); 4226 } 4227 } else { /* any other mode is not valid for ibt_set_alt_path */ 4228 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4229 "invalid mode %x specified", mode); 4230 return (IBT_INVALID_PARAM); 4231 } 4232 4233 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0) 4234 return (IBT_INVALID_PARAM); 4235 4236 /* get the statep */ 4237 IBCM_GET_CHAN_PRIVATE(channel, statep); 4238 if (statep == NULL) { 4239 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL"); 4240 return (IBT_CM_FAILURE); 4241 } 4242 4243 mutex_enter(&statep->state_mutex); 4244 IBCM_RELEASE_CHAN_PRIVATE(channel); 4245 IBCM_REF_CNT_INCR(statep); 4246 mutex_exit(&statep->state_mutex); 4247 4248 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep); 4249 4250 hcap = statep->hcap; 4251 4252 /* HCA must have been in active state. If not, it's a client bug */ 4253 if (!IBCM_ACCESS_HCA_OK(hcap)) 4254 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state"); 4255 4256 ASSERT(statep->cm_handler != NULL); 4257 4258 /* Check Alternate port */ 4259 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL, 4260 &base_lid); 4261 if (status != IBT_SUCCESS) { 4262 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4263 "ibt_get_port_state_byguid status %d ", status); 4264 mutex_enter(&statep->state_mutex); 4265 IBCM_REF_CNT_DECR(statep); 4266 mutex_exit(&statep->state_mutex); 4267 return (status); 4268 } 4269 4270 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 4271 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 4272 != IBT_SUCCESS)) { 4273 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4274 "ibmf reg or callback setup failed during re-initialize"); 4275 mutex_enter(&statep->state_mutex); 4276 IBCM_REF_CNT_DECR(statep); 4277 mutex_exit(&statep->state_mutex); 4278 return (status); 4279 } 4280 4281 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl; 4282 4283 alternate_slid = base_lid + 4284 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path; 4285 4286 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x", 4287 h2b16(alternate_slid)); 4288 4289 ibcm_rc_flow_control_enter(); /* limit how many run simultaneously */ 4290 4291 /* Allocate MAD for LAP */ 4292 if (statep->lapr_msg == NULL) 4293 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg, 4294 MAD_METHOD_SEND)) != IBT_SUCCESS) { 4295 ibcm_rc_flow_control_exit(); 4296 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4297 "chan 0x%p ibcm_alloc_out_msg failed", channel); 4298 mutex_enter(&statep->state_mutex); 4299 IBCM_REF_CNT_DECR(statep); 4300 mutex_exit(&statep->state_mutex); 4301 return (status); 4302 } 4303 4304 mutex_enter(&statep->state_mutex); 4305 4306 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is" 4307 " %x", statep->state); 4308 4309 /* Check state */ 4310 if ((statep->state != IBCM_STATE_ESTABLISHED) || 4311 (statep->ap_state != IBCM_AP_STATE_IDLE)) { 4312 IBCM_REF_CNT_DECR(statep); 4313 mutex_exit(&statep->state_mutex); 4314 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg); 4315 ibcm_rc_flow_control_exit(); 4316 return (IBT_CHAN_STATE_INVALID); 4317 } else { 4318 /* Set to LAP Sent state */ 4319 statep->ap_state = IBCM_AP_STATE_LAP_SENT; 4320 statep->ap_done = B_FALSE; 4321 statep->remaining_retry_cnt = statep->max_cm_retries; 4322 statep->timer_stored_state = statep->state; 4323 statep->timer_stored_ap_state = statep->ap_state; 4324 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */ 4325 } 4326 4327 mutex_exit(&statep->state_mutex); 4328 4329 /* No more failure returns below */ 4330 4331 /* Allocate MAD for LAP */ 4332 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:" 4333 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg)); 4334 4335 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4336 4337 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg); 4338 4339 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid); 4340 lap_msgp->lap_alt_r_port_lid = 4341 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid); 4342 4343 /* Fill in remote port gid */ 4344 lap_msgp->lap_alt_r_port_gid.gid_prefix = 4345 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix); 4346 lap_msgp->lap_alt_r_port_gid.gid_guid = 4347 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid); 4348 4349 /* Fill in local port gid */ 4350 lap_msgp->lap_alt_l_port_gid.gid_prefix = 4351 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix); 4352 lap_msgp->lap_alt_l_port_gid.gid_guid = 4353 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid); 4354 4355 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh; 4356 4357 /* alternate_flow_label, and alternate srate, alternate traffic class */ 4358 lap_msgp->lap_alt_srate_plus = 4359 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f; 4360 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ? 4361 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) | 4362 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass); 4363 4364 /* Alternate hop limit, service level */ 4365 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ? 4366 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 0xff; 4367 lap_msgp->lap_alt_sl_plus = 4368 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 | 4369 ((alt_grh == B_FALSE) ? 0x8 : 0); 4370 4371 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib( 4372 (2 * statep->rc_alt_pkt_lt) + 4373 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 4374 4375 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid); 4376 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid); 4377 4378 lap_msgp->lap_remote_qpn_eecn_plus = 4379 h2b32((statep->remote_qpn << 8) | 4380 ibt_usec2ib(ibcm_remote_response_time) << 3); 4381 4382 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ); 4383 if ((len > 0) && priv_data) { 4384 bcopy(priv_data, lap_msgp->lap_private_data, len); 4385 } 4386 4387 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */ 4388 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 4389 4390 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt); 4391 4392 /* return_data is filled up in the state machine code */ 4393 statep->ap_return_data = ret_args; 4394 4395 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 4396 4397 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID = 4398 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID); 4399 4400 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID = 4401 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid, 4402 0)); 4403 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx", 4404 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID); 4405 4406 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4407 4408 /* Send LAP */ 4409 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete, 4410 statep); 4411 4412 mutex_enter(&statep->state_mutex); 4413 4414 if (mode == IBT_BLOCKING) { 4415 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking"); 4416 4417 /* wait for APR */ 4418 while (statep->ap_done != B_TRUE) { 4419 cv_wait(&statep->block_client_cv, 4420 &statep->state_mutex); 4421 } 4422 4423 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking"); 4424 4425 /* 4426 * In the case that ibt_set_alt_path fails, 4427 * change retval to IBT_CM_FAILURE 4428 */ 4429 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED) 4430 status = IBT_CM_FAILURE; 4431 4432 } 4433 4434 /* decrement the ref-count before leaving here */ 4435 IBCM_REF_CNT_DECR(statep); 4436 4437 mutex_exit(&statep->state_mutex); 4438 4439 /* If this message isn't seen then ibt_set_alt_path failed */ 4440 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done"); 4441 4442 return (status); 4443 } 4444 4445 4446 #ifdef DEBUG 4447 4448 /* 4449 * ibcm_query_classport_info: 4450 * Query classportinfo 4451 * 4452 * INPUTS: 4453 * channel - Channel that is associated with a statep 4454 * 4455 * RETURN VALUE: NONE 4456 * This function is currently used to generate a valid get method classport 4457 * info, and test CM functionality. There is no ibtl client interface to 4458 * generate a classportinfo. It is possible that CM may use classportinfo 4459 * from other nodes in the future, and most of the code below could be re-used. 4460 */ 4461 void 4462 ibcm_query_classport_info(ibt_channel_hdl_t channel) 4463 { 4464 ibcm_state_data_t *statep; 4465 ibmf_msg_t *msgp; 4466 4467 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p)", channel); 4468 4469 /* validate channel, first */ 4470 if (IBCM_INVALID_CHANNEL(channel)) { 4471 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4472 "invalid channel (%p)", channel); 4473 return; 4474 } 4475 4476 /* get the statep */ 4477 IBCM_GET_CHAN_PRIVATE(channel, statep); 4478 4479 /* 4480 * This can happen, if the statep is already gone by a DREQ from 4481 * the remote side 4482 */ 4483 if (statep == NULL) { 4484 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4485 "statep NULL"); 4486 return; 4487 } 4488 4489 mutex_enter(&statep->state_mutex); 4490 IBCM_RELEASE_CHAN_PRIVATE(channel); 4491 IBCM_REF_CNT_INCR(statep); 4492 mutex_exit(&statep->state_mutex); 4493 4494 /* Debug/test code, so don't care about return status */ 4495 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp, 4496 MAD_METHOD_GET); 4497 4498 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid( 4499 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0)); 4500 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO); 4501 4502 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL); 4503 4504 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p) " 4505 "Get method MAD posted ", channel); 4506 4507 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp); 4508 4509 mutex_enter(&statep->state_mutex); 4510 IBCM_REF_CNT_DECR(statep); 4511 mutex_exit(&statep->state_mutex); 4512 } 4513 4514 static void 4515 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr) 4516 { 4517 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, " 4518 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid, 4519 cm_reply_addr->rcvd_addr.ia_remote_lid); 4520 4521 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, " 4522 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key, 4523 cm_reply_addr->rcvd_addr.ia_p_key, 4524 cm_reply_addr->rcvd_addr.ia_remote_qno, 4525 cm_reply_addr->rcvd_addr.ia_service_level); 4526 4527 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ", 4528 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix, 4529 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid); 4530 4531 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX", 4532 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix, 4533 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid); 4534 4535 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x", 4536 cm_reply_addr->grh_hdr.ig_flow_label, 4537 cm_reply_addr->grh_hdr.ig_tclass, 4538 cm_reply_addr->grh_hdr.ig_hop_limit); 4539 } 4540 4541 #endif 4542 4543 typedef struct ibcm_join_mcg_tqarg_s { 4544 ib_gid_t rgid; 4545 ibt_mcg_attr_t mcg_attr; 4546 ibt_mcg_info_t *mcg_infop; 4547 ibt_mcg_handler_t func; 4548 void *arg; 4549 } ibcm_join_mcg_tqarg_t; 4550 4551 _NOTE(READ_ONLY_DATA(ibcm_join_mcg_tqarg_s)) 4552 4553 /* 4554 * Function: 4555 * ibt_join_mcg 4556 * Input: 4557 * rgid The request GID that defines the HCA port from which a 4558 * contact to SA Access is performed to add the specified 4559 * endport GID ((mcg_attr->mc_pgid) to a multicast group. 4560 * If mcg_attr->mc_pgid is null, then this (rgid) will be 4561 * treated as endport GID that is to be added to the 4562 * multicast group. 4563 * 4564 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines 4565 * the attributes of the desired multicast group to be 4566 * created or joined. 4567 * 4568 * func NULL or a pointer to a function to call when 4569 * ibt_join_mcg() completes. If 'func' is not NULL then 4570 * ibt_join_mcg() will return as soon as possible after 4571 * initiating the multicast group join/create process. 4572 * 'func' is then called when the process completes. 4573 * 4574 * arg Argument to the 'func'. 4575 * 4576 * Output: 4577 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated 4578 * by the caller, where the attributes of the created or 4579 * joined multicast group are copied. 4580 * Returns: 4581 * IBT_SUCCESS 4582 * IBT_INVALID_PARAM 4583 * IBT_MCG_RECORDS_NOT_FOUND 4584 * IBT_INSUFF_RESOURCE 4585 * Description: 4586 * Join a multicast group. The first full member "join" causes the MCG 4587 * to be created. 4588 */ 4589 ibt_status_t 4590 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, 4591 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg) 4592 { 4593 ibcm_join_mcg_tqarg_t *mcg_tq; 4594 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP); 4595 4596 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix, 4597 rgid.gid_guid, mcg_attr); 4598 4599 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4600 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required"); 4601 return (IBT_INVALID_PARAM); 4602 } 4603 4604 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) || 4605 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) { 4606 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified"); 4607 return (IBT_INVALID_PARAM); 4608 } 4609 4610 if (mcg_attr->mc_join_state == 0) { 4611 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified"); 4612 return (IBT_INVALID_PARAM); 4613 } 4614 4615 if (mcg_info_p == NULL) { 4616 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL"); 4617 return (IBT_INVALID_PARAM); 4618 } 4619 4620 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4621 4622 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag); 4623 if (mcg_tq == NULL) { 4624 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: " 4625 "Unable to allocate memory for local usage."); 4626 return (IBT_INSUFF_KERNEL_RESOURCE); 4627 } 4628 4629 mcg_tq->rgid = rgid; 4630 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t)); 4631 mcg_tq->mcg_infop = mcg_info_p; 4632 mcg_tq->func = func; 4633 mcg_tq->arg = arg; 4634 4635 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4636 4637 if (func != NULL) { /* Non-Blocking */ 4638 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call"); 4639 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg, 4640 mcg_tq, TQ_NOSLEEP) == 0) { 4641 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to " 4642 "Dispatch the TaskQ"); 4643 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t)); 4644 return (IBT_INSUFF_KERNEL_RESOURCE); 4645 } else 4646 return (IBT_SUCCESS); 4647 } else { /* Blocking */ 4648 return (ibcm_process_join_mcg(mcg_tq)); 4649 } 4650 } 4651 4652 static void 4653 ibcm_process_async_join_mcg(void *tq_arg) 4654 { 4655 (void) ibcm_process_join_mcg(tq_arg); 4656 } 4657 4658 static ibt_status_t 4659 ibcm_process_join_mcg(void *taskq_arg) 4660 { 4661 sa_mcmember_record_t mcg_req; 4662 sa_mcmember_record_t *mcg_resp; 4663 ibmf_saa_access_args_t access_args; 4664 ibmf_saa_handle_t saa_handle; 4665 uint64_t component_mask = 0; 4666 ibt_status_t retval; 4667 ibtl_cm_hca_port_t hca_port; 4668 uint_t num_records; 4669 size_t length; 4670 ibcm_hca_info_t *hcap; 4671 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg; 4672 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop; 4673 4674 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg); 4675 4676 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port); 4677 if (retval != IBT_SUCCESS) { 4678 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get " 4679 "port info from specified RGID: status = %d", retval); 4680 goto ibcm_join_mcg_exit1; 4681 } 4682 4683 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4684 4685 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) || 4686 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) { 4687 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4688 "Request GID is Port GID"); 4689 mcg_req.PortGID = mcg_arg->rgid; 4690 } else { 4691 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid; 4692 } 4693 component_mask |= SA_MC_COMPMASK_PORTGID; 4694 4695 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey; 4696 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey; 4697 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state; 4698 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass; 4699 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow; 4700 mcg_req.SL = mcg_arg->mcg_attr.mc_sl; 4701 4702 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY | 4703 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS | 4704 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL; 4705 4706 /* If client has specified MGID, use it else SA will assign one. */ 4707 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4708 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid; 4709 component_mask |= SA_MC_COMPMASK_MGID; 4710 } 4711 4712 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: PGID=%llX:%llX, " 4713 "MGID=%llX:%llX", mcg_req.PortGID.gid_prefix, 4714 mcg_req.PortGID.gid_guid, mcg_req.MGID.gid_prefix, 4715 mcg_req.MGID.gid_guid); 4716 4717 /* Is MTU specified. */ 4718 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) { 4719 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu; 4720 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector; 4721 4722 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 4723 SA_MC_COMPMASK_MTU; 4724 } 4725 4726 /* Is RATE specified. */ 4727 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) { 4728 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate; 4729 mcg_req.RateSelector = 4730 mcg_arg->mcg_attr.mc_rate_req.r_selector; 4731 4732 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 4733 SA_MC_COMPMASK_RATE; 4734 } 4735 4736 /* Is Packet Life Time specified. */ 4737 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) { 4738 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt; 4739 mcg_req.RateSelector = 4740 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector; 4741 4742 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 4743 SA_MC_COMPMASK_PKTLT; 4744 } 4745 4746 if (mcg_arg->mcg_attr.mc_hop) { 4747 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop; 4748 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 4749 } 4750 4751 if (mcg_arg->mcg_attr.mc_scope) { 4752 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope; 4753 component_mask |= SA_MC_COMPMASK_SCOPE; 4754 } 4755 4756 if (mcg_arg->mcg_attr.mc_mlid) { 4757 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid; 4758 component_mask |= SA_MC_COMPMASK_MLID; 4759 } 4760 4761 /* Get SA Access Handle. */ 4762 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4763 if (hcap == NULL) { 4764 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found"); 4765 4766 retval = IBT_HCA_BUSY_DETACHING; 4767 goto ibcm_join_mcg_exit1; 4768 } 4769 4770 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4771 if (saa_handle == NULL) { 4772 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL"); 4773 4774 retval = IBT_HCA_PORT_NOT_ACTIVE; 4775 goto ibcm_join_mcg_exit; 4776 } 4777 4778 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) && 4779 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) { 4780 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0, 4781 &hca_port); 4782 if (retval != IBT_SUCCESS) { 4783 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed " 4784 "to get PortInfo of specified PGID: status = %d", 4785 retval); 4786 goto ibcm_join_mcg_exit1; 4787 } 4788 } 4789 4790 /* Contact SA Access */ 4791 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4792 access_args.sq_access_type = IBMF_SAA_UPDATE; 4793 access_args.sq_component_mask = component_mask; 4794 access_args.sq_template = &mcg_req; 4795 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4796 access_args.sq_callback = NULL; 4797 access_args.sq_callback_arg = NULL; 4798 4799 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 4800 (void **)&mcg_resp); 4801 if (retval != IBT_SUCCESS) { 4802 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: " 4803 "SA Access Failed"); 4804 goto ibcm_join_mcg_exit; 4805 } 4806 4807 num_records = length/sizeof (sa_mcmember_record_t); 4808 4809 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: " 4810 "Found %d MCMember Records", num_records); 4811 4812 /* Validate the returned number of records. */ 4813 if ((mcg_resp != NULL) && (num_records > 0)) { 4814 /* Update the return values. */ 4815 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID; 4816 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID; 4817 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate; 4818 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL; 4819 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel; 4820 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass; 4821 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit; 4822 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE; 4823 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID; 4824 mcg_info_p->mc_mtu = mcg_resp->MTU; 4825 mcg_info_p->mc_qkey = mcg_resp->Q_Key; 4826 4827 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid, 4828 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix); 4829 if (retval != IBT_SUCCESS) { 4830 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4831 "Pkey2Index Conversion failed<%d>", retval); 4832 mcg_info_p->mc_pkey_ix = 0; 4833 } 4834 4835 mcg_info_p->mc_scope = mcg_resp->Scope; 4836 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime; 4837 4838 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port; 4839 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix; 4840 mcg_info_p->mc_adds_vect.av_src_path = 0; 4841 4842 /* Deallocate the memory allocated by SA for mcg_resp. */ 4843 kmem_free(mcg_resp, length); 4844 retval = IBT_SUCCESS; 4845 } else { 4846 retval = IBT_MCG_RECORDS_NOT_FOUND; 4847 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4848 "MCG RECORDS NOT FOUND"); 4849 } 4850 4851 ibcm_join_mcg_exit: 4852 ibcm_dec_hca_acc_cnt(hcap); 4853 4854 ibcm_join_mcg_exit1: 4855 if (mcg_arg->func) 4856 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p); 4857 4858 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t)); 4859 4860 return (retval); 4861 } 4862 4863 4864 /* 4865 * Function: 4866 * ibt_leave_mcg 4867 * Input: 4868 * rgid The request GID that defines the HCA port upon which 4869 * to send the request to the Subnet Administrator, to 4870 * remove the specified port (port_gid) from the multicast 4871 * group. If 'port_gid' is the Reserved GID (i.e. 4872 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0), 4873 * then the end-port associated with 'rgid' is removed 4874 * from the multicast group. 4875 * 4876 * mc_gid A multicast group GID as returned from ibt_join_mcg() 4877 * call. This is optional, if not specified (i.e. 4878 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to 4879 * identify this as being a multicast GID), then the 4880 * port is removed from all the multicast groups of 4881 * which it is a member. 4882 * 4883 * port_gid This is optional, if not the Reserved GID (gid_prefix 4884 * and gid_guid not equal to 0), then this specifies the 4885 * endport GID of the multicast group member being deleted 4886 * from the group. If it is the Reserved GID (gid_prefix 4887 * and gid_guid equal to 0) then the member endport GID is 4888 * determined from 'rgid'. 4889 * 4890 * mc_join_state The Join State attribute used when the group was joined 4891 * using ibt_join_mcg(). This Join State component must 4892 * contains at least one bit set to 1 in the same position 4893 * as that used during ibt_join_mcg(). i.e. the logical 4894 * AND of the two JoinState components is not all zeros. 4895 * This Join State component must not have some bits set 4896 * which are not set using ibt_join_mcg(). 4897 * Output: 4898 * None. 4899 * Returns: 4900 * IBT_SUCCESS 4901 * IBT_INVALID_PARAM 4902 * IBT_MC_GROUP_INVALID 4903 * IBT_INSUFF_RESOURCE 4904 * Description: 4905 * The port associated with the port GID shall be removed from the 4906 * multicast group specified by MGID (mc_gid) or from all the multicast 4907 * groups of which it is a member if the MGID (mc_gid) is not specified. 4908 * 4909 * The last full member to leave causes the destruction of the Multicast 4910 * Group. 4911 */ 4912 ibt_status_t 4913 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid, 4914 uint8_t mc_join_state) 4915 { 4916 sa_mcmember_record_t mcg_req; 4917 ibmf_saa_access_args_t access_args; 4918 ibmf_saa_handle_t saa_handle; 4919 uint64_t component_mask = 0; 4920 int sa_retval; 4921 ibt_status_t retval; 4922 ibtl_cm_hca_port_t hca_port; 4923 size_t length; 4924 void *results_p; 4925 ibcm_hca_info_t *hcap; 4926 4927 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)", 4928 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid); 4929 4930 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)", 4931 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state); 4932 4933 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4934 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required"); 4935 return (IBT_INVALID_PARAM); 4936 } 4937 4938 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4939 4940 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX", 4941 mc_gid.gid_prefix, mc_gid.gid_guid); 4942 4943 /* Validate MGID */ 4944 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4945 mcg_req.MGID = mc_gid; 4946 component_mask |= SA_MC_COMPMASK_MGID; 4947 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) { 4948 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified"); 4949 return (IBT_MC_MGID_INVALID); 4950 } 4951 4952 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) { 4953 mcg_req.PortGID = rgid; 4954 } else { 4955 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave"); 4956 mcg_req.PortGID = port_gid; 4957 } 4958 component_mask |= SA_MC_COMPMASK_PORTGID; 4959 4960 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>", 4961 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 4962 4963 /* Join State */ 4964 mcg_req.JoinState = mc_join_state; 4965 component_mask |= SA_MC_COMPMASK_JOINSTATE; 4966 4967 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port); 4968 if (retval != IBT_SUCCESS) { 4969 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info " 4970 "from specified RGID : status = %d", retval); 4971 return (retval); 4972 } 4973 4974 /* Get SA Access Handle. */ 4975 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4976 if (hcap == NULL) { 4977 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: " 4978 "NO HCA found"); 4979 return (IBT_HCA_BUSY_DETACHING); 4980 } 4981 4982 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4983 if (saa_handle == NULL) { 4984 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL"); 4985 ibcm_dec_hca_acc_cnt(hcap); 4986 return (IBT_HCA_PORT_NOT_ACTIVE); 4987 } 4988 4989 /* Contact SA Access */ 4990 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4991 access_args.sq_access_type = IBMF_SAA_DELETE; 4992 access_args.sq_component_mask = component_mask; 4993 access_args.sq_template = &mcg_req; 4994 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4995 access_args.sq_callback = NULL; 4996 access_args.sq_callback_arg = NULL; 4997 4998 ibcm_sa_access_enter(); 4999 5000 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length, 5001 &results_p); 5002 if (sa_retval != IBMF_SUCCESS) { 5003 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d", 5004 sa_retval); 5005 (void) ibcm_ibmf_analyze_error(sa_retval); 5006 retval = IBT_MC_GROUP_INVALID; 5007 } 5008 5009 ibcm_sa_access_exit(); 5010 5011 ibcm_dec_hca_acc_cnt(hcap); 5012 5013 return (retval); 5014 } 5015 5016 5017 /* 5018 * Function: 5019 * ibt_query_mcg 5020 * Input: 5021 * rgid The request GID that defines the HCA port upon which 5022 * to send the request to the Subnet Administrator, to 5023 * retrieve Multicast Records matching attributes as 5024 * specified through 'mcg_attr' argument. 5025 * 5026 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that 5027 * specifies MCG attributes that are to be matched. 5028 * Attributes that are not required can be wild carded 5029 * by specifying as '0'. 5030 * 5031 * mcgs_max_num The maximum number of matching multicast groups to 5032 * return. If zero, then all available matching multicast 5033 * groups are returned. 5034 * Output: 5035 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where 5036 * multicast group information is returned. The actual 5037 * number of entries filled in the array is returned in 5038 * entries_p. 5039 * 5040 * entries_p The number of ibt_mcg_attr_t entries returned. 5041 * Returns: 5042 * IBT_SUCCESS 5043 * IBT_INVALID_PARAM 5044 * IBT_MCG_RECORDS_NOT_FOUND 5045 * Description: 5046 * Request information on multicast groups that match the parameters 5047 * specified in mcg_attr. Information on each multicast group is returned 5048 * to the caller in the form of an array of ibt_mcg_info_t. 5049 * ibt_query_mcg() allocates the memory for this array and returns a 5050 * pointer to the array (mcgs_p) and the number of entries in the array 5051 * (entries_p). This memory should be freed by the client using 5052 * ibt_free_mcg_info(). 5053 */ 5054 ibt_status_t 5055 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num, 5056 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p) 5057 { 5058 sa_mcmember_record_t mcg_req; 5059 sa_mcmember_record_t *mcg_resp; 5060 ibt_mcg_info_t *mcg_infop; 5061 ibmf_saa_access_args_t access_args; 5062 ibmf_saa_handle_t saa_handle; 5063 uint64_t component_mask = 0; 5064 ibt_status_t retval; 5065 ibtl_cm_hca_port_t hport; 5066 uint_t num_records; 5067 size_t length; 5068 void *results_p; 5069 ib_gid_t port_gid; 5070 ibcm_hca_info_t *hcap; 5071 5072 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num); 5073 5074 if ((entries_p == NULL) || (mcgs_info_p == NULL)) { 5075 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5076 "entries_p or mcgs_info_p is NULL"); 5077 return (IBT_INVALID_PARAM); 5078 } 5079 5080 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5081 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required"); 5082 return (IBT_INVALID_PARAM); 5083 } 5084 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>", 5085 rgid.gid_prefix, rgid.gid_guid); 5086 5087 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5088 port_gid.gid_prefix = port_gid.gid_guid = 0; 5089 5090 if (mcg_attr != NULL) { 5091 port_gid = mcg_attr->mc_pgid; 5092 5093 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5094 mcg_req.PortGID = mcg_attr->mc_pgid; 5095 component_mask |= SA_MC_COMPMASK_PORTGID; 5096 5097 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX", 5098 port_gid.gid_prefix, port_gid.gid_guid); 5099 } 5100 5101 /* Is Q_Key specified. */ 5102 if (mcg_attr->mc_qkey != 0) { 5103 mcg_req.Q_Key = mcg_attr->mc_qkey; 5104 component_mask |= SA_MC_COMPMASK_QKEY; 5105 } 5106 5107 /* Is P_Key specified. */ 5108 if (mcg_attr->mc_pkey != 0) { 5109 mcg_req.P_Key = mcg_attr->mc_pkey; 5110 component_mask |= SA_MC_COMPMASK_PKEY; 5111 } 5112 5113 /* Is MGID specified. */ 5114 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5115 mcg_req.MGID = mcg_attr->mc_mgid; 5116 component_mask |= SA_MC_COMPMASK_MGID; 5117 } 5118 5119 /* Is MTU specified. */ 5120 if (mcg_attr->mc_mtu_req.r_mtu) { 5121 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu; 5122 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector; 5123 5124 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 5125 SA_MC_COMPMASK_MTU; 5126 } 5127 5128 if (mcg_attr->mc_tclass) { 5129 mcg_req.TClass = mcg_attr->mc_tclass; 5130 component_mask |= SA_MC_COMPMASK_TCLASS; 5131 } 5132 5133 /* Is RATE specified. */ 5134 if (mcg_attr->mc_rate_req.r_srate) { 5135 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate; 5136 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector; 5137 5138 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 5139 SA_MC_COMPMASK_RATE; 5140 } 5141 5142 /* Is Packet Life Time specified. */ 5143 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) { 5144 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt; 5145 mcg_req.RateSelector = 5146 mcg_attr->mc_pkt_lt_req.p_selector; 5147 5148 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5149 SA_MC_COMPMASK_PKTLT; 5150 } 5151 5152 if (mcg_attr->mc_hop) { 5153 mcg_req.HopLimit = mcg_attr->mc_hop; 5154 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5155 } 5156 5157 if (mcg_attr->mc_flow) { 5158 mcg_req.FlowLabel = mcg_attr->mc_flow; 5159 component_mask |= SA_MC_COMPMASK_FLOWLABEL; 5160 } 5161 5162 if (mcg_attr->mc_sl) { 5163 mcg_req.SL = mcg_attr->mc_sl; 5164 component_mask |= SA_MC_COMPMASK_SL; 5165 } 5166 5167 if (mcg_attr->mc_scope) { 5168 mcg_req.Scope = mcg_attr->mc_scope; 5169 component_mask |= SA_MC_COMPMASK_SCOPE; 5170 } 5171 5172 if (mcg_attr->mc_join_state) { 5173 mcg_req.JoinState = mcg_attr->mc_join_state; 5174 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5175 } 5176 5177 if (mcg_attr->mc_mlid) { 5178 mcg_req.MLID = mcg_attr->mc_mlid; 5179 component_mask |= SA_MC_COMPMASK_MLID; 5180 } 5181 } 5182 5183 retval = ibtl_cm_get_hca_port(rgid, 0, &hport); 5184 if (retval != IBT_SUCCESS) { 5185 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info " 5186 "from specified RGID : status = %d", retval); 5187 return (retval); 5188 } 5189 5190 /* Get SA Access Handle. */ 5191 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5192 if (hcap == NULL) { 5193 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found"); 5194 return (IBT_HCA_BUSY_DETACHING); 5195 } 5196 5197 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port); 5198 if (saa_handle == NULL) { 5199 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL"); 5200 ibcm_dec_hca_acc_cnt(hcap); 5201 return (IBT_HCA_PORT_NOT_ACTIVE); 5202 } 5203 5204 /* Contact SA Access */ 5205 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5206 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 5207 access_args.sq_component_mask = component_mask; 5208 access_args.sq_template = &mcg_req; 5209 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5210 access_args.sq_callback = NULL; 5211 access_args.sq_callback_arg = NULL; 5212 5213 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5214 &results_p); 5215 if (retval != IBT_SUCCESS) { 5216 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed"); 5217 ibcm_dec_hca_acc_cnt(hcap); 5218 return (retval); 5219 } 5220 5221 num_records = length/sizeof (sa_mcmember_record_t); 5222 5223 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records", 5224 num_records); 5225 5226 /* Validate the returned number of records. */ 5227 if ((results_p != NULL) && (num_records > 0)) { 5228 uint_t i; 5229 5230 /* 5231 * If mcgs_max_num is zero, then return all records else 5232 * return only requested number of records 5233 */ 5234 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) { 5235 /* we are interested in only mcgs_max_num records */ 5236 num_records = mcgs_max_num; 5237 } 5238 5239 /* 5240 * The SGID returned in "mcg_info_p" buffer should be PortGID, 5241 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified, 5242 * else RequestGID (rgid) should be returned. 5243 */ 5244 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5245 5246 /* Get sgid_ix and port number of 'port_gid' */ 5247 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport); 5248 if (retval != IBT_SUCCESS) { 5249 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5250 "Failed to Get Portinfo for PortGID :" 5251 "status = %d", retval); 5252 return (retval); 5253 } 5254 } else { 5255 /* 5256 * The sgid_ix and port number related to RequestGID 5257 * are already obtained at the beginning. 5258 */ 5259 port_gid = rgid; 5260 } 5261 5262 /* 5263 * Allocate memory for return buffer, to be freed in 5264 * ibt_free_mcg_info(). 5265 */ 5266 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)), 5267 KM_SLEEP); 5268 5269 *mcgs_info_p = mcg_infop; 5270 *entries_p = num_records; 5271 5272 /* Update the return values. */ 5273 for (i = 0; i < num_records; i++) { 5274 5275 mcg_resp = (sa_mcmember_record_t *)((uchar_t *) 5276 results_p + i * sizeof (sa_mcmember_record_t)); 5277 5278 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID; 5279 mcg_infop[i].mc_adds_vect.av_sgid = port_gid; 5280 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate; 5281 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL; 5282 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5283 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass; 5284 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit; 5285 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port; 5286 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE; 5287 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID; 5288 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix; 5289 mcg_infop[i].mc_adds_vect.av_src_path = 0; 5290 mcg_infop[i].mc_mtu = mcg_resp->MTU; 5291 mcg_infop[i].mc_qkey = mcg_resp->Q_Key; 5292 mcg_infop[i].mc_scope = mcg_resp->Scope; 5293 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime; 5294 5295 if (ibt_pkey2index_byguid(hport.hp_hca_guid, 5296 hport.hp_port, mcg_resp->P_Key, 5297 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) { 5298 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: " 5299 "Pkey2Index Conversion failed"); 5300 mcg_infop[i].mc_pkey_ix = 0; 5301 } 5302 } 5303 5304 /* 5305 * Deallocate the memory allocated by SA for results_p. 5306 */ 5307 kmem_free(results_p, length); 5308 retval = IBT_SUCCESS; 5309 5310 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords", 5311 num_records); 5312 5313 } else { 5314 retval = IBT_MCG_RECORDS_NOT_FOUND; 5315 *entries_p = 0; 5316 5317 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND"); 5318 } 5319 5320 ibcm_dec_hca_acc_cnt(hcap); 5321 5322 return (retval); 5323 } 5324 5325 5326 /* 5327 * ibt_free_mcg_info() 5328 * Free the memory allocated by successful ibt_query_mcg() 5329 * 5330 * mcgs_info Pointer returned by ibt_query_mcg(). 5331 * 5332 * entries The number of ibt_mcg_info_t entries to free. 5333 */ 5334 void 5335 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries) 5336 { 5337 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: " 5338 "Free <%d> entries from 0x%p", entries, mcgs_info); 5339 5340 if ((mcgs_info != NULL) && (entries > 0)) 5341 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t)); 5342 else 5343 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: " 5344 "ERROR: NULL buf pointer or length specified."); 5345 } 5346 5347 5348 /* 5349 * Function: 5350 * ibt_gid_to_node_info() 5351 * Input: 5352 * gid Identifies the IB Node and port for which to obtain 5353 * Node information. 5354 * Output: 5355 * node_info_p A pointer to an ibt_node_info_t structure (allocated 5356 * by the caller) in which to return the node information. 5357 * Returns: 5358 * IBT_SUCCESS 5359 * IBT_INVALID_PARAM 5360 * IBT_NODE_RECORDS_NOT_FOUND 5361 * IBT_NO_HCAS_AVAILABLE 5362 * Description: 5363 * Retrieve Node Information for the specified GID. 5364 */ 5365 ibt_status_t 5366 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p) 5367 { 5368 sa_node_record_t nr_req, *nr_resp; 5369 ibmf_saa_handle_t saa_handle; 5370 ibt_status_t retval; 5371 ibcm_hca_info_t *hcap; 5372 ibtl_cm_hca_port_t hport; 5373 int i, j; 5374 uint_t num_rec; 5375 ib_guid_t *guid_array = NULL; 5376 sa_path_record_t *path; 5377 size_t len; 5378 uint8_t npaths; 5379 uint32_t num_hcas = 0; 5380 ib_lid_t node_lid; 5381 boolean_t local_node = B_FALSE; 5382 void *res_p; 5383 uint8_t num_ports = 0; 5384 5385 5386 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)", 5387 gid.gid_prefix, gid.gid_guid, node_info_p); 5388 5389 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) { 5390 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required"); 5391 return (IBT_INVALID_PARAM); 5392 } 5393 5394 if (node_info_p == NULL) { 5395 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5396 "Return Buf (node_info_p) is NULL."); 5397 return (IBT_INVALID_PARAM); 5398 } 5399 5400 /* 5401 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5402 * associated port) info via ibtl_cm_get_hca_port() call. 5403 */ 5404 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5405 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) { 5406 5407 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5408 if (hcap == NULL) { 5409 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5410 "HCA(%llX) info not found", hport.hp_hca_guid); 5411 return (IBT_NO_HCAS_AVAILABLE); 5412 } 5413 num_ports = 1; 5414 num_hcas = 1; 5415 node_lid = hport.hp_base_lid; 5416 local_node = B_TRUE; 5417 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: " 5418 "LID = 0x%X", node_lid); 5419 } else { 5420 /* Get the number of HCAs and their GUIDs */ 5421 num_hcas = ibt_get_hca_list(&guid_array); 5422 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list " 5423 "returned %d hcas", num_hcas); 5424 5425 if (num_hcas == 0) { 5426 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5427 "NO HCA's Found on this system"); 5428 return (IBT_NO_HCAS_AVAILABLE); 5429 } 5430 } 5431 5432 for (i = 0; i < num_hcas; i++) { 5433 if (local_node == B_FALSE) { 5434 hcap = ibcm_find_hca_entry(guid_array[i]); 5435 if (hcap == NULL) { 5436 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5437 "HCA(%llX) info not found", guid_array[i]); 5438 retval = IBT_NO_HCAS_AVAILABLE; 5439 continue; 5440 } 5441 num_ports = hcap->hca_num_ports; 5442 } 5443 5444 for (j = 0; j < num_ports; j++) { 5445 uint8_t port = 0; 5446 5447 if (local_node == B_TRUE) 5448 port = hport.hp_port; 5449 else 5450 port = j + 1; 5451 5452 /* Get SA Access Handle. */ 5453 saa_handle = ibcm_get_saa_handle(hcap, port); 5454 if (saa_handle == NULL) { 5455 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5456 "Port %d of HCA (%llX) is NOT ACTIVE", 5457 port, hport.hp_hca_guid); 5458 retval = IBT_NODE_RECORDS_NOT_FOUND; 5459 continue; 5460 } 5461 5462 if (local_node == B_FALSE) { 5463 ib_gid_t sgid; 5464 int sa_ret; 5465 5466 /* 5467 * Check whether 'gid' and this port has same 5468 * subnet prefix. If not, then there is no use 5469 * in searching from this port. 5470 */ 5471 sgid = hcap->hca_port_info[j].port_sgid0; 5472 if (gid.gid_prefix != sgid.gid_prefix) { 5473 IBTF_DPRINTF_L3(cmlog, 5474 "ibt_gid_to_node_info:Sn_Prefix of " 5475 "GID(%llX) and Port's(%llX) differ", 5476 gid.gid_prefix, sgid.gid_prefix); 5477 retval = IBT_NODE_RECORDS_NOT_FOUND; 5478 continue; 5479 } 5480 5481 /* 5482 * First Get Path Records for the specified DGID 5483 * from this port (SGID). From Path Records, 5484 * note down DLID, then use this DLID as Input 5485 * attribute to get NodeRecords from SA Access. 5486 */ 5487 npaths = 1; 5488 path = NULL; 5489 5490 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5491 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5492 &path); 5493 if (sa_ret != IBMF_SUCCESS) { 5494 IBTF_DPRINTF_L2(cmlog, 5495 "ibt_gid_to_node_info: " 5496 "ibmf_saa_gid_to_pathrecords() " 5497 "returned error: %d ", sa_ret); 5498 retval = 5499 ibcm_ibmf_analyze_error(sa_ret); 5500 continue; 5501 } else if ((npaths == 0) || (path == NULL)) { 5502 IBTF_DPRINTF_L3(cmlog, 5503 "ibt_gid_to_node_info: failed (%d) " 5504 "to get path records for the DGID " 5505 "0x%llX from SGID 0x%llX", sa_ret, 5506 gid.gid_guid, sgid.gid_guid); 5507 retval = IBT_NODE_RECORDS_NOT_FOUND; 5508 continue; 5509 } 5510 node_lid = path->DLID; /* LID */ 5511 5512 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5513 "Remote Node: LID = 0x%X", node_lid); 5514 5515 /* Free SA_Access memory for path record. */ 5516 kmem_free(path, len); 5517 } 5518 5519 /* Retrieve Node Records from SA Access. */ 5520 bzero(&nr_req, sizeof (sa_node_record_t)); 5521 5522 nr_req.LID = node_lid; /* LID */ 5523 5524 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5525 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5526 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5527 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5528 "failed (%d) to get Node records", retval); 5529 continue; 5530 } else if (retval != IBT_SUCCESS) { 5531 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5532 "failed (%d) to get Node records", retval); 5533 ibcm_dec_hca_acc_cnt(hcap); 5534 goto gid_to_ni_exit; 5535 } 5536 5537 num_rec = len/sizeof (sa_node_record_t); 5538 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5539 5540 /* Validate the returned number of records. */ 5541 if ((nr_resp != NULL) && (num_rec > 0)) { 5542 5543 IBCM_DUMP_NODE_REC(nr_resp); 5544 5545 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS( 5546 *node_info_p)) 5547 5548 node_info_p->n_sys_img_guid = 5549 nr_resp->NodeInfo.SystemImageGUID; 5550 node_info_p->n_node_guid = 5551 nr_resp->NodeInfo.NodeGUID; 5552 node_info_p->n_port_guid = 5553 nr_resp->NodeInfo.PortGUID; 5554 node_info_p->n_dev_id = 5555 nr_resp->NodeInfo.DeviceID; 5556 node_info_p->n_revision = 5557 nr_resp->NodeInfo.Revision; 5558 node_info_p->n_vendor_id = 5559 nr_resp->NodeInfo.VendorID; 5560 node_info_p->n_num_ports = 5561 nr_resp->NodeInfo.NumPorts; 5562 node_info_p->n_port_num = 5563 nr_resp->NodeInfo.LocalPortNum; 5564 node_info_p->n_node_type = 5565 nr_resp->NodeInfo.NodeType; 5566 (void) strncpy(node_info_p->n_description, 5567 (char *)&nr_resp->NodeDescription, 64); 5568 5569 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS( 5570 *node_info_p)) 5571 5572 /* 5573 * Deallocate the memory allocated by SA for 5574 * 'nr_resp'. 5575 */ 5576 ibcm_dec_hca_acc_cnt(hcap); 5577 kmem_free(nr_resp, len); 5578 retval = IBT_SUCCESS; 5579 5580 goto gid_to_ni_exit; 5581 } else { 5582 retval = IBT_NODE_RECORDS_NOT_FOUND; 5583 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5584 "Node Records NOT found - PortGUID %016llX", 5585 gid.gid_guid); 5586 } 5587 } 5588 ibcm_dec_hca_acc_cnt(hcap); 5589 5590 if (local_node == B_TRUE) 5591 break; 5592 } 5593 5594 gid_to_ni_exit: 5595 if (guid_array) 5596 ibt_free_hca_list(guid_array, num_hcas); 5597 5598 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval); 5599 5600 return (retval); 5601 } 5602 5603 5604 static ibt_status_t 5605 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req, 5606 uint64_t component_mask, void *result_p, size_t *len) 5607 { 5608 ibmf_saa_access_args_t args; 5609 size_t length; 5610 ibt_status_t retval; 5611 5612 args.sq_attr_id = SA_NODERECORD_ATTRID; 5613 args.sq_template = nr_req; 5614 args.sq_access_type = IBMF_SAA_RETRIEVE; 5615 args.sq_template_length = sizeof (sa_node_record_t); 5616 args.sq_component_mask = component_mask; 5617 args.sq_callback = NULL; 5618 args.sq_callback_arg = NULL; 5619 5620 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p); 5621 if (retval != IBT_SUCCESS) { 5622 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed"); 5623 return (retval); 5624 } 5625 5626 *len = length; 5627 5628 /* Validate the returned number of records. */ 5629 if ((result_p != NULL) && (length > 0)) { 5630 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND"); 5631 5632 /* Got it, done!. */ 5633 return (IBT_SUCCESS); 5634 } else { 5635 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found"); 5636 return (IBT_NODE_RECORDS_NOT_FOUND); 5637 } 5638 } 5639 5640 5641 /* 5642 * Function: 5643 * ibt_get_companion_port_gids() 5644 * Description: 5645 * Get list of GID's available on a companion port(s) of the specified 5646 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID. 5647 */ 5648 ibt_status_t 5649 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid, 5650 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p) 5651 { 5652 sa_node_record_t nr_req, *nr_resp; 5653 void *res_p; 5654 ibmf_saa_handle_t saa_handle; 5655 int sa_ret; 5656 ibt_status_t retval; 5657 ibcm_hca_info_t *hcap; 5658 ibtl_cm_hca_port_t hport; 5659 int i, j; 5660 uint_t num_rec; 5661 ib_guid_t *guid_array = NULL; 5662 sa_path_record_t *path; 5663 size_t len; 5664 uint8_t npaths; 5665 uint32_t num_hcas = 0; 5666 boolean_t local_node = B_FALSE; 5667 boolean_t local_hca = B_FALSE; 5668 ib_guid_t h_guid = hca_guid; 5669 ib_gid_t *gidp = NULL, *t_gidp = NULL; 5670 int multi_hca_loop = 0; 5671 5672 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, " 5673 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid); 5674 5675 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) && 5676 (sysimg_guid == 0)) { 5677 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5678 "Null Input attribute specified."); 5679 return (IBT_INVALID_PARAM); 5680 } 5681 5682 if ((num_gids_p == NULL) || (gids_p == NULL)) { 5683 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5684 "num_gids_p or gids_p is NULL"); 5685 return (IBT_INVALID_PARAM); 5686 } 5687 5688 *num_gids_p = 0; 5689 5690 /* Get the number of HCAs and their GUIDs */ 5691 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) { 5692 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5693 "NO HCA's Found on this system"); 5694 return (IBT_NO_HCAS_AVAILABLE); 5695 } 5696 5697 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5698 "ibt_get_hca_list() returned %d hcas", num_hcas); 5699 5700 /* 5701 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5702 * associated port) info via ibtl_cm_get_hca_port() call. 5703 */ 5704 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5705 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) && 5706 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) { 5707 5708 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) { 5709 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5710 "Invalid GID<->HCAGUID combination specified."); 5711 retval = IBT_INVALID_PARAM; 5712 goto get_comp_pgid_exit; 5713 } 5714 h_guid = hport.hp_hca_guid; 5715 local_node = B_TRUE; 5716 5717 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5718 "Local Node: HCA (0x%llX)", h_guid); 5719 } else if (h_guid) { /* Is specified HCA GUID - local? */ 5720 for (i = 0; i < num_hcas; i++) { 5721 if (h_guid == guid_array[i]) { 5722 local_hca = B_TRUE; 5723 break; 5724 } 5725 } 5726 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */ 5727 for (i = 0; i < num_hcas; i++) { 5728 ibt_status_t ret; 5729 ibt_hca_attr_t hca_attr; 5730 5731 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr); 5732 if (ret != IBT_SUCCESS) { 5733 IBTF_DPRINTF_L2(cmlog, 5734 "ibt_get_companion_port_gids: HCA(%llX) " 5735 "info not found", guid_array[i]); 5736 retval = IBT_NO_HCAS_AVAILABLE; 5737 continue; 5738 } 5739 if (hca_attr.hca_si_guid == sysimg_guid) { 5740 if ((hca_guid != 0) && 5741 (hca_guid != hca_attr.hca_node_guid)) { 5742 IBTF_DPRINTF_L2(cmlog, 5743 "ibt_get_companion_port_gids: " 5744 "Invalid SysImg<->HCA GUID " 5745 "combination specified."); 5746 retval = IBT_INVALID_PARAM; 5747 goto get_comp_pgid_exit; 5748 } 5749 local_hca = B_TRUE; 5750 h_guid = hca_attr.hca_node_guid; 5751 break; 5752 } 5753 } 5754 } 5755 5756 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) { 5757 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p, 5758 num_gids_p); 5759 goto get_comp_pgid_exit; 5760 } 5761 5762 get_comp_for_multihca: 5763 /* We will be here, if request is for remote node */ 5764 for (i = 0; i < num_hcas; i++) { 5765 int multism; 5766 uint8_t count = 0; 5767 int multi_sm_loop = 0; 5768 uint_t k = 0, l; 5769 5770 hcap = ibcm_find_hca_entry(guid_array[i]); 5771 if (hcap == NULL) { 5772 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5773 "HCA(%llX) info not found", guid_array[i]); 5774 retval = IBT_NO_HCAS_AVAILABLE; 5775 continue; 5776 } 5777 5778 /* 1 - MultiSM, 0 - Single SM */ 5779 multism = ibtl_cm_is_multi_sm(guid_array[i]); 5780 5781 for (j = 0; j < hcap->hca_num_ports; j++) { 5782 ib_gid_t sgid; 5783 uint64_t c_mask = 0; 5784 ib_guid_t pg; 5785 uint_t port = j; 5786 5787 get_comp_for_multism: 5788 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5789 "Port %d, HCA %llX, MultiSM= %d, Loop=%d", 5790 port + 1, h_guid, multism, multi_sm_loop); 5791 5792 /* Get SA Access Handle. */ 5793 saa_handle = ibcm_get_saa_handle(hcap, port + 1); 5794 if (saa_handle == NULL) { 5795 IBTF_DPRINTF_L2(cmlog, 5796 "ibt_get_companion_port_gids: " 5797 "Port (%d) - NOT ACTIVE", port); 5798 retval = IBT_HCA_PORT_NOT_ACTIVE; 5799 continue; 5800 } 5801 5802 /* 5803 * Check whether 'gid' and this port has same subnet 5804 * prefix. If not, then there is no use in searching 5805 * from this port. 5806 */ 5807 sgid = hcap->hca_port_info[port].port_sgid0; 5808 if ((h_guid == 0) && (gid.gid_prefix != 0) && 5809 (multi_sm_loop == 0) && 5810 (gid.gid_prefix != sgid.gid_prefix)) { 5811 IBTF_DPRINTF_L2(cmlog, 5812 "ibt_get_companion_port_gids: SnPrefix of " 5813 "GID(%llX) and Port SN_Pfx(%llX) differ", 5814 gid.gid_prefix, sgid.gid_prefix); 5815 retval = IBT_NODE_RECORDS_NOT_FOUND; 5816 continue; 5817 } 5818 5819 /* 5820 * If HCA GUID or System Image GUID is specified, then 5821 * we can achieve our goal sooner!. 5822 */ 5823 if ((h_guid == 0) && (sysimg_guid == 0)) { 5824 /* So only GID info is provided. */ 5825 5826 /* 5827 * First Get Path Records for the specified DGID 5828 * from this port (SGID). From Path Records, 5829 * note down DLID, then use this DLID as Input 5830 * attribute to get NodeRecords. 5831 */ 5832 npaths = 1; 5833 path = NULL; 5834 5835 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5836 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5837 &path); 5838 if (sa_ret != IBMF_SUCCESS) { 5839 IBTF_DPRINTF_L2(cmlog, 5840 "ibt_get_companion_port_gids: " 5841 "ibmf_saa_gid_to_pathrecords() " 5842 "returned error: %d ", sa_ret); 5843 retval = 5844 ibcm_ibmf_analyze_error(sa_ret); 5845 ibcm_dec_hca_acc_cnt(hcap); 5846 goto get_comp_pgid_exit; 5847 } else if ((npaths == 0) || (path == NULL)) { 5848 IBTF_DPRINTF_L2(cmlog, 5849 "ibt_get_companion_port_gids: " 5850 "failed (%d) to get path records " 5851 "for the DGID (0x%llX) from SGID " 5852 "(0x%llX)", sa_ret, gid.gid_guid, 5853 sgid.gid_guid); 5854 retval = IBT_NODE_RECORDS_NOT_FOUND; 5855 continue; 5856 } 5857 5858 bzero(&nr_req, sizeof (sa_node_record_t)); 5859 nr_req.LID = path->DLID; /* LID */ 5860 5861 IBTF_DPRINTF_L3(cmlog, 5862 "ibt_get_companion_port_gids: " 5863 "Remote Node: LID = 0x%X", nr_req.LID); 5864 5865 /* Free SA_Access memory for path record. */ 5866 kmem_free(path, len); 5867 5868 IBTF_DPRINTF_L3(cmlog, 5869 "ibt_get_companion_port_gids: SAA Call: " 5870 "based on LID "); 5871 5872 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5873 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5874 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5875 IBTF_DPRINTF_L2(cmlog, 5876 "ibt_get_companion_port_gids: " 5877 "failed (%d) to get Node records", 5878 retval); 5879 continue; 5880 } else if (retval != IBT_SUCCESS) { 5881 IBTF_DPRINTF_L2(cmlog, 5882 "ibt_get_companion_port_gids: " 5883 "failed (%d) to get Node records", 5884 retval); 5885 ibcm_dec_hca_acc_cnt(hcap); 5886 goto get_comp_pgid_exit; 5887 } 5888 5889 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5890 /* Note down HCA GUID info. */ 5891 h_guid = nr_resp->NodeInfo.NodeGUID; 5892 5893 IBTF_DPRINTF_L3(cmlog, 5894 "ibt_get_companion_port_gids: " 5895 "Remote HCA GUID: 0x%llX", h_guid); 5896 5897 IBCM_DUMP_NODE_REC(nr_resp); 5898 5899 kmem_free(res_p, len); 5900 } 5901 5902 bzero(&nr_req, sizeof (sa_node_record_t)); 5903 if (h_guid != 0) { 5904 nr_req.NodeInfo.NodeGUID = h_guid; 5905 c_mask = SA_NODEINFO_COMPMASK_NODEGUID; 5906 } 5907 5908 if (sysimg_guid != 0) { 5909 nr_req.NodeInfo.SystemImageGUID = sysimg_guid; 5910 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID; 5911 } 5912 5913 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5914 "SAA Call: CMASK= 0x%llX", c_mask); 5915 5916 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask, 5917 &res_p, &len); 5918 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5919 IBTF_DPRINTF_L3(cmlog, 5920 "ibt_get_companion_port_gids: " 5921 "failed (%d) to get Node records", retval); 5922 continue; 5923 } else if (retval != IBT_SUCCESS) { 5924 IBTF_DPRINTF_L2(cmlog, 5925 "ibt_get_companion_port_gids: Error: (%d) " 5926 "while getting Node records", retval); 5927 ibcm_dec_hca_acc_cnt(hcap); 5928 goto get_comp_pgid_exit; 5929 } 5930 5931 num_rec = len/sizeof (sa_node_record_t); 5932 5933 /* We will be here, only if we found some NodeRec */ 5934 if (gid.gid_prefix && gid.gid_guid) { 5935 nr_resp = (sa_node_record_t *)res_p; 5936 for (l = 0; l < num_rec; l++, nr_resp++) { 5937 pg = nr_resp->NodeInfo.PortGUID; 5938 if (gid.gid_guid != pg) 5939 count++; 5940 } 5941 } else { 5942 count = num_rec; 5943 } 5944 5945 if (count != 0) { 5946 if (multi_sm_loop == 1) { 5947 count += k; 5948 t_gidp = kmem_zalloc(count * 5949 sizeof (ib_gid_t), KM_SLEEP); 5950 5951 if ((k != 0) && (gidp != NULL)) { 5952 bcopy(gidp, t_gidp, 5953 k * sizeof (ib_gid_t)); 5954 kmem_free(gidp, 5955 k * sizeof (ib_gid_t)); 5956 } 5957 gidp = t_gidp; 5958 } else { 5959 gidp = kmem_zalloc(count * 5960 sizeof (ib_gid_t), KM_SLEEP); 5961 } 5962 *num_gids_p = count; 5963 *gids_p = gidp; 5964 5965 nr_resp = (sa_node_record_t *)res_p; 5966 for (l = 0; l < num_rec; l++, nr_resp++) { 5967 IBCM_DUMP_NODE_REC(nr_resp); 5968 5969 pg = nr_resp->NodeInfo.PortGUID; 5970 IBTF_DPRINTF_L4(cmlog, 5971 "ibt_get_companion_port_gids: " 5972 "PortGID %llX", pg); 5973 5974 if (pg != gid.gid_guid) { 5975 gidp[k].gid_prefix = 5976 sgid.gid_prefix; 5977 gidp[k].gid_guid = pg; 5978 5979 IBTF_DPRINTF_L3(cmlog, 5980 "ibt_get_companion_pgids: " 5981 "GID[%d] = %llX:%llX", k, 5982 gidp[k].gid_prefix, 5983 gidp[k].gid_guid); 5984 5985 k++; 5986 if (k == count) 5987 break; 5988 } 5989 } 5990 retval = IBT_SUCCESS; /* done!. */ 5991 kmem_free(res_p, len); 5992 ibcm_dec_hca_acc_cnt(hcap); 5993 goto get_comp_pgid_exit; 5994 } else { 5995 IBTF_DPRINTF_L2(cmlog, 5996 "ibt_get_companion_port_gids: " 5997 "Companion PortGIDs not available"); 5998 retval = IBT_GIDS_NOT_FOUND; 5999 } 6000 /* Deallocate the memory for 'res_p'. */ 6001 kmem_free(res_p, len); 6002 6003 /* 6004 * If we are on MultiSM setup, then we need to lookout 6005 * from that subnet port too. 6006 */ 6007 if (multism) { 6008 /* break if already searched both the subnet */ 6009 if (multi_sm_loop == 1) 6010 break; 6011 6012 port = (j == 0) ? 1 : 0; 6013 multi_sm_loop = 1; 6014 goto get_comp_for_multism; 6015 } else { 6016 break; 6017 } 6018 } 6019 ibcm_dec_hca_acc_cnt(hcap); 6020 6021 /* 6022 * We may be on dual HCA with dual SM configured system. And 6023 * the input attr GID was visible from second HCA. So in order 6024 * to get the companion portgid we need to re-look from the 6025 * first HCA ports. 6026 */ 6027 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) && 6028 (multi_hca_loop != 1)) { 6029 multi_hca_loop = 1; 6030 goto get_comp_for_multihca; 6031 } 6032 } 6033 6034 get_comp_pgid_exit: 6035 if (guid_array) 6036 ibt_free_hca_list(guid_array, num_hcas); 6037 6038 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) { 6039 retval = IBT_SUCCESS; 6040 } 6041 6042 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, " 6043 "Found %d GIDs", retval, *num_gids_p); 6044 6045 return (retval); 6046 } 6047 6048 6049 /* Routines for warlock */ 6050 6051 /* ARGSUSED */ 6052 static void 6053 ibcm_dummy_mcg_handler(void *arg, ibt_status_t retval, ibt_mcg_info_t *minfo) 6054 { 6055 ibcm_join_mcg_tqarg_t dummy_mcg; 6056 6057 dummy_mcg.func = ibcm_dummy_mcg_handler; 6058 6059 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_mcg_handler: " 6060 "dummy_mcg.func %p", dummy_mcg.func); 6061 } 6062 6063 6064 /* ARGSUSED */ 6065 static void 6066 ibcm_dummy_recycle_rc_handler(ibt_status_t retval, void *arg) 6067 { 6068 ibcm_taskq_recycle_arg_t dummy_rc_recycle; 6069 6070 dummy_rc_recycle.func = ibcm_dummy_recycle_rc_handler; 6071 6072 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_recycle_rc_handler: " 6073 "dummy_rc_recycle.func %p", dummy_rc_recycle.func); 6074 } 6075 6076 6077 /* ARGSUSED */ 6078 static ibt_cm_status_t 6079 ibcm_dummy_ud_handler(void *priv, ibt_cm_ud_event_t *event, 6080 ibt_cm_ud_return_args_t *ret_args, 6081 void *priv_data, ibt_priv_data_len_t len) 6082 { 6083 /* 6084 * Let warlock see that ibcm_local_handler_s::actual_cm_handler 6085 * points to this routine. 6086 */ 6087 ibcm_local_handler_t p; 6088 ibcm_ud_state_data_t dummy_ud; 6089 6090 p.actual_cm_handler = ibcm_dummy_ud_handler; 6091 dummy_ud.ud_cm_handler = ibcm_dummy_ud_handler; 6092 6093 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_ud_handler: p.actual_cm_handler %p" 6094 "dummy_ud.ud_cm_handler %p", p.actual_cm_handler, 6095 dummy_ud.ud_cm_handler); 6096 /* 6097 * Call all routines that the client's callback routine could call. 6098 */ 6099 6100 return (IBT_CM_ACCEPT); 6101 } 6102 6103 /* ARGSUSED */ 6104 static ibt_cm_status_t 6105 ibcm_dummy_rc_handler(void *priv, ibt_cm_event_t *event, 6106 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 6107 { 6108 ibcm_state_data_t dummy_rc; 6109 6110 dummy_rc.cm_handler = ibcm_dummy_rc_handler; 6111 6112 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_rc_handler: " 6113 "dummy_ud.ud_cm_handler %p", dummy_rc.cm_handler); 6114 /* 6115 * Call all routines that the client's callback routine could call. 6116 */ 6117 6118 return (IBT_CM_ACCEPT); 6119 } 6120