1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * ibdm.c 28 * 29 * This file contains the InifiniBand Device Manager (IBDM) support functions. 30 * IB nexus driver will only be the client for the IBDM module. 31 * 32 * IBDM registers with IBTF for HCA arrival/removal notification. 33 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 34 * the IOU's. 35 * 36 * IB nexus driver registers with IBDM to find the information about the 37 * HCA's and IOC's (behind the IOU) present on the IB fabric. 38 */ 39 40 #include <sys/systm.h> 41 #include <sys/taskq.h> 42 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 43 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 44 #include <sys/ib/ibtl/impl/ibtl_ibnex.h> 45 #include <sys/modctl.h> 46 47 /* Function Prototype declarations */ 48 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *, ibdm_iou_info_t **); 49 static int ibdm_fini(void); 50 static int ibdm_init(void); 51 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 52 ibdm_hca_list_t *); 53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 55 static boolean_t ibdm_is_cisco(ib_guid_t); 56 static boolean_t ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *); 57 static void ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *); 58 static int ibdm_set_classportinfo(ibdm_dp_gidinfo_t *); 59 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 60 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 61 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 62 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 63 ib_guid_t *, ib_guid_t *); 64 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 65 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 66 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 67 static int ibdm_handle_redirection(ibmf_msg_t *, 68 ibdm_dp_gidinfo_t *, int *); 69 static void ibdm_wait_probe_completion(void); 70 static void ibdm_sweep_fabric(int); 71 static void ibdm_probe_gid_thread(void *); 72 static void ibdm_wakeup_probe_gid_cv(void); 73 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 74 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 75 static void ibdm_update_port_attr(ibdm_port_attr_t *); 76 static void ibdm_handle_hca_attach(ib_guid_t); 77 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 78 ibdm_dp_gidinfo_t *, int *); 79 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 80 static void ibdm_recv_incoming_mad(void *); 81 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 82 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 83 static void ibdm_pkt_timeout_hdlr(void *arg); 84 static void ibdm_initialize_port(ibdm_port_attr_t *); 85 static void ibdm_update_port_pkeys(ibdm_port_attr_t *port); 86 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 87 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 88 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 89 static void ibdm_free_send_buffers(ibmf_msg_t *); 90 static void ibdm_handle_hca_detach(ib_guid_t); 91 static void ibdm_handle_port_change_event(ibt_async_event_t *); 92 static int ibdm_fini_port(ibdm_port_attr_t *); 93 static int ibdm_uninit_hca(ibdm_hca_list_t *); 94 static void ibdm_handle_setclassportinfo(ibmf_handle_t, ibmf_msg_t *, 95 ibdm_dp_gidinfo_t *, int *); 96 static void ibdm_handle_iounitinfo(ibmf_handle_t, 97 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 98 static void ibdm_handle_ioc_profile(ibmf_handle_t, 99 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 100 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 101 ibt_async_code_t, ibt_async_event_t *); 102 static void ibdm_handle_classportinfo(ibmf_handle_t, 103 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 104 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 105 ibdm_dp_gidinfo_t *); 106 107 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 108 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 109 ibdm_dp_gidinfo_t *gid_list); 110 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 111 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 112 ibdm_dp_gidinfo_t *, int *); 113 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 114 ibdm_hca_list_t **); 115 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 116 size_t *, ib_guid_t); 117 static int ibdm_get_node_record_by_port(ibmf_saa_handle_t, 118 ib_guid_t, sa_node_record_t **, size_t *); 119 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 120 ib_lid_t); 121 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 122 ib_gid_t, ib_gid_t); 123 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 124 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 125 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 126 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 127 ibmf_saa_event_details_t *, void *); 128 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 129 ibdm_dp_gidinfo_t *); 130 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 131 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 132 ibdm_dp_gidinfo_t *); 133 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 134 static void ibdm_free_gid_list(ibdm_gid_t *); 135 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 136 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 137 static void ibdm_saa_event_taskq(void *); 138 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 139 static void ibdm_get_next_port(ibdm_hca_list_t **, 140 ibdm_port_attr_t **, int); 141 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 142 ibdm_dp_gidinfo_t *); 143 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 144 ibdm_hca_list_t *); 145 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 146 static void ibdm_saa_handle_new_gid(void *); 147 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 148 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 149 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 150 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 151 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 152 static ibdm_ioc_info_t *ibdm_handle_prev_iou(); 153 static int ibdm_serv_cmp(ibdm_srvents_info_t *, ibdm_srvents_info_t *, 154 int); 155 static ibdm_ioc_info_t *ibdm_get_ioc_info_with_gid(ib_guid_t, 156 ibdm_dp_gidinfo_t **); 157 158 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 159 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 160 #ifdef DEBUG 161 int ibdm_ignore_saa_event = 0; 162 #endif 163 164 /* Modload support */ 165 static struct modlmisc ibdm_modlmisc = { 166 &mod_miscops, 167 "InfiniBand Device Manager" 168 }; 169 170 struct modlinkage ibdm_modlinkage = { 171 MODREV_1, 172 (void *)&ibdm_modlmisc, 173 NULL 174 }; 175 176 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 177 IBTI_V_CURR, 178 IBT_DM, 179 ibdm_event_hdlr, 180 NULL, 181 "ibdm" 182 }; 183 184 /* Global variables */ 185 ibdm_t ibdm; 186 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 187 char *ibdm_string = "ibdm"; 188 189 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 190 ibdm.ibdm_dp_gidlist_head)) 191 192 /* 193 * _init 194 * Loadable module init, called before any other module. 195 * Initialize mutex 196 * Register with IBTF 197 */ 198 int 199 _init(void) 200 { 201 int err; 202 203 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 204 205 if ((err = ibdm_init()) != IBDM_SUCCESS) { 206 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 207 (void) ibdm_fini(); 208 return (DDI_FAILURE); 209 } 210 211 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 212 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 213 (void) ibdm_fini(); 214 } 215 return (err); 216 } 217 218 219 int 220 _fini(void) 221 { 222 int err; 223 224 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 225 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 226 (void) ibdm_init(); 227 return (EBUSY); 228 } 229 230 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 231 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 232 (void) ibdm_init(); 233 } 234 return (err); 235 } 236 237 238 int 239 _info(struct modinfo *modinfop) 240 { 241 return (mod_info(&ibdm_modlinkage, modinfop)); 242 } 243 244 245 /* 246 * ibdm_init(): 247 * Register with IBTF 248 * Allocate memory for the HCAs 249 * Allocate minor-nodes for the HCAs 250 */ 251 static int 252 ibdm_init(void) 253 { 254 int i, hca_count; 255 ib_guid_t *hca_guids; 256 ibt_status_t status; 257 258 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 259 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 260 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 261 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 262 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 263 cv_init(&ibdm.ibdm_port_settle_cv, NULL, CV_DRIVER, NULL); 264 mutex_enter(&ibdm.ibdm_mutex); 265 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 266 } 267 268 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 269 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 270 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 271 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 272 "failed %x", status); 273 mutex_exit(&ibdm.ibdm_mutex); 274 return (IBDM_FAILURE); 275 } 276 277 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 278 mutex_exit(&ibdm.ibdm_mutex); 279 } 280 281 282 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 283 hca_count = ibt_get_hca_list(&hca_guids); 284 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 285 for (i = 0; i < hca_count; i++) 286 (void) ibdm_handle_hca_attach(hca_guids[i]); 287 if (hca_count) 288 ibt_free_hca_list(hca_guids, hca_count); 289 290 mutex_enter(&ibdm.ibdm_mutex); 291 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 292 mutex_exit(&ibdm.ibdm_mutex); 293 } 294 295 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 296 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 297 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 298 mutex_enter(&ibdm.ibdm_mutex); 299 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 300 mutex_exit(&ibdm.ibdm_mutex); 301 } 302 return (IBDM_SUCCESS); 303 } 304 305 306 static int 307 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info, ibdm_iou_info_t **ioup) 308 { 309 int ii, k, niocs; 310 size_t size; 311 ibdm_gid_t *delete, *head; 312 timeout_id_t timeout_id; 313 ibdm_ioc_info_t *ioc; 314 ibdm_iou_info_t *gl_iou = *ioup; 315 316 ASSERT(mutex_owned(&gid_info->gl_mutex)); 317 if (gl_iou == NULL) { 318 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 319 return (0); 320 } 321 322 niocs = gl_iou->iou_info.iou_num_ctrl_slots; 323 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 324 gid_info, niocs); 325 326 for (ii = 0; ii < niocs; ii++) { 327 ioc = (ibdm_ioc_info_t *)&gl_iou->iou_ioc_info[ii]; 328 329 /* handle the case where an ioc_timeout_id is scheduled */ 330 if (ioc->ioc_timeout_id) { 331 timeout_id = ioc->ioc_timeout_id; 332 ioc->ioc_timeout_id = 0; 333 mutex_exit(&gid_info->gl_mutex); 334 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 335 "ioc_timeout_id = 0x%x", timeout_id); 336 if (untimeout(timeout_id) == -1) { 337 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 338 "untimeout ioc_timeout_id failed"); 339 mutex_enter(&gid_info->gl_mutex); 340 return (-1); 341 } 342 mutex_enter(&gid_info->gl_mutex); 343 } 344 345 /* handle the case where an ioc_dc_timeout_id is scheduled */ 346 if (ioc->ioc_dc_timeout_id) { 347 timeout_id = ioc->ioc_dc_timeout_id; 348 ioc->ioc_dc_timeout_id = 0; 349 mutex_exit(&gid_info->gl_mutex); 350 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 351 "ioc_dc_timeout_id = 0x%x", timeout_id); 352 if (untimeout(timeout_id) == -1) { 353 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 354 "untimeout ioc_dc_timeout_id failed"); 355 mutex_enter(&gid_info->gl_mutex); 356 return (-1); 357 } 358 mutex_enter(&gid_info->gl_mutex); 359 } 360 361 /* handle the case where serv[k].se_timeout_id is scheduled */ 362 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 363 if (ioc->ioc_serv[k].se_timeout_id) { 364 timeout_id = ioc->ioc_serv[k].se_timeout_id; 365 ioc->ioc_serv[k].se_timeout_id = 0; 366 mutex_exit(&gid_info->gl_mutex); 367 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 368 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 369 k, timeout_id); 370 if (untimeout(timeout_id) == -1) { 371 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 372 " untimeout se_timeout_id failed"); 373 mutex_enter(&gid_info->gl_mutex); 374 return (-1); 375 } 376 mutex_enter(&gid_info->gl_mutex); 377 } 378 } 379 380 /* delete GID list in IOC */ 381 head = ioc->ioc_gid_list; 382 while (head) { 383 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 384 "Deleting gid_list struct %p", head); 385 delete = head; 386 head = head->gid_next; 387 kmem_free(delete, sizeof (ibdm_gid_t)); 388 } 389 ioc->ioc_gid_list = NULL; 390 391 /* delete ioc_serv */ 392 size = ioc->ioc_profile.ioc_service_entries * 393 sizeof (ibdm_srvents_info_t); 394 if (ioc->ioc_serv && size) { 395 kmem_free(ioc->ioc_serv, size); 396 ioc->ioc_serv = NULL; 397 } 398 } 399 /* 400 * Clear the IBDM_CISCO_PROBE_DONE flag to get the IO Unit information 401 * via the switch during the probe process. 402 */ 403 gid_info->gl_flag &= ~IBDM_CISCO_PROBE_DONE; 404 405 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 406 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 407 kmem_free(gl_iou, size); 408 *ioup = NULL; 409 return (0); 410 } 411 412 413 /* 414 * ibdm_fini(): 415 * Un-register with IBTF 416 * De allocate memory for the GID info 417 */ 418 static int 419 ibdm_fini() 420 { 421 int ii; 422 ibdm_hca_list_t *hca_list, *temp; 423 ibdm_dp_gidinfo_t *gid_info, *tmp; 424 ibdm_gid_t *head, *delete; 425 426 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 427 428 mutex_enter(&ibdm.ibdm_hl_mutex); 429 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 430 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 431 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 432 mutex_exit(&ibdm.ibdm_hl_mutex); 433 return (IBDM_FAILURE); 434 } 435 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 436 ibdm.ibdm_ibt_clnt_hdl = NULL; 437 } 438 439 hca_list = ibdm.ibdm_hca_list_head; 440 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 441 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 442 temp = hca_list; 443 hca_list = hca_list->hl_next; 444 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 445 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 446 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 447 "uninit_hca %p failed", temp); 448 mutex_exit(&ibdm.ibdm_hl_mutex); 449 return (IBDM_FAILURE); 450 } 451 } 452 mutex_exit(&ibdm.ibdm_hl_mutex); 453 454 mutex_enter(&ibdm.ibdm_mutex); 455 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 456 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 457 458 gid_info = ibdm.ibdm_dp_gidlist_head; 459 while (gid_info) { 460 mutex_enter(&gid_info->gl_mutex); 461 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 462 mutex_exit(&gid_info->gl_mutex); 463 ibdm_delete_glhca_list(gid_info); 464 465 tmp = gid_info; 466 gid_info = gid_info->gl_next; 467 mutex_destroy(&tmp->gl_mutex); 468 head = tmp->gl_gid; 469 while (head) { 470 IBTF_DPRINTF_L4("ibdm", 471 "\tibdm_fini: Deleting gid structs"); 472 delete = head; 473 head = head->gid_next; 474 kmem_free(delete, sizeof (ibdm_gid_t)); 475 } 476 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 477 } 478 mutex_exit(&ibdm.ibdm_mutex); 479 480 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 481 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 482 mutex_destroy(&ibdm.ibdm_mutex); 483 mutex_destroy(&ibdm.ibdm_hl_mutex); 484 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 485 cv_destroy(&ibdm.ibdm_port_settle_cv); 486 } 487 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 488 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 489 cv_destroy(&ibdm.ibdm_probe_cv); 490 cv_destroy(&ibdm.ibdm_busy_cv); 491 } 492 return (IBDM_SUCCESS); 493 } 494 495 496 /* 497 * ibdm_event_hdlr() 498 * 499 * IBDM registers this asynchronous event handler at the time of 500 * ibt_attach. IBDM support the following async events. For other 501 * event, simply returns success. 502 * IBT_HCA_ATTACH_EVENT: 503 * Retrieves the information about all the port that are 504 * present on this HCA, allocates the port attributes 505 * structure and calls IB nexus callback routine with 506 * the port attributes structure as an input argument. 507 * IBT_HCA_DETACH_EVENT: 508 * Retrieves the information about all the ports that are 509 * present on this HCA and calls IB nexus callback with 510 * port guid as an argument 511 * IBT_EVENT_PORT_UP: 512 * Register with IBMF and SA access 513 * Setup IBMF receive callback routine 514 * IBT_EVENT_PORT_DOWN: 515 * Un-Register with IBMF and SA access 516 * Teardown IBMF receive callback routine 517 */ 518 /*ARGSUSED*/ 519 static void 520 ibdm_event_hdlr(void *clnt_hdl, 521 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 522 { 523 ibdm_hca_list_t *hca_list; 524 ibdm_port_attr_t *port; 525 ibmf_saa_handle_t port_sa_hdl; 526 527 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 528 529 switch (code) { 530 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 531 ibdm_handle_hca_attach(event->ev_hca_guid); 532 break; 533 534 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 535 ibdm_handle_hca_detach(event->ev_hca_guid); 536 mutex_enter(&ibdm.ibdm_ibnex_mutex); 537 if (ibdm.ibdm_ibnex_callback != NULL) { 538 (*ibdm.ibdm_ibnex_callback)((void *) 539 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 540 } 541 mutex_exit(&ibdm.ibdm_ibnex_mutex); 542 break; 543 544 case IBT_EVENT_PORT_UP: 545 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 546 mutex_enter(&ibdm.ibdm_hl_mutex); 547 port = ibdm_get_port_attr(event, &hca_list); 548 if (port == NULL) { 549 IBTF_DPRINTF_L2("ibdm", 550 "\tevent_hdlr: HCA not present"); 551 mutex_exit(&ibdm.ibdm_hl_mutex); 552 break; 553 } 554 ibdm_initialize_port(port); 555 hca_list->hl_nports_active++; 556 cv_broadcast(&ibdm.ibdm_port_settle_cv); 557 mutex_exit(&ibdm.ibdm_hl_mutex); 558 559 /* Inform IB nexus driver */ 560 mutex_enter(&ibdm.ibdm_ibnex_mutex); 561 if (ibdm.ibdm_ibnex_callback != NULL) { 562 (*ibdm.ibdm_ibnex_callback)((void *) 563 &event->ev_hca_guid, IBDM_EVENT_PORT_UP); 564 } 565 mutex_exit(&ibdm.ibdm_ibnex_mutex); 566 break; 567 568 case IBT_ERROR_PORT_DOWN: 569 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 570 mutex_enter(&ibdm.ibdm_hl_mutex); 571 port = ibdm_get_port_attr(event, &hca_list); 572 if (port == NULL) { 573 IBTF_DPRINTF_L2("ibdm", 574 "\tevent_hdlr: HCA not present"); 575 mutex_exit(&ibdm.ibdm_hl_mutex); 576 break; 577 } 578 hca_list->hl_nports_active--; 579 port_sa_hdl = port->pa_sa_hdl; 580 (void) ibdm_fini_port(port); 581 port->pa_state = IBT_PORT_DOWN; 582 cv_broadcast(&ibdm.ibdm_port_settle_cv); 583 mutex_exit(&ibdm.ibdm_hl_mutex); 584 ibdm_reset_all_dgids(port_sa_hdl); 585 break; 586 587 case IBT_PORT_CHANGE_EVENT: 588 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE"); 589 if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY) 590 ibdm_handle_port_change_event(event); 591 break; 592 593 default: /* Ignore all other events/errors */ 594 break; 595 } 596 } 597 598 static void 599 ibdm_handle_port_change_event(ibt_async_event_t *event) 600 { 601 ibdm_port_attr_t *port; 602 ibdm_hca_list_t *hca_list; 603 604 IBTF_DPRINTF_L2("ibdm", "\tibdm_handle_port_change_event:" 605 " HCA guid %llx", event->ev_hca_guid); 606 mutex_enter(&ibdm.ibdm_hl_mutex); 607 port = ibdm_get_port_attr(event, &hca_list); 608 if (port == NULL) { 609 IBTF_DPRINTF_L2("ibdm", "\tevent_hdlr: HCA not present"); 610 mutex_exit(&ibdm.ibdm_hl_mutex); 611 return; 612 } 613 ibdm_update_port_pkeys(port); 614 cv_broadcast(&ibdm.ibdm_port_settle_cv); 615 mutex_exit(&ibdm.ibdm_hl_mutex); 616 617 /* Inform IB nexus driver */ 618 mutex_enter(&ibdm.ibdm_ibnex_mutex); 619 if (ibdm.ibdm_ibnex_callback != NULL) { 620 (*ibdm.ibdm_ibnex_callback)((void *) 621 &event->ev_hca_guid, IBDM_EVENT_PORT_PKEY_CHANGE); 622 } 623 mutex_exit(&ibdm.ibdm_ibnex_mutex); 624 } 625 626 /* 627 * ibdm_update_port_pkeys() 628 * Update the pkey table 629 * Update the port attributes 630 */ 631 static void 632 ibdm_update_port_pkeys(ibdm_port_attr_t *port) 633 { 634 uint_t nports, size; 635 uint_t pkey_idx, opkey_idx; 636 uint16_t npkeys; 637 ibt_hca_portinfo_t *pinfop; 638 ib_pkey_t pkey; 639 ibdm_pkey_tbl_t *pkey_tbl; 640 ibdm_port_attr_t newport; 641 642 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:"); 643 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 644 645 /* Check whether the port is active */ 646 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 647 NULL) != IBT_SUCCESS) 648 return; 649 650 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 651 &pinfop, &nports, &size) != IBT_SUCCESS) { 652 /* This should not occur */ 653 port->pa_npkeys = 0; 654 port->pa_pkey_tbl = NULL; 655 return; 656 } 657 658 npkeys = pinfop->p_pkey_tbl_sz; 659 pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 660 newport.pa_pkey_tbl = pkey_tbl; 661 newport.pa_ibmf_hdl = port->pa_ibmf_hdl; 662 663 for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) { 664 pkey = pkey_tbl[pkey_idx].pt_pkey = 665 pinfop->p_pkey_tbl[pkey_idx]; 666 /* 667 * Is this pkey present in the current table ? 668 */ 669 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) { 670 if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) { 671 pkey_tbl[pkey_idx].pt_qp_hdl = 672 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl; 673 port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL; 674 break; 675 } 676 } 677 678 if (opkey_idx == port->pa_npkeys) { 679 pkey = pkey_tbl[pkey_idx].pt_pkey; 680 if (IBDM_INVALID_PKEY(pkey)) { 681 pkey_tbl[pkey_idx].pt_qp_hdl = NULL; 682 continue; 683 } 684 ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx); 685 } 686 } 687 688 for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) { 689 if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) { 690 if (ibdm_port_attr_ibmf_fini(port, opkey_idx) != 691 IBDM_SUCCESS) { 692 IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: " 693 "ibdm_port_attr_ibmf_fini failed for " 694 "port pkey 0x%x", 695 port->pa_pkey_tbl[opkey_idx].pt_pkey); 696 } 697 } 698 } 699 700 if (port->pa_pkey_tbl != NULL) { 701 kmem_free(port->pa_pkey_tbl, 702 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 703 } 704 705 port->pa_npkeys = npkeys; 706 port->pa_pkey_tbl = pkey_tbl; 707 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 708 port->pa_state = pinfop->p_linkstate; 709 ibt_free_portinfo(pinfop, size); 710 } 711 712 /* 713 * ibdm_initialize_port() 714 * Register with IBMF 715 * Register with SA access 716 * Register a receive callback routine with IBMF. IBMF invokes 717 * this routine whenever a MAD arrives at this port. 718 * Update the port attributes 719 */ 720 static void 721 ibdm_initialize_port(ibdm_port_attr_t *port) 722 { 723 int ii; 724 uint_t nports, size; 725 uint_t pkey_idx; 726 ib_pkey_t pkey; 727 ibt_hca_portinfo_t *pinfop; 728 ibmf_register_info_t ibmf_reg; 729 ibmf_saa_subnet_event_args_t event_args; 730 731 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 732 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 733 734 /* Check whether the port is active */ 735 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 736 NULL) != IBT_SUCCESS) 737 return; 738 739 if (port->pa_sa_hdl != NULL) 740 return; 741 742 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 743 &pinfop, &nports, &size) != IBT_SUCCESS) { 744 /* This should not occur */ 745 port->pa_npkeys = 0; 746 port->pa_pkey_tbl = NULL; 747 return; 748 } 749 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 750 751 port->pa_state = pinfop->p_linkstate; 752 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 753 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 754 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 755 756 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 757 port->pa_pkey_tbl[pkey_idx].pt_pkey = 758 pinfop->p_pkey_tbl[pkey_idx]; 759 760 ibt_free_portinfo(pinfop, size); 761 762 event_args.is_event_callback = ibdm_saa_event_cb; 763 event_args.is_event_callback_arg = port; 764 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 765 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 766 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 767 "sa access registration failed"); 768 return; 769 } 770 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 771 ibmf_reg.ir_port_num = port->pa_port_num; 772 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 773 774 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 775 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 776 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 777 "IBMF registration failed"); 778 (void) ibdm_fini_port(port); 779 return; 780 } 781 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, 782 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 783 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 784 "IBMF setup recv cb failed"); 785 (void) ibdm_fini_port(port); 786 return; 787 } 788 789 for (ii = 0; ii < port->pa_npkeys; ii++) { 790 pkey = port->pa_pkey_tbl[ii].pt_pkey; 791 if (IBDM_INVALID_PKEY(pkey)) { 792 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 793 continue; 794 } 795 ibdm_port_attr_ibmf_init(port, pkey, ii); 796 } 797 } 798 799 800 /* 801 * ibdm_port_attr_ibmf_init: 802 * With IBMF - Alloc QP Handle and Setup Async callback 803 */ 804 static void 805 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 806 { 807 int ret; 808 809 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 810 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 811 IBMF_SUCCESS) { 812 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 813 "IBMF failed to alloc qp %d", ret); 814 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 815 return; 816 } 817 818 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 819 port->pa_ibmf_hdl); 820 821 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 822 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 823 IBMF_SUCCESS) { 824 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 825 "IBMF setup recv cb failed %d", ret); 826 (void) ibmf_free_qp(port->pa_ibmf_hdl, 827 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 828 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 829 } 830 } 831 832 833 /* 834 * ibdm_get_port_attr() 835 * Get port attributes from HCA guid and port number 836 * Return pointer to ibdm_port_attr_t on Success 837 * and NULL on failure 838 */ 839 static ibdm_port_attr_t * 840 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 841 { 842 ibdm_hca_list_t *hca_list; 843 ibdm_port_attr_t *port_attr; 844 int ii; 845 846 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 847 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 848 hca_list = ibdm.ibdm_hca_list_head; 849 while (hca_list) { 850 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 851 for (ii = 0; ii < hca_list->hl_nports; ii++) { 852 port_attr = &hca_list->hl_port_attr[ii]; 853 if (port_attr->pa_port_num == event->ev_port) { 854 *retval = hca_list; 855 return (port_attr); 856 } 857 } 858 } 859 hca_list = hca_list->hl_next; 860 } 861 return (NULL); 862 } 863 864 865 /* 866 * ibdm_update_port_attr() 867 * Update the port attributes 868 */ 869 static void 870 ibdm_update_port_attr(ibdm_port_attr_t *port) 871 { 872 uint_t nports, size; 873 uint_t pkey_idx; 874 ibt_hca_portinfo_t *portinfop; 875 876 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 877 if (ibt_query_hca_ports(port->pa_hca_hdl, 878 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 879 /* This should not occur */ 880 port->pa_npkeys = 0; 881 port->pa_pkey_tbl = NULL; 882 return; 883 } 884 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 885 886 port->pa_state = portinfop->p_linkstate; 887 888 /* 889 * PKey information in portinfo valid only if port is 890 * ACTIVE. Bail out if not. 891 */ 892 if (port->pa_state != IBT_PORT_ACTIVE) { 893 port->pa_npkeys = 0; 894 port->pa_pkey_tbl = NULL; 895 ibt_free_portinfo(portinfop, size); 896 return; 897 } 898 899 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 900 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 901 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 902 903 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 904 port->pa_pkey_tbl[pkey_idx].pt_pkey = 905 portinfop->p_pkey_tbl[pkey_idx]; 906 } 907 ibt_free_portinfo(portinfop, size); 908 } 909 910 911 /* 912 * ibdm_handle_hca_attach() 913 */ 914 static void 915 ibdm_handle_hca_attach(ib_guid_t hca_guid) 916 { 917 uint_t size; 918 uint_t ii, nports; 919 ibt_status_t status; 920 ibt_hca_hdl_t hca_hdl; 921 ibt_hca_attr_t *hca_attr; 922 ibdm_hca_list_t *hca_list, *temp; 923 ibdm_port_attr_t *port_attr; 924 ibt_hca_portinfo_t *portinfop; 925 926 IBTF_DPRINTF_L4("ibdm", 927 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 928 929 /* open the HCA first */ 930 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 931 &hca_hdl)) != IBT_SUCCESS) { 932 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 933 "open_hca failed, status 0x%x", status); 934 return; 935 } 936 937 hca_attr = (ibt_hca_attr_t *) 938 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 939 /* ibt_query_hca always returns IBT_SUCCESS */ 940 (void) ibt_query_hca(hca_hdl, hca_attr); 941 942 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 943 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 944 hca_attr->hca_version_id, hca_attr->hca_nports); 945 946 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 947 &size)) != IBT_SUCCESS) { 948 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 949 "ibt_query_hca_ports failed, status 0x%x", status); 950 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 951 (void) ibt_close_hca(hca_hdl); 952 return; 953 } 954 hca_list = (ibdm_hca_list_t *) 955 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 956 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 957 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 958 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 959 hca_list->hl_nports = hca_attr->hca_nports; 960 hca_list->hl_attach_time = ddi_get_time(); 961 hca_list->hl_hca_hdl = hca_hdl; 962 963 /* 964 * Init a dummy port attribute for the HCA node 965 * This is for Per-HCA Node. Initialize port_attr : 966 * hca_guid & port_guid -> hca_guid 967 * npkeys, pkey_tbl is NULL 968 * port_num, sn_prefix is 0 969 * vendorid, product_id, dev_version from HCA 970 * pa_state is IBT_PORT_ACTIVE 971 */ 972 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 973 sizeof (ibdm_port_attr_t), KM_SLEEP); 974 port_attr = hca_list->hl_hca_port_attr; 975 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 976 port_attr->pa_productid = hca_attr->hca_device_id; 977 port_attr->pa_dev_version = hca_attr->hca_version_id; 978 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 979 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 980 port_attr->pa_port_guid = hca_attr->hca_node_guid; 981 port_attr->pa_state = IBT_PORT_ACTIVE; 982 983 984 for (ii = 0; ii < nports; ii++) { 985 port_attr = &hca_list->hl_port_attr[ii]; 986 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 987 port_attr->pa_productid = hca_attr->hca_device_id; 988 port_attr->pa_dev_version = hca_attr->hca_version_id; 989 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 990 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 991 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 992 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 993 port_attr->pa_port_num = portinfop[ii].p_port_num; 994 port_attr->pa_state = portinfop[ii].p_linkstate; 995 996 /* 997 * Register with IBMF, SA access when the port is in 998 * ACTIVE state. Also register a callback routine 999 * with IBMF to receive incoming DM MAD's. 1000 * The IBDM event handler takes care of registration of 1001 * port which are not active. 1002 */ 1003 IBTF_DPRINTF_L4("ibdm", 1004 "\thandle_hca_attach: port guid %llx Port state 0x%x", 1005 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 1006 1007 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 1008 mutex_enter(&ibdm.ibdm_hl_mutex); 1009 hca_list->hl_nports_active++; 1010 ibdm_initialize_port(port_attr); 1011 cv_broadcast(&ibdm.ibdm_port_settle_cv); 1012 mutex_exit(&ibdm.ibdm_hl_mutex); 1013 } 1014 } 1015 mutex_enter(&ibdm.ibdm_hl_mutex); 1016 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 1017 if (temp->hl_hca_guid == hca_guid) { 1018 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 1019 "already seen by IBDM", hca_guid); 1020 mutex_exit(&ibdm.ibdm_hl_mutex); 1021 (void) ibdm_uninit_hca(hca_list); 1022 return; 1023 } 1024 } 1025 ibdm.ibdm_hca_count++; 1026 if (ibdm.ibdm_hca_list_head == NULL) { 1027 ibdm.ibdm_hca_list_head = hca_list; 1028 ibdm.ibdm_hca_list_tail = hca_list; 1029 } else { 1030 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 1031 ibdm.ibdm_hca_list_tail = hca_list; 1032 } 1033 mutex_exit(&ibdm.ibdm_hl_mutex); 1034 mutex_enter(&ibdm.ibdm_ibnex_mutex); 1035 if (ibdm.ibdm_ibnex_callback != NULL) { 1036 (*ibdm.ibdm_ibnex_callback)((void *) 1037 &hca_guid, IBDM_EVENT_HCA_ADDED); 1038 } 1039 mutex_exit(&ibdm.ibdm_ibnex_mutex); 1040 1041 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1042 ibt_free_portinfo(portinfop, size); 1043 } 1044 1045 1046 /* 1047 * ibdm_handle_hca_detach() 1048 */ 1049 static void 1050 ibdm_handle_hca_detach(ib_guid_t hca_guid) 1051 { 1052 ibdm_hca_list_t *head, *prev = NULL; 1053 size_t len; 1054 ibdm_dp_gidinfo_t *gidinfo; 1055 1056 IBTF_DPRINTF_L4("ibdm", 1057 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 1058 1059 /* Make sure no probes are running */ 1060 mutex_enter(&ibdm.ibdm_mutex); 1061 while (ibdm.ibdm_busy & IBDM_BUSY) 1062 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1063 ibdm.ibdm_busy |= IBDM_BUSY; 1064 mutex_exit(&ibdm.ibdm_mutex); 1065 1066 mutex_enter(&ibdm.ibdm_hl_mutex); 1067 head = ibdm.ibdm_hca_list_head; 1068 while (head) { 1069 if (head->hl_hca_guid == hca_guid) { 1070 if (prev == NULL) 1071 ibdm.ibdm_hca_list_head = head->hl_next; 1072 else 1073 prev->hl_next = head->hl_next; 1074 if (ibdm.ibdm_hca_list_tail == head) 1075 ibdm.ibdm_hca_list_tail = prev; 1076 ibdm.ibdm_hca_count--; 1077 break; 1078 } 1079 prev = head; 1080 head = head->hl_next; 1081 } 1082 mutex_exit(&ibdm.ibdm_hl_mutex); 1083 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 1084 (void) ibdm_handle_hca_attach(hca_guid); 1085 1086 /* 1087 * Now clean up the HCA lists in the gidlist. 1088 */ 1089 for (gidinfo = ibdm.ibdm_dp_gidlist_head; gidinfo; gidinfo = 1090 gidinfo->gl_next) { 1091 prev = NULL; 1092 head = gidinfo->gl_hca_list; 1093 while (head) { 1094 if (head->hl_hca_guid == hca_guid) { 1095 if (prev == NULL) 1096 gidinfo->gl_hca_list = 1097 head->hl_next; 1098 else 1099 prev->hl_next = head->hl_next; 1100 1101 len = sizeof (ibdm_hca_list_t) + 1102 (head->hl_nports * 1103 sizeof (ibdm_port_attr_t)); 1104 kmem_free(head, len); 1105 1106 break; 1107 } 1108 prev = head; 1109 head = head->hl_next; 1110 } 1111 } 1112 1113 mutex_enter(&ibdm.ibdm_mutex); 1114 ibdm.ibdm_busy &= ~IBDM_BUSY; 1115 cv_broadcast(&ibdm.ibdm_busy_cv); 1116 mutex_exit(&ibdm.ibdm_mutex); 1117 } 1118 1119 1120 static int 1121 ibdm_uninit_hca(ibdm_hca_list_t *head) 1122 { 1123 int ii; 1124 ibdm_port_attr_t *port_attr; 1125 1126 for (ii = 0; ii < head->hl_nports; ii++) { 1127 port_attr = &head->hl_port_attr[ii]; 1128 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 1129 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 1130 "ibdm_fini_port() failed", head, ii); 1131 return (IBDM_FAILURE); 1132 } 1133 } 1134 if (head->hl_hca_hdl) 1135 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) { 1136 IBTF_DPRINTF_L2("ibdm", "uninit_hca: " 1137 "ibt_close_hca() failed"); 1138 return (IBDM_FAILURE); 1139 } 1140 kmem_free(head->hl_port_attr, 1141 head->hl_nports * sizeof (ibdm_port_attr_t)); 1142 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 1143 kmem_free(head, sizeof (ibdm_hca_list_t)); 1144 return (IBDM_SUCCESS); 1145 } 1146 1147 1148 /* 1149 * For each port on the HCA, 1150 * 1) Teardown IBMF receive callback function 1151 * 2) Unregister with IBMF 1152 * 3) Unregister with SA access 1153 */ 1154 static int 1155 ibdm_fini_port(ibdm_port_attr_t *port_attr) 1156 { 1157 int ii, ibmf_status; 1158 1159 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 1160 if (port_attr->pa_pkey_tbl == NULL) 1161 break; 1162 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 1163 continue; 1164 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 1165 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1166 "ibdm_port_attr_ibmf_fini failed for " 1167 "port pkey 0x%x", ii); 1168 return (IBDM_FAILURE); 1169 } 1170 } 1171 1172 if (port_attr->pa_ibmf_hdl) { 1173 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1174 IBMF_QP_HANDLE_DEFAULT, 0); 1175 if (ibmf_status != IBMF_SUCCESS) { 1176 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1177 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1178 return (IBDM_FAILURE); 1179 } 1180 1181 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 1182 if (ibmf_status != IBMF_SUCCESS) { 1183 IBTF_DPRINTF_L2("ibdm", "\tfini_port: " 1184 "ibmf_unregister failed %d", ibmf_status); 1185 return (IBDM_FAILURE); 1186 } 1187 1188 port_attr->pa_ibmf_hdl = NULL; 1189 } 1190 1191 if (port_attr->pa_sa_hdl) { 1192 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1193 if (ibmf_status != IBMF_SUCCESS) { 1194 IBTF_DPRINTF_L2("ibdm", "\tfini_port: " 1195 "ibmf_sa_session_close failed %d", ibmf_status); 1196 return (IBDM_FAILURE); 1197 } 1198 port_attr->pa_sa_hdl = NULL; 1199 } 1200 1201 if (port_attr->pa_pkey_tbl != NULL) { 1202 kmem_free(port_attr->pa_pkey_tbl, 1203 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1204 port_attr->pa_pkey_tbl = NULL; 1205 port_attr->pa_npkeys = 0; 1206 } 1207 1208 return (IBDM_SUCCESS); 1209 } 1210 1211 1212 /* 1213 * ibdm_port_attr_ibmf_fini: 1214 * With IBMF - Tear down Async callback and free QP Handle 1215 */ 1216 static int 1217 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1218 { 1219 int ibmf_status; 1220 1221 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1222 1223 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1224 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1225 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1226 if (ibmf_status != IBMF_SUCCESS) { 1227 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1228 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1229 return (IBDM_FAILURE); 1230 } 1231 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1232 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1233 if (ibmf_status != IBMF_SUCCESS) { 1234 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1235 "ibmf_free_qp failed %d", ibmf_status); 1236 return (IBDM_FAILURE); 1237 } 1238 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1239 } 1240 return (IBDM_SUCCESS); 1241 } 1242 1243 1244 /* 1245 * ibdm_gid_decr_pending: 1246 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1247 */ 1248 static void 1249 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1250 { 1251 mutex_enter(&ibdm.ibdm_mutex); 1252 mutex_enter(&gidinfo->gl_mutex); 1253 if (--gidinfo->gl_pending_cmds == 0) { 1254 /* 1255 * Handle DGID getting removed. 1256 */ 1257 if (gidinfo->gl_disconnected) { 1258 mutex_exit(&gidinfo->gl_mutex); 1259 mutex_exit(&ibdm.ibdm_mutex); 1260 1261 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1262 "gidinfo %p hot removal", gidinfo); 1263 ibdm_delete_gidinfo(gidinfo); 1264 1265 mutex_enter(&ibdm.ibdm_mutex); 1266 ibdm.ibdm_ngid_probes_in_progress--; 1267 ibdm_wait_probe_completion(); 1268 mutex_exit(&ibdm.ibdm_mutex); 1269 return; 1270 } 1271 mutex_exit(&gidinfo->gl_mutex); 1272 mutex_exit(&ibdm.ibdm_mutex); 1273 ibdm_notify_newgid_iocs(gidinfo); 1274 mutex_enter(&ibdm.ibdm_mutex); 1275 mutex_enter(&gidinfo->gl_mutex); 1276 1277 ibdm.ibdm_ngid_probes_in_progress--; 1278 ibdm_wait_probe_completion(); 1279 } 1280 mutex_exit(&gidinfo->gl_mutex); 1281 mutex_exit(&ibdm.ibdm_mutex); 1282 } 1283 1284 1285 /* 1286 * ibdm_wait_probe_completion: 1287 * wait for probing to complete 1288 */ 1289 static void 1290 ibdm_wait_probe_completion(void) 1291 { 1292 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1293 if (ibdm.ibdm_ngid_probes_in_progress) { 1294 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1295 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1296 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1297 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1298 } 1299 } 1300 1301 1302 /* 1303 * ibdm_wait_cisco_probe_completion: 1304 * wait for the reply from the Cisco FC GW switch after a setclassportinfo 1305 * request is sent. This wait can be achieved on each gid. 1306 */ 1307 static void 1308 ibdm_wait_cisco_probe_completion(ibdm_dp_gidinfo_t *gidinfo) 1309 { 1310 ASSERT(MUTEX_HELD(&gidinfo->gl_mutex)); 1311 IBTF_DPRINTF_L4("ibdm", "\twait for cisco probe complete"); 1312 gidinfo->gl_flag |= IBDM_CISCO_PROBE; 1313 while (gidinfo->gl_flag & IBDM_CISCO_PROBE) 1314 cv_wait(&gidinfo->gl_probe_cv, &gidinfo->gl_mutex); 1315 } 1316 1317 1318 /* 1319 * ibdm_wakeup_probe_gid_cv: 1320 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1321 */ 1322 static void 1323 ibdm_wakeup_probe_gid_cv(void) 1324 { 1325 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1326 if (!ibdm.ibdm_ngid_probes_in_progress) { 1327 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1328 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1329 cv_broadcast(&ibdm.ibdm_probe_cv); 1330 } 1331 1332 } 1333 1334 1335 /* 1336 * ibdm_sweep_fabric(reprobe_flag) 1337 * Find all possible Managed IOU's and their IOC's that are visible 1338 * to the host. The algorithm used is as follows 1339 * 1340 * Send a "bus walk" request for each port on the host HCA to SA access 1341 * SA returns complete set of GID's that are reachable from 1342 * source port. This is done in parallel. 1343 * 1344 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1345 * 1346 * Sort the GID list and eliminate duplicate GID's 1347 * 1) Use DGID for sorting 1348 * 2) use PortGuid for sorting 1349 * Send SA query to retrieve NodeRecord and 1350 * extract PortGuid from that. 1351 * 1352 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1353 * support DM MAD's 1354 * Send a "Portinfo" query to get the port capabilities and 1355 * then check for DM MAD's support 1356 * 1357 * Send "ClassPortInfo" request for all the GID's in parallel, 1358 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1359 * cv_signal to complete. 1360 * 1361 * When DM agent on the remote GID sends back the response, IBMF 1362 * invokes DM callback routine. 1363 * 1364 * If the response is proper, send "IOUnitInfo" request and set 1365 * GID state to IBDM_GET_IOUNITINFO. 1366 * 1367 * If the response is proper, send "IocProfileInfo" request to 1368 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1369 * 1370 * Send request to get Service entries simultaneously 1371 * 1372 * Signal the waiting thread when received response for all the commands. 1373 * 1374 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1375 * response during the probing period. 1376 * 1377 * Note: 1378 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1379 * keep track of number commands in progress at any point of time. 1380 * MAD transaction ID is used to identify a particular GID 1381 * TBD: Consider registering the IBMF receive callback on demand 1382 * 1383 * Note: This routine must be called with ibdm.ibdm_mutex held 1384 * TBD: Re probe the failure GID (for certain failures) when requested 1385 * for fabric sweep next time 1386 * 1387 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1388 */ 1389 static void 1390 ibdm_sweep_fabric(int reprobe_flag) 1391 { 1392 int ii; 1393 int new_paths = 0; 1394 uint8_t niocs; 1395 taskqid_t tid; 1396 ibdm_ioc_info_t *ioc; 1397 ibdm_hca_list_t *hca_list = NULL; 1398 ibdm_port_attr_t *port = NULL; 1399 ibdm_dp_gidinfo_t *gid_info; 1400 1401 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1402 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1403 1404 /* 1405 * Check whether a sweep already in progress. If so, just 1406 * wait for the fabric sweep to complete 1407 */ 1408 while (ibdm.ibdm_busy & IBDM_BUSY) 1409 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1410 ibdm.ibdm_busy |= IBDM_BUSY; 1411 mutex_exit(&ibdm.ibdm_mutex); 1412 1413 ibdm_dump_sweep_fabric_timestamp(0); 1414 1415 /* Rescan the GID list for any removed GIDs for reprobe */ 1416 if (reprobe_flag) 1417 ibdm_rescan_gidlist(NULL); 1418 1419 /* 1420 * Get list of all the ports reachable from the local known HCA 1421 * ports which are active 1422 */ 1423 mutex_enter(&ibdm.ibdm_hl_mutex); 1424 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1425 ibdm_get_next_port(&hca_list, &port, 1)) { 1426 /* 1427 * Get PATHS to all the reachable ports from 1428 * SGID and update the global ibdm structure. 1429 */ 1430 new_paths = ibdm_get_reachable_ports(port, hca_list); 1431 ibdm.ibdm_ngids += new_paths; 1432 } 1433 mutex_exit(&ibdm.ibdm_hl_mutex); 1434 1435 mutex_enter(&ibdm.ibdm_mutex); 1436 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1437 mutex_exit(&ibdm.ibdm_mutex); 1438 1439 /* Send a request to probe GIDs asynchronously. */ 1440 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1441 gid_info = gid_info->gl_next) { 1442 mutex_enter(&gid_info->gl_mutex); 1443 gid_info->gl_reprobe_flag = reprobe_flag; 1444 mutex_exit(&gid_info->gl_mutex); 1445 1446 /* process newly encountered GIDs */ 1447 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1448 (void *)gid_info, TQ_NOSLEEP); 1449 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1450 " taskq_id = %x", gid_info, tid); 1451 /* taskq failed to dispatch call it directly */ 1452 if (tid == NULL) 1453 ibdm_probe_gid_thread((void *)gid_info); 1454 } 1455 1456 mutex_enter(&ibdm.ibdm_mutex); 1457 ibdm_wait_probe_completion(); 1458 1459 /* 1460 * Update the properties, if reprobe_flag is set 1461 * Skip if gl_reprobe_flag is set, this will be 1462 * a re-inserted / new GID, for which notifications 1463 * have already been send. 1464 */ 1465 if (reprobe_flag) { 1466 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1467 gid_info = gid_info->gl_next) { 1468 if (gid_info->gl_iou == NULL) 1469 continue; 1470 if (gid_info->gl_reprobe_flag) { 1471 gid_info->gl_reprobe_flag = 0; 1472 continue; 1473 } 1474 1475 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1476 for (ii = 0; ii < niocs; ii++) { 1477 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1478 if (ioc) 1479 ibdm_reprobe_update_port_srv(ioc, 1480 gid_info); 1481 } 1482 } 1483 } else if (ibdm.ibdm_prev_iou) { 1484 ibdm_ioc_info_t *ioc_list; 1485 1486 /* 1487 * Get the list of IOCs which have changed. 1488 * If any IOCs have changed, Notify IBNexus 1489 */ 1490 ibdm.ibdm_prev_iou = 0; 1491 ioc_list = ibdm_handle_prev_iou(); 1492 if (ioc_list) { 1493 if (ibdm.ibdm_ibnex_callback != NULL) { 1494 (*ibdm.ibdm_ibnex_callback)( 1495 (void *)ioc_list, 1496 IBDM_EVENT_IOC_PROP_UPDATE); 1497 } 1498 } 1499 } 1500 1501 ibdm_dump_sweep_fabric_timestamp(1); 1502 1503 ibdm.ibdm_busy &= ~IBDM_BUSY; 1504 cv_broadcast(&ibdm.ibdm_busy_cv); 1505 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1506 } 1507 1508 1509 /* 1510 * ibdm_is_cisco: 1511 * Check if this is a Cisco device or not. 1512 */ 1513 static boolean_t 1514 ibdm_is_cisco(ib_guid_t guid) 1515 { 1516 if ((guid >> IBDM_OUI_GUID_SHIFT) == IBDM_CISCO_COMPANY_ID) 1517 return (B_TRUE); 1518 return (B_FALSE); 1519 } 1520 1521 1522 /* 1523 * ibdm_is_cisco_switch: 1524 * Check if this switch is a CISCO switch or not. 1525 * Note that if this switch is already activated, ibdm_is_cisco_switch() 1526 * returns B_FALSE not to re-activate it again. 1527 */ 1528 static boolean_t 1529 ibdm_is_cisco_switch(ibdm_dp_gidinfo_t *gid_info) 1530 { 1531 int company_id, device_id; 1532 ASSERT(gid_info != 0); 1533 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 1534 1535 /* 1536 * If this switch is already activated, don't re-activate it. 1537 */ 1538 if (gid_info->gl_flag & IBDM_CISCO_PROBE_DONE) 1539 return (B_FALSE); 1540 1541 /* 1542 * Check if this switch is a Cisco FC GW or not. 1543 * Use the node guid (the OUI part) instead of the vendor id 1544 * since the vendor id is zero in practice. 1545 */ 1546 company_id = gid_info->gl_nodeguid >> IBDM_OUI_GUID_SHIFT; 1547 device_id = gid_info->gl_devid; 1548 1549 if (company_id == IBDM_CISCO_COMPANY_ID && 1550 device_id == IBDM_CISCO_DEVICE_ID) 1551 return (B_TRUE); 1552 return (B_FALSE); 1553 } 1554 1555 1556 /* 1557 * ibdm_probe_gid_thread: 1558 * thread that does the actual work for sweeping the fabric 1559 * for a given GID 1560 */ 1561 static void 1562 ibdm_probe_gid_thread(void *args) 1563 { 1564 int reprobe_flag; 1565 ib_guid_t node_guid; 1566 ib_guid_t port_guid; 1567 ibdm_dp_gidinfo_t *gid_info; 1568 1569 gid_info = (ibdm_dp_gidinfo_t *)args; 1570 reprobe_flag = gid_info->gl_reprobe_flag; 1571 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1572 gid_info, reprobe_flag); 1573 ASSERT(gid_info != NULL); 1574 ASSERT(gid_info->gl_pending_cmds == 0); 1575 1576 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1577 reprobe_flag == 0) { 1578 /* 1579 * This GID may have been already probed. Send 1580 * in a CLP to check if IOUnitInfo changed? 1581 * Explicitly set gl_reprobe_flag to 0 so that 1582 * IBnex is not notified on completion 1583 */ 1584 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1585 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1586 "get new IOCs information"); 1587 mutex_enter(&gid_info->gl_mutex); 1588 gid_info->gl_pending_cmds++; 1589 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1590 gid_info->gl_reprobe_flag = 0; 1591 mutex_exit(&gid_info->gl_mutex); 1592 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1593 mutex_enter(&gid_info->gl_mutex); 1594 --gid_info->gl_pending_cmds; 1595 mutex_exit(&gid_info->gl_mutex); 1596 mutex_enter(&ibdm.ibdm_mutex); 1597 --ibdm.ibdm_ngid_probes_in_progress; 1598 ibdm_wakeup_probe_gid_cv(); 1599 mutex_exit(&ibdm.ibdm_mutex); 1600 } 1601 } else { 1602 mutex_enter(&ibdm.ibdm_mutex); 1603 --ibdm.ibdm_ngid_probes_in_progress; 1604 ibdm_wakeup_probe_gid_cv(); 1605 mutex_exit(&ibdm.ibdm_mutex); 1606 } 1607 return; 1608 } else if (reprobe_flag && gid_info->gl_state == 1609 IBDM_GID_PROBING_COMPLETE) { 1610 /* 1611 * Reprobe all IOCs for the GID which has completed 1612 * probe. Skip other port GIDs to same IOU. 1613 * Explicitly set gl_reprobe_flag to 0 so that 1614 * IBnex is not notified on completion 1615 */ 1616 ibdm_ioc_info_t *ioc_info; 1617 uint8_t niocs, ii; 1618 1619 ASSERT(gid_info->gl_iou); 1620 mutex_enter(&gid_info->gl_mutex); 1621 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1622 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1623 gid_info->gl_pending_cmds += niocs; 1624 gid_info->gl_reprobe_flag = 0; 1625 mutex_exit(&gid_info->gl_mutex); 1626 for (ii = 0; ii < niocs; ii++) { 1627 uchar_t slot_info; 1628 ib_dm_io_unitinfo_t *giou_info; 1629 1630 /* 1631 * Check whether IOC is present in the slot 1632 * Series of nibbles (in the field 1633 * iou_ctrl_list) represents a slot in the 1634 * IOU. 1635 * Byte format: 76543210 1636 * Bits 0-3 of first byte represent Slot 2 1637 * bits 4-7 of first byte represent slot 1, 1638 * bits 0-3 of second byte represent slot 4 1639 * and so on 1640 * Each 4-bit nibble has the following meaning 1641 * 0x0 : IOC not installed 1642 * 0x1 : IOC is present 1643 * 0xf : Slot does not exist 1644 * and all other values are reserved. 1645 */ 1646 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1647 giou_info = &gid_info->gl_iou->iou_info; 1648 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1649 if ((ii % 2) == 0) 1650 slot_info = (slot_info >> 4); 1651 1652 if ((slot_info & 0xf) != 1) { 1653 ioc_info->ioc_state = 1654 IBDM_IOC_STATE_PROBE_FAILED; 1655 ibdm_gid_decr_pending(gid_info); 1656 continue; 1657 } 1658 1659 if (ibdm_send_ioc_profile(gid_info, ii) != 1660 IBDM_SUCCESS) { 1661 ibdm_gid_decr_pending(gid_info); 1662 } 1663 } 1664 1665 return; 1666 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1667 mutex_enter(&ibdm.ibdm_mutex); 1668 --ibdm.ibdm_ngid_probes_in_progress; 1669 ibdm_wakeup_probe_gid_cv(); 1670 mutex_exit(&ibdm.ibdm_mutex); 1671 return; 1672 } 1673 1674 /* 1675 * Check whether the destination GID supports DM agents. If 1676 * not, stop probing the GID and continue with the next GID 1677 * in the list. 1678 */ 1679 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1680 mutex_enter(&gid_info->gl_mutex); 1681 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1682 gid_info->gl_is_dm_capable = B_FALSE; 1683 mutex_exit(&gid_info->gl_mutex); 1684 ibdm_delete_glhca_list(gid_info); 1685 mutex_enter(&ibdm.ibdm_mutex); 1686 --ibdm.ibdm_ngid_probes_in_progress; 1687 ibdm_wakeup_probe_gid_cv(); 1688 mutex_exit(&ibdm.ibdm_mutex); 1689 return; 1690 } 1691 1692 /* 1693 * This GID is Device management capable 1694 */ 1695 mutex_enter(&gid_info->gl_mutex); 1696 gid_info->gl_is_dm_capable = B_TRUE; 1697 mutex_exit(&gid_info->gl_mutex); 1698 1699 /* Get the nodeguid and portguid of the port */ 1700 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1701 &node_guid, &port_guid) != IBDM_SUCCESS) { 1702 mutex_enter(&gid_info->gl_mutex); 1703 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1704 mutex_exit(&gid_info->gl_mutex); 1705 ibdm_delete_glhca_list(gid_info); 1706 mutex_enter(&ibdm.ibdm_mutex); 1707 --ibdm.ibdm_ngid_probes_in_progress; 1708 ibdm_wakeup_probe_gid_cv(); 1709 mutex_exit(&ibdm.ibdm_mutex); 1710 return; 1711 } 1712 1713 /* 1714 * Check whether we already knew about this NodeGuid 1715 * If so, do not probe the GID and continue with the 1716 * next GID in the gid list. Set the GID state to 1717 * probing done. 1718 */ 1719 mutex_enter(&ibdm.ibdm_mutex); 1720 gid_info->gl_nodeguid = node_guid; 1721 gid_info->gl_portguid = port_guid; 1722 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1723 mutex_exit(&ibdm.ibdm_mutex); 1724 mutex_enter(&gid_info->gl_mutex); 1725 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1726 mutex_exit(&gid_info->gl_mutex); 1727 ibdm_delete_glhca_list(gid_info); 1728 mutex_enter(&ibdm.ibdm_mutex); 1729 --ibdm.ibdm_ngid_probes_in_progress; 1730 ibdm_wakeup_probe_gid_cv(); 1731 mutex_exit(&ibdm.ibdm_mutex); 1732 return; 1733 } 1734 ibdm_add_to_gl_gid(gid_info, gid_info); 1735 mutex_exit(&ibdm.ibdm_mutex); 1736 1737 /* 1738 * New or reinserted GID : Enable notification to IBnex 1739 */ 1740 mutex_enter(&gid_info->gl_mutex); 1741 gid_info->gl_reprobe_flag = 1; 1742 1743 /* 1744 * A Cisco FC GW needs the special handling to get IOUnitInfo. 1745 */ 1746 if (ibdm_is_cisco_switch(gid_info)) { 1747 gid_info->gl_pending_cmds++; 1748 gid_info->gl_state = IBDM_SET_CLASSPORTINFO; 1749 mutex_exit(&gid_info->gl_mutex); 1750 1751 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) { 1752 mutex_enter(&gid_info->gl_mutex); 1753 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1754 --gid_info->gl_pending_cmds; 1755 mutex_exit(&gid_info->gl_mutex); 1756 1757 /* free the hca_list on this gid_info */ 1758 ibdm_delete_glhca_list(gid_info); 1759 1760 mutex_enter(&ibdm.ibdm_mutex); 1761 --ibdm.ibdm_ngid_probes_in_progress; 1762 ibdm_wakeup_probe_gid_cv(); 1763 mutex_exit(&ibdm.ibdm_mutex); 1764 1765 return; 1766 } 1767 1768 mutex_enter(&gid_info->gl_mutex); 1769 ibdm_wait_cisco_probe_completion(gid_info); 1770 1771 IBTF_DPRINTF_L4("ibdm", "\tibdm_probe_gid_thread: " 1772 "CISCO Wakeup signal received"); 1773 } 1774 1775 /* move on to the 'GET_CLASSPORTINFO' stage */ 1776 gid_info->gl_pending_cmds++; 1777 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1778 mutex_exit(&gid_info->gl_mutex); 1779 1780 IBTF_DPRINTF_L3(ibdm_string, "\tibdm_probe_gid_thread: " 1781 "%d: gid_info %p gl_state %d pending_cmds %d", 1782 __LINE__, gid_info, gid_info->gl_state, 1783 gid_info->gl_pending_cmds); 1784 1785 /* 1786 * Send ClassPortInfo request to the GID asynchronously. 1787 */ 1788 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1789 1790 mutex_enter(&gid_info->gl_mutex); 1791 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1792 --gid_info->gl_pending_cmds; 1793 mutex_exit(&gid_info->gl_mutex); 1794 1795 /* free the hca_list on this gid_info */ 1796 ibdm_delete_glhca_list(gid_info); 1797 1798 mutex_enter(&ibdm.ibdm_mutex); 1799 --ibdm.ibdm_ngid_probes_in_progress; 1800 ibdm_wakeup_probe_gid_cv(); 1801 mutex_exit(&ibdm.ibdm_mutex); 1802 1803 return; 1804 } 1805 } 1806 1807 1808 /* 1809 * ibdm_check_dest_nodeguid 1810 * Searches for the NodeGuid in the GID list 1811 * Returns matching gid_info if found and otherwise NULL 1812 * 1813 * This function is called to handle new GIDs discovered 1814 * during device sweep / probe or for GID_AVAILABLE event. 1815 * 1816 * Parameter : 1817 * gid_info GID to check 1818 */ 1819 static ibdm_dp_gidinfo_t * 1820 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1821 { 1822 ibdm_dp_gidinfo_t *gid_list; 1823 ibdm_gid_t *tmp; 1824 1825 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1826 1827 gid_list = ibdm.ibdm_dp_gidlist_head; 1828 while (gid_list) { 1829 if ((gid_list != gid_info) && 1830 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1831 IBTF_DPRINTF_L4("ibdm", 1832 "\tcheck_dest_nodeguid: NodeGuid is present"); 1833 1834 /* Add to gid_list */ 1835 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1836 KM_SLEEP); 1837 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1838 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1839 tmp->gid_next = gid_list->gl_gid; 1840 gid_list->gl_gid = tmp; 1841 gid_list->gl_ngids++; 1842 return (gid_list); 1843 } 1844 1845 gid_list = gid_list->gl_next; 1846 } 1847 1848 return (NULL); 1849 } 1850 1851 1852 /* 1853 * ibdm_is_dev_mgt_supported 1854 * Get the PortInfo attribute (SA Query) 1855 * Check "CompatabilityMask" field in the Portinfo. 1856 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1857 * by the port, otherwise IBDM_FAILURE 1858 */ 1859 static int 1860 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1861 { 1862 int ret; 1863 size_t length = 0; 1864 sa_portinfo_record_t req, *resp = NULL; 1865 ibmf_saa_access_args_t qargs; 1866 1867 bzero(&req, sizeof (sa_portinfo_record_t)); 1868 req.EndportLID = gid_info->gl_dlid; 1869 1870 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1871 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1872 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1873 qargs.sq_template = &req; 1874 qargs.sq_callback = NULL; 1875 qargs.sq_callback_arg = NULL; 1876 1877 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1878 &qargs, 0, &length, (void **)&resp); 1879 1880 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1881 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1882 "failed to get PORTINFO attribute %d", ret); 1883 return (IBDM_FAILURE); 1884 } 1885 1886 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1887 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1888 ret = IBDM_SUCCESS; 1889 } else { 1890 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1891 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1892 ret = IBDM_FAILURE; 1893 } 1894 kmem_free(resp, length); 1895 return (ret); 1896 } 1897 1898 1899 /* 1900 * ibdm_get_node_port_guids() 1901 * Get the NodeInfoRecord of the port 1902 * Save NodeGuid and PortGUID values in the GID list structure. 1903 * Return IBDM_SUCCESS/IBDM_FAILURE 1904 */ 1905 static int 1906 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1907 ib_guid_t *node_guid, ib_guid_t *port_guid) 1908 { 1909 int ret; 1910 size_t length = 0; 1911 sa_node_record_t req, *resp = NULL; 1912 ibmf_saa_access_args_t qargs; 1913 1914 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1915 1916 bzero(&req, sizeof (sa_node_record_t)); 1917 req.LID = dlid; 1918 1919 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1920 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1921 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1922 qargs.sq_template = &req; 1923 qargs.sq_callback = NULL; 1924 qargs.sq_callback_arg = NULL; 1925 1926 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1927 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1928 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1929 " SA Retrieve Failed: %d", ret); 1930 return (IBDM_FAILURE); 1931 } 1932 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1933 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1934 1935 *node_guid = resp->NodeInfo.NodeGUID; 1936 *port_guid = resp->NodeInfo.PortGUID; 1937 kmem_free(resp, length); 1938 return (IBDM_SUCCESS); 1939 } 1940 1941 1942 /* 1943 * ibdm_get_reachable_ports() 1944 * Get list of the destination GID (and its path records) by 1945 * querying the SA access. 1946 * 1947 * Returns Number paths 1948 */ 1949 static int 1950 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1951 { 1952 uint_t ii, jj, nrecs; 1953 uint_t npaths = 0; 1954 size_t length; 1955 ib_gid_t sgid; 1956 ibdm_pkey_tbl_t *pkey_tbl; 1957 sa_path_record_t *result; 1958 sa_path_record_t *precp; 1959 ibdm_dp_gidinfo_t *gid_info; 1960 1961 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1962 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1963 1964 sgid.gid_prefix = portinfo->pa_sn_prefix; 1965 sgid.gid_guid = portinfo->pa_port_guid; 1966 1967 /* get reversible paths */ 1968 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 1969 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 1970 != IBMF_SUCCESS) { 1971 IBTF_DPRINTF_L2("ibdm", 1972 "\tget_reachable_ports: Getting path records failed"); 1973 return (0); 1974 } 1975 1976 for (ii = 0; ii < nrecs; ii++) { 1977 sa_node_record_t *nrec; 1978 size_t length; 1979 1980 precp = &result[ii]; 1981 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 1982 precp->DGID.gid_prefix)) != NULL) { 1983 IBTF_DPRINTF_L5("ibdm", "\tget_reachable_ports: " 1984 "Already exists nrecs %d, ii %d", nrecs, ii); 1985 ibdm_addto_glhcalist(gid_info, hca); 1986 continue; 1987 } 1988 /* 1989 * This is a new GID. Allocate a GID structure and 1990 * initialize the structure 1991 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 1992 * by kmem_zalloc call 1993 */ 1994 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 1995 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 1996 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL); 1997 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 1998 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 1999 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 2000 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 2001 gid_info->gl_p_key = precp->P_Key; 2002 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 2003 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 2004 gid_info->gl_slid = precp->SLID; 2005 gid_info->gl_dlid = precp->DLID; 2006 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 2007 << IBDM_GID_TRANSACTIONID_SHIFT; 2008 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 2009 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 2010 << IBDM_GID_TRANSACTIONID_SHIFT; 2011 gid_info->gl_SL = precp->SL; 2012 2013 /* 2014 * get the node record with this guid if the destination 2015 * device is a Cisco one. 2016 */ 2017 if (ibdm_is_cisco(precp->DGID.gid_guid) && 2018 (gid_info->gl_nodeguid == 0 || gid_info->gl_devid == 0) && 2019 ibdm_get_node_record_by_port(portinfo->pa_sa_hdl, 2020 precp->DGID.gid_guid, &nrec, &length) == IBDM_SUCCESS) { 2021 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID; 2022 gid_info->gl_devid = nrec->NodeInfo.DeviceID; 2023 kmem_free(nrec, length); 2024 } 2025 2026 ibdm_addto_glhcalist(gid_info, hca); 2027 2028 ibdm_dump_path_info(precp); 2029 2030 gid_info->gl_qp_hdl = NULL; 2031 ASSERT(portinfo->pa_pkey_tbl != NULL && 2032 portinfo->pa_npkeys != 0); 2033 2034 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 2035 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 2036 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 2037 (pkey_tbl->pt_qp_hdl != NULL)) { 2038 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 2039 break; 2040 } 2041 } 2042 2043 /* 2044 * QP handle for GID not initialized. No matching Pkey 2045 * was found!! ibdm should *not* hit this case. Flag an 2046 * error and drop the GID if ibdm does encounter this. 2047 */ 2048 if (gid_info->gl_qp_hdl == NULL) { 2049 IBTF_DPRINTF_L2(ibdm_string, 2050 "\tget_reachable_ports: No matching Pkey"); 2051 ibdm_delete_gidinfo(gid_info); 2052 continue; 2053 } 2054 if (ibdm.ibdm_dp_gidlist_head == NULL) { 2055 ibdm.ibdm_dp_gidlist_head = gid_info; 2056 ibdm.ibdm_dp_gidlist_tail = gid_info; 2057 } else { 2058 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 2059 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 2060 ibdm.ibdm_dp_gidlist_tail = gid_info; 2061 } 2062 npaths++; 2063 } 2064 kmem_free(result, length); 2065 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 2066 return (npaths); 2067 } 2068 2069 2070 /* 2071 * ibdm_check_dgid() 2072 * Look in the global list to check whether we know this DGID already 2073 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 2074 */ 2075 static ibdm_dp_gidinfo_t * 2076 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 2077 { 2078 ibdm_dp_gidinfo_t *gid_list; 2079 2080 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 2081 gid_list = gid_list->gl_next) { 2082 if ((guid == gid_list->gl_dgid_lo) && 2083 (prefix == gid_list->gl_dgid_hi)) { 2084 break; 2085 } 2086 } 2087 return (gid_list); 2088 } 2089 2090 2091 /* 2092 * ibdm_find_gid() 2093 * Look in the global list to find a GID entry with matching 2094 * port & node GUID. 2095 * Return pointer to gidinfo if found, else return NULL 2096 */ 2097 static ibdm_dp_gidinfo_t * 2098 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 2099 { 2100 ibdm_dp_gidinfo_t *gid_list; 2101 2102 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 2103 nodeguid, portguid); 2104 2105 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 2106 gid_list = gid_list->gl_next) { 2107 if ((portguid == gid_list->gl_portguid) && 2108 (nodeguid == gid_list->gl_nodeguid)) { 2109 break; 2110 } 2111 } 2112 2113 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 2114 gid_list); 2115 return (gid_list); 2116 } 2117 2118 2119 /* 2120 * ibdm_set_classportinfo() 2121 * ibdm_set_classportinfo() is a function to activate a Cisco FC GW 2122 * by sending the setClassPortInfo request with the trapLID, trapGID 2123 * and etc. to the gateway since the gateway doesn't provide the IO 2124 * Unit Information othewise. This behavior is the Cisco specific one, 2125 * and this function is called to a Cisco FC GW only. 2126 * Returns IBDM_SUCCESS/IBDM_FAILURE 2127 */ 2128 static int 2129 ibdm_set_classportinfo(ibdm_dp_gidinfo_t *gid_info) 2130 { 2131 ibmf_msg_t *msg; 2132 ib_mad_hdr_t *hdr; 2133 ibdm_timeout_cb_args_t *cb_args; 2134 void *data; 2135 ib_mad_classportinfo_t *cpi; 2136 2137 IBTF_DPRINTF_L4("ibdm", 2138 "\tset_classportinfo: gid info 0x%p", gid_info); 2139 2140 /* 2141 * Send command to set classportinfo attribute. Allocate a IBMF 2142 * packet and initialize the packet. 2143 */ 2144 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2145 &msg) != IBMF_SUCCESS) { 2146 IBTF_DPRINTF_L4("ibdm", "\tset_classportinfo: pkt alloc fail"); 2147 return (IBDM_FAILURE); 2148 } 2149 2150 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2151 ibdm_alloc_send_buffers(msg); 2152 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2153 2154 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2155 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2156 msg->im_local_addr.ia_remote_qno = 1; 2157 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2158 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2159 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2160 2161 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2162 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2163 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2164 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2165 hdr->R_Method = IB_DM_DEVMGT_METHOD_SET; 2166 hdr->Status = 0; 2167 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2168 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 2169 hdr->AttributeModifier = 0; 2170 2171 data = msg->im_msgbufs_send.im_bufs_cl_data; 2172 cpi = (ib_mad_classportinfo_t *)data; 2173 2174 /* 2175 * Set the classportinfo values to activate this Cisco FC GW. 2176 */ 2177 cpi->TrapGID_hi = h2b64(gid_info->gl_sgid_hi); 2178 cpi->TrapGID_lo = h2b64(gid_info->gl_sgid_lo); 2179 cpi->TrapLID = h2b16(gid_info->gl_slid); 2180 cpi->TrapSL = gid_info->gl_SL; 2181 cpi->TrapP_Key = h2b16(gid_info->gl_p_key); 2182 cpi->TrapQP = h2b32((((ibmf_alt_qp_t *)gid_info->gl_qp_hdl)->isq_qpn)); 2183 cpi->TrapQ_Key = h2b32((((ibmf_alt_qp_t *) 2184 gid_info->gl_qp_hdl)->isq_qkey)); 2185 2186 cb_args = &gid_info->gl_cpi_cb_args; 2187 cb_args->cb_gid_info = gid_info; 2188 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2189 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 2190 2191 mutex_enter(&gid_info->gl_mutex); 2192 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2193 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2194 mutex_exit(&gid_info->gl_mutex); 2195 2196 IBTF_DPRINTF_L5("ibdm", "\tset_classportinfo: " 2197 "timeout id %x", gid_info->gl_timeout_id); 2198 2199 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2200 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2201 IBTF_DPRINTF_L2("ibdm", 2202 "\tset_classportinfo: ibmf send failed"); 2203 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2204 } 2205 2206 return (IBDM_SUCCESS); 2207 } 2208 2209 2210 /* 2211 * ibdm_send_classportinfo() 2212 * Send classportinfo request. When the request is completed 2213 * IBMF calls ibdm_classportinfo_cb routine to inform about 2214 * the completion. 2215 * Returns IBDM_SUCCESS/IBDM_FAILURE 2216 */ 2217 static int 2218 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 2219 { 2220 ibmf_msg_t *msg; 2221 ib_mad_hdr_t *hdr; 2222 ibdm_timeout_cb_args_t *cb_args; 2223 2224 IBTF_DPRINTF_L4("ibdm", 2225 "\tsend_classportinfo: gid info 0x%p", gid_info); 2226 2227 /* 2228 * Send command to get classportinfo attribute. Allocate a IBMF 2229 * packet and initialize the packet. 2230 */ 2231 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2232 &msg) != IBMF_SUCCESS) { 2233 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 2234 return (IBDM_FAILURE); 2235 } 2236 2237 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2238 ibdm_alloc_send_buffers(msg); 2239 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2240 2241 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2242 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2243 msg->im_local_addr.ia_remote_qno = 1; 2244 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2245 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2246 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2247 2248 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2249 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2250 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2251 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2252 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2253 hdr->Status = 0; 2254 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2255 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 2256 hdr->AttributeModifier = 0; 2257 2258 cb_args = &gid_info->gl_cpi_cb_args; 2259 cb_args->cb_gid_info = gid_info; 2260 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2261 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 2262 2263 mutex_enter(&gid_info->gl_mutex); 2264 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2265 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2266 mutex_exit(&gid_info->gl_mutex); 2267 2268 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 2269 "timeout id %x", gid_info->gl_timeout_id); 2270 2271 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2272 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2273 IBTF_DPRINTF_L2("ibdm", 2274 "\tsend_classportinfo: ibmf send failed"); 2275 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2276 } 2277 2278 return (IBDM_SUCCESS); 2279 } 2280 2281 2282 /* 2283 * ibdm_handle_setclassportinfo() 2284 * Invoked by the IBMF when setClassPortInfo request is completed. 2285 */ 2286 static void 2287 ibdm_handle_setclassportinfo(ibmf_handle_t ibmf_hdl, 2288 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2289 { 2290 void *data; 2291 timeout_id_t timeout_id; 2292 ib_mad_classportinfo_t *cpi; 2293 2294 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo:ibmf hdl " 2295 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2296 2297 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 2298 IBTF_DPRINTF_L4("ibdm", "\thandle_setclassportinfo: " 2299 "Not a ClassPortInfo resp"); 2300 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 2301 return; 2302 } 2303 2304 /* 2305 * Verify whether timeout handler is created/active. 2306 * If created/ active, cancel the timeout handler 2307 */ 2308 mutex_enter(&gid_info->gl_mutex); 2309 if (gid_info->gl_state != IBDM_SET_CLASSPORTINFO) { 2310 IBTF_DPRINTF_L2("ibdm", "\thandle_setclassportinfo:DUP resp"); 2311 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2312 mutex_exit(&gid_info->gl_mutex); 2313 return; 2314 } 2315 ibdm_bump_transactionID(gid_info); 2316 2317 gid_info->gl_iou_cb_args.cb_req_type = 0; 2318 if (gid_info->gl_timeout_id) { 2319 timeout_id = gid_info->gl_timeout_id; 2320 mutex_exit(&gid_info->gl_mutex); 2321 IBTF_DPRINTF_L5("ibdm", "handle_setlassportinfo: " 2322 "gl_timeout_id = 0x%x", timeout_id); 2323 if (untimeout(timeout_id) == -1) { 2324 IBTF_DPRINTF_L2("ibdm", "handle_setclassportinfo: " 2325 "untimeout gl_timeout_id failed"); 2326 } 2327 mutex_enter(&gid_info->gl_mutex); 2328 gid_info->gl_timeout_id = 0; 2329 } 2330 mutex_exit(&gid_info->gl_mutex); 2331 2332 data = msg->im_msgbufs_recv.im_bufs_cl_data; 2333 cpi = (ib_mad_classportinfo_t *)data; 2334 2335 ibdm_dump_classportinfo(cpi); 2336 } 2337 2338 2339 /* 2340 * ibdm_handle_classportinfo() 2341 * Invoked by the IBMF when the classportinfo request is completed. 2342 */ 2343 static void 2344 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 2345 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2346 { 2347 void *data; 2348 timeout_id_t timeout_id; 2349 ib_mad_hdr_t *hdr; 2350 ib_mad_classportinfo_t *cpi; 2351 2352 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 2353 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2354 2355 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 2356 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 2357 "Not a ClassPortInfo resp"); 2358 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 2359 return; 2360 } 2361 2362 /* 2363 * Verify whether timeout handler is created/active. 2364 * If created/ active, cancel the timeout handler 2365 */ 2366 mutex_enter(&gid_info->gl_mutex); 2367 ibdm_bump_transactionID(gid_info); 2368 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 2369 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 2370 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2371 mutex_exit(&gid_info->gl_mutex); 2372 return; 2373 } 2374 gid_info->gl_iou_cb_args.cb_req_type = 0; 2375 if (gid_info->gl_timeout_id) { 2376 timeout_id = gid_info->gl_timeout_id; 2377 mutex_exit(&gid_info->gl_mutex); 2378 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 2379 "gl_timeout_id = 0x%x", timeout_id); 2380 if (untimeout(timeout_id) == -1) { 2381 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 2382 "untimeout gl_timeout_id failed"); 2383 } 2384 mutex_enter(&gid_info->gl_mutex); 2385 gid_info->gl_timeout_id = 0; 2386 } 2387 gid_info->gl_state = IBDM_GET_IOUNITINFO; 2388 gid_info->gl_pending_cmds++; 2389 mutex_exit(&gid_info->gl_mutex); 2390 2391 data = msg->im_msgbufs_recv.im_bufs_cl_data; 2392 cpi = (ib_mad_classportinfo_t *)data; 2393 2394 /* 2395 * Cache the "RespTimeValue" and redirection information in the 2396 * global gid list data structure. This cached information will 2397 * be used to send any further requests to the GID. 2398 */ 2399 gid_info->gl_resp_timeout = 2400 (b2h32(cpi->RespTimeValue) & 0x1F); 2401 2402 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 2403 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 2404 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 2405 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 2406 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 2407 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 2408 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 2409 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 2410 gid_info->gl_redirectSL = cpi->RedirectSL; 2411 2412 ibdm_dump_classportinfo(cpi); 2413 2414 /* 2415 * Send IOUnitInfo request 2416 * Reuse previously allocated IBMF packet for sending ClassPortInfo 2417 * Check whether DM agent on the remote node requested redirection 2418 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 2419 */ 2420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2421 ibdm_alloc_send_buffers(msg); 2422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2423 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2424 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2425 2426 if (gid_info->gl_redirected == B_TRUE) { 2427 if (gid_info->gl_redirect_dlid != 0) { 2428 msg->im_local_addr.ia_remote_lid = 2429 gid_info->gl_redirect_dlid; 2430 } 2431 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2432 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2433 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2434 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 2435 } else { 2436 msg->im_local_addr.ia_remote_qno = 1; 2437 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2438 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2439 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2440 } 2441 2442 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2443 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2444 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2445 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2446 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2447 hdr->Status = 0; 2448 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2449 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2450 hdr->AttributeModifier = 0; 2451 2452 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2453 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2454 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2455 2456 mutex_enter(&gid_info->gl_mutex); 2457 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2458 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2459 mutex_exit(&gid_info->gl_mutex); 2460 2461 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 2462 "timeout %x", gid_info->gl_timeout_id); 2463 2464 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 2465 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 2466 IBTF_DPRINTF_L2("ibdm", 2467 "\thandle_classportinfo: msg transport failed"); 2468 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 2469 } 2470 (*flag) |= IBDM_IBMF_PKT_REUSED; 2471 } 2472 2473 2474 /* 2475 * ibdm_send_iounitinfo: 2476 * Sends a DM request to get IOU unitinfo. 2477 */ 2478 static int 2479 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 2480 { 2481 ibmf_msg_t *msg; 2482 ib_mad_hdr_t *hdr; 2483 2484 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 2485 2486 /* 2487 * Send command to get iounitinfo attribute. Allocate a IBMF 2488 * packet and initialize the packet. 2489 */ 2490 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 2491 IBMF_SUCCESS) { 2492 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2493 return (IBDM_FAILURE); 2494 } 2495 2496 mutex_enter(&gid_info->gl_mutex); 2497 ibdm_bump_transactionID(gid_info); 2498 mutex_exit(&gid_info->gl_mutex); 2499 2500 2501 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2502 ibdm_alloc_send_buffers(msg); 2503 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2504 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2505 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2506 msg->im_local_addr.ia_remote_qno = 1; 2507 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2508 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2509 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2510 2511 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2512 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2513 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2514 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2515 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2516 hdr->Status = 0; 2517 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2518 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2519 hdr->AttributeModifier = 0; 2520 2521 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2522 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2523 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2524 2525 mutex_enter(&gid_info->gl_mutex); 2526 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2527 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2528 mutex_exit(&gid_info->gl_mutex); 2529 2530 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2531 "timeout %x", gid_info->gl_timeout_id); 2532 2533 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2534 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2535 IBMF_SUCCESS) { 2536 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2537 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2538 msg, &gid_info->gl_iou_cb_args); 2539 } 2540 return (IBDM_SUCCESS); 2541 } 2542 2543 /* 2544 * ibdm_handle_iounitinfo() 2545 * Invoked by the IBMF when IO Unitinfo request is completed. 2546 */ 2547 static void 2548 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2549 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2550 { 2551 int ii, first = B_TRUE; 2552 int num_iocs; 2553 size_t size; 2554 uchar_t slot_info; 2555 timeout_id_t timeout_id; 2556 ib_mad_hdr_t *hdr; 2557 ibdm_ioc_info_t *ioc_info; 2558 ib_dm_io_unitinfo_t *iou_info; 2559 ib_dm_io_unitinfo_t *giou_info; 2560 ibdm_timeout_cb_args_t *cb_args; 2561 2562 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2563 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2564 2565 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2566 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2567 "Unexpected response"); 2568 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2569 return; 2570 } 2571 2572 mutex_enter(&gid_info->gl_mutex); 2573 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2574 IBTF_DPRINTF_L4("ibdm", 2575 "\thandle_iounitinfo: DUP resp"); 2576 mutex_exit(&gid_info->gl_mutex); 2577 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2578 return; 2579 } 2580 gid_info->gl_iou_cb_args.cb_req_type = 0; 2581 if (gid_info->gl_timeout_id) { 2582 timeout_id = gid_info->gl_timeout_id; 2583 mutex_exit(&gid_info->gl_mutex); 2584 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2585 "gl_timeout_id = 0x%x", timeout_id); 2586 if (untimeout(timeout_id) == -1) { 2587 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2588 "untimeout gl_timeout_id failed"); 2589 } 2590 mutex_enter(&gid_info->gl_mutex); 2591 gid_info->gl_timeout_id = 0; 2592 } 2593 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2594 2595 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2596 ibdm_dump_iounitinfo(iou_info); 2597 num_iocs = iou_info->iou_num_ctrl_slots; 2598 /* 2599 * check if number of IOCs reported is zero? if yes, return. 2600 * when num_iocs are reported zero internal IOC database needs 2601 * to be updated. To ensure that save the number of IOCs in 2602 * the new field "gl_num_iocs". Use a new field instead of 2603 * "giou_info->iou_num_ctrl_slots" as that would prevent 2604 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2605 */ 2606 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2607 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2608 mutex_exit(&gid_info->gl_mutex); 2609 return; 2610 } 2611 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2612 2613 /* 2614 * if there is an existing gl_iou (IOU has been probed before) 2615 * check if the "iou_changeid" is same as saved entry in 2616 * "giou_info->iou_changeid". 2617 * (note: this logic can prevent IOC enumeration if a given 2618 * vendor doesn't support setting iou_changeid field for its IOU) 2619 * 2620 * if there is an existing gl_iou and iou_changeid has changed : 2621 * free up existing gl_iou info and its related structures. 2622 * reallocate gl_iou info all over again. 2623 * if we donot free this up; then this leads to memory leaks 2624 */ 2625 if (gid_info->gl_iou) { 2626 giou_info = &gid_info->gl_iou->iou_info; 2627 if (b2h16(iou_info->iou_changeid) == 2628 giou_info->iou_changeid) { 2629 IBTF_DPRINTF_L3("ibdm", 2630 "\thandle_iounitinfo: no IOCs changed"); 2631 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2632 mutex_exit(&gid_info->gl_mutex); 2633 return; 2634 } 2635 2636 /* 2637 * Store the iou info as prev_iou to be used after 2638 * sweep is done. 2639 */ 2640 ASSERT(gid_info->gl_prev_iou == NULL); 2641 IBTF_DPRINTF_L4(ibdm_string, 2642 "\thandle_iounitinfo: setting gl_prev_iou %p", 2643 gid_info->gl_prev_iou); 2644 gid_info->gl_prev_iou = gid_info->gl_iou; 2645 ibdm.ibdm_prev_iou = 1; 2646 gid_info->gl_iou = NULL; 2647 } 2648 2649 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2650 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2651 giou_info = &gid_info->gl_iou->iou_info; 2652 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2653 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2654 2655 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2656 giou_info->iou_flag = iou_info->iou_flag; 2657 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2658 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2659 gid_info->gl_pending_cmds++; /* for diag code */ 2660 mutex_exit(&gid_info->gl_mutex); 2661 2662 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2663 mutex_enter(&gid_info->gl_mutex); 2664 gid_info->gl_pending_cmds--; 2665 mutex_exit(&gid_info->gl_mutex); 2666 } 2667 /* 2668 * Parallelize getting IOC controller profiles from here. 2669 * Allocate IBMF packets and send commands to get IOC profile for 2670 * each IOC present on the IOU. 2671 */ 2672 for (ii = 0; ii < num_iocs; ii++) { 2673 /* 2674 * Check whether IOC is present in the slot 2675 * Series of nibbles (in the field iou_ctrl_list) represents 2676 * a slot in the IOU. 2677 * Byte format: 76543210 2678 * Bits 0-3 of first byte represent Slot 2 2679 * bits 4-7 of first byte represent slot 1, 2680 * bits 0-3 of second byte represent slot 4 and so on 2681 * Each 4-bit nibble has the following meaning 2682 * 0x0 : IOC not installed 2683 * 0x1 : IOC is present 2684 * 0xf : Slot does not exist 2685 * and all other values are reserved. 2686 */ 2687 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2688 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2689 if ((ii % 2) == 0) 2690 slot_info = (slot_info >> 4); 2691 2692 if ((slot_info & 0xf) != 1) { 2693 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2694 "No IOC is present in the slot = %d", ii); 2695 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2696 continue; 2697 } 2698 2699 mutex_enter(&gid_info->gl_mutex); 2700 ibdm_bump_transactionID(gid_info); 2701 mutex_exit(&gid_info->gl_mutex); 2702 2703 /* 2704 * Re use the already allocated packet (for IOUnitinfo) to 2705 * send the first IOC controller attribute. Allocate new 2706 * IBMF packets for the rest of the IOC's 2707 */ 2708 if (first != B_TRUE) { 2709 msg = NULL; 2710 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2711 &msg) != IBMF_SUCCESS) { 2712 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2713 "IBMF packet allocation failed"); 2714 continue; 2715 } 2716 2717 } 2718 2719 /* allocate send buffers for all messages */ 2720 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2721 ibdm_alloc_send_buffers(msg); 2722 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2723 2724 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2725 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2726 if (gid_info->gl_redirected == B_TRUE) { 2727 if (gid_info->gl_redirect_dlid != 0) { 2728 msg->im_local_addr.ia_remote_lid = 2729 gid_info->gl_redirect_dlid; 2730 } 2731 msg->im_local_addr.ia_remote_qno = 2732 gid_info->gl_redirect_QP; 2733 msg->im_local_addr.ia_p_key = 2734 gid_info->gl_redirect_pkey; 2735 msg->im_local_addr.ia_q_key = 2736 gid_info->gl_redirect_qkey; 2737 msg->im_local_addr.ia_service_level = 2738 gid_info->gl_redirectSL; 2739 } else { 2740 msg->im_local_addr.ia_remote_qno = 1; 2741 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2742 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2743 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2744 } 2745 2746 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2747 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2748 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2749 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2750 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2751 hdr->Status = 0; 2752 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2753 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2754 hdr->AttributeModifier = h2b32(ii + 1); 2755 2756 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2757 cb_args = &ioc_info->ioc_cb_args; 2758 cb_args->cb_gid_info = gid_info; 2759 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2760 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2761 cb_args->cb_ioc_num = ii; 2762 2763 mutex_enter(&gid_info->gl_mutex); 2764 gid_info->gl_pending_cmds++; /* for diag code */ 2765 2766 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2767 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2768 mutex_exit(&gid_info->gl_mutex); 2769 2770 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2771 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2772 2773 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2774 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2775 IBTF_DPRINTF_L2("ibdm", 2776 "\thandle_iounitinfo: msg transport failed"); 2777 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2778 } 2779 (*flag) |= IBDM_IBMF_PKT_REUSED; 2780 first = B_FALSE; 2781 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2782 } 2783 } 2784 2785 2786 /* 2787 * ibdm_handle_ioc_profile() 2788 * Invoked by the IBMF when the IOCControllerProfile request 2789 * gets completed 2790 */ 2791 static void 2792 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2793 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2794 { 2795 int first = B_TRUE, reprobe = 0; 2796 uint_t ii, ioc_no, srv_start; 2797 uint_t nserv_entries; 2798 timeout_id_t timeout_id; 2799 ib_mad_hdr_t *hdr; 2800 ibdm_ioc_info_t *ioc_info; 2801 ibdm_timeout_cb_args_t *cb_args; 2802 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2803 2804 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2805 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2806 2807 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2808 /* 2809 * Check whether we know this IOC already 2810 * This will return NULL if reprobe is in progress 2811 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2812 * Do not hold mutexes here. 2813 */ 2814 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2815 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2816 "IOC guid %llx is present", ioc->ioc_guid); 2817 return; 2818 } 2819 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2820 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2821 2822 /* Make sure that IOC index is with the valid range */ 2823 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2824 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2825 "IOC index Out of range, index %d", ioc); 2826 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2827 return; 2828 } 2829 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2830 ioc_info->ioc_iou_info = gid_info->gl_iou; 2831 2832 mutex_enter(&gid_info->gl_mutex); 2833 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2834 reprobe = 1; 2835 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2836 ioc_info->ioc_serv = NULL; 2837 ioc_info->ioc_prev_serv_cnt = 2838 ioc_info->ioc_profile.ioc_service_entries; 2839 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2840 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2841 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2842 mutex_exit(&gid_info->gl_mutex); 2843 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2844 return; 2845 } 2846 ioc_info->ioc_cb_args.cb_req_type = 0; 2847 if (ioc_info->ioc_timeout_id) { 2848 timeout_id = ioc_info->ioc_timeout_id; 2849 ioc_info->ioc_timeout_id = 0; 2850 mutex_exit(&gid_info->gl_mutex); 2851 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2852 "ioc_timeout_id = 0x%x", timeout_id); 2853 if (untimeout(timeout_id) == -1) { 2854 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2855 "untimeout ioc_timeout_id failed"); 2856 } 2857 mutex_enter(&gid_info->gl_mutex); 2858 } 2859 2860 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2861 if (reprobe == 0) { 2862 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2863 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2864 } 2865 2866 /* 2867 * Save all the IOC information in the global structures. 2868 * Note the wire format is Big Endian and the Sparc process also 2869 * big endian. So, there is no need to convert the data fields 2870 * The conversion routines used below are ineffective on Sparc 2871 * machines where as they will be effective on little endian 2872 * machines such as Intel processors. 2873 */ 2874 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2875 2876 /* 2877 * Restrict updates to onlyport GIDs and service entries during reprobe 2878 */ 2879 if (reprobe == 0) { 2880 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2881 gioc->ioc_vendorid = 2882 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2883 >> IB_DM_VENDORID_SHIFT); 2884 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2885 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2886 gioc->ioc_subsys_vendorid = 2887 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2888 >> IB_DM_VENDORID_SHIFT); 2889 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2890 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2891 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2892 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2893 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2894 gioc->ioc_send_msg_qdepth = 2895 b2h16(ioc->ioc_send_msg_qdepth); 2896 gioc->ioc_rdma_read_qdepth = 2897 b2h16(ioc->ioc_rdma_read_qdepth); 2898 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2899 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2900 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2901 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2902 IB_DM_IOC_ID_STRING_LEN); 2903 2904 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2905 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2906 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2907 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2908 2909 if (ioc_info->ioc_diagdeviceid == B_TRUE) { 2910 gid_info->gl_pending_cmds++; 2911 IBTF_DPRINTF_L3(ibdm_string, 2912 "\tibdm_handle_ioc_profile: " 2913 "%d: gid_info %p gl_state %d pending_cmds %d", 2914 __LINE__, gid_info, gid_info->gl_state, 2915 gid_info->gl_pending_cmds); 2916 } 2917 } 2918 gioc->ioc_service_entries = ioc->ioc_service_entries; 2919 mutex_exit(&gid_info->gl_mutex); 2920 2921 ibdm_dump_ioc_profile(gioc); 2922 2923 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2924 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2925 mutex_enter(&gid_info->gl_mutex); 2926 gid_info->gl_pending_cmds--; 2927 mutex_exit(&gid_info->gl_mutex); 2928 } 2929 } 2930 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2931 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2932 KM_SLEEP); 2933 2934 /* 2935 * In one single request, maximum number of requests that can be 2936 * obtained is 4. If number of service entries are more than four, 2937 * calculate number requests needed and send them parallelly. 2938 */ 2939 nserv_entries = ioc->ioc_service_entries; 2940 ii = 0; 2941 while (nserv_entries) { 2942 mutex_enter(&gid_info->gl_mutex); 2943 gid_info->gl_pending_cmds++; 2944 ibdm_bump_transactionID(gid_info); 2945 mutex_exit(&gid_info->gl_mutex); 2946 2947 if (first != B_TRUE) { 2948 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2949 &msg) != IBMF_SUCCESS) { 2950 continue; 2951 } 2952 2953 } 2954 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 2955 ibdm_alloc_send_buffers(msg); 2956 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 2957 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2958 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2959 if (gid_info->gl_redirected == B_TRUE) { 2960 if (gid_info->gl_redirect_dlid != 0) { 2961 msg->im_local_addr.ia_remote_lid = 2962 gid_info->gl_redirect_dlid; 2963 } 2964 msg->im_local_addr.ia_remote_qno = 2965 gid_info->gl_redirect_QP; 2966 msg->im_local_addr.ia_p_key = 2967 gid_info->gl_redirect_pkey; 2968 msg->im_local_addr.ia_q_key = 2969 gid_info->gl_redirect_qkey; 2970 msg->im_local_addr.ia_service_level = 2971 gid_info->gl_redirectSL; 2972 } else { 2973 msg->im_local_addr.ia_remote_qno = 1; 2974 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2975 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2976 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 2977 } 2978 2979 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2980 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2981 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2982 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2983 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2984 hdr->Status = 0; 2985 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2986 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 2987 2988 srv_start = ii * 4; 2989 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 2990 cb_args->cb_gid_info = gid_info; 2991 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2992 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 2993 cb_args->cb_srvents_start = srv_start; 2994 cb_args->cb_ioc_num = ioc_no - 1; 2995 2996 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 2997 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 2998 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 2999 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 3000 } else { 3001 cb_args->cb_srvents_end = 3002 (cb_args->cb_srvents_start + nserv_entries - 1); 3003 nserv_entries = 0; 3004 } 3005 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 3006 ibdm_fill_srv_attr_mod(hdr, cb_args); 3007 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 3008 3009 mutex_enter(&gid_info->gl_mutex); 3010 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 3011 ibdm_pkt_timeout_hdlr, cb_args, 3012 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3013 mutex_exit(&gid_info->gl_mutex); 3014 3015 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 3016 "timeout %x, ioc %d srv %d", 3017 ioc_info->ioc_serv[srv_start].se_timeout_id, 3018 ioc_no - 1, srv_start); 3019 3020 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 3021 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3022 IBTF_DPRINTF_L2("ibdm", 3023 "\thandle_ioc_profile: msg send failed"); 3024 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 3025 } 3026 (*flag) |= IBDM_IBMF_PKT_REUSED; 3027 first = B_FALSE; 3028 ii++; 3029 } 3030 } 3031 3032 3033 /* 3034 * ibdm_handle_srventry_mad() 3035 */ 3036 static void 3037 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 3038 ibdm_dp_gidinfo_t *gid_info, int *flag) 3039 { 3040 uint_t ii, ioc_no, attrmod; 3041 uint_t nentries, start, end; 3042 timeout_id_t timeout_id; 3043 ib_dm_srv_t *srv_ents; 3044 ibdm_ioc_info_t *ioc_info; 3045 ibdm_srvents_info_t *gsrv_ents; 3046 3047 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 3048 " IBMF msg %p gid info %p", msg, gid_info); 3049 3050 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 3051 /* 3052 * Get the start and end index of the service entries 3053 * Upper 16 bits identify the IOC 3054 * Lower 16 bits specify the range of service entries 3055 * LSB specifies (Big endian) end of the range 3056 * MSB specifies (Big endian) start of the range 3057 */ 3058 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3059 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3060 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 3061 start = (attrmod & IBDM_8_BIT_MASK); 3062 3063 /* Make sure that IOC index is with the valid range */ 3064 if ((ioc_no < 1) | 3065 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 3066 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3067 "IOC index Out of range, index %d", ioc_no); 3068 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3069 return; 3070 } 3071 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3072 3073 /* 3074 * Make sure that the "start" and "end" service indexes are 3075 * with in the valid range 3076 */ 3077 nentries = ioc_info->ioc_profile.ioc_service_entries; 3078 if ((start > end) | (start >= nentries) | (end >= nentries)) { 3079 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3080 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 3081 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3082 return; 3083 } 3084 gsrv_ents = &ioc_info->ioc_serv[start]; 3085 mutex_enter(&gid_info->gl_mutex); 3086 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 3087 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 3088 "already known, ioc %d, srv %d, se_state %x", 3089 ioc_no - 1, start, gsrv_ents->se_state); 3090 mutex_exit(&gid_info->gl_mutex); 3091 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3092 return; 3093 } 3094 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 3095 if (ioc_info->ioc_serv[start].se_timeout_id) { 3096 IBTF_DPRINTF_L2("ibdm", 3097 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 3098 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 3099 ioc_info->ioc_serv[start].se_timeout_id = 0; 3100 mutex_exit(&gid_info->gl_mutex); 3101 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 3102 "se_timeout_id = 0x%x", timeout_id); 3103 if (untimeout(timeout_id) == -1) { 3104 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 3105 "untimeout se_timeout_id failed"); 3106 } 3107 mutex_enter(&gid_info->gl_mutex); 3108 } 3109 3110 gsrv_ents->se_state = IBDM_SE_VALID; 3111 mutex_exit(&gid_info->gl_mutex); 3112 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 3113 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 3114 bcopy(srv_ents->srv_name, 3115 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 3116 ibdm_dump_service_entries(&gsrv_ents->se_attr); 3117 } 3118 } 3119 3120 3121 /* 3122 * ibdm_get_diagcode: 3123 * Send request to get IOU/IOC diag code 3124 * Returns IBDM_SUCCESS/IBDM_FAILURE 3125 */ 3126 static int 3127 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 3128 { 3129 ibmf_msg_t *msg; 3130 ib_mad_hdr_t *hdr; 3131 ibdm_ioc_info_t *ioc; 3132 ibdm_timeout_cb_args_t *cb_args; 3133 timeout_id_t *timeout_id; 3134 3135 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 3136 gid_info, attr); 3137 3138 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 3139 &msg) != IBMF_SUCCESS) { 3140 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 3141 return (IBDM_FAILURE); 3142 } 3143 3144 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 3145 ibdm_alloc_send_buffers(msg); 3146 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 3147 3148 mutex_enter(&gid_info->gl_mutex); 3149 ibdm_bump_transactionID(gid_info); 3150 mutex_exit(&gid_info->gl_mutex); 3151 3152 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3153 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3154 if (gid_info->gl_redirected == B_TRUE) { 3155 if (gid_info->gl_redirect_dlid != 0) { 3156 msg->im_local_addr.ia_remote_lid = 3157 gid_info->gl_redirect_dlid; 3158 } 3159 3160 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3161 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3162 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3163 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 3164 } else { 3165 msg->im_local_addr.ia_remote_qno = 1; 3166 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3167 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3168 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 3169 } 3170 3171 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3172 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3173 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3174 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3175 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3176 hdr->Status = 0; 3177 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3178 3179 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3180 hdr->AttributeModifier = h2b32(attr); 3181 3182 if (attr == 0) { 3183 cb_args = &gid_info->gl_iou_cb_args; 3184 gid_info->gl_iou->iou_dc_valid = B_FALSE; 3185 cb_args->cb_ioc_num = 0; 3186 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 3187 timeout_id = &gid_info->gl_timeout_id; 3188 } else { 3189 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 3190 ioc->ioc_dc_valid = B_FALSE; 3191 cb_args = &ioc->ioc_dc_cb_args; 3192 cb_args->cb_ioc_num = attr - 1; 3193 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 3194 timeout_id = &ioc->ioc_dc_timeout_id; 3195 } 3196 cb_args->cb_gid_info = gid_info; 3197 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 3198 cb_args->cb_srvents_start = 0; 3199 3200 mutex_enter(&gid_info->gl_mutex); 3201 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3202 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3203 mutex_exit(&gid_info->gl_mutex); 3204 3205 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 3206 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 3207 3208 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3209 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3210 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 3211 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3212 } 3213 return (IBDM_SUCCESS); 3214 } 3215 3216 /* 3217 * ibdm_handle_diagcode: 3218 * Process the DiagCode MAD response and update local DM 3219 * data structure. 3220 */ 3221 static void 3222 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 3223 ibdm_dp_gidinfo_t *gid_info, int *flag) 3224 { 3225 uint16_t attrmod, *diagcode; 3226 ibdm_iou_info_t *iou; 3227 ibdm_ioc_info_t *ioc; 3228 timeout_id_t timeout_id; 3229 ibdm_timeout_cb_args_t *cb_args; 3230 3231 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 3232 3233 mutex_enter(&gid_info->gl_mutex); 3234 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 3235 iou = gid_info->gl_iou; 3236 if (attrmod == 0) { 3237 if (iou->iou_dc_valid != B_FALSE) { 3238 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3239 IBTF_DPRINTF_L4("ibdm", 3240 "\thandle_diagcode: Duplicate IOU DiagCode"); 3241 mutex_exit(&gid_info->gl_mutex); 3242 return; 3243 } 3244 cb_args = &gid_info->gl_iou_cb_args; 3245 cb_args->cb_req_type = 0; 3246 iou->iou_diagcode = b2h16(*diagcode); 3247 iou->iou_dc_valid = B_TRUE; 3248 if (gid_info->gl_timeout_id) { 3249 timeout_id = gid_info->gl_timeout_id; 3250 mutex_exit(&gid_info->gl_mutex); 3251 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 3252 "gl_timeout_id = 0x%x", timeout_id); 3253 if (untimeout(timeout_id) == -1) { 3254 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 3255 "untimeout gl_timeout_id failed"); 3256 } 3257 mutex_enter(&gid_info->gl_mutex); 3258 gid_info->gl_timeout_id = 0; 3259 } 3260 } else { 3261 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 3262 if (ioc->ioc_dc_valid != B_FALSE) { 3263 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 3264 IBTF_DPRINTF_L4("ibdm", 3265 "\thandle_diagcode: Duplicate IOC DiagCode"); 3266 mutex_exit(&gid_info->gl_mutex); 3267 return; 3268 } 3269 cb_args = &ioc->ioc_dc_cb_args; 3270 cb_args->cb_req_type = 0; 3271 ioc->ioc_diagcode = b2h16(*diagcode); 3272 ioc->ioc_dc_valid = B_TRUE; 3273 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 3274 if (timeout_id) { 3275 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 3276 mutex_exit(&gid_info->gl_mutex); 3277 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 3278 "timeout_id = 0x%x", timeout_id); 3279 if (untimeout(timeout_id) == -1) { 3280 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 3281 "untimeout ioc_dc_timeout_id failed"); 3282 } 3283 mutex_enter(&gid_info->gl_mutex); 3284 } 3285 } 3286 mutex_exit(&gid_info->gl_mutex); 3287 3288 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 3289 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 3290 } 3291 3292 3293 /* 3294 * ibdm_is_ioc_present() 3295 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 3296 */ 3297 static ibdm_ioc_info_t * 3298 ibdm_is_ioc_present(ib_guid_t ioc_guid, 3299 ibdm_dp_gidinfo_t *gid_info, int *flag) 3300 { 3301 int ii; 3302 ibdm_ioc_info_t *ioc; 3303 ibdm_dp_gidinfo_t *head; 3304 ib_dm_io_unitinfo_t *iou; 3305 3306 mutex_enter(&ibdm.ibdm_mutex); 3307 head = ibdm.ibdm_dp_gidlist_head; 3308 while (head) { 3309 mutex_enter(&head->gl_mutex); 3310 if (head->gl_iou == NULL) { 3311 mutex_exit(&head->gl_mutex); 3312 head = head->gl_next; 3313 continue; 3314 } 3315 iou = &head->gl_iou->iou_info; 3316 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 3317 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 3318 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 3319 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 3320 if (gid_info == head) { 3321 *flag |= IBDM_IBMF_PKT_DUP_RESP; 3322 } else if (ibdm_check_dgid(head->gl_dgid_lo, 3323 head->gl_dgid_hi) != NULL) { 3324 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 3325 "present: gid not present"); 3326 ibdm_add_to_gl_gid(gid_info, head); 3327 } 3328 mutex_exit(&head->gl_mutex); 3329 mutex_exit(&ibdm.ibdm_mutex); 3330 return (ioc); 3331 } 3332 } 3333 mutex_exit(&head->gl_mutex); 3334 head = head->gl_next; 3335 } 3336 mutex_exit(&ibdm.ibdm_mutex); 3337 return (NULL); 3338 } 3339 3340 3341 /* 3342 * ibdm_ibmf_send_cb() 3343 * IBMF invokes this callback routine after posting the DM MAD to 3344 * the HCA. 3345 */ 3346 /*ARGSUSED*/ 3347 static void 3348 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 3349 { 3350 ibdm_dump_ibmf_msg(ibmf_msg, 1); 3351 ibdm_free_send_buffers(ibmf_msg); 3352 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 3353 IBTF_DPRINTF_L4("ibdm", 3354 "\tibmf_send_cb: IBMF free msg failed"); 3355 } 3356 } 3357 3358 3359 /* 3360 * ibdm_ibmf_recv_cb() 3361 * Invoked by the IBMF when a response to the one of the DM requests 3362 * is received. 3363 */ 3364 /*ARGSUSED*/ 3365 static void 3366 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 3367 { 3368 ibdm_taskq_args_t *taskq_args; 3369 3370 /* 3371 * If the taskq enable is set then dispatch a taskq to process 3372 * the MAD, otherwise just process it on this thread 3373 */ 3374 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 3375 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 3376 return; 3377 } 3378 3379 /* 3380 * create a taskq and dispatch it to process the incoming MAD 3381 */ 3382 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 3383 if (taskq_args == NULL) { 3384 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 3385 "taskq_args"); 3386 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3387 IBTF_DPRINTF_L4("ibmf_recv_cb", 3388 "\tibmf_recv_cb: IBMF free msg failed"); 3389 } 3390 return; 3391 } 3392 taskq_args->tq_ibmf_handle = ibmf_hdl; 3393 taskq_args->tq_ibmf_msg = msg; 3394 taskq_args->tq_args = arg; 3395 3396 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 3397 TQ_NOSLEEP) == 0) { 3398 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 3399 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3400 IBTF_DPRINTF_L4("ibmf_recv_cb", 3401 "\tibmf_recv_cb: IBMF free msg failed"); 3402 } 3403 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 3404 return; 3405 } 3406 3407 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 3408 } 3409 3410 3411 void 3412 ibdm_recv_incoming_mad(void *args) 3413 { 3414 ibdm_taskq_args_t *taskq_args; 3415 3416 taskq_args = (ibdm_taskq_args_t *)args; 3417 3418 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 3419 "Processing incoming MAD via taskq"); 3420 3421 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 3422 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 3423 3424 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 3425 } 3426 3427 3428 /* 3429 * Calls ibdm_process_incoming_mad with all function arguments extracted 3430 * from args 3431 */ 3432 /*ARGSUSED*/ 3433 static void 3434 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 3435 { 3436 int flag = 0; 3437 int ret; 3438 uint64_t transaction_id; 3439 ib_mad_hdr_t *hdr; 3440 ibdm_dp_gidinfo_t *gid_info = NULL; 3441 3442 IBTF_DPRINTF_L4("ibdm", 3443 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 3444 ibdm_dump_ibmf_msg(msg, 0); 3445 3446 /* 3447 * IBMF calls this routine for every DM MAD that arrives at this port. 3448 * But we handle only the responses for requests we sent. We drop all 3449 * the DM packets that does not have response bit set in the MAD 3450 * header(this eliminates all the requests sent to this port). 3451 * We handle only DM class version 1 MAD's 3452 */ 3453 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 3454 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 3455 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3456 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3457 "IBMF free msg failed DM request drop it"); 3458 } 3459 return; 3460 } 3461 3462 transaction_id = b2h64(hdr->TransactionID); 3463 3464 mutex_enter(&ibdm.ibdm_mutex); 3465 gid_info = ibdm.ibdm_dp_gidlist_head; 3466 while (gid_info) { 3467 if ((gid_info->gl_transactionID & 3468 IBDM_GID_TRANSACTIONID_MASK) == 3469 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 3470 break; 3471 gid_info = gid_info->gl_next; 3472 } 3473 mutex_exit(&ibdm.ibdm_mutex); 3474 3475 if (gid_info == NULL) { 3476 /* Drop the packet */ 3477 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 3478 " does not match: 0x%llx", transaction_id); 3479 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3480 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3481 "IBMF free msg failed DM request drop it"); 3482 } 3483 return; 3484 } 3485 3486 /* Handle redirection for all the MAD's, except ClassPortInfo */ 3487 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 3488 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 3489 ret = ibdm_handle_redirection(msg, gid_info, &flag); 3490 if (ret == IBDM_SUCCESS) { 3491 return; 3492 } 3493 } else { 3494 uint_t gl_state; 3495 3496 mutex_enter(&gid_info->gl_mutex); 3497 gl_state = gid_info->gl_state; 3498 mutex_exit(&gid_info->gl_mutex); 3499 3500 switch (gl_state) { 3501 3502 case IBDM_SET_CLASSPORTINFO: 3503 ibdm_handle_setclassportinfo( 3504 ibmf_hdl, msg, gid_info, &flag); 3505 break; 3506 3507 case IBDM_GET_CLASSPORTINFO: 3508 ibdm_handle_classportinfo( 3509 ibmf_hdl, msg, gid_info, &flag); 3510 break; 3511 3512 case IBDM_GET_IOUNITINFO: 3513 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 3514 break; 3515 3516 case IBDM_GET_IOC_DETAILS: 3517 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3518 3519 case IB_DM_ATTR_SERVICE_ENTRIES: 3520 ibdm_handle_srventry_mad(msg, gid_info, &flag); 3521 break; 3522 3523 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3524 ibdm_handle_ioc_profile( 3525 ibmf_hdl, msg, gid_info, &flag); 3526 break; 3527 3528 case IB_DM_ATTR_DIAG_CODE: 3529 ibdm_handle_diagcode(msg, gid_info, &flag); 3530 break; 3531 3532 default: 3533 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3534 "Error state, wrong attribute :-("); 3535 (void) ibmf_free_msg(ibmf_hdl, &msg); 3536 return; 3537 } 3538 break; 3539 default: 3540 IBTF_DPRINTF_L2("ibdm", 3541 "process_incoming_mad: Dropping the packet" 3542 " gl_state %x", gl_state); 3543 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3544 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3545 "IBMF free msg failed DM request drop it"); 3546 } 3547 return; 3548 } 3549 } 3550 3551 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3552 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3553 IBTF_DPRINTF_L2("ibdm", 3554 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3555 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3556 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3557 "IBMF free msg failed DM request drop it"); 3558 } 3559 return; 3560 } 3561 3562 mutex_enter(&gid_info->gl_mutex); 3563 if (gid_info->gl_pending_cmds < 1) { 3564 IBTF_DPRINTF_L2("ibdm", 3565 "\tprocess_incoming_mad: pending commands negative"); 3566 } 3567 if (--gid_info->gl_pending_cmds) { 3568 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3569 "gid_info %p pending cmds %d", 3570 gid_info, gid_info->gl_pending_cmds); 3571 mutex_exit(&gid_info->gl_mutex); 3572 } else { 3573 uint_t prev_state; 3574 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3575 prev_state = gid_info->gl_state; 3576 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3577 if (prev_state == IBDM_SET_CLASSPORTINFO) { 3578 IBTF_DPRINTF_L4("ibdm", 3579 "\tprocess_incoming_mad: " 3580 "Setclassportinfo for Cisco FC GW is done."); 3581 gid_info->gl_flag &= ~IBDM_CISCO_PROBE; 3582 gid_info->gl_flag |= IBDM_CISCO_PROBE_DONE; 3583 mutex_exit(&gid_info->gl_mutex); 3584 cv_broadcast(&gid_info->gl_probe_cv); 3585 } else { 3586 mutex_exit(&gid_info->gl_mutex); 3587 ibdm_notify_newgid_iocs(gid_info); 3588 mutex_enter(&ibdm.ibdm_mutex); 3589 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3590 IBTF_DPRINTF_L4("ibdm", 3591 "\tprocess_incoming_mad: Wakeup"); 3592 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3593 cv_broadcast(&ibdm.ibdm_probe_cv); 3594 } 3595 mutex_exit(&ibdm.ibdm_mutex); 3596 } 3597 } 3598 3599 /* 3600 * Do not deallocate the IBMF packet if atleast one request 3601 * is posted. IBMF packet is reused. 3602 */ 3603 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3604 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3605 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3606 "IBMF free msg failed DM request drop it"); 3607 } 3608 } 3609 } 3610 3611 3612 /* 3613 * ibdm_verify_mad_status() 3614 * Verifies the MAD status 3615 * Returns IBDM_SUCCESS if status is correct 3616 * Returns IBDM_FAILURE for bogus MAD status 3617 */ 3618 static int 3619 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3620 { 3621 int ret = 0; 3622 3623 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3624 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3625 return (IBDM_FAILURE); 3626 } 3627 3628 if (b2h16(hdr->Status) == 0) 3629 ret = IBDM_SUCCESS; 3630 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3631 ret = IBDM_SUCCESS; 3632 else { 3633 IBTF_DPRINTF_L2("ibdm", 3634 "\tverify_mad_status: Status : 0x%x", b2h16(hdr->Status)); 3635 ret = IBDM_FAILURE; 3636 } 3637 return (ret); 3638 } 3639 3640 3641 3642 /* 3643 * ibdm_handle_redirection() 3644 * Returns IBDM_SUCCESS/IBDM_FAILURE 3645 */ 3646 static int 3647 ibdm_handle_redirection(ibmf_msg_t *msg, 3648 ibdm_dp_gidinfo_t *gid_info, int *flag) 3649 { 3650 int attrmod, ioc_no, start; 3651 void *data; 3652 timeout_id_t *timeout_id; 3653 ib_mad_hdr_t *hdr; 3654 ibdm_ioc_info_t *ioc = NULL; 3655 ibdm_timeout_cb_args_t *cb_args; 3656 ib_mad_classportinfo_t *cpi; 3657 3658 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3659 mutex_enter(&gid_info->gl_mutex); 3660 switch (gid_info->gl_state) { 3661 case IBDM_GET_IOUNITINFO: 3662 cb_args = &gid_info->gl_iou_cb_args; 3663 timeout_id = &gid_info->gl_timeout_id; 3664 break; 3665 3666 case IBDM_GET_IOC_DETAILS: 3667 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3668 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3669 3670 case IB_DM_ATTR_DIAG_CODE: 3671 if (attrmod == 0) { 3672 cb_args = &gid_info->gl_iou_cb_args; 3673 timeout_id = &gid_info->gl_timeout_id; 3674 break; 3675 } 3676 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3677 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3678 "IOC# Out of range %d", attrmod); 3679 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3680 mutex_exit(&gid_info->gl_mutex); 3681 return (IBDM_FAILURE); 3682 } 3683 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3684 cb_args = &ioc->ioc_dc_cb_args; 3685 timeout_id = &ioc->ioc_dc_timeout_id; 3686 break; 3687 3688 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3689 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3690 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3691 "IOC# Out of range %d", attrmod); 3692 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3693 mutex_exit(&gid_info->gl_mutex); 3694 return (IBDM_FAILURE); 3695 } 3696 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3697 cb_args = &ioc->ioc_cb_args; 3698 timeout_id = &ioc->ioc_timeout_id; 3699 break; 3700 3701 case IB_DM_ATTR_SERVICE_ENTRIES: 3702 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3703 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3704 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3705 "IOC# Out of range %d", ioc_no); 3706 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3707 mutex_exit(&gid_info->gl_mutex); 3708 return (IBDM_FAILURE); 3709 } 3710 start = (attrmod & IBDM_8_BIT_MASK); 3711 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3712 if (start > ioc->ioc_profile.ioc_service_entries) { 3713 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3714 " SE index Out of range %d", start); 3715 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3716 mutex_exit(&gid_info->gl_mutex); 3717 return (IBDM_FAILURE); 3718 } 3719 cb_args = &ioc->ioc_serv[start].se_cb_args; 3720 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3721 break; 3722 3723 default: 3724 /* ERROR State */ 3725 IBTF_DPRINTF_L2("ibdm", 3726 "\thandle_redirection: wrong attribute :-("); 3727 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3728 mutex_exit(&gid_info->gl_mutex); 3729 return (IBDM_FAILURE); 3730 } 3731 break; 3732 default: 3733 /* ERROR State */ 3734 IBTF_DPRINTF_L2("ibdm", 3735 "\thandle_redirection: Error state :-("); 3736 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3737 mutex_exit(&gid_info->gl_mutex); 3738 return (IBDM_FAILURE); 3739 } 3740 if ((*timeout_id) != 0) { 3741 mutex_exit(&gid_info->gl_mutex); 3742 if (untimeout(*timeout_id) == -1) { 3743 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3744 "untimeout failed %x", *timeout_id); 3745 } else { 3746 IBTF_DPRINTF_L5("ibdm", 3747 "\thandle_redirection: timeout %x", *timeout_id); 3748 } 3749 mutex_enter(&gid_info->gl_mutex); 3750 *timeout_id = 0; 3751 } 3752 3753 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3754 cpi = (ib_mad_classportinfo_t *)data; 3755 3756 gid_info->gl_resp_timeout = 3757 (b2h32(cpi->RespTimeValue) & 0x1F); 3758 3759 gid_info->gl_redirected = B_TRUE; 3760 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3761 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3762 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3763 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3764 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3765 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3766 gid_info->gl_redirectSL = cpi->RedirectSL; 3767 3768 if (gid_info->gl_redirect_dlid != 0) { 3769 msg->im_local_addr.ia_remote_lid = 3770 gid_info->gl_redirect_dlid; 3771 } 3772 ibdm_bump_transactionID(gid_info); 3773 mutex_exit(&gid_info->gl_mutex); 3774 3775 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg, *hdr)) 3776 ibdm_alloc_send_buffers(msg); 3777 3778 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3779 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3780 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3781 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3782 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3783 hdr->Status = 0; 3784 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3785 hdr->AttributeID = 3786 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3787 hdr->AttributeModifier = 3788 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3789 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg, *hdr)) 3790 3791 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3792 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3793 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3794 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 3795 3796 mutex_enter(&gid_info->gl_mutex); 3797 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3798 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3799 mutex_exit(&gid_info->gl_mutex); 3800 3801 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3802 "timeout %x", *timeout_id); 3803 3804 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3805 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3806 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3807 "message transport failed"); 3808 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3809 } 3810 (*flag) |= IBDM_IBMF_PKT_REUSED; 3811 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3812 return (IBDM_SUCCESS); 3813 } 3814 3815 3816 /* 3817 * ibdm_pkt_timeout_hdlr 3818 * This timeout handler is registed for every IBMF packet that is 3819 * sent through the IBMF. It gets called when no response is received 3820 * within the specified time for the packet. No retries for the failed 3821 * commands currently. Drops the failed IBMF packet and update the 3822 * pending list commands. 3823 */ 3824 static void 3825 ibdm_pkt_timeout_hdlr(void *arg) 3826 { 3827 ibdm_iou_info_t *iou; 3828 ibdm_ioc_info_t *ioc; 3829 ibdm_timeout_cb_args_t *cb_args = arg; 3830 ibdm_dp_gidinfo_t *gid_info; 3831 int srv_ent; 3832 uint_t new_gl_state; 3833 3834 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3835 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3836 cb_args->cb_req_type, cb_args->cb_ioc_num, 3837 cb_args->cb_srvents_start); 3838 3839 gid_info = cb_args->cb_gid_info; 3840 mutex_enter(&gid_info->gl_mutex); 3841 3842 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3843 (cb_args->cb_req_type == 0)) { 3844 3845 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3846 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3847 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3848 3849 if (gid_info->gl_timeout_id) 3850 gid_info->gl_timeout_id = 0; 3851 mutex_exit(&gid_info->gl_mutex); 3852 return; 3853 } 3854 if (cb_args->cb_retry_count) { 3855 cb_args->cb_retry_count--; 3856 /* 3857 * A new timeout_id is set inside ibdm_retry_command(). 3858 * When the function returns an error, the timeout_id 3859 * is reset (to zero) in the switch statement below. 3860 */ 3861 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3862 mutex_exit(&gid_info->gl_mutex); 3863 return; 3864 } 3865 cb_args->cb_retry_count = 0; 3866 } 3867 3868 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3869 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3870 cb_args->cb_req_type, cb_args->cb_ioc_num, 3871 cb_args->cb_srvents_start); 3872 3873 switch (cb_args->cb_req_type) { 3874 3875 case IBDM_REQ_TYPE_CLASSPORTINFO: 3876 case IBDM_REQ_TYPE_IOUINFO: 3877 new_gl_state = IBDM_GID_PROBING_FAILED; 3878 if (gid_info->gl_timeout_id) 3879 gid_info->gl_timeout_id = 0; 3880 break; 3881 3882 case IBDM_REQ_TYPE_IOCINFO: 3883 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3884 iou = gid_info->gl_iou; 3885 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3886 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3887 if (ioc->ioc_timeout_id) 3888 ioc->ioc_timeout_id = 0; 3889 break; 3890 3891 case IBDM_REQ_TYPE_SRVENTS: 3892 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3893 iou = gid_info->gl_iou; 3894 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3895 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3896 srv_ent = cb_args->cb_srvents_start; 3897 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3898 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3899 break; 3900 3901 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3902 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3903 iou = gid_info->gl_iou; 3904 iou->iou_dc_valid = B_FALSE; 3905 if (gid_info->gl_timeout_id) 3906 gid_info->gl_timeout_id = 0; 3907 break; 3908 3909 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3910 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3911 iou = gid_info->gl_iou; 3912 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3913 ioc->ioc_dc_valid = B_FALSE; 3914 if (ioc->ioc_dc_timeout_id) 3915 ioc->ioc_dc_timeout_id = 0; 3916 break; 3917 3918 default: /* ERROR State */ 3919 new_gl_state = IBDM_GID_PROBING_FAILED; 3920 if (gid_info->gl_timeout_id) 3921 gid_info->gl_timeout_id = 0; 3922 IBTF_DPRINTF_L2("ibdm", 3923 "\tpkt_timeout_hdlr: wrong request type."); 3924 break; 3925 } 3926 3927 --gid_info->gl_pending_cmds; /* decrease the counter */ 3928 3929 if (gid_info->gl_pending_cmds == 0) { 3930 gid_info->gl_state = new_gl_state; 3931 mutex_exit(&gid_info->gl_mutex); 3932 /* 3933 * Delete this gid_info if the gid probe fails. 3934 */ 3935 if (new_gl_state == IBDM_GID_PROBING_FAILED) { 3936 ibdm_delete_glhca_list(gid_info); 3937 } 3938 ibdm_notify_newgid_iocs(gid_info); 3939 mutex_enter(&ibdm.ibdm_mutex); 3940 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3941 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3942 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3943 cv_broadcast(&ibdm.ibdm_probe_cv); 3944 } 3945 mutex_exit(&ibdm.ibdm_mutex); 3946 } else { 3947 /* 3948 * Reset gl_pending_cmd if the extra timeout happens since 3949 * gl_pending_cmd becomes negative as a result. 3950 */ 3951 if (gid_info->gl_pending_cmds < 0) { 3952 gid_info->gl_pending_cmds = 0; 3953 IBTF_DPRINTF_L2("ibdm", 3954 "\tpkt_timeout_hdlr: extra timeout request." 3955 " reset gl_pending_cmds"); 3956 } 3957 mutex_exit(&gid_info->gl_mutex); 3958 /* 3959 * Delete this gid_info if the gid probe fails. 3960 */ 3961 if (new_gl_state == IBDM_GID_PROBING_FAILED) { 3962 ibdm_delete_glhca_list(gid_info); 3963 } 3964 } 3965 } 3966 3967 3968 /* 3969 * ibdm_retry_command() 3970 * Retries the failed command. 3971 * Returns IBDM_FAILURE/IBDM_SUCCESS 3972 */ 3973 static int 3974 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 3975 { 3976 int ret; 3977 ibmf_msg_t *msg; 3978 ib_mad_hdr_t *hdr; 3979 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 3980 timeout_id_t *timeout_id; 3981 ibdm_ioc_info_t *ioc; 3982 int ioc_no; 3983 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 3984 3985 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 3986 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3987 cb_args->cb_req_type, cb_args->cb_ioc_num, 3988 cb_args->cb_srvents_start); 3989 3990 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 3991 3992 3993 /* 3994 * Reset the gid if alloc_msg failed with BAD_HANDLE 3995 * ibdm_reset_gidinfo reinits the gid_info 3996 */ 3997 if (ret == IBMF_BAD_HANDLE) { 3998 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 3999 gid_info); 4000 4001 mutex_exit(&gid_info->gl_mutex); 4002 ibdm_reset_gidinfo(gid_info); 4003 mutex_enter(&gid_info->gl_mutex); 4004 4005 /* Retry alloc */ 4006 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 4007 &msg); 4008 } 4009 4010 if (ret != IBDM_SUCCESS) { 4011 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 4012 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 4013 cb_args->cb_req_type, cb_args->cb_ioc_num, 4014 cb_args->cb_srvents_start); 4015 return (IBDM_FAILURE); 4016 } 4017 4018 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 4019 ibdm_alloc_send_buffers(msg); 4020 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 4021 4022 ibdm_bump_transactionID(gid_info); 4023 4024 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4025 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4026 if (gid_info->gl_redirected == B_TRUE) { 4027 if (gid_info->gl_redirect_dlid != 0) { 4028 msg->im_local_addr.ia_remote_lid = 4029 gid_info->gl_redirect_dlid; 4030 } 4031 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4032 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4033 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4034 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 4035 } else { 4036 msg->im_local_addr.ia_remote_qno = 1; 4037 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4038 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4039 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 4040 } 4041 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4042 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*hdr)) 4043 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4044 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4045 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4046 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4047 hdr->Status = 0; 4048 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4049 4050 switch (cb_args->cb_req_type) { 4051 case IBDM_REQ_TYPE_CLASSPORTINFO: 4052 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 4053 hdr->AttributeModifier = 0; 4054 timeout_id = &gid_info->gl_timeout_id; 4055 break; 4056 case IBDM_REQ_TYPE_IOUINFO: 4057 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 4058 hdr->AttributeModifier = 0; 4059 timeout_id = &gid_info->gl_timeout_id; 4060 break; 4061 case IBDM_REQ_TYPE_IOCINFO: 4062 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4063 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 4064 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 4065 timeout_id = &ioc->ioc_timeout_id; 4066 break; 4067 case IBDM_REQ_TYPE_SRVENTS: 4068 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 4069 ibdm_fill_srv_attr_mod(hdr, cb_args); 4070 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 4071 timeout_id = 4072 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 4073 break; 4074 case IBDM_REQ_TYPE_IOU_DIAGCODE: 4075 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 4076 hdr->AttributeModifier = 0; 4077 timeout_id = &gid_info->gl_timeout_id; 4078 break; 4079 case IBDM_REQ_TYPE_IOC_DIAGCODE: 4080 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 4081 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 4082 ioc_no = cb_args->cb_ioc_num; 4083 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 4084 timeout_id = &ioc->ioc_dc_timeout_id; 4085 break; 4086 } 4087 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*hdr)) 4088 4089 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4090 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4091 4092 mutex_exit(&gid_info->gl_mutex); 4093 4094 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 4095 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 4096 cb_args->cb_srvents_start, *timeout_id); 4097 4098 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, 4099 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 4100 cb_args, 0) != IBMF_SUCCESS) { 4101 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 4102 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 4103 cb_args->cb_req_type, cb_args->cb_ioc_num, 4104 cb_args->cb_srvents_start); 4105 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4106 } 4107 mutex_enter(&gid_info->gl_mutex); 4108 return (IBDM_SUCCESS); 4109 } 4110 4111 4112 /* 4113 * ibdm_update_ioc_port_gidlist() 4114 */ 4115 static void 4116 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 4117 ibdm_dp_gidinfo_t *gid_info) 4118 { 4119 int ii, ngid_ents; 4120 ibdm_gid_t *tmp; 4121 ibdm_hca_list_t *gid_hca_head, *temp; 4122 ibdm_hca_list_t *ioc_head = NULL; 4123 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 4124 4125 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 4126 4127 ngid_ents = gid_info->gl_ngids; 4128 dest->ioc_nportgids = ngid_ents; 4129 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 4130 ngid_ents, KM_SLEEP); 4131 tmp = gid_info->gl_gid; 4132 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 4133 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 4134 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 4135 tmp = tmp->gid_next; 4136 } 4137 4138 gid_hca_head = gid_info->gl_hca_list; 4139 while (gid_hca_head) { 4140 temp = ibdm_dup_hca_attr(gid_hca_head); 4141 temp->hl_next = ioc_head; 4142 ioc_head = temp; 4143 gid_hca_head = gid_hca_head->hl_next; 4144 } 4145 dest->ioc_hca_list = ioc_head; 4146 } 4147 4148 4149 /* 4150 * ibdm_alloc_send_buffers() 4151 * Allocates memory for the IBMF send buffer to send and/or receive 4152 * the Device Management MAD packet. 4153 */ 4154 static void 4155 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 4156 { 4157 msgp->im_msgbufs_send.im_bufs_mad_hdr = 4158 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 4159 4160 msgp->im_msgbufs_send.im_bufs_cl_hdr = (uchar_t *) 4161 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 4162 msgp->im_msgbufs_send.im_bufs_cl_hdr_len = IBDM_DM_MAD_HDR_SZ; 4163 4164 msgp->im_msgbufs_send.im_bufs_cl_data = 4165 ((char *)msgp->im_msgbufs_send.im_bufs_cl_hdr + IBDM_DM_MAD_HDR_SZ); 4166 msgp->im_msgbufs_send.im_bufs_cl_data_len = 4167 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t) - IBDM_DM_MAD_HDR_SZ; 4168 } 4169 4170 4171 /* 4172 * ibdm_alloc_send_buffers() 4173 * De-allocates memory for the IBMF send buffer 4174 */ 4175 static void 4176 ibdm_free_send_buffers(ibmf_msg_t *msgp) 4177 { 4178 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 4179 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 4180 } 4181 4182 /* 4183 * ibdm_probe_ioc() 4184 * 1. Gets the node records for the port GUID. This detects all the port 4185 * to the IOU. 4186 * 2. Selectively probes all the IOC, given it's node GUID 4187 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 4188 * Controller Profile asynchronously 4189 */ 4190 /*ARGSUSED*/ 4191 static void 4192 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 4193 { 4194 int ii, nrecords; 4195 size_t nr_len = 0, pi_len = 0; 4196 ib_gid_t sgid, dgid; 4197 ibdm_hca_list_t *hca_list = NULL; 4198 sa_node_record_t *nr, *tmp; 4199 ibdm_port_attr_t *port = NULL; 4200 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 4201 ibdm_dp_gidinfo_t *temp_gidinfo; 4202 ibdm_gid_t *temp_gid; 4203 sa_portinfo_record_t *pi; 4204 4205 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%llx, %llx, %x): Begin", 4206 nodeguid, ioc_guid, reprobe_flag); 4207 4208 /* Rescan the GID list for any removed GIDs for reprobe */ 4209 if (reprobe_flag) 4210 ibdm_rescan_gidlist(&ioc_guid); 4211 4212 mutex_enter(&ibdm.ibdm_hl_mutex); 4213 for (ibdm_get_next_port(&hca_list, &port, 1); port; 4214 ibdm_get_next_port(&hca_list, &port, 1)) { 4215 reprobe_gid = new_gid = node_gid = NULL; 4216 4217 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 4218 if (nr == NULL) { 4219 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 4220 continue; 4221 } 4222 nrecords = (nr_len / sizeof (sa_node_record_t)); 4223 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 4224 if ((pi = ibdm_get_portinfo( 4225 port->pa_sa_hdl, &pi_len, tmp->LID)) == NULL) { 4226 IBTF_DPRINTF_L4("ibdm", 4227 "\tibdm_get_portinfo: no portinfo recs"); 4228 continue; 4229 } 4230 4231 /* 4232 * If Device Management is not supported on 4233 * this port, skip the rest. 4234 */ 4235 if (!(pi->PortInfo.CapabilityMask & 4236 SM_CAP_MASK_IS_DM_SUPPD)) { 4237 kmem_free(pi, pi_len); 4238 continue; 4239 } 4240 4241 /* 4242 * For reprobes: Check if GID, already in 4243 * the list. If so, set the state to SKIPPED 4244 */ 4245 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 4246 tmp->NodeInfo.PortGUID)) != NULL) && 4247 temp_gidinfo->gl_state == 4248 IBDM_GID_PROBING_COMPLETE) { 4249 ASSERT(reprobe_gid == NULL); 4250 ibdm_addto_glhcalist(temp_gidinfo, 4251 hca_list); 4252 reprobe_gid = temp_gidinfo; 4253 kmem_free(pi, pi_len); 4254 continue; 4255 } else if (temp_gidinfo != NULL) { 4256 kmem_free(pi, pi_len); 4257 ibdm_addto_glhcalist(temp_gidinfo, 4258 hca_list); 4259 continue; 4260 } 4261 4262 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 4263 "create_gid : prefix %llx, guid %llx\n", 4264 pi->PortInfo.GidPrefix, 4265 tmp->NodeInfo.PortGUID); 4266 4267 sgid.gid_prefix = port->pa_sn_prefix; 4268 sgid.gid_guid = port->pa_port_guid; 4269 dgid.gid_prefix = pi->PortInfo.GidPrefix; 4270 dgid.gid_guid = tmp->NodeInfo.PortGUID; 4271 new_gid = ibdm_create_gid_info(port, sgid, 4272 dgid); 4273 if (new_gid == NULL) { 4274 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4275 "create_gid_info failed\n"); 4276 kmem_free(pi, pi_len); 4277 continue; 4278 } 4279 if (node_gid == NULL) { 4280 node_gid = new_gid; 4281 ibdm_add_to_gl_gid(node_gid, node_gid); 4282 } else { 4283 IBTF_DPRINTF_L4("ibdm", 4284 "\tprobe_ioc: new gid"); 4285 temp_gid = kmem_zalloc( 4286 sizeof (ibdm_gid_t), KM_SLEEP); 4287 temp_gid->gid_dgid_hi = 4288 new_gid->gl_dgid_hi; 4289 temp_gid->gid_dgid_lo = 4290 new_gid->gl_dgid_lo; 4291 temp_gid->gid_next = node_gid->gl_gid; 4292 node_gid->gl_gid = temp_gid; 4293 node_gid->gl_ngids++; 4294 } 4295 new_gid->gl_is_dm_capable = B_TRUE; 4296 new_gid->gl_nodeguid = nodeguid; 4297 new_gid->gl_portguid = dgid.gid_guid; 4298 ibdm_addto_glhcalist(new_gid, hca_list); 4299 4300 /* 4301 * Set the state to skipped as all these 4302 * gids point to the same node. 4303 * We (re)probe only one GID below and reset 4304 * state appropriately 4305 */ 4306 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 4307 new_gid->gl_devid = (*tmp).NodeInfo.DeviceID; 4308 kmem_free(pi, pi_len); 4309 } 4310 kmem_free(nr, nr_len); 4311 4312 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 4313 "reprobe_gid %p new_gid %p node_gid %p", 4314 reprobe_flag, reprobe_gid, new_gid, node_gid); 4315 4316 if (reprobe_flag != 0 && reprobe_gid != NULL) { 4317 int niocs, jj; 4318 ibdm_ioc_info_t *tmp_ioc; 4319 int ioc_matched = 0; 4320 4321 mutex_exit(&ibdm.ibdm_hl_mutex); 4322 mutex_enter(&reprobe_gid->gl_mutex); 4323 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 4324 niocs = 4325 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4326 reprobe_gid->gl_pending_cmds++; 4327 mutex_exit(&reprobe_gid->gl_mutex); 4328 4329 for (jj = 0; jj < niocs; jj++) { 4330 tmp_ioc = 4331 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 4332 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 4333 continue; 4334 4335 ioc_matched = 1; 4336 4337 /* 4338 * Explicitly set gl_reprobe_flag to 0 so that 4339 * IBnex is not notified on completion 4340 */ 4341 mutex_enter(&reprobe_gid->gl_mutex); 4342 reprobe_gid->gl_reprobe_flag = 0; 4343 mutex_exit(&reprobe_gid->gl_mutex); 4344 4345 mutex_enter(&ibdm.ibdm_mutex); 4346 ibdm.ibdm_ngid_probes_in_progress++; 4347 mutex_exit(&ibdm.ibdm_mutex); 4348 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 4349 IBDM_SUCCESS) { 4350 IBTF_DPRINTF_L4("ibdm", 4351 "\tprobe_ioc: " 4352 "send_ioc_profile failed " 4353 "for ioc %d", jj); 4354 ibdm_gid_decr_pending(reprobe_gid); 4355 break; 4356 } 4357 mutex_enter(&ibdm.ibdm_mutex); 4358 ibdm_wait_probe_completion(); 4359 mutex_exit(&ibdm.ibdm_mutex); 4360 break; 4361 } 4362 if (ioc_matched == 0) 4363 ibdm_gid_decr_pending(reprobe_gid); 4364 else { 4365 mutex_enter(&ibdm.ibdm_hl_mutex); 4366 break; 4367 } 4368 } else if (new_gid != NULL) { 4369 mutex_exit(&ibdm.ibdm_hl_mutex); 4370 node_gid = node_gid ? node_gid : new_gid; 4371 4372 /* 4373 * New or reinserted GID : Enable notification 4374 * to IBnex 4375 */ 4376 mutex_enter(&node_gid->gl_mutex); 4377 node_gid->gl_reprobe_flag = 1; 4378 mutex_exit(&node_gid->gl_mutex); 4379 4380 ibdm_probe_gid(node_gid); 4381 4382 mutex_enter(&ibdm.ibdm_hl_mutex); 4383 } 4384 } 4385 mutex_exit(&ibdm.ibdm_hl_mutex); 4386 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 4387 } 4388 4389 4390 /* 4391 * ibdm_probe_gid() 4392 * Selectively probes the GID 4393 */ 4394 static void 4395 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 4396 { 4397 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 4398 4399 /* 4400 * A Cisco FC GW needs the special handling to get IOUnitInfo. 4401 */ 4402 mutex_enter(&gid_info->gl_mutex); 4403 if (ibdm_is_cisco_switch(gid_info)) { 4404 gid_info->gl_pending_cmds++; 4405 gid_info->gl_state = IBDM_SET_CLASSPORTINFO; 4406 mutex_exit(&gid_info->gl_mutex); 4407 4408 if (ibdm_set_classportinfo(gid_info) != IBDM_SUCCESS) { 4409 4410 mutex_enter(&gid_info->gl_mutex); 4411 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 4412 --gid_info->gl_pending_cmds; 4413 mutex_exit(&gid_info->gl_mutex); 4414 4415 /* free the hca_list on this gid_info */ 4416 ibdm_delete_glhca_list(gid_info); 4417 gid_info = gid_info->gl_next; 4418 return; 4419 } 4420 4421 mutex_enter(&gid_info->gl_mutex); 4422 ibdm_wait_cisco_probe_completion(gid_info); 4423 4424 IBTF_DPRINTF_L4("ibdm", 4425 "\tprobe_gid: CISCO Wakeup signal received"); 4426 } 4427 4428 /* move on to the 'GET_CLASSPORTINFO' stage */ 4429 gid_info->gl_pending_cmds++; 4430 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 4431 mutex_exit(&gid_info->gl_mutex); 4432 4433 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 4434 4435 mutex_enter(&gid_info->gl_mutex); 4436 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 4437 --gid_info->gl_pending_cmds; 4438 mutex_exit(&gid_info->gl_mutex); 4439 4440 /* free the hca_list on this gid_info */ 4441 ibdm_delete_glhca_list(gid_info); 4442 gid_info = gid_info->gl_next; 4443 return; 4444 } 4445 4446 mutex_enter(&ibdm.ibdm_mutex); 4447 ibdm.ibdm_ngid_probes_in_progress++; 4448 gid_info = gid_info->gl_next; 4449 ibdm_wait_probe_completion(); 4450 mutex_exit(&ibdm.ibdm_mutex); 4451 4452 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 4453 } 4454 4455 4456 /* 4457 * ibdm_create_gid_info() 4458 * Allocates a gid_info structure and initializes 4459 * Returns pointer to the structure on success 4460 * and NULL on failure 4461 */ 4462 static ibdm_dp_gidinfo_t * 4463 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 4464 { 4465 uint8_t ii, npaths; 4466 sa_path_record_t *path; 4467 size_t len; 4468 ibdm_pkey_tbl_t *pkey_tbl; 4469 ibdm_dp_gidinfo_t *gid_info = NULL; 4470 int ret; 4471 4472 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 4473 npaths = 1; 4474 4475 /* query for reversible paths */ 4476 if (port->pa_sa_hdl) 4477 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 4478 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 4479 &len, &path); 4480 else 4481 return (NULL); 4482 4483 if (ret == IBMF_SUCCESS && path) { 4484 ibdm_dump_path_info(path); 4485 4486 gid_info = kmem_zalloc( 4487 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 4488 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 4489 cv_init(&gid_info->gl_probe_cv, NULL, CV_DRIVER, NULL); 4490 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 4491 gid_info->gl_dgid_lo = path->DGID.gid_guid; 4492 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 4493 gid_info->gl_sgid_lo = path->SGID.gid_guid; 4494 gid_info->gl_p_key = path->P_Key; 4495 gid_info->gl_sa_hdl = port->pa_sa_hdl; 4496 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 4497 gid_info->gl_slid = path->SLID; 4498 gid_info->gl_dlid = path->DLID; 4499 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 4500 << IBDM_GID_TRANSACTIONID_SHIFT; 4501 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 4502 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 4503 << IBDM_GID_TRANSACTIONID_SHIFT; 4504 gid_info->gl_SL = path->SL; 4505 4506 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 4507 for (ii = 0; ii < port->pa_npkeys; ii++) { 4508 if (port->pa_pkey_tbl == NULL) 4509 break; 4510 4511 pkey_tbl = &port->pa_pkey_tbl[ii]; 4512 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 4513 (pkey_tbl->pt_qp_hdl != NULL)) { 4514 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 4515 break; 4516 } 4517 } 4518 kmem_free(path, len); 4519 4520 /* 4521 * QP handle for GID not initialized. No matching Pkey 4522 * was found!! ibdm should *not* hit this case. Flag an 4523 * error and drop the GID if ibdm does encounter this. 4524 */ 4525 if (gid_info->gl_qp_hdl == NULL) { 4526 IBTF_DPRINTF_L2(ibdm_string, 4527 "\tcreate_gid_info: No matching Pkey"); 4528 ibdm_delete_gidinfo(gid_info); 4529 return (NULL); 4530 } 4531 4532 ibdm.ibdm_ngids++; 4533 if (ibdm.ibdm_dp_gidlist_head == NULL) { 4534 ibdm.ibdm_dp_gidlist_head = gid_info; 4535 ibdm.ibdm_dp_gidlist_tail = gid_info; 4536 } else { 4537 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 4538 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 4539 ibdm.ibdm_dp_gidlist_tail = gid_info; 4540 } 4541 } 4542 4543 return (gid_info); 4544 } 4545 4546 4547 /* 4548 * ibdm_get_node_records 4549 * Sends a SA query to get the NODE record 4550 * Returns pointer to the sa_node_record_t on success 4551 * and NULL on failure 4552 */ 4553 static sa_node_record_t * 4554 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 4555 { 4556 sa_node_record_t req, *resp = NULL; 4557 ibmf_saa_access_args_t args; 4558 int ret; 4559 4560 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 4561 4562 bzero(&req, sizeof (sa_node_record_t)); 4563 req.NodeInfo.NodeGUID = guid; 4564 4565 args.sq_attr_id = SA_NODERECORD_ATTRID; 4566 args.sq_access_type = IBMF_SAA_RETRIEVE; 4567 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 4568 args.sq_template = &req; 4569 args.sq_callback = NULL; 4570 args.sq_callback_arg = NULL; 4571 4572 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4573 if (ret != IBMF_SUCCESS) { 4574 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 4575 " SA Retrieve Failed: %d", ret); 4576 return (NULL); 4577 } 4578 if ((resp == NULL) || (*length == 0)) { 4579 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 4580 return (NULL); 4581 } 4582 4583 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 4584 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 4585 4586 return (resp); 4587 } 4588 4589 4590 /* 4591 * ibdm_get_portinfo() 4592 * Sends a SA query to get the PortInfo record 4593 * Returns pointer to the sa_portinfo_record_t on success 4594 * and NULL on failure 4595 */ 4596 static sa_portinfo_record_t * 4597 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 4598 { 4599 sa_portinfo_record_t req, *resp = NULL; 4600 ibmf_saa_access_args_t args; 4601 int ret; 4602 4603 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 4604 4605 bzero(&req, sizeof (sa_portinfo_record_t)); 4606 req.EndportLID = lid; 4607 4608 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 4609 args.sq_access_type = IBMF_SAA_RETRIEVE; 4610 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 4611 args.sq_template = &req; 4612 args.sq_callback = NULL; 4613 args.sq_callback_arg = NULL; 4614 4615 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 4616 if (ret != IBMF_SUCCESS) { 4617 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 4618 " SA Retrieve Failed: 0x%X", ret); 4619 return (NULL); 4620 } 4621 if ((*length == 0) || (resp == NULL)) 4622 return (NULL); 4623 4624 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 4625 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 4626 return (resp); 4627 } 4628 4629 4630 /* 4631 * ibdm_ibnex_register_callback 4632 * IB nexus callback routine for HCA attach and detach notification 4633 */ 4634 void 4635 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4636 { 4637 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4638 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4639 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4640 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4641 } 4642 4643 4644 /* 4645 * ibdm_ibnex_unregister_callbacks 4646 */ 4647 void 4648 ibdm_ibnex_unregister_callback() 4649 { 4650 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4651 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4652 ibdm.ibdm_ibnex_callback = NULL; 4653 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4654 } 4655 4656 /* 4657 * ibdm_get_waittime() 4658 * Calculates the wait time based on the last HCA attach time 4659 */ 4660 static time_t 4661 ibdm_get_waittime(ib_guid_t hca_guid, int dft_wait) 4662 { 4663 int ii; 4664 time_t temp, wait_time = 0; 4665 ibdm_hca_list_t *hca; 4666 4667 IBTF_DPRINTF_L4("ibdm", "\tget_waittime hcaguid:%llx" 4668 "\tport settling time %d", hca_guid, dft_wait); 4669 4670 ASSERT(mutex_owned(&ibdm.ibdm_hl_mutex)); 4671 4672 hca = ibdm.ibdm_hca_list_head; 4673 4674 if (hca_guid) { 4675 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4676 if ((hca_guid == hca->hl_hca_guid) && 4677 (hca->hl_nports != hca->hl_nports_active)) { 4678 wait_time = 4679 ddi_get_time() - hca->hl_attach_time; 4680 wait_time = ((wait_time >= dft_wait) ? 4681 0 : (dft_wait - wait_time)); 4682 break; 4683 } 4684 hca = hca->hl_next; 4685 } 4686 IBTF_DPRINTF_L4("ibdm", "\tget_waittime %llx", wait_time); 4687 return (wait_time); 4688 } 4689 4690 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4691 if (hca->hl_nports != hca->hl_nports_active) { 4692 temp = ddi_get_time() - hca->hl_attach_time; 4693 temp = ((temp >= dft_wait) ? 0 : (dft_wait - temp)); 4694 wait_time = (temp > wait_time) ? temp : wait_time; 4695 } 4696 } 4697 IBTF_DPRINTF_L4("ibdm", "\tget_waittime %llx", wait_time); 4698 return (wait_time); 4699 } 4700 4701 void 4702 ibdm_ibnex_port_settle_wait(ib_guid_t hca_guid, int dft_wait) 4703 { 4704 time_t wait_time; 4705 clock_t delta; 4706 4707 mutex_enter(&ibdm.ibdm_hl_mutex); 4708 4709 while ((wait_time = ibdm_get_waittime(hca_guid, dft_wait)) > 0) { 4710 delta = wait_time * drv_usectohz(wait_time * 1000000); 4711 (void) cv_reltimedwait(&ibdm.ibdm_port_settle_cv, 4712 &ibdm.ibdm_hl_mutex, delta, TR_CLOCK_TICK); 4713 } 4714 4715 mutex_exit(&ibdm.ibdm_hl_mutex); 4716 } 4717 4718 4719 /* 4720 * ibdm_ibnex_probe_hcaport 4721 * Probes the presence of HCA port (with HCA dip and port number) 4722 * Returns port attributes structure on SUCCESS 4723 */ 4724 ibdm_port_attr_t * 4725 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4726 { 4727 int ii, jj; 4728 ibdm_hca_list_t *hca_list; 4729 ibdm_port_attr_t *port_attr; 4730 4731 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4732 4733 mutex_enter(&ibdm.ibdm_hl_mutex); 4734 hca_list = ibdm.ibdm_hca_list_head; 4735 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4736 if (hca_list->hl_hca_guid == hca_guid) { 4737 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4738 if (hca_list->hl_port_attr[jj].pa_port_num == 4739 port_num) { 4740 break; 4741 } 4742 } 4743 if (jj != hca_list->hl_nports) 4744 break; 4745 } 4746 hca_list = hca_list->hl_next; 4747 } 4748 if (ii == ibdm.ibdm_hca_count) { 4749 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4750 mutex_exit(&ibdm.ibdm_hl_mutex); 4751 return (NULL); 4752 } 4753 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4754 sizeof (ibdm_port_attr_t), KM_SLEEP); 4755 bcopy((char *)&hca_list->hl_port_attr[jj], 4756 port_attr, sizeof (ibdm_port_attr_t)); 4757 ibdm_update_port_attr(port_attr); 4758 4759 mutex_exit(&ibdm.ibdm_hl_mutex); 4760 return (port_attr); 4761 } 4762 4763 4764 /* 4765 * ibdm_ibnex_get_port_attrs 4766 * Scan all HCAs for a matching port_guid. 4767 * Returns "port attributes" structure on success. 4768 */ 4769 ibdm_port_attr_t * 4770 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4771 { 4772 int ii, jj; 4773 ibdm_hca_list_t *hca_list; 4774 ibdm_port_attr_t *port_attr; 4775 4776 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4777 4778 mutex_enter(&ibdm.ibdm_hl_mutex); 4779 hca_list = ibdm.ibdm_hca_list_head; 4780 4781 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4782 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4783 if (hca_list->hl_port_attr[jj].pa_port_guid == 4784 port_guid) { 4785 break; 4786 } 4787 } 4788 if (jj != hca_list->hl_nports) 4789 break; 4790 hca_list = hca_list->hl_next; 4791 } 4792 4793 if (ii == ibdm.ibdm_hca_count) { 4794 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4795 mutex_exit(&ibdm.ibdm_hl_mutex); 4796 return (NULL); 4797 } 4798 4799 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4800 KM_SLEEP); 4801 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4802 sizeof (ibdm_port_attr_t)); 4803 ibdm_update_port_attr(port_attr); 4804 4805 mutex_exit(&ibdm.ibdm_hl_mutex); 4806 return (port_attr); 4807 } 4808 4809 4810 /* 4811 * ibdm_ibnex_free_port_attr() 4812 */ 4813 void 4814 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4815 { 4816 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4817 if (port_attr) { 4818 if (port_attr->pa_pkey_tbl != NULL) { 4819 kmem_free(port_attr->pa_pkey_tbl, 4820 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4821 } 4822 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4823 } 4824 } 4825 4826 4827 /* 4828 * ibdm_ibnex_get_hca_list() 4829 * Returns portinfo for all the port for all the HCA's 4830 */ 4831 void 4832 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4833 { 4834 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4835 int ii; 4836 4837 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4838 4839 mutex_enter(&ibdm.ibdm_hl_mutex); 4840 temp = ibdm.ibdm_hca_list_head; 4841 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4842 temp1 = ibdm_dup_hca_attr(temp); 4843 temp1->hl_next = head; 4844 head = temp1; 4845 temp = temp->hl_next; 4846 } 4847 *count = ibdm.ibdm_hca_count; 4848 *hca = head; 4849 mutex_exit(&ibdm.ibdm_hl_mutex); 4850 } 4851 4852 4853 /* 4854 * ibdm_ibnex_get_hca_info_by_guid() 4855 */ 4856 ibdm_hca_list_t * 4857 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4858 { 4859 ibdm_hca_list_t *head = NULL, *hca = NULL; 4860 4861 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4862 4863 mutex_enter(&ibdm.ibdm_hl_mutex); 4864 head = ibdm.ibdm_hca_list_head; 4865 while (head) { 4866 if (head->hl_hca_guid == hca_guid) { 4867 hca = ibdm_dup_hca_attr(head); 4868 hca->hl_next = NULL; 4869 break; 4870 } 4871 head = head->hl_next; 4872 } 4873 mutex_exit(&ibdm.ibdm_hl_mutex); 4874 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4875 return (hca); 4876 } 4877 4878 4879 /* 4880 * ibdm_dup_hca_attr() 4881 * Allocate a new HCA attribute strucuture and initialize 4882 * hca attribute structure with the incoming HCA attributes 4883 * returned the allocated hca attributes. 4884 */ 4885 static ibdm_hca_list_t * 4886 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4887 { 4888 int len; 4889 ibdm_hca_list_t *out_hca; 4890 4891 len = sizeof (ibdm_hca_list_t) + 4892 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4893 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4894 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4895 bcopy((char *)in_hca, 4896 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4897 if (in_hca->hl_nports) { 4898 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4899 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4900 bcopy((char *)in_hca->hl_port_attr, 4901 (char *)out_hca->hl_port_attr, 4902 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4903 for (len = 0; len < out_hca->hl_nports; len++) 4904 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4905 } 4906 return (out_hca); 4907 } 4908 4909 4910 /* 4911 * ibdm_ibnex_free_hca_list() 4912 * Free one/more HCA lists 4913 */ 4914 void 4915 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4916 { 4917 int ii; 4918 size_t len; 4919 ibdm_hca_list_t *temp; 4920 ibdm_port_attr_t *port; 4921 4922 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4923 ASSERT(hca_list); 4924 while (hca_list) { 4925 temp = hca_list; 4926 hca_list = hca_list->hl_next; 4927 for (ii = 0; ii < temp->hl_nports; ii++) { 4928 port = &temp->hl_port_attr[ii]; 4929 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4930 if (len != 0) 4931 kmem_free(port->pa_pkey_tbl, len); 4932 } 4933 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4934 sizeof (ibdm_port_attr_t)); 4935 kmem_free(temp, len); 4936 } 4937 } 4938 4939 4940 /* 4941 * ibdm_ibnex_probe_iocguid() 4942 * Probes the IOC on the fabric and returns the IOC information 4943 * if present. Otherwise, NULL is returned 4944 */ 4945 /* ARGSUSED */ 4946 ibdm_ioc_info_t * 4947 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4948 { 4949 int k; 4950 ibdm_ioc_info_t *ioc_info; 4951 ibdm_dp_gidinfo_t *gid_info; /* used as index and arg */ 4952 timeout_id_t *timeout_id; 4953 4954 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4955 iou, ioc_guid, reprobe_flag); 4956 /* Check whether we know this already */ 4957 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 4958 if (ioc_info == NULL) { 4959 mutex_enter(&ibdm.ibdm_mutex); 4960 while (ibdm.ibdm_busy & IBDM_BUSY) 4961 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4962 ibdm.ibdm_busy |= IBDM_BUSY; 4963 mutex_exit(&ibdm.ibdm_mutex); 4964 ibdm_probe_ioc(iou, ioc_guid, 0); 4965 mutex_enter(&ibdm.ibdm_mutex); 4966 ibdm.ibdm_busy &= ~IBDM_BUSY; 4967 cv_broadcast(&ibdm.ibdm_busy_cv); 4968 mutex_exit(&ibdm.ibdm_mutex); 4969 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 4970 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 4971 ASSERT(gid_info != NULL); 4972 /* Free the ioc_list before reprobe; and cancel any timers */ 4973 mutex_enter(&ibdm.ibdm_mutex); 4974 mutex_enter(&gid_info->gl_mutex); 4975 if (ioc_info->ioc_timeout_id) { 4976 timeout_id = ioc_info->ioc_timeout_id; 4977 ioc_info->ioc_timeout_id = 0; 4978 mutex_exit(&gid_info->gl_mutex); 4979 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4980 "ioc_timeout_id = 0x%x", timeout_id); 4981 if (untimeout(timeout_id) == -1) { 4982 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4983 "untimeout ioc_timeout_id failed"); 4984 } 4985 mutex_enter(&gid_info->gl_mutex); 4986 } 4987 if (ioc_info->ioc_dc_timeout_id) { 4988 timeout_id = ioc_info->ioc_dc_timeout_id; 4989 ioc_info->ioc_dc_timeout_id = 0; 4990 mutex_exit(&gid_info->gl_mutex); 4991 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4992 "ioc_dc_timeout_id = 0x%x", timeout_id); 4993 if (untimeout(timeout_id) == -1) { 4994 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4995 "untimeout ioc_dc_timeout_id failed"); 4996 } 4997 mutex_enter(&gid_info->gl_mutex); 4998 } 4999 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 5000 if (ioc_info->ioc_serv[k].se_timeout_id) { 5001 timeout_id = ioc_info->ioc_serv[k]. 5002 se_timeout_id; 5003 ioc_info->ioc_serv[k].se_timeout_id = 0; 5004 mutex_exit(&gid_info->gl_mutex); 5005 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 5006 "ioc_info->ioc_serv[k].se_timeout_id = %x", 5007 k, timeout_id); 5008 if (untimeout(timeout_id) == -1) { 5009 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 5010 "untimeout se_timeout_id %d " 5011 "failed", k); 5012 } 5013 mutex_enter(&gid_info->gl_mutex); 5014 } 5015 mutex_exit(&gid_info->gl_mutex); 5016 mutex_exit(&ibdm.ibdm_mutex); 5017 ibdm_ibnex_free_ioc_list(ioc_info); 5018 5019 mutex_enter(&ibdm.ibdm_mutex); 5020 while (ibdm.ibdm_busy & IBDM_BUSY) 5021 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5022 ibdm.ibdm_busy |= IBDM_BUSY; 5023 mutex_exit(&ibdm.ibdm_mutex); 5024 5025 ibdm_probe_ioc(iou, ioc_guid, 1); 5026 5027 /* 5028 * Skip if gl_reprobe_flag is set, this will be 5029 * a re-inserted / new GID, for which notifications 5030 * have already been send. 5031 */ 5032 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 5033 gid_info = gid_info->gl_next) { 5034 uint8_t ii, niocs; 5035 ibdm_ioc_info_t *ioc; 5036 5037 if (gid_info->gl_iou == NULL) 5038 continue; 5039 5040 if (gid_info->gl_reprobe_flag) { 5041 gid_info->gl_reprobe_flag = 0; 5042 continue; 5043 } 5044 5045 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5046 for (ii = 0; ii < niocs; ii++) { 5047 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5048 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 5049 mutex_enter(&ibdm.ibdm_mutex); 5050 ibdm_reprobe_update_port_srv(ioc, 5051 gid_info); 5052 mutex_exit(&ibdm.ibdm_mutex); 5053 } 5054 } 5055 } 5056 mutex_enter(&ibdm.ibdm_mutex); 5057 ibdm.ibdm_busy &= ~IBDM_BUSY; 5058 cv_broadcast(&ibdm.ibdm_busy_cv); 5059 mutex_exit(&ibdm.ibdm_mutex); 5060 5061 ioc_info = ibdm_get_ioc_info_with_gid(ioc_guid, &gid_info); 5062 } 5063 return (ioc_info); 5064 } 5065 5066 5067 /* 5068 * ibdm_get_ioc_info_with_gid() 5069 * Returns pointer to ibdm_ioc_info_t if it finds 5070 * matching record for the ioc_guid. Otherwise NULL is returned. 5071 * The pointer to gid_info is set to the second argument in case that 5072 * the non-NULL value returns (and the second argument is not NULL). 5073 * 5074 * Note. use the same strings as "ibnex_get_ioc_info" in 5075 * IBTF_DPRINTF() to keep compatibility. 5076 */ 5077 static ibdm_ioc_info_t * 5078 ibdm_get_ioc_info_with_gid(ib_guid_t ioc_guid, 5079 ibdm_dp_gidinfo_t **gid_info) 5080 { 5081 int ii; 5082 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 5083 ibdm_dp_gidinfo_t *gid_list; 5084 ib_dm_io_unitinfo_t *iou; 5085 5086 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 5087 5088 mutex_enter(&ibdm.ibdm_mutex); 5089 while (ibdm.ibdm_busy & IBDM_BUSY) 5090 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5091 ibdm.ibdm_busy |= IBDM_BUSY; 5092 5093 if (gid_info) 5094 *gid_info = NULL; /* clear the value of gid_info */ 5095 5096 gid_list = ibdm.ibdm_dp_gidlist_head; 5097 while (gid_list) { 5098 mutex_enter(&gid_list->gl_mutex); 5099 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 5100 mutex_exit(&gid_list->gl_mutex); 5101 gid_list = gid_list->gl_next; 5102 continue; 5103 } 5104 if (gid_list->gl_iou == NULL) { 5105 IBTF_DPRINTF_L2("ibdm", 5106 "\tget_ioc_info: No IOU info"); 5107 mutex_exit(&gid_list->gl_mutex); 5108 gid_list = gid_list->gl_next; 5109 continue; 5110 } 5111 iou = &gid_list->gl_iou->iou_info; 5112 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 5113 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 5114 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 5115 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 5116 ioc = ibdm_dup_ioc_info(tmp, gid_list); 5117 if (gid_info) 5118 *gid_info = gid_list; /* set this ptr */ 5119 mutex_exit(&gid_list->gl_mutex); 5120 ibdm.ibdm_busy &= ~IBDM_BUSY; 5121 cv_broadcast(&ibdm.ibdm_busy_cv); 5122 mutex_exit(&ibdm.ibdm_mutex); 5123 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 5124 return (ioc); 5125 } 5126 } 5127 if (ii == iou->iou_num_ctrl_slots) 5128 ioc = NULL; 5129 5130 mutex_exit(&gid_list->gl_mutex); 5131 gid_list = gid_list->gl_next; 5132 } 5133 5134 ibdm.ibdm_busy &= ~IBDM_BUSY; 5135 cv_broadcast(&ibdm.ibdm_busy_cv); 5136 mutex_exit(&ibdm.ibdm_mutex); 5137 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 5138 return (ioc); 5139 } 5140 5141 /* 5142 * ibdm_ibnex_get_ioc_info() 5143 * Returns pointer to ibdm_ioc_info_t if it finds 5144 * matching record for the ioc_guid, otherwise NULL 5145 * is returned 5146 * 5147 * Note. this is a wrapper function to ibdm_get_ioc_info_with_gid() now. 5148 */ 5149 ibdm_ioc_info_t * 5150 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 5151 { 5152 /* will not use the gid_info pointer, so the second arg is NULL */ 5153 return (ibdm_get_ioc_info_with_gid(ioc_guid, NULL)); 5154 } 5155 5156 /* 5157 * ibdm_ibnex_get_ioc_count() 5158 * Returns number of ibdm_ioc_info_t it finds 5159 */ 5160 int 5161 ibdm_ibnex_get_ioc_count(void) 5162 { 5163 int count = 0, k; 5164 ibdm_ioc_info_t *ioc; 5165 ibdm_dp_gidinfo_t *gid_list; 5166 5167 mutex_enter(&ibdm.ibdm_mutex); 5168 ibdm_sweep_fabric(0); 5169 5170 while (ibdm.ibdm_busy & IBDM_BUSY) 5171 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5172 ibdm.ibdm_busy |= IBDM_BUSY; 5173 5174 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5175 gid_list = gid_list->gl_next) { 5176 mutex_enter(&gid_list->gl_mutex); 5177 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 5178 (gid_list->gl_iou == NULL)) { 5179 mutex_exit(&gid_list->gl_mutex); 5180 continue; 5181 } 5182 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 5183 k++) { 5184 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 5185 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 5186 ++count; 5187 } 5188 mutex_exit(&gid_list->gl_mutex); 5189 } 5190 ibdm.ibdm_busy &= ~IBDM_BUSY; 5191 cv_broadcast(&ibdm.ibdm_busy_cv); 5192 mutex_exit(&ibdm.ibdm_mutex); 5193 5194 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 5195 return (count); 5196 } 5197 5198 5199 /* 5200 * ibdm_ibnex_get_ioc_list() 5201 * Returns information about all the IOCs present on the fabric. 5202 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 5203 * Does not sweep fabric if DONOT_PROBE is set 5204 */ 5205 ibdm_ioc_info_t * 5206 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 5207 { 5208 int ii; 5209 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 5210 ibdm_dp_gidinfo_t *gid_list; 5211 ib_dm_io_unitinfo_t *iou; 5212 5213 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 5214 5215 mutex_enter(&ibdm.ibdm_mutex); 5216 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 5217 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 5218 5219 while (ibdm.ibdm_busy & IBDM_BUSY) 5220 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5221 ibdm.ibdm_busy |= IBDM_BUSY; 5222 5223 gid_list = ibdm.ibdm_dp_gidlist_head; 5224 while (gid_list) { 5225 mutex_enter(&gid_list->gl_mutex); 5226 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 5227 mutex_exit(&gid_list->gl_mutex); 5228 gid_list = gid_list->gl_next; 5229 continue; 5230 } 5231 if (gid_list->gl_iou == NULL) { 5232 IBTF_DPRINTF_L2("ibdm", 5233 "\tget_ioc_list: No IOU info"); 5234 mutex_exit(&gid_list->gl_mutex); 5235 gid_list = gid_list->gl_next; 5236 continue; 5237 } 5238 iou = &gid_list->gl_iou->iou_info; 5239 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 5240 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 5241 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 5242 tmp = ibdm_dup_ioc_info(ioc, gid_list); 5243 tmp->ioc_next = ioc_list; 5244 ioc_list = tmp; 5245 } 5246 } 5247 mutex_exit(&gid_list->gl_mutex); 5248 gid_list = gid_list->gl_next; 5249 } 5250 ibdm.ibdm_busy &= ~IBDM_BUSY; 5251 cv_broadcast(&ibdm.ibdm_busy_cv); 5252 mutex_exit(&ibdm.ibdm_mutex); 5253 5254 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 5255 return (ioc_list); 5256 } 5257 5258 /* 5259 * ibdm_dup_ioc_info() 5260 * Duplicate the IOC information and return the IOC 5261 * information. 5262 */ 5263 static ibdm_ioc_info_t * 5264 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 5265 { 5266 ibdm_ioc_info_t *out_ioc; 5267 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 5268 ASSERT(MUTEX_HELD(&gid_list->gl_mutex)); 5269 5270 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 5271 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 5272 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 5273 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 5274 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 5275 5276 return (out_ioc); 5277 } 5278 5279 5280 /* 5281 * ibdm_free_ioc_list() 5282 * Deallocate memory for IOC list structure 5283 */ 5284 void 5285 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 5286 { 5287 ibdm_ioc_info_t *temp; 5288 5289 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 5290 while (ioc) { 5291 temp = ioc; 5292 ioc = ioc->ioc_next; 5293 kmem_free(temp->ioc_gid_list, 5294 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 5295 if (temp->ioc_hca_list) 5296 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 5297 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 5298 } 5299 } 5300 5301 5302 /* 5303 * ibdm_ibnex_update_pkey_tbls 5304 * Updates the DM P_Key database. 5305 * NOTE: Two cases are handled here: P_Key being added or removed. 5306 * 5307 * Arguments : NONE 5308 * Return Values : NONE 5309 */ 5310 void 5311 ibdm_ibnex_update_pkey_tbls(void) 5312 { 5313 int h, pp, pidx; 5314 uint_t nports; 5315 uint_t size; 5316 ib_pkey_t new_pkey; 5317 ib_pkey_t *orig_pkey; 5318 ibdm_hca_list_t *hca_list; 5319 ibdm_port_attr_t *port; 5320 ibt_hca_portinfo_t *pinfop; 5321 5322 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 5323 5324 mutex_enter(&ibdm.ibdm_hl_mutex); 5325 hca_list = ibdm.ibdm_hca_list_head; 5326 5327 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 5328 5329 /* This updates P_Key Tables for all ports of this HCA */ 5330 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 5331 &nports, &size); 5332 5333 /* number of ports shouldn't have changed */ 5334 ASSERT(nports == hca_list->hl_nports); 5335 5336 for (pp = 0; pp < hca_list->hl_nports; pp++) { 5337 port = &hca_list->hl_port_attr[pp]; 5338 5339 /* 5340 * First figure out the P_Keys from IBTL. 5341 * Three things could have happened: 5342 * New P_Keys added 5343 * Existing P_Keys removed 5344 * Both of the above two 5345 * 5346 * Loop through the P_Key Indices and check if a 5347 * give P_Key_Ix matches that of the one seen by 5348 * IBDM. If they match no action is needed. 5349 * 5350 * If they don't match: 5351 * 1. if orig_pkey is invalid and new_pkey is valid 5352 * ---> add new_pkey to DM database 5353 * 2. if orig_pkey is valid and new_pkey is invalid 5354 * ---> remove orig_pkey from DM database 5355 * 3. if orig_pkey and new_pkey are both valid: 5356 * ---> remov orig_pkey from DM database 5357 * ---> add new_pkey to DM database 5358 * 4. if orig_pkey and new_pkey are both invalid: 5359 * ---> do nothing. Updated DM database. 5360 */ 5361 5362 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 5363 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 5364 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 5365 5366 /* keys match - do nothing */ 5367 if (*orig_pkey == new_pkey) 5368 continue; 5369 5370 if (IBDM_INVALID_PKEY(*orig_pkey) && 5371 !IBDM_INVALID_PKEY(new_pkey)) { 5372 /* P_Key was added */ 5373 IBTF_DPRINTF_L5("ibdm", 5374 "\tibnex_update_pkey_tbls: new " 5375 "P_Key added = 0x%x", new_pkey); 5376 *orig_pkey = new_pkey; 5377 ibdm_port_attr_ibmf_init(port, 5378 new_pkey, pp); 5379 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 5380 IBDM_INVALID_PKEY(new_pkey)) { 5381 /* P_Key was removed */ 5382 IBTF_DPRINTF_L5("ibdm", 5383 "\tibnex_update_pkey_tbls: P_Key " 5384 "removed = 0x%x", *orig_pkey); 5385 *orig_pkey = new_pkey; 5386 (void) ibdm_port_attr_ibmf_fini(port, 5387 pidx); 5388 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 5389 !IBDM_INVALID_PKEY(new_pkey)) { 5390 /* P_Key were replaced */ 5391 IBTF_DPRINTF_L5("ibdm", 5392 "\tibnex_update_pkey_tbls: P_Key " 5393 "replaced 0x%x with 0x%x", 5394 *orig_pkey, new_pkey); 5395 (void) ibdm_port_attr_ibmf_fini(port, 5396 pidx); 5397 *orig_pkey = new_pkey; 5398 ibdm_port_attr_ibmf_init(port, 5399 new_pkey, pp); 5400 } else { 5401 /* 5402 * P_Keys are invalid 5403 * set anyway to reflect if 5404 * INVALID_FULL was changed to 5405 * INVALID_LIMITED or vice-versa. 5406 */ 5407 *orig_pkey = new_pkey; 5408 } /* end of else */ 5409 5410 } /* loop of p_key index */ 5411 5412 } /* loop of #ports of HCA */ 5413 5414 ibt_free_portinfo(pinfop, size); 5415 hca_list = hca_list->hl_next; 5416 5417 } /* loop for all HCAs in the system */ 5418 5419 mutex_exit(&ibdm.ibdm_hl_mutex); 5420 } 5421 5422 5423 /* 5424 * ibdm_send_ioc_profile() 5425 * Send IOC Controller Profile request. When the request is completed 5426 * IBMF calls ibdm_process_incoming_mad routine to inform about 5427 * the completion. 5428 */ 5429 static int 5430 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 5431 { 5432 ibmf_msg_t *msg; 5433 ib_mad_hdr_t *hdr; 5434 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 5435 ibdm_timeout_cb_args_t *cb_args; 5436 5437 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 5438 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 5439 5440 /* 5441 * Send command to get IOC profile. 5442 * Allocate a IBMF packet and initialize the packet. 5443 */ 5444 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 5445 &msg) != IBMF_SUCCESS) { 5446 IBTF_DPRINTF_L2("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 5447 return (IBDM_FAILURE); 5448 } 5449 5450 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*msg)) 5451 ibdm_alloc_send_buffers(msg); 5452 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*msg)) 5453 5454 mutex_enter(&gid_info->gl_mutex); 5455 ibdm_bump_transactionID(gid_info); 5456 mutex_exit(&gid_info->gl_mutex); 5457 5458 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 5459 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 5460 if (gid_info->gl_redirected == B_TRUE) { 5461 if (gid_info->gl_redirect_dlid != 0) { 5462 msg->im_local_addr.ia_remote_lid = 5463 gid_info->gl_redirect_dlid; 5464 } 5465 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 5466 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 5467 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 5468 msg->im_local_addr.ia_service_level = gid_info->gl_redirectSL; 5469 } else { 5470 msg->im_local_addr.ia_remote_qno = 1; 5471 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 5472 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 5473 msg->im_local_addr.ia_service_level = gid_info->gl_SL; 5474 } 5475 5476 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 5477 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 5478 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 5479 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 5480 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 5481 hdr->Status = 0; 5482 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 5483 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 5484 hdr->AttributeModifier = h2b32(ioc_no + 1); 5485 5486 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 5487 cb_args = &ioc_info->ioc_cb_args; 5488 cb_args->cb_gid_info = gid_info; 5489 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 5490 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 5491 cb_args->cb_ioc_num = ioc_no; 5492 5493 mutex_enter(&gid_info->gl_mutex); 5494 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 5495 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 5496 mutex_exit(&gid_info->gl_mutex); 5497 5498 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 5499 "timeout %x", ioc_info->ioc_timeout_id); 5500 5501 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 5502 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 5503 IBTF_DPRINTF_L2("ibdm", 5504 "\tsend_ioc_profile: msg transport failed"); 5505 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 5506 } 5507 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 5508 return (IBDM_SUCCESS); 5509 } 5510 5511 5512 /* 5513 * ibdm_port_reachable 5514 * Returns B_TRUE if the port GID is reachable by sending 5515 * a SA query to get the NODE record for this port GUID. 5516 */ 5517 static boolean_t 5518 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid) 5519 { 5520 sa_node_record_t *resp; 5521 size_t length; 5522 5523 /* 5524 * Verify if it's reachable by getting the node record. 5525 */ 5526 if (ibdm_get_node_record_by_port(sa_hdl, guid, &resp, &length) == 5527 IBDM_SUCCESS) { 5528 kmem_free(resp, length); 5529 return (B_TRUE); 5530 } 5531 return (B_FALSE); 5532 } 5533 5534 /* 5535 * ibdm_get_node_record_by_port 5536 * Sends a SA query to get the NODE record for port GUID 5537 * Returns IBDM_SUCCESS if the port GID is reachable. 5538 * 5539 * Note: the caller must be responsible for freeing the resource 5540 * by calling kmem_free(resp, length) later. 5541 */ 5542 static int 5543 ibdm_get_node_record_by_port(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 5544 sa_node_record_t **resp, size_t *length) 5545 { 5546 sa_node_record_t req; 5547 ibmf_saa_access_args_t args; 5548 int ret; 5549 ASSERT(resp != NULL && length != NULL); 5550 5551 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 5552 guid); 5553 5554 bzero(&req, sizeof (sa_node_record_t)); 5555 req.NodeInfo.PortGUID = guid; 5556 5557 args.sq_attr_id = SA_NODERECORD_ATTRID; 5558 args.sq_access_type = IBMF_SAA_RETRIEVE; 5559 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 5560 args.sq_template = &req; 5561 args.sq_callback = NULL; 5562 args.sq_callback_arg = NULL; 5563 5564 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) resp); 5565 if (ret != IBMF_SUCCESS) { 5566 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 5567 " SA Retrieve Failed: %d", ret); 5568 return (IBDM_FAILURE); 5569 } 5570 if (*resp == NULL || *length == 0) { 5571 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 5572 return (IBDM_FAILURE); 5573 } 5574 /* 5575 * There is one NodeRecord on each endport on a subnet. 5576 */ 5577 ASSERT(*length == sizeof (sa_node_record_t)); 5578 5579 return (IBDM_SUCCESS); 5580 } 5581 5582 5583 /* 5584 * Update the gidlist for all affected IOCs when GID becomes 5585 * available/unavailable. 5586 * 5587 * Parameters : 5588 * gidinfo - Incoming / Outgoing GID. 5589 * add_flag - 1 for GID added, 0 for GID removed. 5590 * - (-1) : IOC gid list updated, ioc_list required. 5591 * 5592 * This function gets the GID for the node GUID corresponding to the 5593 * port GID. Gets the IOU info 5594 */ 5595 static ibdm_ioc_info_t * 5596 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 5597 { 5598 ibdm_dp_gidinfo_t *node_gid = NULL; 5599 uint8_t niocs, ii; 5600 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 5601 5602 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 5603 5604 switch (avail_flag) { 5605 case 1 : 5606 node_gid = ibdm_check_dest_nodeguid(gid_info); 5607 break; 5608 case 0 : 5609 node_gid = ibdm_handle_gid_rm(gid_info); 5610 break; 5611 case -1 : 5612 node_gid = gid_info; 5613 break; 5614 default : 5615 break; 5616 } 5617 5618 if (node_gid == NULL) { 5619 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 5620 "No node GID found, port gid 0x%p, avail_flag %d", 5621 gid_info, avail_flag); 5622 return (NULL); 5623 } 5624 5625 mutex_enter(&node_gid->gl_mutex); 5626 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 5627 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 5628 node_gid->gl_iou == NULL) { 5629 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 5630 "gl_state %x, gl_iou %p", node_gid->gl_state, 5631 node_gid->gl_iou); 5632 mutex_exit(&node_gid->gl_mutex); 5633 return (NULL); 5634 } 5635 5636 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 5637 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 5638 niocs); 5639 for (ii = 0; ii < niocs; ii++) { 5640 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 5641 /* 5642 * Skip IOCs for which probe is not complete or 5643 * reprobe is progress 5644 */ 5645 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 5646 tmp = ibdm_dup_ioc_info(ioc, node_gid); 5647 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 5648 tmp->ioc_next = ioc_list; 5649 ioc_list = tmp; 5650 } 5651 } 5652 mutex_exit(&node_gid->gl_mutex); 5653 5654 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 5655 ioc_list); 5656 return (ioc_list); 5657 } 5658 5659 /* 5660 * ibdm_saa_event_cb : 5661 * Event handling which does *not* require ibdm_hl_mutex to be 5662 * held are executed in the same thread. This is to prevent 5663 * deadlocks with HCA port down notifications which hold the 5664 * ibdm_hl_mutex. 5665 * 5666 * GID_AVAILABLE event is handled here. A taskq is spawned to 5667 * handle GID_UNAVAILABLE. 5668 * 5669 * A new mutex ibdm_ibnex_mutex has been introduced to protect 5670 * ibnex_callback. This has been done to prevent any possible 5671 * deadlock (described above) while handling GID_AVAILABLE. 5672 * 5673 * IBMF calls the event callback for a HCA port. The SA handle 5674 * for this port would be valid, till the callback returns. 5675 * IBDM calling IBDM using the above SA handle should be valid. 5676 * 5677 * IBDM will additionally check (SA handle != NULL), before 5678 * calling IBMF. 5679 */ 5680 /*ARGSUSED*/ 5681 static void 5682 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 5683 ibmf_saa_subnet_event_t ibmf_saa_event, 5684 ibmf_saa_event_details_t *event_details, void *callback_arg) 5685 { 5686 ibdm_saa_event_arg_t *event_arg; 5687 ib_gid_t sgid, dgid; 5688 ibdm_port_attr_t *hca_port; 5689 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 5690 sa_node_record_t *nrec; 5691 size_t length; 5692 5693 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5694 5695 hca_port = (ibdm_port_attr_t *)callback_arg; 5696 5697 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 5698 ibmf_saa_handle, ibmf_saa_event, event_details, 5699 callback_arg); 5700 #ifdef DEBUG 5701 if (ibdm_ignore_saa_event) 5702 return; 5703 #endif 5704 5705 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 5706 /* 5707 * Ensure no other probe / sweep fabric is in 5708 * progress. 5709 */ 5710 mutex_enter(&ibdm.ibdm_mutex); 5711 while (ibdm.ibdm_busy & IBDM_BUSY) 5712 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5713 ibdm.ibdm_busy |= IBDM_BUSY; 5714 mutex_exit(&ibdm.ibdm_mutex); 5715 5716 /* 5717 * If we already know about this GID, return. 5718 * GID_AVAILABLE may be reported for multiple HCA 5719 * ports. 5720 */ 5721 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5722 event_details->ie_gid.gid_prefix)) != NULL) { 5723 mutex_enter(&ibdm.ibdm_mutex); 5724 ibdm.ibdm_busy &= ~IBDM_BUSY; 5725 cv_broadcast(&ibdm.ibdm_busy_cv); 5726 mutex_exit(&ibdm.ibdm_mutex); 5727 return; 5728 } 5729 5730 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5731 "Insertion notified", 5732 event_details->ie_gid.gid_prefix, 5733 event_details->ie_gid.gid_guid); 5734 5735 /* This is a new gid, insert it to GID list */ 5736 sgid.gid_prefix = hca_port->pa_sn_prefix; 5737 sgid.gid_guid = hca_port->pa_port_guid; 5738 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5739 dgid.gid_guid = event_details->ie_gid.gid_guid; 5740 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5741 if (gid_info == NULL) { 5742 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5743 "create_gid_info returned NULL"); 5744 mutex_enter(&ibdm.ibdm_mutex); 5745 ibdm.ibdm_busy &= ~IBDM_BUSY; 5746 cv_broadcast(&ibdm.ibdm_busy_cv); 5747 mutex_exit(&ibdm.ibdm_mutex); 5748 return; 5749 } 5750 mutex_enter(&gid_info->gl_mutex); 5751 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5752 mutex_exit(&gid_info->gl_mutex); 5753 5754 /* Get the node GUID */ 5755 if (ibdm_get_node_record_by_port(ibmf_saa_handle, dgid.gid_guid, 5756 &nrec, &length) != IBDM_SUCCESS) { 5757 /* 5758 * Set the state to PROBE_NOT_DONE for the 5759 * next sweep to probe it 5760 */ 5761 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5762 "Skipping GID : port GUID not found"); 5763 mutex_enter(&gid_info->gl_mutex); 5764 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5765 mutex_exit(&gid_info->gl_mutex); 5766 mutex_enter(&ibdm.ibdm_mutex); 5767 ibdm.ibdm_busy &= ~IBDM_BUSY; 5768 cv_broadcast(&ibdm.ibdm_busy_cv); 5769 mutex_exit(&ibdm.ibdm_mutex); 5770 return; 5771 } 5772 gid_info->gl_nodeguid = nrec->NodeInfo.NodeGUID; 5773 gid_info->gl_devid = nrec->NodeInfo.DeviceID; 5774 kmem_free(nrec, length); 5775 gid_info->gl_portguid = dgid.gid_guid; 5776 5777 /* 5778 * Get the gid info with the same node GUID. 5779 */ 5780 mutex_enter(&ibdm.ibdm_mutex); 5781 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5782 while (node_gid_info) { 5783 if (node_gid_info->gl_nodeguid == 5784 gid_info->gl_nodeguid && 5785 node_gid_info->gl_iou != NULL) { 5786 break; 5787 } 5788 node_gid_info = node_gid_info->gl_next; 5789 } 5790 mutex_exit(&ibdm.ibdm_mutex); 5791 5792 /* 5793 * Handling a new GID requires filling of gl_hca_list. 5794 * This require ibdm hca_list to be parsed and hence 5795 * holding the ibdm_hl_mutex. Spawning a new thread to 5796 * handle this. 5797 */ 5798 if (node_gid_info == NULL) { 5799 if (taskq_dispatch(system_taskq, 5800 ibdm_saa_handle_new_gid, (void *)gid_info, 5801 TQ_NOSLEEP) == NULL) { 5802 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5803 "new_gid taskq_dispatch failed"); 5804 return; 5805 } 5806 } 5807 5808 mutex_enter(&ibdm.ibdm_mutex); 5809 ibdm.ibdm_busy &= ~IBDM_BUSY; 5810 cv_broadcast(&ibdm.ibdm_busy_cv); 5811 mutex_exit(&ibdm.ibdm_mutex); 5812 return; 5813 } 5814 5815 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5816 return; 5817 5818 /* 5819 * GID UNAVAIL EVENT: Try to locate the GID in the GID list. 5820 * If we don't find it we just return. 5821 */ 5822 mutex_enter(&ibdm.ibdm_mutex); 5823 gid_info = ibdm.ibdm_dp_gidlist_head; 5824 while (gid_info) { 5825 if (gid_info->gl_portguid == 5826 event_details->ie_gid.gid_guid) { 5827 break; 5828 } 5829 gid_info = gid_info->gl_next; 5830 } 5831 mutex_exit(&ibdm.ibdm_mutex); 5832 if (gid_info == NULL) { 5833 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5834 "GID for GUID %llX not found during GID UNAVAIL event", 5835 event_details->ie_gid.gid_guid); 5836 return; 5837 } 5838 5839 /* 5840 * If this GID is DM capable, we'll have to check whether this DGID 5841 * is reachable via another port. 5842 */ 5843 if (gid_info->gl_is_dm_capable == B_TRUE) { 5844 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5845 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5846 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5847 event_arg->ibmf_saa_event = ibmf_saa_event; 5848 bcopy(event_details, &event_arg->event_details, 5849 sizeof (ibmf_saa_event_details_t)); 5850 event_arg->callback_arg = callback_arg; 5851 5852 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5853 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5854 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5855 "taskq_dispatch failed"); 5856 ibdm_free_saa_event_arg(event_arg); 5857 return; 5858 } 5859 } 5860 } 5861 5862 /* 5863 * Handle a new GID discovered by GID_AVAILABLE saa event. 5864 */ 5865 void 5866 ibdm_saa_handle_new_gid(void *arg) 5867 { 5868 ibdm_dp_gidinfo_t *gid_info; 5869 ibdm_hca_list_t *hca_list = NULL; 5870 ibdm_port_attr_t *port = NULL; 5871 ibdm_ioc_info_t *ioc_list = NULL; 5872 5873 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5874 5875 gid_info = (ibdm_dp_gidinfo_t *)arg; 5876 5877 /* 5878 * Ensure that no other sweep / probe has completed 5879 * probing this gid. 5880 */ 5881 mutex_enter(&gid_info->gl_mutex); 5882 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5883 mutex_exit(&gid_info->gl_mutex); 5884 return; 5885 } 5886 mutex_exit(&gid_info->gl_mutex); 5887 5888 /* 5889 * Parse HCAs to fill gl_hca_list 5890 */ 5891 mutex_enter(&ibdm.ibdm_hl_mutex); 5892 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5893 ibdm_get_next_port(&hca_list, &port, 1)) { 5894 if (ibdm_port_reachable(port->pa_sa_hdl, 5895 gid_info->gl_portguid) == B_TRUE) { 5896 ibdm_addto_glhcalist(gid_info, hca_list); 5897 } 5898 } 5899 mutex_exit(&ibdm.ibdm_hl_mutex); 5900 5901 /* 5902 * Ensure no other probe / sweep fabric is in 5903 * progress. 5904 */ 5905 mutex_enter(&ibdm.ibdm_mutex); 5906 while (ibdm.ibdm_busy & IBDM_BUSY) 5907 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5908 ibdm.ibdm_busy |= IBDM_BUSY; 5909 mutex_exit(&ibdm.ibdm_mutex); 5910 5911 /* 5912 * New IOU probe it, to check if new IOCs 5913 */ 5914 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5915 "new GID : probing"); 5916 mutex_enter(&ibdm.ibdm_mutex); 5917 ibdm.ibdm_ngid_probes_in_progress++; 5918 mutex_exit(&ibdm.ibdm_mutex); 5919 mutex_enter(&gid_info->gl_mutex); 5920 gid_info->gl_reprobe_flag = 0; 5921 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5922 mutex_exit(&gid_info->gl_mutex); 5923 ibdm_probe_gid_thread((void *)gid_info); 5924 5925 mutex_enter(&ibdm.ibdm_mutex); 5926 ibdm_wait_probe_completion(); 5927 mutex_exit(&ibdm.ibdm_mutex); 5928 5929 if (gid_info->gl_iou == NULL) { 5930 mutex_enter(&ibdm.ibdm_mutex); 5931 ibdm.ibdm_busy &= ~IBDM_BUSY; 5932 cv_broadcast(&ibdm.ibdm_busy_cv); 5933 mutex_exit(&ibdm.ibdm_mutex); 5934 return; 5935 } 5936 5937 /* 5938 * Update GID list in all IOCs affected by this 5939 */ 5940 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5941 5942 /* 5943 * Pass on the IOCs with updated GIDs to IBnexus 5944 */ 5945 if (ioc_list) { 5946 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5947 if (ibdm.ibdm_ibnex_callback != NULL) { 5948 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5949 IBDM_EVENT_IOC_PROP_UPDATE); 5950 } 5951 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5952 } 5953 5954 mutex_enter(&ibdm.ibdm_mutex); 5955 ibdm.ibdm_busy &= ~IBDM_BUSY; 5956 cv_broadcast(&ibdm.ibdm_busy_cv); 5957 mutex_exit(&ibdm.ibdm_mutex); 5958 } 5959 5960 /* 5961 * ibdm_saa_event_taskq : 5962 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 5963 * held. The GID_UNAVAILABLE handling is done in a taskq to 5964 * prevent deadlocks with HCA port down notifications which hold 5965 * ibdm_hl_mutex. 5966 */ 5967 void 5968 ibdm_saa_event_taskq(void *arg) 5969 { 5970 ibdm_saa_event_arg_t *event_arg; 5971 ibmf_saa_handle_t ibmf_saa_handle; 5972 ibmf_saa_subnet_event_t ibmf_saa_event; 5973 ibmf_saa_event_details_t *event_details; 5974 void *callback_arg; 5975 5976 ibdm_dp_gidinfo_t *gid_info; 5977 ibdm_port_attr_t *hca_port, *port = NULL; 5978 ibdm_hca_list_t *hca_list = NULL; 5979 int sa_handle_valid = 0; 5980 ibdm_ioc_info_t *ioc_list = NULL; 5981 5982 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5983 5984 event_arg = (ibdm_saa_event_arg_t *)arg; 5985 ibmf_saa_handle = event_arg->ibmf_saa_handle; 5986 ibmf_saa_event = event_arg->ibmf_saa_event; 5987 event_details = &event_arg->event_details; 5988 callback_arg = event_arg->callback_arg; 5989 5990 ASSERT(callback_arg != NULL); 5991 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 5992 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 5993 ibmf_saa_handle, ibmf_saa_event, event_details, 5994 callback_arg); 5995 5996 hca_port = (ibdm_port_attr_t *)callback_arg; 5997 5998 /* Check if the port_attr is still valid */ 5999 mutex_enter(&ibdm.ibdm_hl_mutex); 6000 for (ibdm_get_next_port(&hca_list, &port, 0); port; 6001 ibdm_get_next_port(&hca_list, &port, 0)) { 6002 if (port == hca_port && port->pa_port_guid == 6003 hca_port->pa_port_guid) { 6004 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 6005 sa_handle_valid = 1; 6006 break; 6007 } 6008 } 6009 mutex_exit(&ibdm.ibdm_hl_mutex); 6010 if (sa_handle_valid == 0) { 6011 ibdm_free_saa_event_arg(event_arg); 6012 return; 6013 } 6014 6015 if (hca_port && (hca_port->pa_sa_hdl == NULL || 6016 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 6017 ibdm_free_saa_event_arg(event_arg); 6018 return; 6019 } 6020 hca_list = NULL; 6021 port = NULL; 6022 6023 /* 6024 * Check if the GID is visible to other HCA ports. 6025 * Return if so. 6026 */ 6027 mutex_enter(&ibdm.ibdm_hl_mutex); 6028 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6029 ibdm_get_next_port(&hca_list, &port, 1)) { 6030 if (ibdm_port_reachable(port->pa_sa_hdl, 6031 event_details->ie_gid.gid_guid) == B_TRUE) { 6032 mutex_exit(&ibdm.ibdm_hl_mutex); 6033 ibdm_free_saa_event_arg(event_arg); 6034 return; 6035 } 6036 } 6037 mutex_exit(&ibdm.ibdm_hl_mutex); 6038 6039 /* 6040 * Ensure no other probe / sweep fabric is in 6041 * progress. 6042 */ 6043 mutex_enter(&ibdm.ibdm_mutex); 6044 while (ibdm.ibdm_busy & IBDM_BUSY) 6045 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 6046 ibdm.ibdm_busy |= IBDM_BUSY; 6047 mutex_exit(&ibdm.ibdm_mutex); 6048 6049 /* 6050 * If this GID is no longer in GID list, return 6051 * GID_UNAVAILABLE may be reported for multiple HCA 6052 * ports. 6053 */ 6054 mutex_enter(&ibdm.ibdm_mutex); 6055 gid_info = ibdm.ibdm_dp_gidlist_head; 6056 while (gid_info) { 6057 if (gid_info->gl_portguid == 6058 event_details->ie_gid.gid_guid) { 6059 break; 6060 } 6061 gid_info = gid_info->gl_next; 6062 } 6063 mutex_exit(&ibdm.ibdm_mutex); 6064 if (gid_info == NULL) { 6065 mutex_enter(&ibdm.ibdm_mutex); 6066 ibdm.ibdm_busy &= ~IBDM_BUSY; 6067 cv_broadcast(&ibdm.ibdm_busy_cv); 6068 mutex_exit(&ibdm.ibdm_mutex); 6069 ibdm_free_saa_event_arg(event_arg); 6070 return; 6071 } 6072 6073 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 6074 "Unavailable notification", 6075 event_details->ie_gid.gid_prefix, 6076 event_details->ie_gid.gid_guid); 6077 6078 /* 6079 * Update GID list in all IOCs affected by this 6080 */ 6081 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 6082 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 6083 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 6084 6085 /* 6086 * Remove GID from the global GID list 6087 * Handle the case where all port GIDs for an 6088 * IOU have been hot-removed. Check both gid_info 6089 * & ioc_info for checking ngids. 6090 */ 6091 mutex_enter(&ibdm.ibdm_mutex); 6092 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 6093 mutex_enter(&gid_info->gl_mutex); 6094 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 6095 mutex_exit(&gid_info->gl_mutex); 6096 } 6097 if (gid_info->gl_prev != NULL) 6098 gid_info->gl_prev->gl_next = gid_info->gl_next; 6099 if (gid_info->gl_next != NULL) 6100 gid_info->gl_next->gl_prev = gid_info->gl_prev; 6101 6102 if (gid_info == ibdm.ibdm_dp_gidlist_head) 6103 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 6104 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 6105 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 6106 ibdm.ibdm_ngids--; 6107 6108 ibdm.ibdm_busy &= ~IBDM_BUSY; 6109 cv_broadcast(&ibdm.ibdm_busy_cv); 6110 mutex_exit(&ibdm.ibdm_mutex); 6111 6112 /* free the hca_list on this gid_info */ 6113 ibdm_delete_glhca_list(gid_info); 6114 6115 mutex_destroy(&gid_info->gl_mutex); 6116 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 6117 6118 /* 6119 * Pass on the IOCs with updated GIDs to IBnexus 6120 */ 6121 if (ioc_list) { 6122 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 6123 "IOC_PROP_UPDATE for %p\n", ioc_list); 6124 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6125 if (ibdm.ibdm_ibnex_callback != NULL) { 6126 (*ibdm.ibdm_ibnex_callback)((void *) 6127 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6128 } 6129 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6130 } 6131 6132 ibdm_free_saa_event_arg(event_arg); 6133 } 6134 6135 6136 static int 6137 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 6138 { 6139 ibdm_gid_t *scan_new, *scan_prev; 6140 int cmp_failed = 0; 6141 6142 ASSERT(new != NULL); 6143 ASSERT(prev != NULL); 6144 6145 /* 6146 * Search for each new gid anywhere in the prev GID list. 6147 * Note that the gid list could have been re-ordered. 6148 */ 6149 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 6150 for (scan_prev = prev, cmp_failed = 1; scan_prev; 6151 scan_prev = scan_prev->gid_next) { 6152 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 6153 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 6154 cmp_failed = 0; 6155 break; 6156 } 6157 } 6158 6159 if (cmp_failed) 6160 return (1); 6161 } 6162 return (0); 6163 } 6164 6165 /* 6166 * This is always called in a single thread 6167 * This function updates the gid_list and serv_list of IOC 6168 * The current gid_list is in ioc_info_t(contains only port 6169 * guids for which probe is done) & gidinfo_t(other port gids) 6170 * The gids in both locations are used for comparision. 6171 */ 6172 static void 6173 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 6174 { 6175 ibdm_gid_t *cur_gid_list; 6176 uint_t cur_nportgids; 6177 6178 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 6179 6180 ioc->ioc_info_updated.ib_prop_updated = 0; 6181 6182 6183 /* Current GID list in gid_info only */ 6184 cur_gid_list = gidinfo->gl_gid; 6185 cur_nportgids = gidinfo->gl_ngids; 6186 6187 if (ioc->ioc_prev_serv_cnt != 6188 ioc->ioc_profile.ioc_service_entries || 6189 ibdm_serv_cmp(&ioc->ioc_serv[0], &ioc->ioc_prev_serv[0], 6190 ioc->ioc_prev_serv_cnt)) 6191 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 6192 6193 if (ioc->ioc_prev_nportgids != cur_nportgids || 6194 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 6195 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 6196 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 6197 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 6198 } 6199 6200 /* Zero out previous entries */ 6201 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 6202 if (ioc->ioc_prev_serv) 6203 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 6204 sizeof (ibdm_srvents_info_t)); 6205 ioc->ioc_prev_serv_cnt = 0; 6206 ioc->ioc_prev_nportgids = 0; 6207 ioc->ioc_prev_serv = NULL; 6208 ioc->ioc_prev_gid_list = NULL; 6209 } 6210 6211 /* 6212 * Handle GID removal. This returns gid_info of an GID for the same 6213 * node GUID, if found. For an GID with IOU information, the same 6214 * gid_info is returned if no gid_info with same node_guid is found. 6215 */ 6216 static ibdm_dp_gidinfo_t * 6217 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 6218 { 6219 ibdm_dp_gidinfo_t *gid_list; 6220 6221 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 6222 6223 if (rm_gid->gl_iou == NULL) { 6224 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 6225 /* 6226 * Search for a GID with same node_guid and 6227 * gl_iou != NULL 6228 */ 6229 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 6230 gid_list = gid_list->gl_next) { 6231 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 6232 == rm_gid->gl_nodeguid)) 6233 break; 6234 } 6235 6236 if (gid_list) 6237 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 6238 6239 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 6240 return (gid_list); 6241 } else { 6242 /* 6243 * Search for a GID with same node_guid and 6244 * gl_iou == NULL 6245 */ 6246 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 6247 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 6248 gid_list = gid_list->gl_next) { 6249 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 6250 == rm_gid->gl_nodeguid)) 6251 break; 6252 } 6253 6254 if (gid_list) { 6255 /* 6256 * Copy the following fields from rm_gid : 6257 * 1. gl_state 6258 * 2. gl_iou 6259 * 3. gl_gid & gl_ngids 6260 * 6261 * Note : Function is synchronized by 6262 * ibdm_busy flag. 6263 * 6264 * Note : Redirect info is initialized if 6265 * any MADs for the GID fail 6266 */ 6267 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 6268 "copying info to GID with gl_iou != NULl"); 6269 gid_list->gl_state = rm_gid->gl_state; 6270 gid_list->gl_iou = rm_gid->gl_iou; 6271 gid_list->gl_gid = rm_gid->gl_gid; 6272 gid_list->gl_ngids = rm_gid->gl_ngids; 6273 6274 /* Remove the GID from gl_gid list */ 6275 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 6276 } else { 6277 /* 6278 * Handle a case where all GIDs to the IOU have 6279 * been removed. 6280 */ 6281 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 6282 "to IOU"); 6283 6284 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 6285 return (rm_gid); 6286 } 6287 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 6288 return (gid_list); 6289 } 6290 } 6291 6292 static void 6293 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 6294 ibdm_dp_gidinfo_t *rm_gid) 6295 { 6296 ibdm_gid_t *tmp, *prev; 6297 6298 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 6299 gid_info, rm_gid); 6300 6301 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 6302 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 6303 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 6304 if (prev == NULL) 6305 gid_info->gl_gid = tmp->gid_next; 6306 else 6307 prev->gid_next = tmp->gid_next; 6308 6309 kmem_free(tmp, sizeof (ibdm_gid_t)); 6310 gid_info->gl_ngids--; 6311 break; 6312 } else { 6313 prev = tmp; 6314 tmp = tmp->gid_next; 6315 } 6316 } 6317 } 6318 6319 static void 6320 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 6321 { 6322 ibdm_gid_t *head = NULL, *new, *tail; 6323 6324 /* First copy the destination */ 6325 for (; dest; dest = dest->gid_next) { 6326 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 6327 new->gid_dgid_hi = dest->gid_dgid_hi; 6328 new->gid_dgid_lo = dest->gid_dgid_lo; 6329 new->gid_next = head; 6330 head = new; 6331 } 6332 6333 /* Insert this to the source */ 6334 if (*src_ptr == NULL) 6335 *src_ptr = head; 6336 else { 6337 for (tail = *src_ptr; tail->gid_next != NULL; 6338 tail = tail->gid_next) 6339 ; 6340 6341 tail->gid_next = head; 6342 } 6343 } 6344 6345 static void 6346 ibdm_free_gid_list(ibdm_gid_t *head) 6347 { 6348 ibdm_gid_t *delete; 6349 6350 for (delete = head; delete; ) { 6351 head = delete->gid_next; 6352 kmem_free(delete, sizeof (ibdm_gid_t)); 6353 delete = head; 6354 } 6355 } 6356 6357 /* 6358 * This function rescans the DM capable GIDs (gl_state is 6359 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 6360 * basically checks if the DM capable GID is reachable. If 6361 * not this is handled the same way as GID_UNAVAILABLE, 6362 * except that notifications are not send to IBnexus. 6363 * 6364 * This function also initializes the ioc_prev_list for 6365 * a particular IOC (when called from probe_ioc, with 6366 * ioc_guidp != NULL) or all IOCs for the gid (called from 6367 * sweep_fabric, ioc_guidp == NULL). 6368 */ 6369 static void 6370 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 6371 { 6372 ibdm_dp_gidinfo_t *gid_info, *tmp; 6373 int ii, niocs, found; 6374 ibdm_hca_list_t *hca_list = NULL; 6375 ibdm_port_attr_t *port = NULL; 6376 ibdm_ioc_info_t *ioc_list; 6377 6378 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 6379 found = 0; 6380 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 6381 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 6382 gid_info = gid_info->gl_next; 6383 continue; 6384 } 6385 6386 /* 6387 * Check if the GID is visible to any HCA ports. 6388 * Return if so. 6389 */ 6390 mutex_enter(&ibdm.ibdm_hl_mutex); 6391 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6392 ibdm_get_next_port(&hca_list, &port, 1)) { 6393 if (ibdm_port_reachable(port->pa_sa_hdl, 6394 gid_info->gl_dgid_lo) == B_TRUE) { 6395 found = 1; 6396 break; 6397 } 6398 } 6399 mutex_exit(&ibdm.ibdm_hl_mutex); 6400 6401 if (found) { 6402 if (gid_info->gl_iou == NULL) { 6403 gid_info = gid_info->gl_next; 6404 continue; 6405 } 6406 6407 /* Intialize the ioc_prev_gid_list */ 6408 niocs = 6409 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 6410 for (ii = 0; ii < niocs; ii++) { 6411 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 6412 6413 if (ioc_guidp == NULL || (*ioc_guidp == 6414 ioc_list->ioc_profile.ioc_guid)) { 6415 /* Add info of GIDs in gid_info also */ 6416 ibdm_addto_gidlist( 6417 &ioc_list->ioc_prev_gid_list, 6418 gid_info->gl_gid); 6419 ioc_list->ioc_prev_nportgids = 6420 gid_info->gl_ngids; 6421 } 6422 } 6423 gid_info = gid_info->gl_next; 6424 continue; 6425 } 6426 6427 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 6428 "deleted port GUID %llx", 6429 gid_info->gl_dgid_lo); 6430 6431 /* 6432 * Update GID list in all IOCs affected by this 6433 */ 6434 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 6435 6436 /* 6437 * Remove GID from the global GID list 6438 * Handle the case where all port GIDs for an 6439 * IOU have been hot-removed. 6440 */ 6441 mutex_enter(&ibdm.ibdm_mutex); 6442 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 6443 mutex_enter(&gid_info->gl_mutex); 6444 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_iou); 6445 mutex_exit(&gid_info->gl_mutex); 6446 } 6447 6448 tmp = gid_info->gl_next; 6449 if (gid_info->gl_prev != NULL) 6450 gid_info->gl_prev->gl_next = gid_info->gl_next; 6451 if (gid_info->gl_next != NULL) 6452 gid_info->gl_next->gl_prev = gid_info->gl_prev; 6453 6454 if (gid_info == ibdm.ibdm_dp_gidlist_head) 6455 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 6456 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 6457 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 6458 ibdm.ibdm_ngids--; 6459 mutex_exit(&ibdm.ibdm_mutex); 6460 6461 /* free the hca_list on this gid_info */ 6462 ibdm_delete_glhca_list(gid_info); 6463 6464 mutex_destroy(&gid_info->gl_mutex); 6465 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 6466 6467 gid_info = tmp; 6468 6469 /* 6470 * Pass on the IOCs with updated GIDs to IBnexus 6471 */ 6472 if (ioc_list) { 6473 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 6474 "IOC_PROP_UPDATE for %p\n", ioc_list); 6475 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6476 if (ibdm.ibdm_ibnex_callback != NULL) { 6477 (*ibdm.ibdm_ibnex_callback)((void *) 6478 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6479 } 6480 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6481 } 6482 } 6483 } 6484 6485 /* 6486 * This function notifies IBnex of IOCs on this GID. 6487 * Notification is for GIDs with gl_reprobe_flag set. 6488 * The flag is set when IOC probe / fabric sweep 6489 * probes a GID starting from CLASS port info. 6490 * 6491 * IBnexus will have information of a reconnected IOC 6492 * if it had probed it before. If this is a new IOC, 6493 * IBnexus ignores the notification. 6494 * 6495 * This function should be called with no locks held. 6496 */ 6497 static void 6498 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 6499 { 6500 ibdm_ioc_info_t *ioc_list; 6501 6502 if (gid_info->gl_reprobe_flag == 0 || 6503 gid_info->gl_iou == NULL) 6504 return; 6505 6506 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 6507 6508 /* 6509 * Pass on the IOCs with updated GIDs to IBnexus 6510 */ 6511 if (ioc_list) { 6512 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6513 if (ibdm.ibdm_ibnex_callback != NULL) { 6514 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 6515 IBDM_EVENT_IOC_PROP_UPDATE); 6516 } 6517 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6518 } 6519 } 6520 6521 6522 static void 6523 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 6524 { 6525 if (arg != NULL) 6526 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 6527 } 6528 6529 /* 6530 * This function parses the list of HCAs and HCA ports 6531 * to return the port_attr of the next HCA port. A port 6532 * connected to IB fabric (port_state active) is returned, 6533 * if connected_flag is set. 6534 */ 6535 static void 6536 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 6537 ibdm_port_attr_t **inp_portp, int connect_flag) 6538 { 6539 int ii; 6540 ibdm_port_attr_t *port, *next_port = NULL; 6541 ibdm_port_attr_t *inp_port; 6542 ibdm_hca_list_t *hca_list; 6543 int found = 0; 6544 6545 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 6546 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 6547 inp_hcap, inp_portp, connect_flag); 6548 6549 hca_list = *inp_hcap; 6550 inp_port = *inp_portp; 6551 6552 if (hca_list == NULL) 6553 hca_list = ibdm.ibdm_hca_list_head; 6554 6555 for (; hca_list; hca_list = hca_list->hl_next) { 6556 for (ii = 0; ii < hca_list->hl_nports; ii++) { 6557 port = &hca_list->hl_port_attr[ii]; 6558 6559 /* 6560 * inp_port != NULL; 6561 * Skip till we find the matching port 6562 */ 6563 if (inp_port && !found) { 6564 if (inp_port == port) 6565 found = 1; 6566 continue; 6567 } 6568 6569 if (!connect_flag) { 6570 next_port = port; 6571 break; 6572 } 6573 6574 if (port->pa_sa_hdl == NULL) 6575 ibdm_initialize_port(port); 6576 if (port->pa_sa_hdl == NULL) 6577 (void) ibdm_fini_port(port); 6578 else if (next_port == NULL && 6579 port->pa_sa_hdl != NULL && 6580 port->pa_state == IBT_PORT_ACTIVE) { 6581 next_port = port; 6582 break; 6583 } 6584 } 6585 6586 if (next_port) 6587 break; 6588 } 6589 6590 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 6591 "returns hca_list %p port %p", hca_list, next_port); 6592 *inp_hcap = hca_list; 6593 *inp_portp = next_port; 6594 } 6595 6596 static void 6597 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 6598 { 6599 ibdm_gid_t *tmp; 6600 6601 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 6602 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 6603 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 6604 6605 mutex_enter(&nodegid->gl_mutex); 6606 tmp->gid_next = nodegid->gl_gid; 6607 nodegid->gl_gid = tmp; 6608 nodegid->gl_ngids++; 6609 mutex_exit(&nodegid->gl_mutex); 6610 } 6611 6612 static void 6613 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 6614 ibdm_hca_list_t *hca) 6615 { 6616 ibdm_hca_list_t *head, *prev = NULL, *temp; 6617 6618 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 6619 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 6620 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 6621 6622 mutex_enter(&gid_info->gl_mutex); 6623 head = gid_info->gl_hca_list; 6624 if (head == NULL) { 6625 head = ibdm_dup_hca_attr(hca); 6626 head->hl_next = NULL; 6627 gid_info->gl_hca_list = head; 6628 mutex_exit(&gid_info->gl_mutex); 6629 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 6630 "gid %p, gl_hca_list %p", gid_info, 6631 gid_info->gl_hca_list); 6632 return; 6633 } 6634 6635 /* Check if already in the list */ 6636 while (head) { 6637 if (head->hl_hca_guid == hca->hl_hca_guid) { 6638 mutex_exit(&gid_info->gl_mutex); 6639 IBTF_DPRINTF_L4(ibdm_string, 6640 "\taddto_glhcalist : gid %p hca %p dup", 6641 gid_info, hca); 6642 return; 6643 } 6644 prev = head; 6645 head = head->hl_next; 6646 } 6647 6648 /* Add this HCA to gl_hca_list */ 6649 temp = ibdm_dup_hca_attr(hca); 6650 temp->hl_next = NULL; 6651 prev->hl_next = temp; 6652 mutex_exit(&gid_info->gl_mutex); 6653 6654 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 6655 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 6656 } 6657 6658 static void 6659 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 6660 { 6661 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 6662 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 6663 6664 mutex_enter(&gid_info->gl_mutex); 6665 if (gid_info->gl_hca_list) 6666 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 6667 gid_info->gl_hca_list = NULL; 6668 mutex_exit(&gid_info->gl_mutex); 6669 } 6670 6671 6672 static void 6673 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 6674 { 6675 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 6676 port_sa_hdl); 6677 6678 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 6679 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 6680 6681 /* Check : Not busy in another probe / sweep */ 6682 mutex_enter(&ibdm.ibdm_mutex); 6683 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 6684 ibdm_dp_gidinfo_t *gid_info; 6685 6686 ibdm.ibdm_busy |= IBDM_BUSY; 6687 mutex_exit(&ibdm.ibdm_mutex); 6688 6689 /* 6690 * Check if any GID is using the SA & IBMF handle 6691 * of HCA port going down. Reset ibdm_dp_gidinfo_t 6692 * using another HCA port which can reach the GID. 6693 * This is for DM capable GIDs only, no need to do 6694 * this for others 6695 * 6696 * Delete the GID if no alternate HCA port to reach 6697 * it is found. 6698 */ 6699 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 6700 ibdm_dp_gidinfo_t *tmp; 6701 6702 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 6703 "checking gidinfo %p", gid_info); 6704 6705 if (gid_info->gl_sa_hdl == port_sa_hdl) { 6706 IBTF_DPRINTF_L3(ibdm_string, 6707 "\tevent_hdlr: down HCA port hdl " 6708 "matches gid %p", gid_info); 6709 6710 /* 6711 * The non-DM GIDs can come back 6712 * with a new subnet prefix, when 6713 * the HCA port commes up again. To 6714 * avoid issues, delete non-DM 6715 * capable GIDs, if the gid was 6716 * discovered using the HCA port 6717 * going down. This is ensured by 6718 * setting gl_disconnected to 1. 6719 */ 6720 if (gid_info->gl_is_dm_capable == B_FALSE) 6721 gid_info->gl_disconnected = 1; 6722 else 6723 ibdm_reset_gidinfo(gid_info); 6724 6725 if (gid_info->gl_disconnected) { 6726 IBTF_DPRINTF_L3(ibdm_string, 6727 "\tevent_hdlr: deleting" 6728 " gid %p", gid_info); 6729 tmp = gid_info; 6730 gid_info = gid_info->gl_next; 6731 ibdm_delete_gidinfo(tmp); 6732 } else 6733 gid_info = gid_info->gl_next; 6734 } else 6735 gid_info = gid_info->gl_next; 6736 } 6737 6738 mutex_enter(&ibdm.ibdm_mutex); 6739 ibdm.ibdm_busy &= ~IBDM_BUSY; 6740 cv_signal(&ibdm.ibdm_busy_cv); 6741 } 6742 mutex_exit(&ibdm.ibdm_mutex); 6743 } 6744 6745 static void 6746 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6747 { 6748 ibdm_hca_list_t *hca_list = NULL; 6749 ibdm_port_attr_t *port = NULL; 6750 int gid_reinited = 0; 6751 sa_node_record_t *nr, *tmp; 6752 sa_portinfo_record_t *pi; 6753 size_t nr_len = 0, pi_len = 0; 6754 size_t path_len; 6755 ib_gid_t sgid, dgid; 6756 int ret, ii, nrecords; 6757 sa_path_record_t *path; 6758 uint8_t npaths = 1; 6759 ibdm_pkey_tbl_t *pkey_tbl; 6760 6761 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6762 6763 /* 6764 * Get list of all the ports reachable from the local known HCA 6765 * ports which are active 6766 */ 6767 mutex_enter(&ibdm.ibdm_hl_mutex); 6768 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6769 ibdm_get_next_port(&hca_list, &port, 1)) { 6770 6771 6772 /* 6773 * Get the path and re-populate the gidinfo. 6774 * Getting the path is the same probe_ioc 6775 * Init the gid info as in ibdm_create_gidinfo() 6776 */ 6777 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6778 gidinfo->gl_nodeguid); 6779 if (nr == NULL) { 6780 IBTF_DPRINTF_L4(ibdm_string, 6781 "\treset_gidinfo : no records"); 6782 continue; 6783 } 6784 6785 nrecords = (nr_len / sizeof (sa_node_record_t)); 6786 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6787 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6788 break; 6789 } 6790 6791 if (ii == nrecords) { 6792 IBTF_DPRINTF_L4(ibdm_string, 6793 "\treset_gidinfo : no record for portguid"); 6794 kmem_free(nr, nr_len); 6795 continue; 6796 } 6797 6798 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6799 if (pi == NULL) { 6800 IBTF_DPRINTF_L4(ibdm_string, 6801 "\treset_gidinfo : no portinfo"); 6802 kmem_free(nr, nr_len); 6803 continue; 6804 } 6805 6806 sgid.gid_prefix = port->pa_sn_prefix; 6807 sgid.gid_guid = port->pa_port_guid; 6808 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6809 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6810 6811 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6812 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6813 6814 if ((ret != IBMF_SUCCESS) || path == NULL) { 6815 IBTF_DPRINTF_L4(ibdm_string, 6816 "\treset_gidinfo : no paths"); 6817 kmem_free(pi, pi_len); 6818 kmem_free(nr, nr_len); 6819 continue; 6820 } 6821 6822 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6823 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6824 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6825 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6826 gidinfo->gl_p_key = path->P_Key; 6827 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6828 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6829 gidinfo->gl_slid = path->SLID; 6830 gidinfo->gl_dlid = path->DLID; 6831 /* Reset redirect info, next MAD will set if redirected */ 6832 gidinfo->gl_redirected = 0; 6833 gidinfo->gl_devid = (*tmp).NodeInfo.DeviceID; 6834 gidinfo->gl_SL = path->SL; 6835 6836 gidinfo->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 6837 for (ii = 0; ii < port->pa_npkeys; ii++) { 6838 if (port->pa_pkey_tbl == NULL) 6839 break; 6840 6841 pkey_tbl = &port->pa_pkey_tbl[ii]; 6842 if ((gidinfo->gl_p_key == pkey_tbl->pt_pkey) && 6843 (pkey_tbl->pt_qp_hdl != NULL)) { 6844 gidinfo->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 6845 break; 6846 } 6847 } 6848 6849 if (gidinfo->gl_qp_hdl == NULL) 6850 IBTF_DPRINTF_L2(ibdm_string, 6851 "\treset_gid_info: No matching Pkey"); 6852 else 6853 gid_reinited = 1; 6854 6855 kmem_free(path, path_len); 6856 kmem_free(pi, pi_len); 6857 kmem_free(nr, nr_len); 6858 break; 6859 } 6860 mutex_exit(&ibdm.ibdm_hl_mutex); 6861 6862 if (!gid_reinited) 6863 gidinfo->gl_disconnected = 1; 6864 } 6865 6866 static void 6867 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6868 { 6869 ibdm_ioc_info_t *ioc_list; 6870 int in_gidlist = 0; 6871 6872 /* 6873 * Check if gidinfo has been inserted into the 6874 * ibdm_dp_gidlist_head list. gl_next or gl_prev 6875 * != NULL, if gidinfo is the list. 6876 */ 6877 if (gidinfo->gl_prev != NULL || 6878 gidinfo->gl_next != NULL || 6879 ibdm.ibdm_dp_gidlist_head == gidinfo) 6880 in_gidlist = 1; 6881 6882 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6883 6884 /* 6885 * Remove GID from the global GID list 6886 * Handle the case where all port GIDs for an 6887 * IOU have been hot-removed. 6888 */ 6889 mutex_enter(&ibdm.ibdm_mutex); 6890 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6891 mutex_enter(&gidinfo->gl_mutex); 6892 (void) ibdm_free_iou_info(gidinfo, &gidinfo->gl_iou); 6893 mutex_exit(&gidinfo->gl_mutex); 6894 } 6895 6896 /* Delete gl_hca_list */ 6897 mutex_exit(&ibdm.ibdm_mutex); 6898 ibdm_delete_glhca_list(gidinfo); 6899 mutex_enter(&ibdm.ibdm_mutex); 6900 6901 if (in_gidlist) { 6902 if (gidinfo->gl_prev != NULL) 6903 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6904 if (gidinfo->gl_next != NULL) 6905 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6906 6907 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6908 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6909 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6910 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6911 ibdm.ibdm_ngids--; 6912 } 6913 mutex_exit(&ibdm.ibdm_mutex); 6914 6915 mutex_destroy(&gidinfo->gl_mutex); 6916 cv_destroy(&gidinfo->gl_probe_cv); 6917 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6918 6919 /* 6920 * Pass on the IOCs with updated GIDs to IBnexus 6921 */ 6922 if (ioc_list) { 6923 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6924 "IOC_PROP_UPDATE for %p\n", ioc_list); 6925 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6926 if (ibdm.ibdm_ibnex_callback != NULL) { 6927 (*ibdm.ibdm_ibnex_callback)((void *) 6928 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6929 } 6930 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6931 } 6932 } 6933 6934 6935 static void 6936 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6937 { 6938 uint32_t attr_mod; 6939 6940 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 6941 attr_mod |= cb_args->cb_srvents_start; 6942 attr_mod |= (cb_args->cb_srvents_end) << 8; 6943 hdr->AttributeModifier = h2b32(attr_mod); 6944 } 6945 6946 static void 6947 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 6948 { 6949 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 6950 gid_info->gl_transactionID++; 6951 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 6952 IBTF_DPRINTF_L4(ibdm_string, 6953 "\tbump_transactionID(%p), wrapup", gid_info); 6954 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 6955 } 6956 } 6957 6958 /* 6959 * gl_prev_iou is set for *non-reprobe* sweeep requests, which 6960 * detected that ChangeID in IOU info has changed. The service 6961 * entry also may have changed. Check if service entry in IOC 6962 * has changed wrt the prev iou, if so notify to IB Nexus. 6963 */ 6964 static ibdm_ioc_info_t * 6965 ibdm_handle_prev_iou() 6966 { 6967 ibdm_dp_gidinfo_t *gid_info; 6968 ibdm_ioc_info_t *ioc_list_head = NULL, *ioc_list; 6969 ibdm_ioc_info_t *prev_ioc, *ioc; 6970 int ii, jj, niocs, prev_niocs; 6971 6972 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 6973 6974 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou enter"); 6975 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 6976 gid_info = gid_info->gl_next) { 6977 if (gid_info->gl_prev_iou == NULL) 6978 continue; 6979 6980 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iou gid %p", 6981 gid_info); 6982 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 6983 prev_niocs = 6984 gid_info->gl_prev_iou->iou_info.iou_num_ctrl_slots; 6985 for (ii = 0; ii < niocs; ii++) { 6986 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 6987 6988 /* Find matching IOC */ 6989 for (jj = 0; jj < prev_niocs; jj++) { 6990 prev_ioc = (ibdm_ioc_info_t *) 6991 &gid_info->gl_prev_iou->iou_ioc_info[jj]; 6992 if (prev_ioc->ioc_profile.ioc_guid == 6993 ioc->ioc_profile.ioc_guid) 6994 break; 6995 } 6996 if (jj == prev_niocs) 6997 prev_ioc = NULL; 6998 if (ioc == NULL || prev_ioc == NULL) 6999 continue; 7000 if ((ioc->ioc_profile.ioc_service_entries != 7001 prev_ioc->ioc_profile.ioc_service_entries) || 7002 ibdm_serv_cmp(&ioc->ioc_serv[0], 7003 &prev_ioc->ioc_serv[0], 7004 ioc->ioc_profile.ioc_service_entries) != 0) { 7005 IBTF_DPRINTF_L4(ibdm_string, 7006 "/thandle_prev_iou modified IOC: " 7007 "current ioc %p, old ioc %p", 7008 ioc, prev_ioc); 7009 mutex_enter(&gid_info->gl_mutex); 7010 ioc_list = ibdm_dup_ioc_info(ioc, gid_info); 7011 mutex_exit(&gid_info->gl_mutex); 7012 ioc_list->ioc_info_updated.ib_prop_updated 7013 = 0; 7014 ioc_list->ioc_info_updated.ib_srv_prop_updated 7015 = 1; 7016 7017 if (ioc_list_head == NULL) 7018 ioc_list_head = ioc_list; 7019 else { 7020 ioc_list_head->ioc_next = ioc_list; 7021 ioc_list_head = ioc_list; 7022 } 7023 } 7024 } 7025 7026 mutex_enter(&gid_info->gl_mutex); 7027 (void) ibdm_free_iou_info(gid_info, &gid_info->gl_prev_iou); 7028 mutex_exit(&gid_info->gl_mutex); 7029 } 7030 IBTF_DPRINTF_L4(ibdm_string, "\thandle_prev_iouret %p", 7031 ioc_list_head); 7032 return (ioc_list_head); 7033 } 7034 7035 /* 7036 * Compares two service entries lists, returns 0 if same, returns 1 7037 * if no match. 7038 */ 7039 static int 7040 ibdm_serv_cmp(ibdm_srvents_info_t *serv1, ibdm_srvents_info_t *serv2, 7041 int nserv) 7042 { 7043 int ii; 7044 7045 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: enter"); 7046 for (ii = 0; ii < nserv; ii++, serv1++, serv2++) { 7047 if (serv1->se_attr.srv_id != serv2->se_attr.srv_id || 7048 bcmp(serv1->se_attr.srv_name, 7049 serv2->se_attr.srv_name, 7050 IB_DM_MAX_SVC_NAME_LEN) != 0) { 7051 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 1"); 7052 return (1); 7053 } 7054 } 7055 IBTF_DPRINTF_L4(ibdm_string, "\tserv_cmp: ret 0"); 7056 return (0); 7057 } 7058 7059 /* For debugging purpose only */ 7060 #ifdef DEBUG 7061 void 7062 ibdm_dump_mad_hdr(ib_mad_hdr_t *mad_hdr) 7063 { 7064 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 7065 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 7066 7067 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 7068 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 7069 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 7070 "\tR Method : 0x%x", 7071 mad_hdr->ClassVersion, mad_hdr->R_Method); 7072 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 7073 "\tTransaction ID : 0x%llx", 7074 b2h16(mad_hdr->Status), b2h64(mad_hdr->TransactionID)); 7075 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 7076 "\tAttribute Modified : 0x%lx", 7077 b2h16(mad_hdr->AttributeID), b2h32(mad_hdr->AttributeModifier)); 7078 } 7079 7080 7081 void 7082 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 7083 { 7084 ib_mad_hdr_t *mad_hdr; 7085 7086 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 7087 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 7088 7089 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 7090 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 7091 ibmf_msg->im_local_addr.ia_remote_lid, 7092 ibmf_msg->im_local_addr.ia_remote_qno); 7093 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x" 7094 " SL : 0x%x", ibmf_msg->im_local_addr.ia_p_key, 7095 ibmf_msg->im_local_addr.ia_q_key, 7096 ibmf_msg->im_local_addr.ia_service_level); 7097 7098 if (flag) 7099 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 7100 else 7101 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 7102 7103 ibdm_dump_mad_hdr(mad_hdr); 7104 } 7105 7106 7107 void 7108 ibdm_dump_path_info(sa_path_record_t *path) 7109 { 7110 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 7111 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 7112 7113 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 7114 path->DGID.gid_prefix, path->DGID.gid_guid); 7115 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 7116 path->SGID.gid_prefix, path->SGID.gid_guid); 7117 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\t\tDlID : %x", 7118 path->SLID, path->DLID); 7119 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x\t\tSL : %x", 7120 path->P_Key, path->SL); 7121 } 7122 7123 7124 void 7125 ibdm_dump_classportinfo(ib_mad_classportinfo_t *classportinfo) 7126 { 7127 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 7128 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 7129 7130 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 7131 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 7132 7133 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%llx", 7134 b2h64(classportinfo->RedirectGID_hi)); 7135 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%llx", 7136 b2h64(classportinfo->RedirectGID_lo)); 7137 IBTF_DPRINTF_L4("ibdm", "\t Redirected TC : 0x%x", 7138 classportinfo->RedirectTC); 7139 IBTF_DPRINTF_L4("ibdm", "\t Redirected SL : 0x%x", 7140 classportinfo->RedirectSL); 7141 IBTF_DPRINTF_L4("ibdm", "\t Redirected FL : 0x%x", 7142 classportinfo->RedirectFL); 7143 IBTF_DPRINTF_L4("ibdm", "\t Redirected LID : 0x%x", 7144 b2h16(classportinfo->RedirectLID)); 7145 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 7146 b2h16(classportinfo->RedirectP_Key)); 7147 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 7148 classportinfo->RedirectQP); 7149 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 7150 b2h32(classportinfo->RedirectQ_Key)); 7151 IBTF_DPRINTF_L4("ibdm", "\t Trap GID hi : 0x%llx", 7152 b2h64(classportinfo->TrapGID_hi)); 7153 IBTF_DPRINTF_L4("ibdm", "\t Trap GID lo : 0x%llx", 7154 b2h64(classportinfo->TrapGID_lo)); 7155 IBTF_DPRINTF_L4("ibdm", "\t Trap TC : 0x%x", 7156 classportinfo->TrapTC); 7157 IBTF_DPRINTF_L4("ibdm", "\t Trap SL : 0x%x", 7158 classportinfo->TrapSL); 7159 IBTF_DPRINTF_L4("ibdm", "\t Trap FL : 0x%x", 7160 classportinfo->TrapFL); 7161 IBTF_DPRINTF_L4("ibdm", "\t Trap LID : 0x%x", 7162 b2h16(classportinfo->TrapLID)); 7163 IBTF_DPRINTF_L4("ibdm", "\t Trap P_Key : 0x%x", 7164 b2h16(classportinfo->TrapP_Key)); 7165 IBTF_DPRINTF_L4("ibdm", "\t Trap HL : 0x%x", 7166 classportinfo->TrapHL); 7167 IBTF_DPRINTF_L4("ibdm", "\t Trap QP : 0x%x", 7168 classportinfo->TrapQP); 7169 IBTF_DPRINTF_L4("ibdm", "\t Trap Q_Key : 0x%x", 7170 b2h32(classportinfo->TrapQ_Key)); 7171 } 7172 7173 7174 void 7175 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 7176 { 7177 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 7178 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 7179 7180 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 7181 b2h16(iou_info->iou_changeid)); 7182 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 7183 iou_info->iou_num_ctrl_slots); 7184 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 7185 iou_info->iou_flag); 7186 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 7187 iou_info->iou_ctrl_list[0]); 7188 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 7189 iou_info->iou_ctrl_list[1]); 7190 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 7191 iou_info->iou_ctrl_list[2]); 7192 } 7193 7194 7195 void 7196 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 7197 { 7198 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 7199 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 7200 7201 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 7202 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 7203 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 7204 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 7205 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 7206 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 7207 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 7208 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 7209 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 7210 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 7211 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 7212 ioc->ioc_rdma_read_qdepth); 7213 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 7214 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 7215 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 7216 ioc->ioc_ctrl_opcap_mask); 7217 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 7218 } 7219 7220 7221 void 7222 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 7223 { 7224 IBTF_DPRINTF_L4("ibdm", 7225 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 7226 7227 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 7228 "Service Name : %s", srv_ents->srv_name); 7229 } 7230 7231 int ibdm_allow_sweep_fabric_timestamp = 1; 7232 7233 void 7234 ibdm_dump_sweep_fabric_timestamp(int flag) 7235 { 7236 static hrtime_t x; 7237 if (flag) { 7238 if (ibdm_allow_sweep_fabric_timestamp) { 7239 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 7240 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 7241 } 7242 x = 0; 7243 } else 7244 x = gethrtime(); 7245 } 7246 #endif 7247