1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * ibdm.c 30 * 31 * This file contains the InifiniBand Device Manager (IBDM) support functions. 32 * IB nexus driver will only be the client for the IBDM module. 33 * 34 * IBDM registers with IBTF for HCA arrival/removal notification. 35 * IBDM registers with SA access to send DM MADs to discover the IOC's behind 36 * the IOU's. 37 * 38 * IB nexus driver registers with IBDM to find the information about the 39 * HCA's and IOC's (behind the IOU) present on the IB fabric. 40 */ 41 42 #include <sys/systm.h> 43 #include <sys/taskq.h> 44 #include <sys/ib/mgt/ibdm/ibdm_impl.h> 45 #include <sys/modctl.h> 46 47 /* Function Prototype declarations */ 48 static int ibdm_free_iou_info(ibdm_dp_gidinfo_t *); 49 static int ibdm_fini(void); 50 static int ibdm_init(void); 51 static int ibdm_get_reachable_ports(ibdm_port_attr_t *, 52 ibdm_hca_list_t *); 53 static ibdm_dp_gidinfo_t *ibdm_check_dgid(ib_guid_t, ib_sn_prefix_t); 54 static ibdm_dp_gidinfo_t *ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *); 55 static int ibdm_send_classportinfo(ibdm_dp_gidinfo_t *); 56 static int ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *); 57 static int ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *); 58 static int ibdm_get_node_port_guids(ibmf_saa_handle_t, ib_lid_t, 59 ib_guid_t *, ib_guid_t *); 60 static int ibdm_retry_command(ibdm_timeout_cb_args_t *); 61 static int ibdm_get_diagcode(ibdm_dp_gidinfo_t *, int); 62 static int ibdm_verify_mad_status(ib_mad_hdr_t *); 63 static int ibdm_handle_redirection(ibmf_msg_t *, 64 ibdm_dp_gidinfo_t *, int *); 65 static void ibdm_wait_probe_completion(void); 66 static void ibdm_sweep_fabric(int); 67 static void ibdm_probe_gid_thread(void *); 68 static void ibdm_wakeup_probe_gid_cv(void); 69 static void ibdm_port_attr_ibmf_init(ibdm_port_attr_t *, ib_pkey_t, int); 70 static int ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *, int); 71 static void ibdm_update_port_attr(ibdm_port_attr_t *); 72 static void ibdm_handle_hca_attach(ib_guid_t); 73 static void ibdm_handle_srventry_mad(ibmf_msg_t *, 74 ibdm_dp_gidinfo_t *, int *); 75 static void ibdm_ibmf_recv_cb(ibmf_handle_t, ibmf_msg_t *, void *); 76 static void ibdm_recv_incoming_mad(void *); 77 static void ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *); 78 static void ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *); 79 static void ibdm_pkt_timeout_hdlr(void *arg); 80 static void ibdm_initialize_port(ibdm_port_attr_t *); 81 static void ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 82 static void ibdm_probe_gid(ibdm_dp_gidinfo_t *); 83 static void ibdm_alloc_send_buffers(ibmf_msg_t *); 84 static void ibdm_free_send_buffers(ibmf_msg_t *); 85 static void ibdm_handle_hca_detach(ib_guid_t); 86 static int ibdm_fini_port(ibdm_port_attr_t *); 87 static int ibdm_uninit_hca(ibdm_hca_list_t *); 88 static void ibdm_handle_iounitinfo(ibmf_handle_t, 89 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 90 static void ibdm_handle_ioc_profile(ibmf_handle_t, 91 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 92 static void ibdm_event_hdlr(void *, ibt_hca_hdl_t, 93 ibt_async_code_t, ibt_async_event_t *); 94 static void ibdm_handle_classportinfo(ibmf_handle_t, 95 ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *); 96 static void ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *, 97 ibdm_dp_gidinfo_t *); 98 99 static ibdm_hca_list_t *ibdm_dup_hca_attr(ibdm_hca_list_t *); 100 static ibdm_ioc_info_t *ibdm_dup_ioc_info(ibdm_ioc_info_t *, 101 ibdm_dp_gidinfo_t *gid_list); 102 static void ibdm_probe_ioc(ib_guid_t, ib_guid_t, int); 103 static ibdm_ioc_info_t *ibdm_is_ioc_present(ib_guid_t, 104 ibdm_dp_gidinfo_t *, int *); 105 static ibdm_port_attr_t *ibdm_get_port_attr(ibt_async_event_t *, 106 ibdm_hca_list_t **); 107 static sa_node_record_t *ibdm_get_node_records(ibmf_saa_handle_t, 108 size_t *, ib_guid_t); 109 static sa_portinfo_record_t *ibdm_get_portinfo(ibmf_saa_handle_t, size_t *, 110 ib_lid_t); 111 static ibdm_dp_gidinfo_t *ibdm_create_gid_info(ibdm_port_attr_t *, 112 ib_gid_t, ib_gid_t); 113 static ibdm_dp_gidinfo_t *ibdm_find_gid(ib_guid_t, ib_guid_t); 114 static int ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *, uint8_t); 115 static ibdm_ioc_info_t *ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *, int); 116 static void ibdm_saa_event_cb(ibmf_saa_handle_t, ibmf_saa_subnet_event_t, 117 ibmf_saa_event_details_t *, void *); 118 static void ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *, 119 ibdm_dp_gidinfo_t *); 120 static ibdm_dp_gidinfo_t *ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *); 121 static void ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *, 122 ibdm_dp_gidinfo_t *); 123 static void ibdm_addto_gidlist(ibdm_gid_t **, ibdm_gid_t *); 124 static void ibdm_free_gid_list(ibdm_gid_t *); 125 static void ibdm_rescan_gidlist(ib_guid_t *ioc_guid); 126 static void ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *); 127 static void ibdm_saa_event_taskq(void *); 128 static void ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *); 129 static void ibdm_get_next_port(ibdm_hca_list_t **, 130 ibdm_port_attr_t **, int); 131 static void ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *, 132 ibdm_dp_gidinfo_t *); 133 static void ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *, 134 ibdm_hca_list_t *); 135 static void ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *); 136 static void ibdm_saa_handle_new_gid(void *); 137 static void ibdm_reset_all_dgids(ibmf_saa_handle_t); 138 static void ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *); 139 static void ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *); 140 static void ibdm_fill_srv_attr_mod(ib_mad_hdr_t *, ibdm_timeout_cb_args_t *); 141 static void ibdm_bump_transactionID(ibdm_dp_gidinfo_t *); 142 143 int ibdm_dft_timeout = IBDM_DFT_TIMEOUT; 144 int ibdm_dft_retry_cnt = IBDM_DFT_NRETRIES; 145 #ifdef DEBUG 146 int ibdm_ignore_saa_event = 0; 147 #endif 148 149 /* Modload support */ 150 static struct modlmisc ibdm_modlmisc = { 151 &mod_miscops, 152 "InfiniBand Device Manager %I%", 153 }; 154 155 struct modlinkage ibdm_modlinkage = { 156 MODREV_1, 157 (void *)&ibdm_modlmisc, 158 NULL 159 }; 160 161 static ibt_clnt_modinfo_t ibdm_ibt_modinfo = { 162 IBTI_V2, 163 IBT_DM, 164 ibdm_event_hdlr, 165 NULL, 166 "ibdm" 167 }; 168 169 /* Global variables */ 170 ibdm_t ibdm; 171 int ibdm_taskq_enable = IBDM_ENABLE_TASKQ_HANDLING; 172 char *ibdm_string = "ibdm"; 173 174 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", 175 ibdm.ibdm_dp_gidlist_head)) 176 177 /* 178 * _init 179 * Loadable module init, called before any other module. 180 * Initialize mutex 181 * Register with IBTF 182 */ 183 int 184 _init(void) 185 { 186 int err; 187 188 IBTF_DPRINTF_L4("ibdm", "\t_init: addr of ibdm %p", &ibdm); 189 190 if ((err = ibdm_init()) != IBDM_SUCCESS) { 191 IBTF_DPRINTF_L2("ibdm", "_init: ibdm_init failed 0x%x", err); 192 (void) ibdm_fini(); 193 return (DDI_FAILURE); 194 } 195 196 if ((err = mod_install(&ibdm_modlinkage)) != 0) { 197 IBTF_DPRINTF_L2("ibdm", "_init: mod_install failed 0x%x", err); 198 (void) ibdm_fini(); 199 } 200 return (err); 201 } 202 203 204 int 205 _fini(void) 206 { 207 int err; 208 209 if ((err = ibdm_fini()) != IBDM_SUCCESS) { 210 IBTF_DPRINTF_L2("ibdm", "_fini: ibdm_fini failed 0x%x", err); 211 (void) ibdm_init(); 212 return (EBUSY); 213 } 214 215 if ((err = mod_remove(&ibdm_modlinkage)) != 0) { 216 IBTF_DPRINTF_L2("ibdm", "_fini: mod_remove failed 0x%x", err); 217 (void) ibdm_init(); 218 } 219 return (err); 220 } 221 222 223 int 224 _info(struct modinfo *modinfop) 225 { 226 return (mod_info(&ibdm_modlinkage, modinfop)); 227 } 228 229 230 /* 231 * ibdm_init(): 232 * Register with IBTF 233 * Allocate memory for the HCAs 234 * Allocate minor-nodes for the HCAs 235 */ 236 static int 237 ibdm_init(void) 238 { 239 int i, hca_count; 240 ib_guid_t *hca_guids; 241 ibt_status_t status; 242 243 IBTF_DPRINTF_L4("ibdm", "\tibdm_init:"); 244 if (!(ibdm.ibdm_state & IBDM_LOCKS_ALLOCED)) { 245 mutex_init(&ibdm.ibdm_mutex, NULL, MUTEX_DEFAULT, NULL); 246 mutex_init(&ibdm.ibdm_hl_mutex, NULL, MUTEX_DEFAULT, NULL); 247 mutex_init(&ibdm.ibdm_ibnex_mutex, NULL, MUTEX_DEFAULT, NULL); 248 mutex_enter(&ibdm.ibdm_mutex); 249 ibdm.ibdm_state |= IBDM_LOCKS_ALLOCED; 250 } 251 252 if (!(ibdm.ibdm_state & IBDM_IBT_ATTACHED)) { 253 if ((status = ibt_attach(&ibdm_ibt_modinfo, NULL, NULL, 254 (void *)&ibdm.ibdm_ibt_clnt_hdl)) != IBT_SUCCESS) { 255 IBTF_DPRINTF_L2("ibdm", "ibdm_init: ibt_attach " 256 "failed %x", status); 257 mutex_exit(&ibdm.ibdm_mutex); 258 return (IBDM_FAILURE); 259 } 260 261 ibdm.ibdm_state |= IBDM_IBT_ATTACHED; 262 mutex_exit(&ibdm.ibdm_mutex); 263 } 264 265 266 if (!(ibdm.ibdm_state & IBDM_HCA_ATTACHED)) { 267 hca_count = ibt_get_hca_list(&hca_guids); 268 IBTF_DPRINTF_L4("ibdm", "ibdm_init: num_hcas = %d", hca_count); 269 for (i = 0; i < hca_count; i++) 270 (void) ibdm_handle_hca_attach(hca_guids[i]); 271 if (hca_count) 272 ibt_free_hca_list(hca_guids, hca_count); 273 274 mutex_enter(&ibdm.ibdm_mutex); 275 ibdm.ibdm_state |= IBDM_HCA_ATTACHED; 276 mutex_exit(&ibdm.ibdm_mutex); 277 } 278 279 if (!(ibdm.ibdm_state & IBDM_CVS_ALLOCED)) { 280 cv_init(&ibdm.ibdm_probe_cv, NULL, CV_DRIVER, NULL); 281 cv_init(&ibdm.ibdm_busy_cv, NULL, CV_DRIVER, NULL); 282 mutex_enter(&ibdm.ibdm_mutex); 283 ibdm.ibdm_state |= IBDM_CVS_ALLOCED; 284 mutex_exit(&ibdm.ibdm_mutex); 285 } 286 return (IBDM_SUCCESS); 287 } 288 289 290 static int 291 ibdm_free_iou_info(ibdm_dp_gidinfo_t *gid_info) 292 { 293 int ii, k, niocs; 294 size_t size; 295 ibdm_gid_t *delete, *head; 296 timeout_id_t timeout_id; 297 ibdm_ioc_info_t *ioc; 298 299 ASSERT(mutex_owned(&gid_info->gl_mutex)); 300 if (gid_info->gl_iou == NULL) { 301 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: No IOU"); 302 return (0); 303 } 304 305 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 306 IBTF_DPRINTF_L4("ibdm", "\tfree_iou_info: gid_info = %p, niocs %d", 307 gid_info, niocs); 308 309 for (ii = 0; ii < niocs; ii++) { 310 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 311 312 /* handle the case where an ioc_timeout_id is scheduled */ 313 if (ioc->ioc_timeout_id) { 314 timeout_id = ioc->ioc_timeout_id; 315 mutex_exit(&gid_info->gl_mutex); 316 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 317 "ioc_timeout_id = 0x%x", timeout_id); 318 if (untimeout(timeout_id) == -1) { 319 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 320 "untimeout ioc_timeout_id failed"); 321 mutex_enter(&gid_info->gl_mutex); 322 return (-1); 323 } 324 mutex_enter(&gid_info->gl_mutex); 325 ioc->ioc_timeout_id = 0; 326 } 327 328 /* handle the case where an ioc_dc_timeout_id is scheduled */ 329 if (ioc->ioc_dc_timeout_id) { 330 timeout_id = ioc->ioc_dc_timeout_id; 331 mutex_exit(&gid_info->gl_mutex); 332 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 333 "ioc_dc_timeout_id = 0x%x", timeout_id); 334 if (untimeout(timeout_id) == -1) { 335 IBTF_DPRINTF_L2("ibdm", "free_iou_info: " 336 "untimeout ioc_dc_timeout_id failed"); 337 mutex_enter(&gid_info->gl_mutex); 338 return (-1); 339 } 340 mutex_enter(&gid_info->gl_mutex); 341 ioc->ioc_dc_timeout_id = 0; 342 } 343 344 /* handle the case where serv[k].se_timeout_id is scheduled */ 345 for (k = 0; k < ioc->ioc_profile.ioc_service_entries; k++) { 346 if (ioc->ioc_serv[k].se_timeout_id) { 347 timeout_id = ioc->ioc_serv[k].se_timeout_id; 348 mutex_exit(&gid_info->gl_mutex); 349 IBTF_DPRINTF_L5("ibdm", "free_iou_info: " 350 "ioc->ioc_serv[%d].se_timeout_id = 0x%x", 351 k, timeout_id); 352 if (untimeout(timeout_id) == -1) { 353 IBTF_DPRINTF_L2("ibdm", "free_iou_info:" 354 " untimeout se_timeout_id failed"); 355 mutex_enter(&gid_info->gl_mutex); 356 return (-1); 357 } 358 mutex_enter(&gid_info->gl_mutex); 359 ioc->ioc_serv[k].se_timeout_id = 0; 360 } 361 } 362 363 /* delete GID list */ 364 head = ioc->ioc_gid_list; 365 while (head) { 366 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: " 367 "Deleting gid_list struct %p", head); 368 delete = head; 369 head = head->gid_next; 370 kmem_free(delete, sizeof (ibdm_gid_t)); 371 } 372 ioc->ioc_gid_list = NULL; 373 374 /* delete ioc_serv */ 375 size = ioc->ioc_profile.ioc_service_entries * 376 sizeof (ibdm_srvents_info_t); 377 if (ioc->ioc_serv && size) { 378 kmem_free(ioc->ioc_serv, size); 379 ioc->ioc_serv = NULL; 380 } 381 } 382 383 IBTF_DPRINTF_L4("ibdm", "\tibdm_free_iou_info: deleting IOU & IOC"); 384 size = sizeof (ibdm_iou_info_t) + niocs * sizeof (ibdm_ioc_info_t); 385 kmem_free(gid_info->gl_iou, size); 386 gid_info->gl_iou = NULL; 387 return (0); 388 } 389 390 391 /* 392 * ibdm_fini(): 393 * Un-register with IBTF 394 * De allocate memory for the GID info 395 */ 396 static int 397 ibdm_fini() 398 { 399 int ii; 400 ibdm_hca_list_t *hca_list, *temp; 401 ibdm_dp_gidinfo_t *gid_info, *tmp; 402 ibdm_gid_t *head, *delete; 403 404 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini"); 405 406 mutex_enter(&ibdm.ibdm_hl_mutex); 407 if (ibdm.ibdm_state & IBDM_IBT_ATTACHED) { 408 if (ibt_detach(ibdm.ibdm_ibt_clnt_hdl) != IBT_SUCCESS) { 409 IBTF_DPRINTF_L2("ibdm", "\t_fini: ibt_detach failed"); 410 mutex_exit(&ibdm.ibdm_hl_mutex); 411 return (IBDM_FAILURE); 412 } 413 ibdm.ibdm_state &= ~IBDM_IBT_ATTACHED; 414 ibdm.ibdm_ibt_clnt_hdl = NULL; 415 } 416 417 hca_list = ibdm.ibdm_hca_list_head; 418 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: nhcas %d", ibdm.ibdm_hca_count); 419 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 420 temp = hca_list; 421 hca_list = hca_list->hl_next; 422 IBTF_DPRINTF_L4("ibdm", "\tibdm_fini: hca %p", temp); 423 if (ibdm_uninit_hca(temp) != IBDM_SUCCESS) { 424 IBTF_DPRINTF_L2("ibdm", "\tibdm_fini: " 425 "uninit_hca %p failed", temp); 426 mutex_exit(&ibdm.ibdm_hl_mutex); 427 return (IBDM_FAILURE); 428 } 429 } 430 mutex_exit(&ibdm.ibdm_hl_mutex); 431 432 mutex_enter(&ibdm.ibdm_mutex); 433 if (ibdm.ibdm_state & IBDM_HCA_ATTACHED) 434 ibdm.ibdm_state &= ~IBDM_HCA_ATTACHED; 435 436 gid_info = ibdm.ibdm_dp_gidlist_head; 437 while (gid_info) { 438 mutex_enter(&gid_info->gl_mutex); 439 (void) ibdm_free_iou_info(gid_info); 440 mutex_exit(&gid_info->gl_mutex); 441 ibdm_delete_glhca_list(gid_info); 442 443 tmp = gid_info; 444 gid_info = gid_info->gl_next; 445 mutex_destroy(&tmp->gl_mutex); 446 head = tmp->gl_gid; 447 while (head) { 448 IBTF_DPRINTF_L4("ibdm", 449 "\tibdm_fini: Deleting gid structs"); 450 delete = head; 451 head = head->gid_next; 452 kmem_free(delete, sizeof (ibdm_gid_t)); 453 } 454 kmem_free(tmp, sizeof (ibdm_dp_gidinfo_t)); 455 } 456 mutex_exit(&ibdm.ibdm_mutex); 457 458 if (ibdm.ibdm_state & IBDM_LOCKS_ALLOCED) { 459 ibdm.ibdm_state &= ~IBDM_LOCKS_ALLOCED; 460 mutex_destroy(&ibdm.ibdm_mutex); 461 mutex_destroy(&ibdm.ibdm_hl_mutex); 462 mutex_destroy(&ibdm.ibdm_ibnex_mutex); 463 } 464 if (ibdm.ibdm_state & IBDM_CVS_ALLOCED) { 465 ibdm.ibdm_state &= ~IBDM_CVS_ALLOCED; 466 cv_destroy(&ibdm.ibdm_probe_cv); 467 cv_destroy(&ibdm.ibdm_busy_cv); 468 } 469 return (IBDM_SUCCESS); 470 } 471 472 473 /* 474 * ibdm_event_hdlr() 475 * 476 * IBDM registers this asynchronous event handler at the time of 477 * ibt_attach. IBDM support the following async events. For other 478 * event, simply returns success. 479 * IBT_HCA_ATTACH_EVENT: 480 * Retrieves the information about all the port that are 481 * present on this HCA, allocates the port attributes 482 * structure and calls IB nexus callback routine with 483 * the port attributes structure as an input argument. 484 * IBT_HCA_DETACH_EVENT: 485 * Retrieves the information about all the ports that are 486 * present on this HCA and calls IB nexus callback with 487 * port guid as an argument 488 * IBT_EVENT_PORT_UP: 489 * Register with IBMF and SA access 490 * Setup IBMF receive callback routine 491 * IBT_EVENT_PORT_DOWN: 492 * Un-Register with IBMF and SA access 493 * Teardown IBMF receive callback routine 494 */ 495 /*ARGSUSED*/ 496 static void 497 ibdm_event_hdlr(void *clnt_hdl, 498 ibt_hca_hdl_t hca_hdl, ibt_async_code_t code, ibt_async_event_t *event) 499 { 500 ibdm_hca_list_t *hca_list; 501 ibdm_port_attr_t *port; 502 ibmf_saa_handle_t port_sa_hdl; 503 504 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: async code 0x%x", code); 505 506 switch (code) { 507 case IBT_HCA_ATTACH_EVENT: /* New HCA registered with IBTF */ 508 ibdm_handle_hca_attach(event->ev_hca_guid); 509 break; 510 511 case IBT_HCA_DETACH_EVENT: /* HCA unregistered with IBTF */ 512 ibdm_handle_hca_detach(event->ev_hca_guid); 513 mutex_enter(&ibdm.ibdm_ibnex_mutex); 514 if (ibdm.ibdm_ibnex_callback != NULL) { 515 (*ibdm.ibdm_ibnex_callback)((void *) 516 &event->ev_hca_guid, IBDM_EVENT_HCA_REMOVED); 517 } 518 mutex_exit(&ibdm.ibdm_ibnex_mutex); 519 break; 520 521 case IBT_EVENT_PORT_UP: 522 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_UP"); 523 mutex_enter(&ibdm.ibdm_hl_mutex); 524 port = ibdm_get_port_attr(event, &hca_list); 525 if (port == NULL) { 526 IBTF_DPRINTF_L2("ibdm", 527 "\tevent_hdlr: HCA not present"); 528 mutex_exit(&ibdm.ibdm_hl_mutex); 529 break; 530 } 531 ibdm_initialize_port(port); 532 hca_list->hl_nports_active++; 533 mutex_exit(&ibdm.ibdm_hl_mutex); 534 break; 535 536 case IBT_ERROR_PORT_DOWN: 537 IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_DOWN"); 538 mutex_enter(&ibdm.ibdm_hl_mutex); 539 port = ibdm_get_port_attr(event, &hca_list); 540 if (port == NULL) { 541 IBTF_DPRINTF_L2("ibdm", 542 "\tevent_hdlr: HCA not present"); 543 mutex_exit(&ibdm.ibdm_hl_mutex); 544 break; 545 } 546 hca_list->hl_nports_active--; 547 port_sa_hdl = port->pa_sa_hdl; 548 (void) ibdm_fini_port(port); 549 mutex_exit(&ibdm.ibdm_hl_mutex); 550 ibdm_reset_all_dgids(port_sa_hdl); 551 break; 552 553 default: /* Ignore all other events/errors */ 554 break; 555 } 556 } 557 558 559 /* 560 * ibdm_initialize_port() 561 * Register with IBMF 562 * Register with SA access 563 * Register a receive callback routine with IBMF. IBMF invokes 564 * this routine whenever a MAD arrives at this port. 565 * Update the port attributes 566 */ 567 static void 568 ibdm_initialize_port(ibdm_port_attr_t *port) 569 { 570 int ii; 571 uint_t nports, size; 572 uint_t pkey_idx; 573 ib_pkey_t pkey; 574 ibt_hca_portinfo_t *pinfop; 575 ibmf_register_info_t ibmf_reg; 576 ibmf_saa_subnet_event_args_t event_args; 577 578 IBTF_DPRINTF_L4("ibdm", "\tinitialize_port:"); 579 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 580 581 /* Check whether the port is active */ 582 if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL, 583 NULL) != IBT_SUCCESS) 584 return; 585 586 if (port->pa_sa_hdl != NULL) 587 return; 588 589 if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num, 590 &pinfop, &nports, &size) != IBT_SUCCESS) { 591 /* This should not occur */ 592 port->pa_npkeys = 0; 593 port->pa_pkey_tbl = NULL; 594 return; 595 } 596 port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix; 597 598 port->pa_state = pinfop->p_linkstate; 599 port->pa_npkeys = pinfop->p_pkey_tbl_sz; 600 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 601 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 602 603 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) 604 port->pa_pkey_tbl[pkey_idx].pt_pkey = 605 pinfop->p_pkey_tbl[pkey_idx]; 606 607 ibt_free_portinfo(pinfop, size); 608 609 event_args.is_event_callback = ibdm_saa_event_cb; 610 event_args.is_event_callback_arg = port; 611 if (ibmf_sa_session_open(port->pa_port_guid, 0, &event_args, 612 IBMF_VERSION, 0, &port->pa_sa_hdl) != IBMF_SUCCESS) { 613 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 614 "sa access registration failed"); 615 return; 616 } 617 ibmf_reg.ir_ci_guid = port->pa_hca_guid; 618 ibmf_reg.ir_port_num = port->pa_port_num; 619 ibmf_reg.ir_client_class = DEV_MGT_MANAGER; 620 621 if (ibmf_register(&ibmf_reg, IBMF_VERSION, 0, NULL, NULL, 622 &port->pa_ibmf_hdl, &port->pa_ibmf_caps) != IBMF_SUCCESS) { 623 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 624 "IBMF registration failed"); 625 (void) ibdm_fini_port(port); 626 return; 627 } 628 if (ibmf_setup_async_cb(port->pa_ibmf_hdl, IBMF_QP_HANDLE_DEFAULT, 629 ibdm_ibmf_recv_cb, 0, 0) != IBMF_SUCCESS) { 630 IBTF_DPRINTF_L2("ibdm", "\tinitialize_port: " 631 "IBMF setup recv cb failed"); 632 (void) ibdm_fini_port(port); 633 return; 634 } 635 636 for (ii = 0; ii < port->pa_npkeys; ii++) { 637 pkey = port->pa_pkey_tbl[ii].pt_pkey; 638 if (IBDM_INVALID_PKEY(pkey)) { 639 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 640 continue; 641 } 642 ibdm_port_attr_ibmf_init(port, pkey, ii); 643 } 644 } 645 646 647 /* 648 * ibdm_port_attr_ibmf_init: 649 * With IBMF - Alloc QP Handle and Setup Async callback 650 */ 651 static void 652 ibdm_port_attr_ibmf_init(ibdm_port_attr_t *port, ib_pkey_t pkey, int ii) 653 { 654 int ret; 655 656 if ((ret = ibmf_alloc_qp(port->pa_ibmf_hdl, pkey, IB_GSI_QKEY, 657 IBMF_ALT_QP_MAD_NO_RMPP, &port->pa_pkey_tbl[ii].pt_qp_hdl)) != 658 IBMF_SUCCESS) { 659 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 660 "IBMF failed to alloc qp %d", ret); 661 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 662 return; 663 } 664 665 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_init: QP handle is %p", 666 port->pa_ibmf_hdl); 667 668 if ((ret = ibmf_setup_async_cb(port->pa_ibmf_hdl, 669 port->pa_pkey_tbl[ii].pt_qp_hdl, ibdm_ibmf_recv_cb, 0, 0)) != 670 IBMF_SUCCESS) { 671 IBTF_DPRINTF_L2("ibdm", "\tport_attr_ibmf_init: " 672 "IBMF setup recv cb failed %d", ret); 673 (void) ibmf_free_qp(port->pa_ibmf_hdl, 674 &port->pa_pkey_tbl[ii].pt_qp_hdl, 0); 675 port->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 676 } 677 } 678 679 680 /* 681 * ibdm_get_port_attr() 682 * Get port attributes from HCA guid and port number 683 * Return pointer to ibdm_port_attr_t on Success 684 * and NULL on failure 685 */ 686 static ibdm_port_attr_t * 687 ibdm_get_port_attr(ibt_async_event_t *event, ibdm_hca_list_t **retval) 688 { 689 ibdm_hca_list_t *hca_list; 690 ibdm_port_attr_t *port_attr; 691 int ii; 692 693 IBTF_DPRINTF_L4("ibdm", "\tget_port_attr: port# %d", event->ev_port); 694 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 695 hca_list = ibdm.ibdm_hca_list_head; 696 while (hca_list) { 697 if (hca_list->hl_hca_guid == event->ev_hca_guid) { 698 for (ii = 0; ii < hca_list->hl_nports; ii++) { 699 port_attr = &hca_list->hl_port_attr[ii]; 700 if (port_attr->pa_port_num == event->ev_port) { 701 *retval = hca_list; 702 return (port_attr); 703 } 704 } 705 } 706 hca_list = hca_list->hl_next; 707 } 708 return (NULL); 709 } 710 711 712 /* 713 * ibdm_update_port_attr() 714 * Update the port attributes 715 */ 716 static void 717 ibdm_update_port_attr(ibdm_port_attr_t *port) 718 { 719 uint_t nports, size; 720 uint_t pkey_idx; 721 ibt_hca_portinfo_t *portinfop; 722 723 IBTF_DPRINTF_L4("ibdm", "\tupdate_port_attr: Begin"); 724 if (ibt_query_hca_ports(port->pa_hca_hdl, 725 port->pa_port_num, &portinfop, &nports, &size) != IBT_SUCCESS) { 726 /* This should not occur */ 727 port->pa_npkeys = 0; 728 port->pa_pkey_tbl = NULL; 729 return; 730 } 731 port->pa_sn_prefix = portinfop->p_sgid_tbl[0].gid_prefix; 732 733 port->pa_state = portinfop->p_linkstate; 734 735 /* 736 * PKey information in portinfo valid only if port is 737 * ACTIVE. Bail out if not. 738 */ 739 if (port->pa_state != IBT_PORT_ACTIVE) { 740 port->pa_npkeys = 0; 741 port->pa_pkey_tbl = NULL; 742 ibt_free_portinfo(portinfop, size); 743 return; 744 } 745 746 port->pa_npkeys = portinfop->p_pkey_tbl_sz; 747 port->pa_pkey_tbl = (ibdm_pkey_tbl_t *)kmem_zalloc( 748 port->pa_npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP); 749 750 for (pkey_idx = 0; pkey_idx < port->pa_npkeys; pkey_idx++) { 751 port->pa_pkey_tbl[pkey_idx].pt_pkey = 752 portinfop->p_pkey_tbl[pkey_idx]; 753 } 754 ibt_free_portinfo(portinfop, size); 755 } 756 757 758 /* 759 * ibdm_handle_hca_attach() 760 */ 761 static void 762 ibdm_handle_hca_attach(ib_guid_t hca_guid) 763 { 764 uint_t size; 765 uint_t ii, nports; 766 ibt_status_t status; 767 ibt_hca_hdl_t hca_hdl; 768 ibt_hca_attr_t *hca_attr; 769 ibdm_hca_list_t *hca_list, *temp; 770 ibdm_port_attr_t *port_attr; 771 ibt_hca_portinfo_t *portinfop; 772 773 IBTF_DPRINTF_L4("ibdm", 774 "\thandle_hca_attach: hca_guid = 0x%llX", hca_guid); 775 776 /* open the HCA first */ 777 if ((status = ibt_open_hca(ibdm.ibdm_ibt_clnt_hdl, hca_guid, 778 &hca_hdl)) != IBT_SUCCESS) { 779 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 780 "open_hca failed, status 0x%x", status); 781 return; 782 } 783 784 hca_attr = (ibt_hca_attr_t *) 785 kmem_alloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 786 /* ibt_query_hca always returns IBT_SUCCESS */ 787 (void) ibt_query_hca(hca_hdl, hca_attr); 788 789 IBTF_DPRINTF_L4("ibdm", "\tvid: 0x%x, pid: 0x%x, ver: 0x%x," 790 " #ports: %d", hca_attr->hca_vendor_id, hca_attr->hca_device_id, 791 hca_attr->hca_version_id, hca_attr->hca_nports); 792 793 if ((status = ibt_query_hca_ports(hca_hdl, 0, &portinfop, &nports, 794 &size)) != IBT_SUCCESS) { 795 IBTF_DPRINTF_L2("ibdm", "\thandle_hca_attach: " 796 "ibt_query_hca_ports failed, status 0x%x", status); 797 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 798 (void) ibt_close_hca(hca_hdl); 799 return; 800 } 801 hca_list = (ibdm_hca_list_t *) 802 kmem_zalloc((sizeof (ibdm_hca_list_t)), KM_SLEEP); 803 hca_list->hl_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 804 (sizeof (ibdm_port_attr_t) * hca_attr->hca_nports), KM_SLEEP); 805 hca_list->hl_hca_guid = hca_attr->hca_node_guid; 806 hca_list->hl_nports = hca_attr->hca_nports; 807 hca_list->hl_attach_time = ddi_get_time(); 808 hca_list->hl_hca_hdl = hca_hdl; 809 810 /* 811 * Init a dummy port attribute for the HCA node 812 * This is for Per-HCA Node. Initialize port_attr : 813 * hca_guid & port_guid -> hca_guid 814 * npkeys, pkey_tbl is NULL 815 * port_num, sn_prefix is 0 816 * vendorid, product_id, dev_version from HCA 817 * pa_state is IBT_PORT_ACTIVE 818 */ 819 hca_list->hl_hca_port_attr = (ibdm_port_attr_t *)kmem_zalloc( 820 sizeof (ibdm_port_attr_t), KM_SLEEP); 821 port_attr = hca_list->hl_hca_port_attr; 822 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 823 port_attr->pa_productid = hca_attr->hca_device_id; 824 port_attr->pa_dev_version = hca_attr->hca_version_id; 825 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 826 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 827 port_attr->pa_port_guid = hca_attr->hca_node_guid; 828 port_attr->pa_state = IBT_PORT_ACTIVE; 829 830 831 for (ii = 0; ii < nports; ii++) { 832 port_attr = &hca_list->hl_port_attr[ii]; 833 port_attr->pa_vendorid = hca_attr->hca_vendor_id; 834 port_attr->pa_productid = hca_attr->hca_device_id; 835 port_attr->pa_dev_version = hca_attr->hca_version_id; 836 port_attr->pa_hca_guid = hca_attr->hca_node_guid; 837 port_attr->pa_hca_hdl = hca_list->hl_hca_hdl; 838 port_attr->pa_port_guid = portinfop[ii].p_sgid_tbl->gid_guid; 839 port_attr->pa_sn_prefix = portinfop[ii].p_sgid_tbl->gid_prefix; 840 port_attr->pa_port_num = portinfop[ii].p_port_num; 841 port_attr->pa_state = portinfop[ii].p_linkstate; 842 843 /* 844 * Register with IBMF, SA access when the port is in 845 * ACTIVE state. Also register a callback routine 846 * with IBMF to receive incoming DM MAD's. 847 * The IBDM event handler takes care of registration of 848 * port which are not active. 849 */ 850 IBTF_DPRINTF_L4("ibdm", 851 "\thandle_hca_attach: port guid %llx Port state 0x%x", 852 port_attr->pa_port_guid, portinfop[ii].p_linkstate); 853 854 if (portinfop[ii].p_linkstate == IBT_PORT_ACTIVE) { 855 mutex_enter(&ibdm.ibdm_hl_mutex); 856 hca_list->hl_nports_active++; 857 ibdm_initialize_port(port_attr); 858 mutex_exit(&ibdm.ibdm_hl_mutex); 859 } 860 } 861 mutex_enter(&ibdm.ibdm_hl_mutex); 862 for (temp = ibdm.ibdm_hca_list_head; temp; temp = temp->hl_next) { 863 if (temp->hl_hca_guid == hca_guid) { 864 IBTF_DPRINTF_L2("ibdm", "hca_attach: HCA %llX " 865 "already seen by IBDM", hca_guid); 866 mutex_exit(&ibdm.ibdm_hl_mutex); 867 (void) ibdm_uninit_hca(hca_list); 868 return; 869 } 870 } 871 ibdm.ibdm_hca_count++; 872 if (ibdm.ibdm_hca_list_head == NULL) { 873 ibdm.ibdm_hca_list_head = hca_list; 874 ibdm.ibdm_hca_list_tail = hca_list; 875 } else { 876 ibdm.ibdm_hca_list_tail->hl_next = hca_list; 877 ibdm.ibdm_hca_list_tail = hca_list; 878 } 879 mutex_exit(&ibdm.ibdm_hl_mutex); 880 mutex_enter(&ibdm.ibdm_ibnex_mutex); 881 if (ibdm.ibdm_ibnex_callback != NULL) { 882 (*ibdm.ibdm_ibnex_callback)((void *) 883 &hca_guid, IBDM_EVENT_HCA_ADDED); 884 } 885 mutex_exit(&ibdm.ibdm_ibnex_mutex); 886 887 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 888 ibt_free_portinfo(portinfop, size); 889 } 890 891 892 /* 893 * ibdm_handle_hca_detach() 894 */ 895 static void 896 ibdm_handle_hca_detach(ib_guid_t hca_guid) 897 { 898 ibdm_hca_list_t *head, *prev = NULL; 899 900 IBTF_DPRINTF_L4("ibdm", 901 "\thandle_hca_detach: hca_guid = 0x%llx", hca_guid); 902 903 /* Make sure no probes are running */ 904 mutex_enter(&ibdm.ibdm_mutex); 905 while (ibdm.ibdm_busy & IBDM_BUSY) 906 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 907 ibdm.ibdm_busy |= IBDM_BUSY; 908 mutex_exit(&ibdm.ibdm_mutex); 909 910 mutex_enter(&ibdm.ibdm_hl_mutex); 911 head = ibdm.ibdm_hca_list_head; 912 while (head) { 913 if (head->hl_hca_guid == hca_guid) { 914 if (prev == NULL) 915 ibdm.ibdm_hca_list_head = head->hl_next; 916 else 917 prev->hl_next = head->hl_next; 918 ibdm.ibdm_hca_count--; 919 break; 920 } 921 prev = head; 922 head = head->hl_next; 923 } 924 mutex_exit(&ibdm.ibdm_hl_mutex); 925 if (ibdm_uninit_hca(head) != IBDM_SUCCESS) 926 (void) ibdm_handle_hca_attach(hca_guid); 927 928 mutex_enter(&ibdm.ibdm_mutex); 929 ibdm.ibdm_busy &= ~IBDM_BUSY; 930 cv_broadcast(&ibdm.ibdm_busy_cv); 931 mutex_exit(&ibdm.ibdm_mutex); 932 } 933 934 935 static int 936 ibdm_uninit_hca(ibdm_hca_list_t *head) 937 { 938 int ii; 939 ibdm_port_attr_t *port_attr; 940 941 for (ii = 0; ii < head->hl_nports; ii++) { 942 port_attr = &head->hl_port_attr[ii]; 943 if (ibdm_fini_port(port_attr) != IBDM_SUCCESS) { 944 IBTF_DPRINTF_L2("ibdm", "uninit_hca: HCA %p port 0x%x " 945 "ibdm_fini_port() failed", head, ii); 946 return (IBDM_FAILURE); 947 } 948 } 949 if (head->hl_hca_hdl) 950 if (ibt_close_hca(head->hl_hca_hdl) != IBT_SUCCESS) 951 return (IBDM_FAILURE); 952 kmem_free(head->hl_port_attr, 953 head->hl_nports * sizeof (ibdm_port_attr_t)); 954 kmem_free(head->hl_hca_port_attr, sizeof (ibdm_port_attr_t)); 955 kmem_free(head, sizeof (ibdm_hca_list_t)); 956 return (IBDM_SUCCESS); 957 } 958 959 960 /* 961 * For each port on the HCA, 962 * 1) Teardown IBMF receive callback function 963 * 2) Unregister with IBMF 964 * 3) Unregister with SA access 965 */ 966 static int 967 ibdm_fini_port(ibdm_port_attr_t *port_attr) 968 { 969 int ii, ibmf_status; 970 971 for (ii = 0; ii < port_attr->pa_npkeys; ii++) { 972 if (port_attr->pa_pkey_tbl == NULL) 973 break; 974 if (!port_attr->pa_pkey_tbl[ii].pt_qp_hdl) 975 continue; 976 if (ibdm_port_attr_ibmf_fini(port_attr, ii) != IBDM_SUCCESS) { 977 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 978 "ibdm_port_attr_ibmf_fini failed for " 979 "port pkey 0x%x", ii); 980 return (IBDM_FAILURE); 981 } 982 } 983 984 if (port_attr->pa_ibmf_hdl) { 985 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 986 IBMF_QP_HANDLE_DEFAULT, 0); 987 if (ibmf_status != IBMF_SUCCESS) { 988 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 989 "ibmf_tear_down_async_cb failed %d", ibmf_status); 990 return (IBDM_FAILURE); 991 } 992 993 ibmf_status = ibmf_unregister(&port_attr->pa_ibmf_hdl, 0); 994 if (ibmf_status != IBMF_SUCCESS) { 995 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 996 "ibmf_unregister failed %d", ibmf_status); 997 return (IBDM_FAILURE); 998 } 999 1000 port_attr->pa_ibmf_hdl = NULL; 1001 } 1002 1003 if (port_attr->pa_sa_hdl) { 1004 ibmf_status = ibmf_sa_session_close(&port_attr->pa_sa_hdl, 0); 1005 if (ibmf_status != IBMF_SUCCESS) { 1006 IBTF_DPRINTF_L4("ibdm", "\tfini_port: " 1007 "ibmf_sa_session_close failed %d", ibmf_status); 1008 return (IBDM_FAILURE); 1009 } 1010 port_attr->pa_sa_hdl = NULL; 1011 } 1012 1013 if (port_attr->pa_pkey_tbl != NULL) { 1014 kmem_free(port_attr->pa_pkey_tbl, 1015 port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 1016 port_attr->pa_pkey_tbl = NULL; 1017 port_attr->pa_npkeys = 0; 1018 } 1019 1020 return (IBDM_SUCCESS); 1021 } 1022 1023 1024 /* 1025 * ibdm_port_attr_ibmf_fini: 1026 * With IBMF - Tear down Async callback and free QP Handle 1027 */ 1028 static int 1029 ibdm_port_attr_ibmf_fini(ibdm_port_attr_t *port_attr, int ii) 1030 { 1031 int ibmf_status; 1032 1033 IBTF_DPRINTF_L5("ibdm", "\tport_attr_ibmf_fini:"); 1034 1035 if (port_attr->pa_pkey_tbl[ii].pt_qp_hdl) { 1036 ibmf_status = ibmf_tear_down_async_cb(port_attr->pa_ibmf_hdl, 1037 port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1038 if (ibmf_status != IBMF_SUCCESS) { 1039 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1040 "ibmf_tear_down_async_cb failed %d", ibmf_status); 1041 return (IBDM_FAILURE); 1042 } 1043 ibmf_status = ibmf_free_qp(port_attr->pa_ibmf_hdl, 1044 &port_attr->pa_pkey_tbl[ii].pt_qp_hdl, 0); 1045 if (ibmf_status != IBMF_SUCCESS) { 1046 IBTF_DPRINTF_L4("ibdm", "\tport_attr_ibmf_fini: " 1047 "ibmf_free_qp failed %d", ibmf_status); 1048 return (IBDM_FAILURE); 1049 } 1050 port_attr->pa_pkey_tbl[ii].pt_qp_hdl = NULL; 1051 } 1052 return (IBDM_SUCCESS); 1053 } 1054 1055 1056 /* 1057 * ibdm_gid_decr_pending: 1058 * decrement gl_pending_cmds. If zero wakeup sleeping threads 1059 */ 1060 static void 1061 ibdm_gid_decr_pending(ibdm_dp_gidinfo_t *gidinfo) 1062 { 1063 mutex_enter(&ibdm.ibdm_mutex); 1064 mutex_enter(&gidinfo->gl_mutex); 1065 if (--gidinfo->gl_pending_cmds == 0) { 1066 /* 1067 * Handle DGID getting removed. 1068 */ 1069 if (gidinfo->gl_disconnected) { 1070 mutex_exit(&gidinfo->gl_mutex); 1071 mutex_exit(&ibdm.ibdm_mutex); 1072 1073 IBTF_DPRINTF_L3(ibdm_string, "\tgid_decr_pending: " 1074 "gidinfo %p hot removal", gidinfo); 1075 ibdm_delete_gidinfo(gidinfo); 1076 1077 mutex_enter(&ibdm.ibdm_mutex); 1078 ibdm.ibdm_ngid_probes_in_progress--; 1079 ibdm_wait_probe_completion(); 1080 mutex_exit(&ibdm.ibdm_mutex); 1081 return; 1082 } 1083 mutex_exit(&gidinfo->gl_mutex); 1084 mutex_exit(&ibdm.ibdm_mutex); 1085 ibdm_notify_newgid_iocs(gidinfo); 1086 mutex_enter(&ibdm.ibdm_mutex); 1087 mutex_enter(&gidinfo->gl_mutex); 1088 1089 ibdm.ibdm_ngid_probes_in_progress--; 1090 ibdm_wait_probe_completion(); 1091 } 1092 mutex_exit(&gidinfo->gl_mutex); 1093 mutex_exit(&ibdm.ibdm_mutex); 1094 } 1095 1096 1097 /* 1098 * ibdm_wait_probe_completion: 1099 * wait for probing to complete 1100 */ 1101 static void 1102 ibdm_wait_probe_completion(void) 1103 { 1104 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1105 if (ibdm.ibdm_ngid_probes_in_progress) { 1106 IBTF_DPRINTF_L4("ibdm", "\twait for probe complete"); 1107 ibdm.ibdm_busy |= IBDM_PROBE_IN_PROGRESS; 1108 while (ibdm.ibdm_busy & IBDM_PROBE_IN_PROGRESS) 1109 cv_wait(&ibdm.ibdm_probe_cv, &ibdm.ibdm_mutex); 1110 } 1111 } 1112 1113 1114 /* 1115 * ibdm_wakeup_probe_gid_cv: 1116 * wakeup waiting threads (based on ibdm_ngid_probes_in_progress) 1117 */ 1118 static void 1119 ibdm_wakeup_probe_gid_cv(void) 1120 { 1121 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1122 if (!ibdm.ibdm_ngid_probes_in_progress) { 1123 IBTF_DPRINTF_L4("ibdm", "wakeup_probe_gid_thread: Wakeup"); 1124 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 1125 cv_broadcast(&ibdm.ibdm_probe_cv); 1126 } 1127 1128 } 1129 1130 1131 /* 1132 * ibdm_sweep_fabric(reprobe_flag) 1133 * Find all possible Managed IOU's and their IOC's that are visible 1134 * to the host. The algorithm used is as follows 1135 * 1136 * Send a "bus walk" request for each port on the host HCA to SA access 1137 * SA returns complete set of GID's that are reachable from 1138 * source port. This is done in parallel. 1139 * 1140 * Initialize GID state to IBDM_GID_PROBE_NOT_DONE 1141 * 1142 * Sort the GID list and eliminate duplicate GID's 1143 * 1) Use DGID for sorting 1144 * 2) use PortGuid for sorting 1145 * Send SA query to retrieve NodeRecord and 1146 * extract PortGuid from that. 1147 * 1148 * Set GID state to IBDM_GID_PROBE_FAILED to all the ports that dont 1149 * support DM MAD's 1150 * Send a "Portinfo" query to get the port capabilities and 1151 * then check for DM MAD's support 1152 * 1153 * Send "ClassPortInfo" request for all the GID's in parallel, 1154 * set the GID state to IBDM_GET_CLASSPORTINFO and wait on the 1155 * cv_signal to complete. 1156 * 1157 * When DM agent on the remote GID sends back the response, IBMF 1158 * invokes DM callback routine. 1159 * 1160 * If the response is proper, send "IOUnitInfo" request and set 1161 * GID state to IBDM_GET_IOUNITINFO. 1162 * 1163 * If the response is proper, send "IocProfileInfo" request to 1164 * all the IOC simultaneously and set GID state to IBDM_GET_IOC_DETAILS. 1165 * 1166 * Send request to get Service entries simultaneously 1167 * 1168 * Signal the waiting thread when received response for all the commands. 1169 * 1170 * Set the GID state to IBDM_GID_PROBE_FAILED when received a error 1171 * response during the probing period. 1172 * 1173 * Note: 1174 * ibdm.ibdm_ngid_probes_in_progress and ibdm_gid_list_t:gl_pending_cmds 1175 * keep track of number commands in progress at any point of time. 1176 * MAD transaction ID is used to identify a particular GID 1177 * TBD: Consider registering the IBMF receive callback on demand 1178 * 1179 * Note: This routine must be called with ibdm.ibdm_mutex held 1180 * TBD: Re probe the failure GID (for certain failures) when requested 1181 * for fabric sweep next time 1182 * 1183 * Parameters : If reprobe_flag is set, All IOCs will be reprobed. 1184 */ 1185 static void 1186 ibdm_sweep_fabric(int reprobe_flag) 1187 { 1188 int ii; 1189 int new_paths = 0; 1190 uint8_t niocs; 1191 taskqid_t tid; 1192 ibdm_ioc_info_t *ioc; 1193 ibdm_hca_list_t *hca_list = NULL; 1194 ibdm_port_attr_t *port = NULL; 1195 ibdm_dp_gidinfo_t *gid_info; 1196 1197 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: Enter"); 1198 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 1199 1200 /* 1201 * Check whether a sweep already in progress. If so, just 1202 * wait for the fabric sweep to complete 1203 */ 1204 while (ibdm.ibdm_busy & IBDM_BUSY) 1205 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 1206 ibdm.ibdm_busy |= IBDM_BUSY; 1207 mutex_exit(&ibdm.ibdm_mutex); 1208 1209 ibdm_dump_sweep_fabric_timestamp(0); 1210 1211 /* Rescan the GID list for any removed GIDs for reprobe */ 1212 if (reprobe_flag) 1213 ibdm_rescan_gidlist(NULL); 1214 1215 /* 1216 * Get list of all the ports reachable from the local known HCA 1217 * ports which are active 1218 */ 1219 mutex_enter(&ibdm.ibdm_hl_mutex); 1220 for (ibdm_get_next_port(&hca_list, &port, 1); port; 1221 ibdm_get_next_port(&hca_list, &port, 1)) { 1222 /* 1223 * Get PATHS to all the reachable ports from 1224 * SGID and update the global ibdm structure. 1225 */ 1226 new_paths = ibdm_get_reachable_ports(port, hca_list); 1227 ibdm.ibdm_ngids += new_paths; 1228 } 1229 mutex_exit(&ibdm.ibdm_hl_mutex); 1230 1231 mutex_enter(&ibdm.ibdm_mutex); 1232 ibdm.ibdm_ngid_probes_in_progress += ibdm.ibdm_ngids; 1233 mutex_exit(&ibdm.ibdm_mutex); 1234 1235 /* Send a request to probe GIDs asynchronously. */ 1236 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1237 gid_info = gid_info->gl_next) { 1238 mutex_enter(&gid_info->gl_mutex); 1239 gid_info->gl_reprobe_flag = reprobe_flag; 1240 mutex_exit(&gid_info->gl_mutex); 1241 1242 /* process newly encountered GIDs */ 1243 tid = taskq_dispatch(system_taskq, ibdm_probe_gid_thread, 1244 (void *)gid_info, TQ_NOSLEEP); 1245 IBTF_DPRINTF_L4("ibdm", "\tsweep_fabric: gid_info = %p" 1246 " taskq_id = %x", gid_info, tid); 1247 /* taskq failed to dispatch call it directly */ 1248 if (tid == NULL) 1249 ibdm_probe_gid_thread((void *)gid_info); 1250 } 1251 1252 mutex_enter(&ibdm.ibdm_mutex); 1253 ibdm_wait_probe_completion(); 1254 1255 /* 1256 * Update the properties, if reprobe_flag is set 1257 * Skip if gl_reprobe_flag is set, this will be 1258 * a re-inserted / new GID, for which notifications 1259 * have already been send. 1260 */ 1261 if (reprobe_flag) { 1262 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 1263 gid_info = gid_info->gl_next) { 1264 if (gid_info->gl_iou == NULL) 1265 continue; 1266 if (gid_info->gl_reprobe_flag) { 1267 gid_info->gl_reprobe_flag = 0; 1268 continue; 1269 } 1270 1271 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1272 for (ii = 0; ii < niocs; ii++) { 1273 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1274 if (ioc) 1275 ibdm_reprobe_update_port_srv(ioc, 1276 gid_info); 1277 } 1278 } 1279 } 1280 ibdm_dump_sweep_fabric_timestamp(1); 1281 1282 ibdm.ibdm_busy &= ~IBDM_BUSY; 1283 cv_broadcast(&ibdm.ibdm_busy_cv); 1284 IBTF_DPRINTF_L5("ibdm", "\tsweep_fabric: EXIT"); 1285 } 1286 1287 1288 /* 1289 * ibdm_probe_gid_thread: 1290 * thread that does the actual work for sweeping the fabric 1291 * for a given GID 1292 */ 1293 static void 1294 ibdm_probe_gid_thread(void *args) 1295 { 1296 int reprobe_flag; 1297 ib_guid_t node_guid; 1298 ib_guid_t port_guid; 1299 ibdm_dp_gidinfo_t *gid_info; 1300 1301 gid_info = (ibdm_dp_gidinfo_t *)args; 1302 reprobe_flag = gid_info->gl_reprobe_flag; 1303 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: gid_info = %p, flag = %d", 1304 gid_info, reprobe_flag); 1305 ASSERT(gid_info != NULL); 1306 ASSERT(gid_info->gl_pending_cmds == 0); 1307 1308 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE && 1309 reprobe_flag == 0) { 1310 /* 1311 * This GID may have been already probed. Send 1312 * in a CLP to check if IOUnitInfo changed? 1313 * Explicitly set gl_reprobe_flag to 0 so that 1314 * IBnex is not notified on completion 1315 */ 1316 if (gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) { 1317 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid_thread: " 1318 "get new IOCs information"); 1319 mutex_enter(&gid_info->gl_mutex); 1320 gid_info->gl_pending_cmds++; 1321 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1322 gid_info->gl_reprobe_flag = 0; 1323 mutex_exit(&gid_info->gl_mutex); 1324 if (ibdm_send_iounitinfo(gid_info) != IBDM_SUCCESS) { 1325 mutex_enter(&gid_info->gl_mutex); 1326 gid_info->gl_pending_cmds = 0; 1327 mutex_exit(&gid_info->gl_mutex); 1328 mutex_enter(&ibdm.ibdm_mutex); 1329 --ibdm.ibdm_ngid_probes_in_progress; 1330 ibdm_wakeup_probe_gid_cv(); 1331 mutex_exit(&ibdm.ibdm_mutex); 1332 } 1333 } else { 1334 mutex_enter(&ibdm.ibdm_mutex); 1335 --ibdm.ibdm_ngid_probes_in_progress; 1336 ibdm_wakeup_probe_gid_cv(); 1337 mutex_exit(&ibdm.ibdm_mutex); 1338 } 1339 return; 1340 } else if (reprobe_flag && gid_info->gl_state == 1341 IBDM_GID_PROBING_COMPLETE) { 1342 /* 1343 * Reprobe all IOCs for the GID which has completed 1344 * probe. Skip other port GIDs to same IOU. 1345 * Explicitly set gl_reprobe_flag to 0 so that 1346 * IBnex is not notified on completion 1347 */ 1348 ibdm_ioc_info_t *ioc_info; 1349 uint8_t niocs, ii; 1350 1351 ASSERT(gid_info->gl_iou); 1352 mutex_enter(&gid_info->gl_mutex); 1353 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 1354 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 1355 gid_info->gl_pending_cmds += niocs; 1356 gid_info->gl_reprobe_flag = 0; 1357 mutex_exit(&gid_info->gl_mutex); 1358 for (ii = 0; ii < niocs; ii++) { 1359 uchar_t slot_info; 1360 ib_dm_io_unitinfo_t *giou_info; 1361 1362 /* 1363 * Check whether IOC is present in the slot 1364 * Series of nibbles (in the field 1365 * iou_ctrl_list) represents a slot in the 1366 * IOU. 1367 * Byte format: 76543210 1368 * Bits 0-3 of first byte represent Slot 2 1369 * bits 4-7 of first byte represent slot 1, 1370 * bits 0-3 of second byte represent slot 4 1371 * and so on 1372 * Each 4-bit nibble has the following meaning 1373 * 0x0 : IOC not installed 1374 * 0x1 : IOC is present 1375 * 0xf : Slot does not exist 1376 * and all other values are reserved. 1377 */ 1378 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 1379 giou_info = &gid_info->gl_iou->iou_info; 1380 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 1381 if ((ii % 2) == 0) 1382 slot_info = (slot_info >> 4); 1383 1384 if ((slot_info & 0xf) != 1) { 1385 ioc_info->ioc_state = 1386 IBDM_IOC_STATE_PROBE_FAILED; 1387 ibdm_gid_decr_pending(gid_info); 1388 continue; 1389 } 1390 1391 if (ibdm_send_ioc_profile(gid_info, ii) != 1392 IBDM_SUCCESS) { 1393 ibdm_gid_decr_pending(gid_info); 1394 } 1395 } 1396 1397 return; 1398 } else if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 1399 mutex_enter(&ibdm.ibdm_mutex); 1400 --ibdm.ibdm_ngid_probes_in_progress; 1401 ibdm_wakeup_probe_gid_cv(); 1402 mutex_exit(&ibdm.ibdm_mutex); 1403 return; 1404 } 1405 1406 mutex_enter(&gid_info->gl_mutex); 1407 gid_info->gl_pending_cmds++; 1408 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 1409 mutex_exit(&gid_info->gl_mutex); 1410 1411 /* 1412 * Check whether the destination GID supports DM agents. If 1413 * not, stop probing the GID and continue with the next GID 1414 * in the list. 1415 */ 1416 if (ibdm_is_dev_mgt_supported(gid_info) != IBDM_SUCCESS) { 1417 mutex_enter(&gid_info->gl_mutex); 1418 gid_info->gl_pending_cmds = 0; 1419 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1420 mutex_exit(&gid_info->gl_mutex); 1421 ibdm_delete_glhca_list(gid_info); 1422 mutex_enter(&ibdm.ibdm_mutex); 1423 --ibdm.ibdm_ngid_probes_in_progress; 1424 ibdm_wakeup_probe_gid_cv(); 1425 mutex_exit(&ibdm.ibdm_mutex); 1426 return; 1427 } 1428 1429 /* Get the nodeguid and portguid of the port */ 1430 if (ibdm_get_node_port_guids(gid_info->gl_sa_hdl, gid_info->gl_dlid, 1431 &node_guid, &port_guid) != IBDM_SUCCESS) { 1432 mutex_enter(&gid_info->gl_mutex); 1433 gid_info->gl_pending_cmds = 0; 1434 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1435 mutex_exit(&gid_info->gl_mutex); 1436 ibdm_delete_glhca_list(gid_info); 1437 mutex_enter(&ibdm.ibdm_mutex); 1438 --ibdm.ibdm_ngid_probes_in_progress; 1439 ibdm_wakeup_probe_gid_cv(); 1440 mutex_exit(&ibdm.ibdm_mutex); 1441 return; 1442 } 1443 1444 /* 1445 * Check whether we already knew about this NodeGuid 1446 * If so, do not probe the GID and continue with the 1447 * next GID in the gid list. Set the GID state to 1448 * probing done. 1449 */ 1450 mutex_enter(&ibdm.ibdm_mutex); 1451 gid_info->gl_nodeguid = node_guid; 1452 gid_info->gl_portguid = port_guid; 1453 if (ibdm_check_dest_nodeguid(gid_info) != NULL) { 1454 mutex_exit(&ibdm.ibdm_mutex); 1455 mutex_enter(&gid_info->gl_mutex); 1456 gid_info->gl_pending_cmds = 0; 1457 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 1458 mutex_exit(&gid_info->gl_mutex); 1459 ibdm_delete_glhca_list(gid_info); 1460 mutex_enter(&ibdm.ibdm_mutex); 1461 --ibdm.ibdm_ngid_probes_in_progress; 1462 ibdm_wakeup_probe_gid_cv(); 1463 mutex_exit(&ibdm.ibdm_mutex); 1464 return; 1465 } 1466 ibdm_add_to_gl_gid(gid_info, gid_info); 1467 mutex_exit(&ibdm.ibdm_mutex); 1468 1469 /* 1470 * New or reinserted GID : Enable notification to IBnex 1471 */ 1472 mutex_enter(&gid_info->gl_mutex); 1473 gid_info->gl_reprobe_flag = 1; 1474 mutex_exit(&gid_info->gl_mutex); 1475 1476 /* 1477 * Send ClassPortInfo request to the GID asynchronously. 1478 */ 1479 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 1480 mutex_enter(&gid_info->gl_mutex); 1481 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 1482 gid_info->gl_pending_cmds = 0; 1483 mutex_exit(&gid_info->gl_mutex); 1484 ibdm_delete_glhca_list(gid_info); 1485 mutex_enter(&ibdm.ibdm_mutex); 1486 --ibdm.ibdm_ngid_probes_in_progress; 1487 ibdm_wakeup_probe_gid_cv(); 1488 mutex_exit(&ibdm.ibdm_mutex); 1489 return; 1490 } 1491 } 1492 1493 1494 /* 1495 * ibdm_check_dest_nodeguid 1496 * Searches for the NodeGuid in the GID list 1497 * Returns matching gid_info if found and otherwise NULL 1498 * 1499 * This function is called to handle new GIDs discovered 1500 * during device sweep / probe or for GID_AVAILABLE event. 1501 * 1502 * Parameter : 1503 * gid_info GID to check 1504 */ 1505 static ibdm_dp_gidinfo_t * 1506 ibdm_check_dest_nodeguid(ibdm_dp_gidinfo_t *gid_info) 1507 { 1508 ibdm_dp_gidinfo_t *gid_list; 1509 ibdm_gid_t *tmp; 1510 1511 IBTF_DPRINTF_L4("ibdm", "\tcheck_dest_nodeguid"); 1512 1513 gid_list = ibdm.ibdm_dp_gidlist_head; 1514 while (gid_list) { 1515 if ((gid_list != gid_info) && 1516 (gid_info->gl_nodeguid == gid_list->gl_nodeguid)) { 1517 IBTF_DPRINTF_L4("ibdm", 1518 "\tcheck_dest_nodeguid: NodeGuid is present"); 1519 1520 /* Add to gid_list */ 1521 tmp = kmem_zalloc(sizeof (ibdm_gid_t), 1522 KM_SLEEP); 1523 tmp->gid_dgid_hi = gid_info->gl_dgid_hi; 1524 tmp->gid_dgid_lo = gid_info->gl_dgid_lo; 1525 tmp->gid_next = gid_list->gl_gid; 1526 gid_list->gl_gid = tmp; 1527 gid_list->gl_ngids++; 1528 return (gid_list); 1529 } 1530 1531 gid_list = gid_list->gl_next; 1532 } 1533 1534 return (NULL); 1535 } 1536 1537 1538 /* 1539 * ibdm_is_dev_mgt_supported 1540 * Get the PortInfo attribute (SA Query) 1541 * Check "CompatabilityMask" field in the Portinfo. 1542 * Return IBDM_SUCCESS if DM MAD's supported (if bit 19 set) 1543 * by the port, otherwise IBDM_FAILURE 1544 */ 1545 static int 1546 ibdm_is_dev_mgt_supported(ibdm_dp_gidinfo_t *gid_info) 1547 { 1548 int ret; 1549 size_t length = 0; 1550 sa_portinfo_record_t req, *resp = NULL; 1551 ibmf_saa_access_args_t qargs; 1552 1553 bzero(&req, sizeof (sa_portinfo_record_t)); 1554 req.EndportLID = gid_info->gl_dlid; 1555 1556 qargs.sq_attr_id = SA_PORTINFORECORD_ATTRID; 1557 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1558 qargs.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 1559 qargs.sq_template = &req; 1560 qargs.sq_callback = NULL; 1561 qargs.sq_callback_arg = NULL; 1562 1563 ret = ibmf_sa_access(gid_info->gl_sa_hdl, 1564 &qargs, 0, &length, (void **)&resp); 1565 1566 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1567 IBTF_DPRINTF_L2("ibdm", "\tis_dev_mgt_supported:" 1568 "failed to get PORTINFO attribute %d", ret); 1569 return (IBDM_FAILURE); 1570 } 1571 1572 if (resp->PortInfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD) { 1573 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: SUPPD !!"); 1574 ret = IBDM_SUCCESS; 1575 } else { 1576 IBTF_DPRINTF_L4("ibdm", "\tis_dev_mgt_supported: " 1577 "Not SUPPD !!, cap 0x%x", resp->PortInfo.CapabilityMask); 1578 ret = IBDM_FAILURE; 1579 } 1580 kmem_free(resp, length); 1581 return (ret); 1582 } 1583 1584 1585 /* 1586 * ibdm_get_node_port_guids() 1587 * Get the NodeInfoRecord of the port 1588 * Save NodeGuid and PortGUID values in the GID list structure. 1589 * Return IBDM_SUCCESS/IBDM_FAILURE 1590 */ 1591 static int 1592 ibdm_get_node_port_guids(ibmf_saa_handle_t sa_hdl, ib_lid_t dlid, 1593 ib_guid_t *node_guid, ib_guid_t *port_guid) 1594 { 1595 int ret; 1596 size_t length = 0; 1597 sa_node_record_t req, *resp = NULL; 1598 ibmf_saa_access_args_t qargs; 1599 1600 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids"); 1601 1602 bzero(&req, sizeof (sa_node_record_t)); 1603 req.LID = dlid; 1604 1605 qargs.sq_attr_id = SA_NODERECORD_ATTRID; 1606 qargs.sq_access_type = IBMF_SAA_RETRIEVE; 1607 qargs.sq_component_mask = SA_NODEINFO_COMPMASK_NODELID; 1608 qargs.sq_template = &req; 1609 qargs.sq_callback = NULL; 1610 qargs.sq_callback_arg = NULL; 1611 1612 ret = ibmf_sa_access(sa_hdl, &qargs, 0, &length, (void **)&resp); 1613 if ((ret != IBMF_SUCCESS) || (length == 0) || (resp == NULL)) { 1614 IBTF_DPRINTF_L2("ibdm", "\tget_node_port_guids:" 1615 " SA Retrieve Failed: %d", ret); 1616 return (IBDM_FAILURE); 1617 } 1618 IBTF_DPRINTF_L4("ibdm", "\tget_node_port_guids: NodeGuid %llx Port" 1619 "GUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.NodeGUID); 1620 1621 *node_guid = resp->NodeInfo.NodeGUID; 1622 *port_guid = resp->NodeInfo.PortGUID; 1623 kmem_free(resp, length); 1624 return (IBDM_SUCCESS); 1625 } 1626 1627 1628 /* 1629 * ibdm_get_reachable_ports() 1630 * Get list of the destination GID (and its path records) by 1631 * querying the SA access. 1632 * 1633 * Returns Number paths 1634 */ 1635 static int 1636 ibdm_get_reachable_ports(ibdm_port_attr_t *portinfo, ibdm_hca_list_t *hca) 1637 { 1638 uint_t ii, jj, nrecs; 1639 uint_t npaths = 0; 1640 size_t length; 1641 ib_gid_t sgid; 1642 ibdm_pkey_tbl_t *pkey_tbl; 1643 sa_path_record_t *result; 1644 sa_path_record_t *precp; 1645 ibdm_dp_gidinfo_t *gid_info; 1646 1647 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 1648 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: portinfo %p", portinfo); 1649 1650 sgid.gid_prefix = portinfo->pa_sn_prefix; 1651 sgid.gid_guid = portinfo->pa_port_guid; 1652 1653 /* get reversible paths */ 1654 if (portinfo->pa_sa_hdl && ibmf_saa_paths_from_gid(portinfo->pa_sa_hdl, 1655 sgid, IBMF_SAA_PKEY_WC, B_TRUE, 0, &nrecs, &length, &result) 1656 != IBMF_SUCCESS) { 1657 IBTF_DPRINTF_L2("ibdm", 1658 "\tget_reachable_ports: Getting path records failed"); 1659 return (0); 1660 } 1661 1662 for (ii = 0; ii < nrecs; ii++) { 1663 precp = &result[ii]; 1664 if ((gid_info = ibdm_check_dgid(precp->DGID.gid_guid, 1665 precp->DGID.gid_prefix)) != NULL) { 1666 IBTF_DPRINTF_L2("ibdm", "\tget_reachable_ports: " 1667 "Already exists nrecs %d, ii %d", nrecs, ii); 1668 ibdm_addto_glhcalist(gid_info, hca); 1669 continue; 1670 } 1671 /* 1672 * This is a new GID. Allocate a GID structure and 1673 * initialize the structure 1674 * gl_state is initialized to IBDM_GID_PROBE_NOT_DONE (0) 1675 * by kmem_zalloc call 1676 */ 1677 gid_info = kmem_zalloc(sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 1678 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 1679 gid_info->gl_dgid_hi = precp->DGID.gid_prefix; 1680 gid_info->gl_dgid_lo = precp->DGID.gid_guid; 1681 gid_info->gl_sgid_hi = precp->SGID.gid_prefix; 1682 gid_info->gl_sgid_lo = precp->SGID.gid_guid; 1683 gid_info->gl_p_key = precp->P_Key; 1684 gid_info->gl_sa_hdl = portinfo->pa_sa_hdl; 1685 gid_info->gl_ibmf_hdl = portinfo->pa_ibmf_hdl; 1686 gid_info->gl_slid = precp->SLID; 1687 gid_info->gl_dlid = precp->DLID; 1688 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 1689 << IBDM_GID_TRANSACTIONID_SHIFT; 1690 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 1691 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 1692 << IBDM_GID_TRANSACTIONID_SHIFT; 1693 ibdm_addto_glhcalist(gid_info, hca); 1694 1695 ibdm_dump_path_info(precp); 1696 1697 gid_info->gl_qp_hdl = NULL; 1698 ASSERT(portinfo->pa_pkey_tbl != NULL && 1699 portinfo->pa_npkeys != 0); 1700 1701 for (jj = 0; jj < portinfo->pa_npkeys; jj++) { 1702 pkey_tbl = &portinfo->pa_pkey_tbl[jj]; 1703 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 1704 (pkey_tbl->pt_qp_hdl != NULL)) { 1705 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 1706 break; 1707 } 1708 } 1709 1710 /* 1711 * QP handle for GID not initialized. No matching Pkey 1712 * was found!! ibdm should *not* hit this case. Flag an 1713 * error and drop the GID if ibdm does encounter this. 1714 */ 1715 if (gid_info->gl_qp_hdl == NULL) { 1716 IBTF_DPRINTF_L2(ibdm_string, 1717 "\tget_reachable_ports: No matching Pkey"); 1718 ibdm_delete_gidinfo(gid_info); 1719 continue; 1720 } 1721 if (ibdm.ibdm_dp_gidlist_head == NULL) { 1722 ibdm.ibdm_dp_gidlist_head = gid_info; 1723 ibdm.ibdm_dp_gidlist_tail = gid_info; 1724 } else { 1725 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 1726 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 1727 ibdm.ibdm_dp_gidlist_tail = gid_info; 1728 } 1729 npaths++; 1730 } 1731 kmem_free(result, length); 1732 IBTF_DPRINTF_L4("ibdm", "\tget_reachable_ports: npaths = %d", npaths); 1733 return (npaths); 1734 } 1735 1736 1737 /* 1738 * ibdm_check_dgid() 1739 * Look in the global list to check whether we know this DGID already 1740 * Return IBDM_GID_PRESENT/IBDM_GID_NOT_PRESENT 1741 */ 1742 static ibdm_dp_gidinfo_t * 1743 ibdm_check_dgid(ib_guid_t guid, ib_sn_prefix_t prefix) 1744 { 1745 ibdm_dp_gidinfo_t *gid_list; 1746 1747 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1748 gid_list = gid_list->gl_next) { 1749 if ((guid == gid_list->gl_dgid_lo) && 1750 (prefix == gid_list->gl_dgid_hi)) { 1751 break; 1752 } 1753 } 1754 return (gid_list); 1755 } 1756 1757 1758 /* 1759 * ibdm_find_gid() 1760 * Look in the global list to find a GID entry with matching 1761 * port & node GUID. 1762 * Return pointer to gidinfo if found, else return NULL 1763 */ 1764 static ibdm_dp_gidinfo_t * 1765 ibdm_find_gid(ib_guid_t nodeguid, ib_guid_t portguid) 1766 { 1767 ibdm_dp_gidinfo_t *gid_list; 1768 1769 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid(%llx, %llx)\n", 1770 nodeguid, portguid); 1771 1772 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 1773 gid_list = gid_list->gl_next) { 1774 if ((portguid == gid_list->gl_portguid) && 1775 (nodeguid == gid_list->gl_nodeguid)) { 1776 break; 1777 } 1778 } 1779 1780 IBTF_DPRINTF_L4("ibdm", "ibdm_find_gid : returned %p\n", 1781 gid_list); 1782 return (gid_list); 1783 } 1784 1785 1786 /* 1787 * ibdm_send_classportinfo() 1788 * Send classportinfo request. When the request is completed 1789 * IBMF calls ibdm_classportinfo_cb routine to inform about 1790 * the completion. 1791 * Returns IBDM_SUCCESS/IBDM_FAILURE 1792 */ 1793 static int 1794 ibdm_send_classportinfo(ibdm_dp_gidinfo_t *gid_info) 1795 { 1796 ibmf_msg_t *msg; 1797 ib_mad_hdr_t *hdr; 1798 ibdm_timeout_cb_args_t *cb_args; 1799 1800 IBTF_DPRINTF_L4("ibdm", 1801 "\tsend_classportinfo: gid info 0x%p", gid_info); 1802 1803 /* 1804 * Send command to get classportinfo attribute. Allocate a IBMF 1805 * packet and initialize the packet. 1806 */ 1807 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 1808 &msg) != IBMF_SUCCESS) { 1809 IBTF_DPRINTF_L4("ibdm", "\tsend_classportinfo: pkt alloc fail"); 1810 return (IBDM_FAILURE); 1811 } 1812 1813 ibdm_alloc_send_buffers(msg); 1814 1815 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1816 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1817 msg->im_local_addr.ia_remote_qno = 1; 1818 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1819 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1820 1821 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1822 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1823 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1824 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1825 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1826 hdr->Status = 0; 1827 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1828 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 1829 hdr->AttributeModifier = 0; 1830 1831 cb_args = &gid_info->gl_cpi_cb_args; 1832 cb_args->cb_gid_info = gid_info; 1833 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 1834 cb_args->cb_req_type = IBDM_REQ_TYPE_CLASSPORTINFO; 1835 1836 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1837 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1838 1839 IBTF_DPRINTF_L5("ibdm", "\tsend_classportinfo: " 1840 "timeout id %x", gid_info->gl_timeout_id); 1841 1842 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 1843 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 1844 IBTF_DPRINTF_L2("ibdm", 1845 "\tsend_classportinfo: ibmf send failed"); 1846 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 1847 } 1848 1849 return (IBDM_SUCCESS); 1850 } 1851 1852 1853 /* 1854 * ibdm_handle_classportinfo() 1855 * Invoked by the IBMF when the classportinfo request is completed. 1856 */ 1857 static void 1858 ibdm_handle_classportinfo(ibmf_handle_t ibmf_hdl, 1859 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 1860 { 1861 void *data; 1862 timeout_id_t timeout_id; 1863 ib_mad_hdr_t *hdr; 1864 ibdm_mad_classportinfo_t *cpi; 1865 1866 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo:ibmf hdl " 1867 "%p msg %p gid info %p", ibmf_hdl, msg, gid_info); 1868 1869 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO) { 1870 IBTF_DPRINTF_L4("ibdm", "\thandle_classportinfo: " 1871 "Not a ClassPortInfo resp"); 1872 *flag |= IBDM_IBMF_PKT_UNEXP_RESP; 1873 return; 1874 } 1875 1876 /* 1877 * Verify whether timeout handler is created/active. 1878 * If created/ active, cancel the timeout handler 1879 */ 1880 mutex_enter(&gid_info->gl_mutex); 1881 ibdm_bump_transactionID(gid_info); 1882 if (gid_info->gl_state != IBDM_GET_CLASSPORTINFO) { 1883 IBTF_DPRINTF_L2("ibdm", "\thandle_classportinfo:DUP resp"); 1884 *flag |= IBDM_IBMF_PKT_DUP_RESP; 1885 mutex_exit(&gid_info->gl_mutex); 1886 return; 1887 } 1888 gid_info->gl_iou_cb_args.cb_req_type = 0; 1889 if (gid_info->gl_timeout_id) { 1890 timeout_id = gid_info->gl_timeout_id; 1891 mutex_exit(&gid_info->gl_mutex); 1892 IBTF_DPRINTF_L5("ibdm", "handle_ioclassportinfo: " 1893 "gl_timeout_id = 0x%x", timeout_id); 1894 if (untimeout(timeout_id) == -1) { 1895 IBTF_DPRINTF_L2("ibdm", "handle_classportinfo: " 1896 "untimeout gl_timeout_id failed"); 1897 } 1898 mutex_enter(&gid_info->gl_mutex); 1899 gid_info->gl_timeout_id = 0; 1900 } 1901 gid_info->gl_state = IBDM_GET_IOUNITINFO; 1902 gid_info->gl_pending_cmds++; 1903 mutex_exit(&gid_info->gl_mutex); 1904 1905 data = msg->im_msgbufs_recv.im_bufs_cl_data; 1906 cpi = (ibdm_mad_classportinfo_t *)data; 1907 1908 /* 1909 * Cache the "RespTimeValue" and redirection information in the 1910 * global gid list data structure. This cached information will 1911 * be used to send any further requests to the GID. 1912 */ 1913 gid_info->gl_resp_timeout = 1914 (b2h32(cpi->RespTimeValue) & 0x1F); 1915 1916 gid_info->gl_redirected = ((IBDM_IN_IBMFMSG_STATUS(msg) & 1917 MAD_STATUS_REDIRECT_REQUIRED) ? B_TRUE: B_FALSE); 1918 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 1919 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 1920 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 1921 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 1922 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 1923 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 1924 1925 ibdm_dump_classportinfo(cpi); 1926 1927 /* 1928 * Send IOUnitInfo request 1929 * Reuse previously allocated IBMF packet for sending ClassPortInfo 1930 * Check whether DM agent on the remote node requested redirection 1931 * If so, send the request to the redirect DGID/DLID/PKEY/QP. 1932 */ 1933 ibdm_alloc_send_buffers(msg); 1934 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 1935 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 1936 1937 if (gid_info->gl_redirected == B_TRUE) { 1938 if (gid_info->gl_redirect_dlid != 0) { 1939 msg->im_local_addr.ia_remote_lid = 1940 gid_info->gl_redirect_dlid; 1941 } 1942 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 1943 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 1944 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 1945 } else { 1946 msg->im_local_addr.ia_remote_qno = 1; 1947 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 1948 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 1949 } 1950 1951 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 1952 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 1953 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 1954 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 1955 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 1956 hdr->Status = 0; 1957 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 1958 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 1959 hdr->AttributeModifier = 0; 1960 1961 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 1962 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 1963 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 1964 1965 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 1966 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 1967 1968 IBTF_DPRINTF_L5("ibdm", "handle_classportinfo:" 1969 "timeout %x", gid_info->gl_timeout_id); 1970 1971 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, NULL, 1972 ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != IBMF_SUCCESS) { 1973 IBTF_DPRINTF_L2("ibdm", 1974 "\thandle_classportinfo: msg transport failed"); 1975 ibdm_ibmf_send_cb(ibmf_hdl, msg, &gid_info->gl_iou_cb_args); 1976 } 1977 (*flag) |= IBDM_IBMF_PKT_REUSED; 1978 } 1979 1980 1981 /* 1982 * ibdm_send_iounitinfo: 1983 * Sends a DM request to get IOU unitinfo. 1984 */ 1985 static int 1986 ibdm_send_iounitinfo(ibdm_dp_gidinfo_t *gid_info) 1987 { 1988 ibmf_msg_t *msg; 1989 ib_mad_hdr_t *hdr; 1990 1991 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: gid info 0x%p", gid_info); 1992 1993 /* 1994 * Send command to get iounitinfo attribute. Allocate a IBMF 1995 * packet and initialize the packet. 1996 */ 1997 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, &msg) != 1998 IBMF_SUCCESS) { 1999 IBTF_DPRINTF_L4("ibdm", "\tsend_iounitinfo: pkt alloc fail"); 2000 return (IBDM_FAILURE); 2001 } 2002 2003 mutex_enter(&gid_info->gl_mutex); 2004 ibdm_bump_transactionID(gid_info); 2005 mutex_exit(&gid_info->gl_mutex); 2006 2007 2008 ibdm_alloc_send_buffers(msg); 2009 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2010 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2011 msg->im_local_addr.ia_remote_qno = 1; 2012 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2013 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2014 2015 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2016 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2017 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2018 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2019 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2020 hdr->Status = 0; 2021 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2022 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 2023 hdr->AttributeModifier = 0; 2024 2025 gid_info->gl_iou_cb_args.cb_gid_info = gid_info; 2026 gid_info->gl_iou_cb_args.cb_retry_count = ibdm_dft_retry_cnt; 2027 gid_info->gl_iou_cb_args.cb_req_type = IBDM_REQ_TYPE_IOUINFO; 2028 2029 gid_info->gl_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2030 &gid_info->gl_iou_cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2031 2032 IBTF_DPRINTF_L5("ibdm", "send_iouunitinfo:" 2033 "timeout %x", gid_info->gl_timeout_id); 2034 2035 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 2036 NULL, ibdm_ibmf_send_cb, &gid_info->gl_iou_cb_args, 0) != 2037 IBMF_SUCCESS) { 2038 IBTF_DPRINTF_L2("ibdm", "\tsend_iounitinfo: ibmf send failed"); 2039 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, 2040 msg, &gid_info->gl_iou_cb_args); 2041 } 2042 return (IBDM_SUCCESS); 2043 } 2044 2045 /* 2046 * ibdm_handle_iounitinfo() 2047 * Invoked by the IBMF when IO Unitinfo request is completed. 2048 */ 2049 static void 2050 ibdm_handle_iounitinfo(ibmf_handle_t ibmf_hdl, 2051 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2052 { 2053 int ii, first = B_TRUE; 2054 int num_iocs; 2055 size_t size; 2056 uchar_t slot_info; 2057 timeout_id_t timeout_id; 2058 ib_mad_hdr_t *hdr; 2059 ibdm_ioc_info_t *ioc_info; 2060 ib_dm_io_unitinfo_t *iou_info; 2061 ib_dm_io_unitinfo_t *giou_info; 2062 ibdm_timeout_cb_args_t *cb_args; 2063 2064 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo:" 2065 " ibmf hdl %p pkt %p gid info %p", ibmf_hdl, msg, gid_info); 2066 2067 if (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_IO_UNITINFO) { 2068 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: " 2069 "Unexpected response"); 2070 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2071 return; 2072 } 2073 2074 mutex_enter(&gid_info->gl_mutex); 2075 if (gid_info->gl_state != IBDM_GET_IOUNITINFO) { 2076 IBTF_DPRINTF_L4("ibdm", 2077 "\thandle_iounitinfo: DUP resp"); 2078 mutex_exit(&gid_info->gl_mutex); 2079 (*flag) = IBDM_IBMF_PKT_DUP_RESP; 2080 return; 2081 } 2082 gid_info->gl_iou_cb_args.cb_req_type = 0; 2083 if (gid_info->gl_timeout_id) { 2084 timeout_id = gid_info->gl_timeout_id; 2085 mutex_exit(&gid_info->gl_mutex); 2086 IBTF_DPRINTF_L5("ibdm", "handle_iounitinfo: " 2087 "gl_timeout_id = 0x%x", timeout_id); 2088 if (untimeout(timeout_id) == -1) { 2089 IBTF_DPRINTF_L2("ibdm", "handle_iounitinfo: " 2090 "untimeout gl_timeout_id failed"); 2091 } 2092 mutex_enter(&gid_info->gl_mutex); 2093 gid_info->gl_timeout_id = 0; 2094 } 2095 gid_info->gl_state = IBDM_GET_IOC_DETAILS; 2096 2097 iou_info = IBDM_IN_IBMFMSG2IOU(msg); 2098 ibdm_dump_iounitinfo(iou_info); 2099 num_iocs = iou_info->iou_num_ctrl_slots; 2100 /* 2101 * check if number of IOCs reported is zero? if yes, return. 2102 * when num_iocs are reported zero internal IOC database needs 2103 * to be updated. To ensure that save the number of IOCs in 2104 * the new field "gl_num_iocs". Use a new field instead of 2105 * "giou_info->iou_num_ctrl_slots" as that would prevent 2106 * an unnecessary kmem_alloc/kmem_free when num_iocs is 0. 2107 */ 2108 if (num_iocs == 0 && gid_info->gl_num_iocs == 0) { 2109 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: no IOC's"); 2110 mutex_exit(&gid_info->gl_mutex); 2111 return; 2112 } 2113 IBTF_DPRINTF_L4("ibdm", "\thandle_iounitinfo: num_iocs = %d", num_iocs); 2114 2115 /* 2116 * if there is an existing gl_iou (IOU has been probed before) 2117 * check if the "iou_changeid" is same as saved entry in 2118 * "giou_info->iou_changeid". 2119 * (note: this logic can prevent IOC enumeration if a given 2120 * vendor doesn't support setting iou_changeid field for its IOU) 2121 * 2122 * if there is an existing gl_iou and iou_changeid has changed : 2123 * free up existing gl_iou info and its related structures. 2124 * reallocate gl_iou info all over again. 2125 * if we donot free this up; then this leads to memory leaks 2126 */ 2127 if (gid_info->gl_iou) { 2128 giou_info = &gid_info->gl_iou->iou_info; 2129 if (iou_info->iou_changeid == giou_info->iou_changeid) { 2130 IBTF_DPRINTF_L3("ibdm", 2131 "\thandle_iounitinfo: no IOCs changed"); 2132 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2133 mutex_exit(&gid_info->gl_mutex); 2134 return; 2135 } 2136 if (ibdm_free_iou_info(gid_info)) { 2137 IBTF_DPRINTF_L3("ibdm", 2138 "\thandle_iounitinfo: failed to cleanup resources"); 2139 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 2140 mutex_exit(&gid_info->gl_mutex); 2141 return; 2142 } 2143 } 2144 2145 size = sizeof (ibdm_iou_info_t) + num_iocs * sizeof (ibdm_ioc_info_t); 2146 gid_info->gl_iou = (ibdm_iou_info_t *)kmem_zalloc(size, KM_SLEEP); 2147 giou_info = &gid_info->gl_iou->iou_info; 2148 gid_info->gl_iou->iou_ioc_info = (ibdm_ioc_info_t *) 2149 ((char *)gid_info->gl_iou + sizeof (ibdm_iou_info_t)); 2150 2151 giou_info->iou_num_ctrl_slots = gid_info->gl_num_iocs = num_iocs; 2152 giou_info->iou_flag = iou_info->iou_flag; 2153 bcopy(iou_info->iou_ctrl_list, giou_info->iou_ctrl_list, 128); 2154 giou_info->iou_changeid = b2h16(iou_info->iou_changeid); 2155 gid_info->gl_pending_cmds += num_iocs; 2156 gid_info->gl_pending_cmds += 1; /* for diag code */ 2157 mutex_exit(&gid_info->gl_mutex); 2158 2159 if (ibdm_get_diagcode(gid_info, 0) != IBDM_SUCCESS) { 2160 mutex_enter(&gid_info->gl_mutex); 2161 gid_info->gl_pending_cmds--; 2162 mutex_exit(&gid_info->gl_mutex); 2163 } 2164 /* 2165 * Parallelize getting IOC controller profiles from here. 2166 * Allocate IBMF packets and send commands to get IOC profile for 2167 * each IOC present on the IOU. 2168 */ 2169 for (ii = 0; ii < num_iocs; ii++) { 2170 /* 2171 * Check whether IOC is present in the slot 2172 * Series of nibbles (in the field iou_ctrl_list) represents 2173 * a slot in the IOU. 2174 * Byte format: 76543210 2175 * Bits 0-3 of first byte represent Slot 2 2176 * bits 4-7 of first byte represent slot 1, 2177 * bits 0-3 of second byte represent slot 4 and so on 2178 * Each 4-bit nibble has the following meaning 2179 * 0x0 : IOC not installed 2180 * 0x1 : IOC is present 2181 * 0xf : Slot does not exist 2182 * and all other values are reserved. 2183 */ 2184 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, ii); 2185 slot_info = giou_info->iou_ctrl_list[(ii/2)]; 2186 if ((ii % 2) == 0) 2187 slot_info = (slot_info >> 4); 2188 2189 if ((slot_info & 0xf) != 1) { 2190 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2191 "No IOC is present in the slot = %d", ii); 2192 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 2193 mutex_enter(&gid_info->gl_mutex); 2194 gid_info->gl_pending_cmds--; 2195 mutex_exit(&gid_info->gl_mutex); 2196 continue; 2197 } 2198 2199 mutex_enter(&gid_info->gl_mutex); 2200 ibdm_bump_transactionID(gid_info); 2201 mutex_exit(&gid_info->gl_mutex); 2202 2203 /* 2204 * Re use the already allocated packet (for IOUnitinfo) to 2205 * send the first IOC controller attribute. Allocate new 2206 * IBMF packets for the rest of the IOC's 2207 */ 2208 if (first != B_TRUE) { 2209 msg = NULL; 2210 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2211 &msg) != IBMF_SUCCESS) { 2212 IBTF_DPRINTF_L4("ibdm", "\thandle_iouintinfo: " 2213 "IBMF packet allocation failed"); 2214 mutex_enter(&gid_info->gl_mutex); 2215 gid_info->gl_pending_cmds--; 2216 mutex_exit(&gid_info->gl_mutex); 2217 continue; 2218 } 2219 2220 } 2221 2222 /* allocate send buffers for all messages */ 2223 ibdm_alloc_send_buffers(msg); 2224 2225 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2226 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2227 if (gid_info->gl_redirected == B_TRUE) { 2228 if (gid_info->gl_redirect_dlid != 0) { 2229 msg->im_local_addr.ia_remote_lid = 2230 gid_info->gl_redirect_dlid; 2231 } 2232 msg->im_local_addr.ia_remote_qno = 2233 gid_info->gl_redirect_QP; 2234 msg->im_local_addr.ia_p_key = 2235 gid_info->gl_redirect_pkey; 2236 msg->im_local_addr.ia_q_key = 2237 gid_info->gl_redirect_qkey; 2238 } else { 2239 msg->im_local_addr.ia_remote_qno = 1; 2240 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2241 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2242 } 2243 2244 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2245 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2246 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2247 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2248 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2249 hdr->Status = 0; 2250 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2251 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 2252 hdr->AttributeModifier = h2b32(ii + 1); 2253 2254 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_INVALID; 2255 cb_args = &ioc_info->ioc_cb_args; 2256 cb_args->cb_gid_info = gid_info; 2257 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2258 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 2259 cb_args->cb_ioc_num = ii; 2260 2261 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2262 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2263 2264 IBTF_DPRINTF_L5("ibdm", "\thandle_iounitinfo:" 2265 "timeout 0x%x, ioc_num %d", ioc_info->ioc_timeout_id, ii); 2266 2267 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2268 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2269 IBTF_DPRINTF_L2("ibdm", 2270 "\thandle_iounitinfo: msg transport failed"); 2271 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2272 } 2273 (*flag) |= IBDM_IBMF_PKT_REUSED; 2274 first = B_FALSE; 2275 gid_info->gl_iou->iou_niocs_probe_in_progress++; 2276 } 2277 } 2278 2279 2280 /* 2281 * ibdm_handle_ioc_profile() 2282 * Invoked by the IBMF when the IOCControllerProfile request 2283 * gets completed 2284 */ 2285 static void 2286 ibdm_handle_ioc_profile(ibmf_handle_t ibmf_hdl, 2287 ibmf_msg_t *msg, ibdm_dp_gidinfo_t *gid_info, int *flag) 2288 { 2289 int first = B_TRUE, reprobe = 0; 2290 uint_t ii, ioc_no, srv_start; 2291 uint_t nserv_entries; 2292 timeout_id_t timeout_id; 2293 ib_mad_hdr_t *hdr; 2294 ibdm_ioc_info_t *ioc_info; 2295 ibdm_timeout_cb_args_t *cb_args; 2296 ib_dm_ioc_ctrl_profile_t *ioc, *gioc; 2297 2298 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2299 " ibmf hdl %p msg %p gid info %p", ibmf_hdl, msg, gid_info); 2300 2301 ioc = IBDM_IN_IBMFMSG2IOC(msg); 2302 /* 2303 * Check whether we know this IOC already 2304 * This will return NULL if reprobe is in progress 2305 * IBDM_IOC_STATE_REPROBE_PROGRESS will be set. 2306 * Do not hold mutexes here. 2307 */ 2308 if (ibdm_is_ioc_present(ioc->ioc_guid, gid_info, flag) != NULL) { 2309 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile:" 2310 "IOC guid %llx is present", ioc->ioc_guid); 2311 return; 2312 } 2313 ioc_no = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2314 IBTF_DPRINTF_L4("ibdm", "\thandle_ioc_profile: ioc_no = %d", ioc_no-1); 2315 2316 /* Make sure that IOC index is with the valid range */ 2317 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 2318 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: " 2319 "IOC index Out of range, index %d", ioc); 2320 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2321 return; 2322 } 2323 ioc_info = &gid_info->gl_iou->iou_ioc_info[ioc_no - 1]; 2324 ioc_info->ioc_iou_info = gid_info->gl_iou; 2325 2326 mutex_enter(&gid_info->gl_mutex); 2327 if (ioc_info->ioc_state == IBDM_IOC_STATE_REPROBE_PROGRESS) { 2328 reprobe = 1; 2329 ioc_info->ioc_prev_serv = ioc_info->ioc_serv; 2330 ioc_info->ioc_serv = NULL; 2331 ioc_info->ioc_prev_serv_cnt = 2332 ioc_info->ioc_profile.ioc_service_entries; 2333 } else if (ioc_info->ioc_state != IBDM_IOC_STATE_PROBE_INVALID) { 2334 IBTF_DPRINTF_L2("ibdm", "\thandle_ioc_profile: DUP response" 2335 "ioc %d, ioc_state %x", ioc_no - 1, ioc_info->ioc_state); 2336 mutex_exit(&gid_info->gl_mutex); 2337 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2338 return; 2339 } 2340 ioc_info->ioc_cb_args.cb_req_type = 0; 2341 if (ioc_info->ioc_timeout_id) { 2342 timeout_id = ioc_info->ioc_timeout_id; 2343 mutex_exit(&gid_info->gl_mutex); 2344 IBTF_DPRINTF_L5("ibdm", "handle_ioc_profile: " 2345 "ioc_timeout_id = 0x%x", timeout_id); 2346 if (untimeout(timeout_id) == -1) { 2347 IBTF_DPRINTF_L2("ibdm", "handle_ioc_profile: " 2348 "untimeout ioc_timeout_id failed"); 2349 } 2350 mutex_enter(&gid_info->gl_mutex); 2351 ioc_info->ioc_timeout_id = 0; 2352 } 2353 2354 ioc_info->ioc_state = IBDM_IOC_STATE_PROBE_SUCCESS; 2355 if (reprobe == 0) { 2356 ioc_info->ioc_iou_guid = gid_info->gl_nodeguid; 2357 ioc_info->ioc_nodeguid = gid_info->gl_nodeguid; 2358 } 2359 2360 /* 2361 * Save all the IOC information in the global structures. 2362 * Note the wire format is Big Endian and the Sparc process also 2363 * big endian. So, there is no need to convert the data fields 2364 * The conversion routines used below are ineffective on Sparc 2365 * machines where as they will be effective on little endian 2366 * machines such as Intel processors. 2367 */ 2368 gioc = (ib_dm_ioc_ctrl_profile_t *)&ioc_info->ioc_profile; 2369 2370 /* 2371 * Restrict updates to onlyport GIDs and service entries during reprobe 2372 */ 2373 if (reprobe == 0) { 2374 gioc->ioc_guid = b2h64(ioc->ioc_guid); 2375 gioc->ioc_vendorid = 2376 ((b2h32(ioc->ioc_vendorid) & IB_DM_VENDORID_MASK) 2377 >> IB_DM_VENDORID_SHIFT); 2378 gioc->ioc_deviceid = b2h32(ioc->ioc_deviceid); 2379 gioc->ioc_device_ver = b2h16(ioc->ioc_device_ver); 2380 gioc->ioc_subsys_vendorid = 2381 ((b2h32(ioc->ioc_subsys_vendorid) & IB_DM_VENDORID_MASK) 2382 >> IB_DM_VENDORID_SHIFT); 2383 gioc->ioc_subsys_id = b2h32(ioc->ioc_subsys_id); 2384 gioc->ioc_io_class = b2h16(ioc->ioc_io_class); 2385 gioc->ioc_io_subclass = b2h16(ioc->ioc_io_subclass); 2386 gioc->ioc_protocol = b2h16(ioc->ioc_protocol); 2387 gioc->ioc_protocol_ver = b2h16(ioc->ioc_protocol_ver); 2388 gioc->ioc_send_msg_qdepth = 2389 b2h16(ioc->ioc_send_msg_qdepth); 2390 gioc->ioc_rdma_read_qdepth = 2391 b2h16(ioc->ioc_rdma_read_qdepth); 2392 gioc->ioc_send_msg_sz = b2h32(ioc->ioc_send_msg_sz); 2393 gioc->ioc_rdma_xfer_sz = b2h32(ioc->ioc_rdma_xfer_sz); 2394 gioc->ioc_ctrl_opcap_mask = ioc->ioc_ctrl_opcap_mask; 2395 bcopy(ioc->ioc_id_string, gioc->ioc_id_string, 2396 IB_DM_IOC_ID_STRING_LEN); 2397 2398 ioc_info->ioc_iou_diagcode = gid_info->gl_iou->iou_diagcode; 2399 ioc_info->ioc_iou_dc_valid = gid_info->gl_iou->iou_dc_valid; 2400 ioc_info->ioc_diagdeviceid = (IB_DM_IOU_DEVICEID_MASK & 2401 gid_info->gl_iou->iou_info.iou_flag) ? B_TRUE : B_FALSE; 2402 2403 if (ioc_info->ioc_diagdeviceid == B_TRUE) 2404 gid_info->gl_pending_cmds++; 2405 } 2406 gioc->ioc_service_entries = ioc->ioc_service_entries; 2407 gid_info->gl_pending_cmds += (gioc->ioc_service_entries/4); 2408 if (gioc->ioc_service_entries % 4) 2409 gid_info->gl_pending_cmds++; 2410 2411 mutex_exit(&gid_info->gl_mutex); 2412 2413 ibdm_dump_ioc_profile(gioc); 2414 2415 if ((ioc_info->ioc_diagdeviceid == B_TRUE) && (reprobe == 0)) { 2416 if (ibdm_get_diagcode(gid_info, ioc_no) != IBDM_SUCCESS) { 2417 mutex_enter(&gid_info->gl_mutex); 2418 gid_info->gl_pending_cmds--; 2419 mutex_exit(&gid_info->gl_mutex); 2420 } 2421 } 2422 ioc_info->ioc_serv = (ibdm_srvents_info_t *)kmem_zalloc( 2423 (gioc->ioc_service_entries * sizeof (ibdm_srvents_info_t)), 2424 KM_SLEEP); 2425 2426 /* 2427 * In one single request, maximum number of requests that can be 2428 * obtained is 4. If number of service entries are more than four, 2429 * calculate number requests needed and send them parallelly. 2430 */ 2431 nserv_entries = ioc->ioc_service_entries; 2432 ii = 0; 2433 while (nserv_entries) { 2434 mutex_enter(&gid_info->gl_mutex); 2435 ibdm_bump_transactionID(gid_info); 2436 mutex_exit(&gid_info->gl_mutex); 2437 2438 if (first != B_TRUE) { 2439 if (ibmf_alloc_msg(ibmf_hdl, IBMF_ALLOC_SLEEP, 2440 &msg) != IBMF_SUCCESS) { 2441 continue; 2442 } 2443 2444 } 2445 ibdm_alloc_send_buffers(msg); 2446 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2447 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2448 if (gid_info->gl_redirected == B_TRUE) { 2449 if (gid_info->gl_redirect_dlid != 0) { 2450 msg->im_local_addr.ia_remote_lid = 2451 gid_info->gl_redirect_dlid; 2452 } 2453 msg->im_local_addr.ia_remote_qno = 2454 gid_info->gl_redirect_QP; 2455 msg->im_local_addr.ia_p_key = 2456 gid_info->gl_redirect_pkey; 2457 msg->im_local_addr.ia_q_key = 2458 gid_info->gl_redirect_qkey; 2459 } else { 2460 msg->im_local_addr.ia_remote_qno = 1; 2461 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2462 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2463 } 2464 2465 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2466 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2467 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2468 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2469 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2470 hdr->Status = 0; 2471 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2472 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 2473 2474 srv_start = ii * 4; 2475 cb_args = &ioc_info->ioc_serv[srv_start].se_cb_args; 2476 cb_args->cb_gid_info = gid_info; 2477 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2478 cb_args->cb_req_type = IBDM_REQ_TYPE_SRVENTS; 2479 cb_args->cb_srvents_start = srv_start; 2480 cb_args->cb_ioc_num = ioc_no - 1; 2481 2482 if (nserv_entries >= IBDM_MAX_SERV_ENTRIES_PER_REQ) { 2483 nserv_entries -= IBDM_MAX_SERV_ENTRIES_PER_REQ; 2484 cb_args->cb_srvents_end = (cb_args->cb_srvents_start + 2485 IBDM_MAX_SERV_ENTRIES_PER_REQ - 1); 2486 } else { 2487 cb_args->cb_srvents_end = 2488 (cb_args->cb_srvents_start + nserv_entries - 1); 2489 nserv_entries = 0; 2490 } 2491 ibdm_fill_srv_attr_mod(hdr, cb_args); 2492 2493 ioc_info->ioc_serv[srv_start].se_timeout_id = timeout( 2494 ibdm_pkt_timeout_hdlr, cb_args, 2495 IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2496 2497 IBTF_DPRINTF_L5("ibdm", "\thandle_ioc_profile:" 2498 "timeout %x, ioc %d srv %d", 2499 ioc_info->ioc_serv[srv_start].se_timeout_id, 2500 ioc_no - 1, srv_start); 2501 2502 if (ibmf_msg_transport(ibmf_hdl, gid_info->gl_qp_hdl, msg, 2503 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2504 IBTF_DPRINTF_L2("ibdm", 2505 "\thandle_ioc_profile: msg send failed"); 2506 ibdm_ibmf_send_cb(ibmf_hdl, msg, cb_args); 2507 } 2508 (*flag) |= IBDM_IBMF_PKT_REUSED; 2509 first = B_FALSE; 2510 ii++; 2511 } 2512 } 2513 2514 2515 /* 2516 * ibdm_handle_srventry_mad() 2517 */ 2518 static void 2519 ibdm_handle_srventry_mad(ibmf_msg_t *msg, 2520 ibdm_dp_gidinfo_t *gid_info, int *flag) 2521 { 2522 uint_t ii, ioc_no, attrmod; 2523 uint_t nentries, start, end; 2524 timeout_id_t timeout_id; 2525 ib_dm_srv_t *srv_ents; 2526 ibdm_ioc_info_t *ioc_info; 2527 ibdm_srvents_info_t *gsrv_ents; 2528 2529 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad:" 2530 " IBMF msg %p gid info %p", msg, gid_info); 2531 2532 srv_ents = IBDM_IN_IBMFMSG2SRVENT(msg); 2533 /* 2534 * Get the start and end index of the service entries 2535 * Upper 16 bits identify the IOC 2536 * Lower 16 bits specify the range of service entries 2537 * LSB specifies (Big endian) end of the range 2538 * MSB specifies (Big endian) start of the range 2539 */ 2540 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 2541 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 2542 end = ((attrmod >> 8) & IBDM_8_BIT_MASK); 2543 start = (attrmod & IBDM_8_BIT_MASK); 2544 2545 /* Make sure that IOC index is with the valid range */ 2546 if ((ioc_no < 1) | 2547 (ioc_no > gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) { 2548 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2549 "IOC index Out of range, index %d", ioc_no); 2550 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2551 return; 2552 } 2553 ioc_info = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 2554 2555 /* 2556 * Make sure that the "start" and "end" service indexes are 2557 * with in the valid range 2558 */ 2559 nentries = ioc_info->ioc_profile.ioc_service_entries; 2560 if ((start > end) | (start >= nentries) | (end >= nentries)) { 2561 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2562 "Attr modifier 0x%x, #Serv entries %d", attrmod, nentries); 2563 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 2564 return; 2565 } 2566 gsrv_ents = &ioc_info->ioc_serv[start]; 2567 mutex_enter(&gid_info->gl_mutex); 2568 if (gsrv_ents->se_state != IBDM_SE_INVALID) { 2569 IBTF_DPRINTF_L2("ibdm", "\thandle_srventry_mad: " 2570 "already known, ioc %d, srv %d, se_state %x", 2571 ioc_no - 1, start, gsrv_ents->se_state); 2572 mutex_exit(&gid_info->gl_mutex); 2573 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2574 return; 2575 } 2576 ioc_info->ioc_serv[start].se_cb_args.cb_req_type = 0; 2577 if (ioc_info->ioc_serv[start].se_timeout_id) { 2578 IBTF_DPRINTF_L2("ibdm", 2579 "\thandle_srventry_mad: ioc %d start %d", ioc_no, start); 2580 timeout_id = ioc_info->ioc_serv[start].se_timeout_id; 2581 mutex_exit(&gid_info->gl_mutex); 2582 IBTF_DPRINTF_L5("ibdm", "handle_srverntry_mad: " 2583 "se_timeout_id = 0x%x", timeout_id); 2584 if (untimeout(timeout_id) == -1) { 2585 IBTF_DPRINTF_L2("ibdm", "handle_srventry_mad: " 2586 "untimeout se_timeout_id failed"); 2587 } 2588 mutex_enter(&gid_info->gl_mutex); 2589 ioc_info->ioc_serv[start].se_timeout_id = 0; 2590 } 2591 2592 gsrv_ents->se_state = IBDM_SE_VALID; 2593 mutex_exit(&gid_info->gl_mutex); 2594 for (ii = start; ii <= end; ii++, srv_ents++, gsrv_ents++) { 2595 gsrv_ents->se_attr.srv_id = b2h64(srv_ents->srv_id); 2596 bcopy(srv_ents->srv_name, 2597 gsrv_ents->se_attr.srv_name, IB_DM_MAX_SVC_NAME_LEN); 2598 ibdm_dump_service_entries(&gsrv_ents->se_attr); 2599 } 2600 } 2601 2602 2603 /* 2604 * ibdm_get_diagcode: 2605 * Send request to get IOU/IOC diag code 2606 * Returns IBDM_SUCCESS/IBDM_FAILURE 2607 */ 2608 static int 2609 ibdm_get_diagcode(ibdm_dp_gidinfo_t *gid_info, int attr) 2610 { 2611 ibmf_msg_t *msg; 2612 ib_mad_hdr_t *hdr; 2613 ibdm_ioc_info_t *ioc; 2614 ibdm_timeout_cb_args_t *cb_args; 2615 timeout_id_t *timeout_id; 2616 2617 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: gid info %p, attr = %d", 2618 gid_info, attr); 2619 2620 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 2621 &msg) != IBMF_SUCCESS) { 2622 IBTF_DPRINTF_L4("ibdm", "\tget_diagcode: pkt alloc fail"); 2623 return (IBDM_FAILURE); 2624 } 2625 2626 ibdm_alloc_send_buffers(msg); 2627 2628 mutex_enter(&gid_info->gl_mutex); 2629 ibdm_bump_transactionID(gid_info); 2630 mutex_exit(&gid_info->gl_mutex); 2631 2632 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 2633 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 2634 if (gid_info->gl_redirected == B_TRUE) { 2635 if (gid_info->gl_redirect_dlid != 0) { 2636 msg->im_local_addr.ia_remote_lid = 2637 gid_info->gl_redirect_dlid; 2638 } 2639 2640 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 2641 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 2642 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 2643 } else { 2644 msg->im_local_addr.ia_remote_qno = 1; 2645 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 2646 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 2647 } 2648 2649 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 2650 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 2651 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 2652 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 2653 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 2654 hdr->Status = 0; 2655 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 2656 2657 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 2658 hdr->AttributeModifier = h2b32(attr); 2659 2660 if (attr == 0) { 2661 cb_args = &gid_info->gl_iou_cb_args; 2662 gid_info->gl_iou->iou_dc_valid = B_FALSE; 2663 cb_args->cb_ioc_num = 0; 2664 cb_args->cb_req_type = IBDM_REQ_TYPE_IOU_DIAGCODE; 2665 timeout_id = &gid_info->gl_timeout_id; 2666 } else { 2667 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attr - 1)); 2668 ioc->ioc_dc_valid = B_FALSE; 2669 cb_args = &ioc->ioc_dc_cb_args; 2670 cb_args->cb_ioc_num = attr - 1; 2671 cb_args->cb_req_type = IBDM_REQ_TYPE_IOC_DIAGCODE; 2672 timeout_id = &ioc->ioc_dc_timeout_id; 2673 } 2674 cb_args->cb_gid_info = gid_info; 2675 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 2676 cb_args->cb_srvents_start = 0; 2677 2678 2679 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 2680 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 2681 2682 IBTF_DPRINTF_L5("ibdm", "\tget_diagcode:" 2683 "timeout %x, ioc %d", *timeout_id, cb_args->cb_ioc_num); 2684 2685 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 2686 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 2687 IBTF_DPRINTF_L2("ibdm", "\tget_diagcode: ibmf send failed"); 2688 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 2689 } 2690 return (IBDM_SUCCESS); 2691 } 2692 2693 /* 2694 * ibdm_handle_diagcode: 2695 * Process the DiagCode MAD response and update local DM 2696 * data structure. 2697 */ 2698 static void 2699 ibdm_handle_diagcode(ibmf_msg_t *ibmf_msg, 2700 ibdm_dp_gidinfo_t *gid_info, int *flag) 2701 { 2702 uint16_t attrmod, *diagcode; 2703 ibdm_iou_info_t *iou; 2704 ibdm_ioc_info_t *ioc; 2705 timeout_id_t timeout_id; 2706 ibdm_timeout_cb_args_t *cb_args; 2707 2708 diagcode = (uint16_t *)ibmf_msg->im_msgbufs_recv.im_bufs_cl_data; 2709 2710 mutex_enter(&gid_info->gl_mutex); 2711 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(ibmf_msg); 2712 iou = gid_info->gl_iou; 2713 if (attrmod == 0) { 2714 if (iou->iou_dc_valid != B_FALSE) { 2715 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2716 IBTF_DPRINTF_L4("ibdm", 2717 "\thandle_diagcode: Duplicate IOU DiagCode"); 2718 mutex_exit(&gid_info->gl_mutex); 2719 return; 2720 } 2721 cb_args = &gid_info->gl_iou_cb_args; 2722 cb_args->cb_req_type = 0; 2723 iou->iou_diagcode = b2h16(*diagcode); 2724 iou->iou_dc_valid = B_TRUE; 2725 if (gid_info->gl_timeout_id) { 2726 timeout_id = gid_info->gl_timeout_id; 2727 mutex_exit(&gid_info->gl_mutex); 2728 IBTF_DPRINTF_L5("ibdm", "\thandle_diagcode: " 2729 "gl_timeout_id = 0x%x", timeout_id); 2730 if (untimeout(timeout_id) == -1) { 2731 IBTF_DPRINTF_L2("ibdm", "handle_diagcode: " 2732 "untimeout gl_timeout_id failed"); 2733 } 2734 mutex_enter(&gid_info->gl_mutex); 2735 gid_info->gl_timeout_id = 0; 2736 } 2737 } else { 2738 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod - 1)); 2739 if (ioc->ioc_dc_valid != B_FALSE) { 2740 (*flag) |= IBDM_IBMF_PKT_DUP_RESP; 2741 IBTF_DPRINTF_L4("ibdm", 2742 "\thandle_diagcode: Duplicate IOC DiagCode"); 2743 mutex_exit(&gid_info->gl_mutex); 2744 return; 2745 } 2746 cb_args = &ioc->ioc_dc_cb_args; 2747 cb_args->cb_req_type = 0; 2748 ioc->ioc_diagcode = b2h16(*diagcode); 2749 ioc->ioc_dc_valid = B_TRUE; 2750 timeout_id = iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id; 2751 if (timeout_id) { 2752 mutex_exit(&gid_info->gl_mutex); 2753 IBTF_DPRINTF_L5("ibdm", "handle_diagcode: " 2754 "timeout_id = 0x%x", timeout_id); 2755 if (untimeout(timeout_id) == -1) { 2756 IBTF_DPRINTF_L2("ibdm", "\thandle_diagcode: " 2757 "untimeout ioc_dc_timeout_id failed"); 2758 } 2759 mutex_enter(&gid_info->gl_mutex); 2760 iou->iou_ioc_info[attrmod - 1].ioc_dc_timeout_id = 0; 2761 } 2762 } 2763 mutex_exit(&gid_info->gl_mutex); 2764 2765 IBTF_DPRINTF_L4("ibdm", "\thandle_diagcode: DiagCode : 0x%x" 2766 "attrmod : 0x%x", b2h16(*diagcode), attrmod); 2767 } 2768 2769 2770 /* 2771 * ibdm_is_ioc_present() 2772 * Return ibdm_ioc_info_t if IOC guid is found in the global gid list 2773 */ 2774 static ibdm_ioc_info_t * 2775 ibdm_is_ioc_present(ib_guid_t ioc_guid, 2776 ibdm_dp_gidinfo_t *gid_info, int *flag) 2777 { 2778 int ii; 2779 ibdm_ioc_info_t *ioc; 2780 ibdm_dp_gidinfo_t *head; 2781 ib_dm_io_unitinfo_t *iou; 2782 2783 mutex_enter(&ibdm.ibdm_mutex); 2784 head = ibdm.ibdm_dp_gidlist_head; 2785 while (head) { 2786 mutex_enter(&head->gl_mutex); 2787 if (head->gl_iou == NULL) { 2788 mutex_exit(&head->gl_mutex); 2789 head = head->gl_next; 2790 continue; 2791 } 2792 iou = &head->gl_iou->iou_info; 2793 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 2794 ioc = IBDM_GIDINFO2IOCINFO(head, ii); 2795 if ((ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) && 2796 (ioc->ioc_profile.ioc_guid == ioc_guid)) { 2797 if (gid_info == head) { 2798 *flag |= IBDM_IBMF_PKT_DUP_RESP; 2799 } else if (ibdm_check_dgid(head->gl_dgid_lo, 2800 head->gl_dgid_hi) != NULL) { 2801 IBTF_DPRINTF_L4("ibdm", "\tis_ioc_" 2802 "present: gid not present"); 2803 ibdm_add_to_gl_gid(gid_info, head); 2804 } 2805 mutex_exit(&head->gl_mutex); 2806 mutex_exit(&ibdm.ibdm_mutex); 2807 return (ioc); 2808 } 2809 } 2810 mutex_exit(&head->gl_mutex); 2811 head = head->gl_next; 2812 } 2813 mutex_exit(&ibdm.ibdm_mutex); 2814 return (NULL); 2815 } 2816 2817 2818 /* 2819 * ibdm_ibmf_send_cb() 2820 * IBMF invokes this callback routine after posting the DM MAD to 2821 * the HCA. 2822 */ 2823 /*ARGSUSED*/ 2824 static void 2825 ibdm_ibmf_send_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *ibmf_msg, void *arg) 2826 { 2827 ibdm_dump_ibmf_msg(ibmf_msg, 1); 2828 ibdm_free_send_buffers(ibmf_msg); 2829 if (ibmf_free_msg(ibmf_hdl, &ibmf_msg) != IBMF_SUCCESS) { 2830 IBTF_DPRINTF_L4("ibdm", 2831 "\tibmf_send_cb: IBMF free msg failed"); 2832 } 2833 } 2834 2835 2836 /* 2837 * ibdm_ibmf_recv_cb() 2838 * Invoked by the IBMF when a response to the one of the DM requests 2839 * is received. 2840 */ 2841 /*ARGSUSED*/ 2842 static void 2843 ibdm_ibmf_recv_cb(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2844 { 2845 ibdm_taskq_args_t *taskq_args; 2846 2847 /* 2848 * If the taskq enable is set then dispatch a taskq to process 2849 * the MAD, otherwise just process it on this thread 2850 */ 2851 if (ibdm_taskq_enable != IBDM_ENABLE_TASKQ_HANDLING) { 2852 ibdm_process_incoming_mad(ibmf_hdl, msg, arg); 2853 return; 2854 } 2855 2856 /* 2857 * create a taskq and dispatch it to process the incoming MAD 2858 */ 2859 taskq_args = kmem_alloc(sizeof (ibdm_taskq_args_t), KM_NOSLEEP); 2860 if (taskq_args == NULL) { 2861 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: kmem_alloc failed for" 2862 "taskq_args"); 2863 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2864 IBTF_DPRINTF_L4("ibmf_recv_cb", 2865 "\tibmf_recv_cb: IBMF free msg failed"); 2866 } 2867 return; 2868 } 2869 taskq_args->tq_ibmf_handle = ibmf_hdl; 2870 taskq_args->tq_ibmf_msg = msg; 2871 taskq_args->tq_args = arg; 2872 2873 if (taskq_dispatch(system_taskq, ibdm_recv_incoming_mad, taskq_args, 2874 TQ_NOSLEEP) == 0) { 2875 IBTF_DPRINTF_L2("ibdm", "ibmf_recv_cb: taskq_dispatch failed"); 2876 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2877 IBTF_DPRINTF_L4("ibmf_recv_cb", 2878 "\tibmf_recv_cb: IBMF free msg failed"); 2879 } 2880 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2881 return; 2882 } 2883 2884 /* taskq_args are deleted in ibdm_recv_incoming_mad() */ 2885 } 2886 2887 2888 void 2889 ibdm_recv_incoming_mad(void *args) 2890 { 2891 ibdm_taskq_args_t *taskq_args; 2892 2893 taskq_args = (ibdm_taskq_args_t *)args; 2894 2895 IBTF_DPRINTF_L4("ibdm", "\tibdm_recv_incoming_mad: " 2896 "Processing incoming MAD via taskq"); 2897 2898 ibdm_process_incoming_mad(taskq_args->tq_ibmf_handle, 2899 taskq_args->tq_ibmf_msg, taskq_args->tq_args); 2900 2901 kmem_free(taskq_args, sizeof (ibdm_taskq_args_t)); 2902 } 2903 2904 2905 /* 2906 * Calls ibdm_process_incoming_mad with all function arguments extracted 2907 * from args 2908 */ 2909 /*ARGSUSED*/ 2910 static void 2911 ibdm_process_incoming_mad(ibmf_handle_t ibmf_hdl, ibmf_msg_t *msg, void *arg) 2912 { 2913 int flag = 0; 2914 int ret; 2915 uint64_t transaction_id; 2916 ib_mad_hdr_t *hdr; 2917 ibdm_dp_gidinfo_t *gid_info = NULL; 2918 2919 IBTF_DPRINTF_L4("ibdm", 2920 "\tprocess_incoming_mad: ibmf hdl %p pkt %p", ibmf_hdl, msg); 2921 ibdm_dump_ibmf_msg(msg, 0); 2922 2923 /* 2924 * IBMF calls this routine for every DM MAD that arrives at this port. 2925 * But we handle only the responses for requests we sent. We drop all 2926 * the DM packets that does not have response bit set in the MAD 2927 * header(this eliminates all the requests sent to this port). 2928 * We handle only DM class version 1 MAD's 2929 */ 2930 hdr = IBDM_IN_IBMFMSG_MADHDR(msg); 2931 if (ibdm_verify_mad_status(hdr) != IBDM_SUCCESS) { 2932 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2933 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 2934 "IBMF free msg failed DM request drop it"); 2935 } 2936 return; 2937 } 2938 2939 transaction_id = b2h64(hdr->TransactionID); 2940 2941 mutex_enter(&ibdm.ibdm_mutex); 2942 gid_info = ibdm.ibdm_dp_gidlist_head; 2943 while (gid_info) { 2944 if ((gid_info->gl_transactionID & 2945 IBDM_GID_TRANSACTIONID_MASK) == 2946 (transaction_id & IBDM_GID_TRANSACTIONID_MASK)) 2947 break; 2948 gid_info = gid_info->gl_next; 2949 } 2950 mutex_exit(&ibdm.ibdm_mutex); 2951 2952 if (gid_info == NULL) { 2953 /* Drop the packet */ 2954 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: transaction ID" 2955 " does not match: 0x%llx", transaction_id); 2956 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 2957 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 2958 "IBMF free msg failed DM request drop it"); 2959 } 2960 return; 2961 } 2962 2963 /* Handle redirection for all the MAD's, except ClassPortInfo */ 2964 if (((IBDM_IN_IBMFMSG_STATUS(msg) & MAD_STATUS_REDIRECT_REQUIRED)) && 2965 (IBDM_IN_IBMFMSG_ATTR(msg) != IB_DM_ATTR_CLASSPORTINFO)) { 2966 ret = ibdm_handle_redirection(msg, gid_info, &flag); 2967 if (ret == IBDM_SUCCESS) { 2968 return; 2969 } 2970 } else { 2971 uint_t gl_state; 2972 2973 mutex_enter(&gid_info->gl_mutex); 2974 gl_state = gid_info->gl_state; 2975 mutex_exit(&gid_info->gl_mutex); 2976 2977 switch (gl_state) { 2978 case IBDM_GET_CLASSPORTINFO: 2979 ibdm_handle_classportinfo( 2980 ibmf_hdl, msg, gid_info, &flag); 2981 break; 2982 2983 case IBDM_GET_IOUNITINFO: 2984 ibdm_handle_iounitinfo(ibmf_hdl, msg, gid_info, &flag); 2985 break; 2986 2987 case IBDM_GET_IOC_DETAILS: 2988 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 2989 2990 case IB_DM_ATTR_SERVICE_ENTRIES: 2991 ibdm_handle_srventry_mad(msg, gid_info, &flag); 2992 break; 2993 2994 case IB_DM_ATTR_IOC_CTRL_PROFILE: 2995 ibdm_handle_ioc_profile( 2996 ibmf_hdl, msg, gid_info, &flag); 2997 break; 2998 2999 case IB_DM_ATTR_DIAG_CODE: 3000 ibdm_handle_diagcode(msg, gid_info, &flag); 3001 break; 3002 3003 default: 3004 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3005 "Error state, wrong attribute :-("); 3006 (void) ibmf_free_msg(ibmf_hdl, &msg); 3007 return; 3008 } 3009 break; 3010 default: 3011 IBTF_DPRINTF_L2("ibdm", 3012 "process_incoming_mad: Dropping the packet" 3013 " gl_state %x", gl_state); 3014 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3015 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3016 "IBMF free msg failed DM request drop it"); 3017 } 3018 return; 3019 } 3020 } 3021 3022 if ((flag & IBDM_IBMF_PKT_DUP_RESP) || 3023 (flag & IBDM_IBMF_PKT_UNEXP_RESP)) { 3024 IBTF_DPRINTF_L2("ibdm", 3025 "\tprocess_incoming_mad:Dup/unexp resp : 0x%x", flag); 3026 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3027 IBTF_DPRINTF_L2("ibdm", "process_incoming_mad: " 3028 "IBMF free msg failed DM request drop it"); 3029 } 3030 return; 3031 } 3032 3033 mutex_enter(&gid_info->gl_mutex); 3034 if (gid_info->gl_pending_cmds < 1) { 3035 IBTF_DPRINTF_L2("ibdm", 3036 "\tprocess_incoming_mad: pending commands negative"); 3037 } 3038 if (--gid_info->gl_pending_cmds) { 3039 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: " 3040 "gid_info %p pending cmds %d", 3041 gid_info, gid_info->gl_pending_cmds); 3042 mutex_exit(&gid_info->gl_mutex); 3043 } else { 3044 IBTF_DPRINTF_L4("ibdm", "\tprocess_incoming_mad: Probing DONE"); 3045 gid_info->gl_state = IBDM_GID_PROBING_COMPLETE; 3046 mutex_exit(&gid_info->gl_mutex); 3047 ibdm_notify_newgid_iocs(gid_info); 3048 mutex_enter(&ibdm.ibdm_mutex); 3049 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3050 IBTF_DPRINTF_L4("ibdm", 3051 "\tprocess_incoming_mad: Wakeup"); 3052 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3053 cv_broadcast(&ibdm.ibdm_probe_cv); 3054 } 3055 mutex_exit(&ibdm.ibdm_mutex); 3056 } 3057 3058 /* 3059 * Do not deallocate the IBMF packet if atleast one request 3060 * is posted. IBMF packet is reused. 3061 */ 3062 if (!(flag & IBDM_IBMF_PKT_REUSED)) { 3063 if (ibmf_free_msg(ibmf_hdl, &msg) != IBMF_SUCCESS) { 3064 IBTF_DPRINTF_L2("ibdm", "\tprocess_incoming_mad: " 3065 "IBMF free msg failed DM request drop it"); 3066 } 3067 } 3068 } 3069 3070 3071 /* 3072 * ibdm_verify_mad_status() 3073 * Verifies the MAD status 3074 * Returns IBDM_SUCCESS if status is correct 3075 * Returns IBDM_FAILURE for bogus MAD status 3076 */ 3077 static int 3078 ibdm_verify_mad_status(ib_mad_hdr_t *hdr) 3079 { 3080 int ret = 0; 3081 3082 if ((hdr->R_Method != IB_DM_DEVMGT_METHOD_GET_RESP) || 3083 (hdr->ClassVersion != IB_DM_CLASS_VERSION_1)) { 3084 return (IBDM_FAILURE); 3085 } 3086 3087 if (b2h16(hdr->Status) == 0) 3088 ret = IBDM_SUCCESS; 3089 else if ((b2h16(hdr->Status) & 0x1f) == MAD_STATUS_REDIRECT_REQUIRED) 3090 ret = IBDM_SUCCESS; 3091 else { 3092 IBTF_DPRINTF_L4("ibdm", 3093 "\tverify_mad_status: Stauts : 0x%x", b2h16(hdr->Status)); 3094 ret = IBDM_FAILURE; 3095 } 3096 return (ret); 3097 } 3098 3099 3100 3101 /* 3102 * ibdm_handle_redirection() 3103 * Returns IBDM_SUCCESS/IBDM_FAILURE 3104 */ 3105 static int 3106 ibdm_handle_redirection(ibmf_msg_t *msg, 3107 ibdm_dp_gidinfo_t *gid_info, int *flag) 3108 { 3109 int attrmod, ioc_no, start; 3110 void *data; 3111 timeout_id_t *timeout_id; 3112 ib_mad_hdr_t *hdr; 3113 ibdm_ioc_info_t *ioc = NULL; 3114 ibdm_timeout_cb_args_t *cb_args; 3115 ibdm_mad_classportinfo_t *cpi; 3116 3117 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Enter"); 3118 mutex_enter(&gid_info->gl_mutex); 3119 switch (gid_info->gl_state) { 3120 case IBDM_GET_IOUNITINFO: 3121 cb_args = &gid_info->gl_iou_cb_args; 3122 timeout_id = &gid_info->gl_timeout_id; 3123 break; 3124 3125 case IBDM_GET_IOC_DETAILS: 3126 attrmod = IBDM_IN_IBMFMSG_ATTRMOD(msg); 3127 switch (IBDM_IN_IBMFMSG_ATTR(msg)) { 3128 3129 case IB_DM_ATTR_DIAG_CODE: 3130 if (attrmod == 0) { 3131 cb_args = &gid_info->gl_iou_cb_args; 3132 timeout_id = &gid_info->gl_timeout_id; 3133 break; 3134 } 3135 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3136 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3137 "IOC# Out of range %d", attrmod); 3138 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3139 mutex_exit(&gid_info->gl_mutex); 3140 return (IBDM_FAILURE); 3141 } 3142 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3143 cb_args = &ioc->ioc_dc_cb_args; 3144 timeout_id = &ioc->ioc_dc_timeout_id; 3145 break; 3146 3147 case IB_DM_ATTR_IOC_CTRL_PROFILE: 3148 if (IBDM_IS_IOC_NUM_INVALID(attrmod, gid_info)) { 3149 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3150 "IOC# Out of range %d", attrmod); 3151 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3152 mutex_exit(&gid_info->gl_mutex); 3153 return (IBDM_FAILURE); 3154 } 3155 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (attrmod -1)); 3156 cb_args = &ioc->ioc_cb_args; 3157 timeout_id = &ioc->ioc_timeout_id; 3158 break; 3159 3160 case IB_DM_ATTR_SERVICE_ENTRIES: 3161 ioc_no = ((attrmod >> 16) & IBDM_16_BIT_MASK); 3162 if (IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)) { 3163 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3164 "IOC# Out of range %d", ioc_no); 3165 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3166 mutex_exit(&gid_info->gl_mutex); 3167 return (IBDM_FAILURE); 3168 } 3169 start = (attrmod & IBDM_8_BIT_MASK); 3170 ioc = IBDM_GIDINFO2IOCINFO(gid_info, (ioc_no -1)); 3171 if (start > ioc->ioc_profile.ioc_service_entries) { 3172 IBTF_DPRINTF_L2("ibdm", "\thandle_redirction:" 3173 " SE index Out of range %d", start); 3174 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3175 mutex_exit(&gid_info->gl_mutex); 3176 return (IBDM_FAILURE); 3177 } 3178 cb_args = &ioc->ioc_serv[start].se_cb_args; 3179 timeout_id = &ioc->ioc_serv[start].se_timeout_id; 3180 break; 3181 3182 default: 3183 /* ERROR State */ 3184 IBTF_DPRINTF_L2("ibdm", 3185 "\thandle_redirection: wrong attribute :-("); 3186 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3187 mutex_exit(&gid_info->gl_mutex); 3188 return (IBDM_FAILURE); 3189 } 3190 break; 3191 default: 3192 /* ERROR State */ 3193 IBTF_DPRINTF_L2("ibdm", 3194 "\thandle_redirection: Error state :-("); 3195 (*flag) |= IBDM_IBMF_PKT_UNEXP_RESP; 3196 mutex_exit(&gid_info->gl_mutex); 3197 return (IBDM_FAILURE); 3198 } 3199 if ((*timeout_id) != 0) { 3200 mutex_exit(&gid_info->gl_mutex); 3201 if (untimeout(*timeout_id) == -1) { 3202 IBTF_DPRINTF_L2("ibdm", "\thandle_redirection: " 3203 "untimeout failed %x", *timeout_id); 3204 } else { 3205 IBTF_DPRINTF_L5("ibdm", 3206 "\thandle_redirection: timeout %x", *timeout_id); 3207 } 3208 mutex_enter(&gid_info->gl_mutex); 3209 *timeout_id = 0; 3210 } 3211 3212 data = msg->im_msgbufs_recv.im_bufs_cl_data; 3213 cpi = (ibdm_mad_classportinfo_t *)data; 3214 3215 gid_info->gl_resp_timeout = 3216 (b2h32(cpi->RespTimeValue) & 0x1F); 3217 3218 gid_info->gl_redirected = B_TRUE; 3219 gid_info->gl_redirect_dlid = b2h16(cpi->RedirectLID); 3220 gid_info->gl_redirect_QP = (b2h32(cpi->RedirectQP) & 0xffffff); 3221 gid_info->gl_redirect_pkey = b2h16(cpi->RedirectP_Key); 3222 gid_info->gl_redirect_qkey = b2h32(cpi->RedirectQ_Key); 3223 gid_info->gl_redirectGID_hi = b2h64(cpi->RedirectGID_hi); 3224 gid_info->gl_redirectGID_lo = b2h64(cpi->RedirectGID_lo); 3225 3226 if (gid_info->gl_redirect_dlid != 0) { 3227 msg->im_local_addr.ia_remote_lid = 3228 gid_info->gl_redirect_dlid; 3229 } 3230 ibdm_bump_transactionID(gid_info); 3231 mutex_exit(&gid_info->gl_mutex); 3232 3233 ibdm_alloc_send_buffers(msg); 3234 3235 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3236 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3237 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3238 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3239 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3240 hdr->Status = 0; 3241 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3242 hdr->AttributeID = 3243 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID; 3244 hdr->AttributeModifier = 3245 msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier; 3246 3247 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3248 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3249 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3250 3251 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3252 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3253 3254 IBTF_DPRINTF_L5("ibdm", "\thandle_redirect:" 3255 "timeout %x", *timeout_id); 3256 3257 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, 3258 msg, NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 3259 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection:" 3260 "message transport failed"); 3261 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3262 } 3263 (*flag) |= IBDM_IBMF_PKT_REUSED; 3264 IBTF_DPRINTF_L4("ibdm", "\thandle_redirection: Exit"); 3265 return (IBDM_SUCCESS); 3266 } 3267 3268 3269 /* 3270 * ibdm_pkt_timeout_hdlr 3271 * This timeout handler is registed for every IBMF packet that is 3272 * sent through the IBMF. It gets called when no response is received 3273 * within the specified time for the packet. No retries for the failed 3274 * commands currently. Drops the failed IBMF packet and update the 3275 * pending list commands. 3276 */ 3277 static void 3278 ibdm_pkt_timeout_hdlr(void *arg) 3279 { 3280 int probe_done = B_FALSE; 3281 ibdm_iou_info_t *iou; 3282 ibdm_ioc_info_t *ioc; 3283 ibdm_timeout_cb_args_t *cb_args = arg; 3284 ibdm_dp_gidinfo_t *gid_info; 3285 int srv_ent; 3286 uint_t new_gl_state; 3287 3288 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: gid_info: %p " 3289 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3290 cb_args->cb_req_type, cb_args->cb_ioc_num, 3291 cb_args->cb_srvents_start); 3292 3293 gid_info = cb_args->cb_gid_info; 3294 mutex_enter(&gid_info->gl_mutex); 3295 3296 if ((gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) || 3297 (cb_args->cb_req_type == 0)) { 3298 3299 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: req completed" 3300 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_req_type, 3301 cb_args->cb_ioc_num, cb_args->cb_srvents_start); 3302 3303 if (gid_info->gl_timeout_id) 3304 gid_info->gl_timeout_id = 0; 3305 mutex_exit(&gid_info->gl_mutex); 3306 return; 3307 } 3308 if (cb_args->cb_retry_count) { 3309 cb_args->cb_retry_count--; 3310 if (ibdm_retry_command(cb_args) == IBDM_SUCCESS) { 3311 if (gid_info->gl_timeout_id) 3312 gid_info->gl_timeout_id = 0; 3313 mutex_exit(&gid_info->gl_mutex); 3314 return; 3315 } 3316 cb_args->cb_retry_count = 0; 3317 } 3318 3319 IBTF_DPRINTF_L2("ibdm", "\tpkt_timeout_hdlr: command failed: gid %p" 3320 " rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3321 cb_args->cb_req_type, cb_args->cb_ioc_num, 3322 cb_args->cb_srvents_start); 3323 3324 new_gl_state = IBDM_GID_PROBING_COMPLETE; 3325 switch (cb_args->cb_req_type) { 3326 3327 case IBDM_REQ_TYPE_CLASSPORTINFO: 3328 case IBDM_REQ_TYPE_IOUINFO: 3329 new_gl_state = IBDM_GID_PROBING_FAILED; 3330 if (--gid_info->gl_pending_cmds == 0) 3331 probe_done = B_TRUE; 3332 if (gid_info->gl_timeout_id) 3333 gid_info->gl_timeout_id = 0; 3334 mutex_exit(&gid_info->gl_mutex); 3335 ibdm_delete_glhca_list(gid_info); 3336 mutex_enter(&gid_info->gl_mutex); 3337 break; 3338 case IBDM_REQ_TYPE_IOCINFO: 3339 iou = gid_info->gl_iou; 3340 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3341 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3342 if (--gid_info->gl_pending_cmds == 0) 3343 probe_done = B_TRUE; 3344 #ifndef __lock_lint 3345 if (ioc->ioc_timeout_id) 3346 ioc->ioc_timeout_id = 0; 3347 #endif 3348 break; 3349 case IBDM_REQ_TYPE_SRVENTS: 3350 iou = gid_info->gl_iou; 3351 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3352 ioc->ioc_state = IBDM_IOC_STATE_PROBE_FAILED; 3353 if (--gid_info->gl_pending_cmds == 0) 3354 probe_done = B_TRUE; 3355 srv_ent = cb_args->cb_srvents_start; 3356 #ifndef __lock_lint 3357 if (ioc->ioc_serv[srv_ent].se_timeout_id) 3358 ioc->ioc_serv[srv_ent].se_timeout_id = 0; 3359 #endif 3360 break; 3361 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3362 iou = gid_info->gl_iou; 3363 iou->iou_dc_valid = B_FALSE; 3364 if (--gid_info->gl_pending_cmds == 0) 3365 probe_done = B_TRUE; 3366 if (gid_info->gl_timeout_id) 3367 gid_info->gl_timeout_id = 0; 3368 break; 3369 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3370 iou = gid_info->gl_iou; 3371 ioc = &iou->iou_ioc_info[cb_args->cb_ioc_num]; 3372 ioc->ioc_dc_valid = B_FALSE; 3373 if (--gid_info->gl_pending_cmds == 0) 3374 probe_done = B_TRUE; 3375 #ifndef __lock_lint 3376 if (ioc->ioc_dc_timeout_id) 3377 ioc->ioc_dc_timeout_id = 0; 3378 #endif 3379 break; 3380 } 3381 if (probe_done == B_TRUE) { 3382 gid_info->gl_state = new_gl_state; 3383 mutex_exit(&gid_info->gl_mutex); 3384 ibdm_notify_newgid_iocs(gid_info); 3385 mutex_enter(&ibdm.ibdm_mutex); 3386 if (--ibdm.ibdm_ngid_probes_in_progress == 0) { 3387 IBTF_DPRINTF_L4("ibdm", "\tpkt_timeout_hdlr: Wakeup"); 3388 ibdm.ibdm_busy &= ~IBDM_PROBE_IN_PROGRESS; 3389 cv_broadcast(&ibdm.ibdm_probe_cv); 3390 } 3391 mutex_exit(&ibdm.ibdm_mutex); 3392 } else 3393 mutex_exit(&gid_info->gl_mutex); 3394 } 3395 3396 3397 /* 3398 * ibdm_retry_command() 3399 * Retries the failed command. 3400 * Returns IBDM_FAILURE/IBDM_SUCCESS 3401 */ 3402 static int 3403 ibdm_retry_command(ibdm_timeout_cb_args_t *cb_args) 3404 { 3405 int ret, rval = IBDM_SUCCESS; 3406 ibmf_msg_t *msg; 3407 ib_mad_hdr_t *hdr; 3408 ibdm_dp_gidinfo_t *gid_info = cb_args->cb_gid_info; 3409 timeout_id_t *timeout_id; 3410 ibdm_ioc_info_t *ioc; 3411 int ioc_no; 3412 3413 IBTF_DPRINTF_L2("ibdm", "\tretry_command: gid_info: %p " 3414 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3415 cb_args->cb_req_type, cb_args->cb_ioc_num, 3416 cb_args->cb_srvents_start); 3417 3418 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, &msg); 3419 3420 3421 /* 3422 * Reset the gid if alloc_msg failed with BAD_HANDLE 3423 * ibdm_reset_gidinfo reinits the gid_info 3424 */ 3425 if (ret == IBMF_BAD_HANDLE) { 3426 IBTF_DPRINTF_L3(ibdm_string, "\tretry_command: gid %p hdl bad", 3427 gid_info); 3428 3429 mutex_exit(&gid_info->gl_mutex); 3430 ibdm_reset_gidinfo(gid_info); 3431 mutex_enter(&gid_info->gl_mutex); 3432 3433 /* Retry alloc */ 3434 ret = ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_NOSLEEP, 3435 &msg); 3436 } 3437 3438 if (ret != IBDM_SUCCESS) { 3439 IBTF_DPRINTF_L2("ibdm", "\tretry_command: alloc failed: %p " 3440 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3441 cb_args->cb_req_type, cb_args->cb_ioc_num, 3442 cb_args->cb_srvents_start); 3443 return (IBDM_FAILURE); 3444 } 3445 3446 ibdm_alloc_send_buffers(msg); 3447 3448 ibdm_bump_transactionID(gid_info); 3449 3450 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 3451 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 3452 if (gid_info->gl_redirected == B_TRUE) { 3453 if (gid_info->gl_redirect_dlid != 0) { 3454 msg->im_local_addr.ia_remote_lid = 3455 gid_info->gl_redirect_dlid; 3456 } 3457 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 3458 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 3459 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 3460 } else { 3461 msg->im_local_addr.ia_remote_qno = 1; 3462 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 3463 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 3464 } 3465 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 3466 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 3467 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 3468 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 3469 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 3470 hdr->Status = 0; 3471 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 3472 3473 switch (cb_args->cb_req_type) { 3474 case IBDM_REQ_TYPE_CLASSPORTINFO: 3475 hdr->AttributeID = h2b16(IB_DM_ATTR_CLASSPORTINFO); 3476 hdr->AttributeModifier = 0; 3477 timeout_id = &gid_info->gl_timeout_id; 3478 break; 3479 case IBDM_REQ_TYPE_IOUINFO: 3480 hdr->AttributeID = h2b16(IB_DM_ATTR_IO_UNITINFO); 3481 hdr->AttributeModifier = 0; 3482 timeout_id = &gid_info->gl_timeout_id; 3483 break; 3484 case IBDM_REQ_TYPE_IOCINFO: 3485 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 3486 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3487 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3488 timeout_id = &ioc->ioc_timeout_id; 3489 break; 3490 case IBDM_REQ_TYPE_SRVENTS: 3491 hdr->AttributeID = h2b16(IB_DM_ATTR_SERVICE_ENTRIES); 3492 ibdm_fill_srv_attr_mod(hdr, cb_args); 3493 ioc = IBDM_GIDINFO2IOCINFO(gid_info, cb_args->cb_ioc_num); 3494 timeout_id = 3495 &ioc->ioc_serv[cb_args->cb_srvents_start].se_timeout_id; 3496 break; 3497 case IBDM_REQ_TYPE_IOU_DIAGCODE: 3498 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3499 hdr->AttributeModifier = 0; 3500 timeout_id = &gid_info->gl_timeout_id; 3501 break; 3502 case IBDM_REQ_TYPE_IOC_DIAGCODE: 3503 hdr->AttributeID = h2b16(IB_DM_ATTR_DIAG_CODE); 3504 hdr->AttributeModifier = h2b32(cb_args->cb_ioc_num + 1); 3505 ioc_no = cb_args->cb_ioc_num; 3506 ioc = &gid_info->gl_iou->iou_ioc_info[ioc_no]; 3507 timeout_id = &ioc->ioc_dc_timeout_id; 3508 break; 3509 } 3510 3511 mutex_exit(&gid_info->gl_mutex); 3512 3513 *timeout_id = timeout(ibdm_pkt_timeout_hdlr, 3514 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 3515 3516 IBTF_DPRINTF_L5("ibdm", "\tretry_command: %p,%x,%d,%d:" 3517 "timeout %x", cb_args->cb_req_type, cb_args->cb_ioc_num, 3518 cb_args->cb_srvents_start, *timeout_id); 3519 3520 if ((rval = ibmf_msg_transport(gid_info->gl_ibmf_hdl, 3521 gid_info->gl_qp_hdl, msg, NULL, ibdm_ibmf_send_cb, 3522 cb_args, 0)) != IBMF_SUCCESS) { 3523 IBTF_DPRINTF_L2("ibdm", "\tretry_command: send failed: %p " 3524 "rtype 0x%x iocidx 0x%x srvidx %d", cb_args->cb_gid_info, 3525 cb_args->cb_req_type, cb_args->cb_ioc_num, 3526 cb_args->cb_srvents_start); 3527 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 3528 rval = IBDM_FAILURE; 3529 } 3530 mutex_enter(&gid_info->gl_mutex); 3531 return (rval); 3532 } 3533 3534 3535 /* 3536 * ibdm_update_ioc_port_gidlist() 3537 */ 3538 static void 3539 ibdm_update_ioc_port_gidlist(ibdm_ioc_info_t *dest, 3540 ibdm_dp_gidinfo_t *gid_info) 3541 { 3542 int ii, ngid_ents; 3543 ibdm_gid_t *tmp; 3544 ibdm_hca_list_t *gid_hca_head, *temp; 3545 ibdm_hca_list_t *ioc_head = NULL; 3546 3547 IBTF_DPRINTF_L5("ibdm", "\tupdate_ioc_port_gidlist: Enter"); 3548 3549 ngid_ents = gid_info->gl_ngids; 3550 dest->ioc_nportgids = ngid_ents; 3551 dest->ioc_gid_list = kmem_zalloc(sizeof (ibdm_gid_t) * 3552 ngid_ents, KM_SLEEP); 3553 tmp = gid_info->gl_gid; 3554 for (ii = 0; (ii < ngid_ents) && (tmp); ii++) { 3555 dest->ioc_gid_list[ii].gid_dgid_hi = tmp->gid_dgid_hi; 3556 dest->ioc_gid_list[ii].gid_dgid_lo = tmp->gid_dgid_lo; 3557 tmp = tmp->gid_next; 3558 } 3559 3560 gid_hca_head = gid_info->gl_hca_list; 3561 while (gid_hca_head) { 3562 temp = ibdm_dup_hca_attr(gid_hca_head); 3563 temp->hl_next = ioc_head; 3564 ioc_head = temp; 3565 gid_hca_head = gid_hca_head->hl_next; 3566 } 3567 dest->ioc_hca_list = ioc_head; 3568 } 3569 3570 3571 /* 3572 * ibdm_alloc_send_buffers() 3573 * Allocates memory for the IBMF send buffer 3574 */ 3575 static void 3576 ibdm_alloc_send_buffers(ibmf_msg_t *msgp) 3577 { 3578 msgp->im_msgbufs_send.im_bufs_mad_hdr = 3579 kmem_zalloc(IBDM_MAD_SIZE, KM_SLEEP); 3580 msgp->im_msgbufs_send.im_bufs_cl_data = (uchar_t *) 3581 msgp->im_msgbufs_send.im_bufs_mad_hdr + sizeof (ib_mad_hdr_t); 3582 msgp->im_msgbufs_send.im_bufs_cl_data_len = 3583 IBDM_MAD_SIZE - sizeof (ib_mad_hdr_t); 3584 } 3585 3586 3587 /* 3588 * ibdm_alloc_send_buffers() 3589 * De-allocates memory for the IBMF send buffer 3590 */ 3591 static void 3592 ibdm_free_send_buffers(ibmf_msg_t *msgp) 3593 { 3594 if (msgp->im_msgbufs_send.im_bufs_mad_hdr != NULL) 3595 kmem_free(msgp->im_msgbufs_send.im_bufs_mad_hdr, IBDM_MAD_SIZE); 3596 } 3597 3598 /* 3599 * ibdm_probe_ioc() 3600 * 1. Gets the node records for the port GUID. This detects all the port 3601 * to the IOU. 3602 * 2. Selectively probes all the IOC, given it's node GUID 3603 * 3. In case of reprobe, only the IOC to be reprobed is send the IOC 3604 * Controller Profile asynchronously 3605 */ 3606 /*ARGSUSED*/ 3607 static void 3608 ibdm_probe_ioc(ib_guid_t nodeguid, ib_guid_t ioc_guid, int reprobe_flag) 3609 { 3610 int ii, nrecords; 3611 size_t nr_len = 0, pi_len = 0; 3612 ib_gid_t sgid, dgid; 3613 ibdm_hca_list_t *hca_list = NULL; 3614 sa_node_record_t *nr, *tmp; 3615 ibdm_port_attr_t *port = NULL; 3616 ibdm_dp_gidinfo_t *reprobe_gid, *new_gid, *node_gid; 3617 ibdm_dp_gidinfo_t *temp_gidinfo; 3618 ibdm_gid_t *temp_gid; 3619 sa_portinfo_record_t *pi; 3620 3621 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc(%x, %x, %x): Begin", 3622 nodeguid, ioc_guid, reprobe_flag); 3623 3624 /* Rescan the GID list for any removed GIDs for reprobe */ 3625 if (reprobe_flag) 3626 ibdm_rescan_gidlist(&ioc_guid); 3627 3628 mutex_enter(&ibdm.ibdm_hl_mutex); 3629 for (ibdm_get_next_port(&hca_list, &port, 1); port; 3630 ibdm_get_next_port(&hca_list, &port, 1)) { 3631 reprobe_gid = new_gid = node_gid = NULL; 3632 3633 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, nodeguid); 3634 if (nr == NULL) { 3635 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc: no records"); 3636 continue; 3637 } 3638 nrecords = (nr_len / sizeof (sa_node_record_t)); 3639 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 3640 pi = ibdm_get_portinfo( 3641 port->pa_sa_hdl, &pi_len, tmp->LID); 3642 3643 if ((pi) && (pi->PortInfo.CapabilityMask & 3644 SM_CAP_MASK_IS_DM_SUPPD)) { 3645 /* 3646 * For reprobes: Check if GID, already in 3647 * the list. If so, set the state to SKIPPED 3648 */ 3649 if (((temp_gidinfo = ibdm_find_gid(nodeguid, 3650 tmp->NodeInfo.PortGUID)) != NULL) && 3651 temp_gidinfo->gl_state == 3652 IBDM_GID_PROBING_COMPLETE) { 3653 ASSERT(reprobe_gid == NULL); 3654 ibdm_addto_glhcalist(temp_gidinfo, 3655 hca_list); 3656 reprobe_gid = temp_gidinfo; 3657 kmem_free(pi, pi_len); 3658 continue; 3659 } else if (temp_gidinfo != NULL) { 3660 kmem_free(pi, pi_len); 3661 ibdm_addto_glhcalist(temp_gidinfo, 3662 hca_list); 3663 continue; 3664 } 3665 3666 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : " 3667 "create_gid : prefix %llx, guid %llx\n", 3668 pi->PortInfo.GidPrefix, 3669 tmp->NodeInfo.PortGUID); 3670 3671 sgid.gid_prefix = port->pa_sn_prefix; 3672 sgid.gid_guid = port->pa_port_guid; 3673 dgid.gid_prefix = pi->PortInfo.GidPrefix; 3674 dgid.gid_guid = tmp->NodeInfo.PortGUID; 3675 new_gid = ibdm_create_gid_info(port, sgid, 3676 dgid); 3677 if (new_gid == NULL) { 3678 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 3679 "create_gid_info failed\n"); 3680 kmem_free(pi, pi_len); 3681 continue; 3682 } 3683 if (node_gid == NULL) { 3684 node_gid = new_gid; 3685 ibdm_add_to_gl_gid(node_gid, node_gid); 3686 } else { 3687 IBTF_DPRINTF_L4("ibdm", 3688 "\tprobe_ioc: new gid"); 3689 temp_gid = kmem_zalloc( 3690 sizeof (ibdm_gid_t), KM_SLEEP); 3691 temp_gid->gid_dgid_hi = 3692 new_gid->gl_dgid_hi; 3693 temp_gid->gid_dgid_lo = 3694 new_gid->gl_dgid_lo; 3695 temp_gid->gid_next = node_gid->gl_gid; 3696 node_gid->gl_gid = temp_gid; 3697 node_gid->gl_ngids++; 3698 } 3699 new_gid->gl_nodeguid = nodeguid; 3700 new_gid->gl_portguid = dgid.gid_guid; 3701 ibdm_addto_glhcalist(new_gid, hca_list); 3702 3703 /* 3704 * Set the state to skipped as all these 3705 * gids point to the same node. 3706 * We (re)probe only one GID below and reset 3707 * state appropriately 3708 */ 3709 new_gid->gl_state = IBDM_GID_PROBING_SKIPPED; 3710 kmem_free(pi, pi_len); 3711 } 3712 } 3713 kmem_free(nr, nr_len); 3714 3715 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : reprobe_flag %d " 3716 "reprobe_gid %p new_gid %p node_gid %p", 3717 reprobe_flag, reprobe_gid, new_gid, node_gid); 3718 3719 if (reprobe_flag != 0 && reprobe_gid != NULL) { 3720 int niocs, jj; 3721 ibdm_ioc_info_t *tmp_ioc; 3722 int ioc_matched = 0; 3723 3724 mutex_exit(&ibdm.ibdm_hl_mutex); 3725 mutex_enter(&reprobe_gid->gl_mutex); 3726 reprobe_gid->gl_state = IBDM_GET_IOC_DETAILS; 3727 niocs = 3728 reprobe_gid->gl_iou->iou_info.iou_num_ctrl_slots; 3729 reprobe_gid->gl_pending_cmds++; 3730 mutex_exit(&reprobe_gid->gl_mutex); 3731 3732 for (jj = 0; jj < niocs; jj++) { 3733 tmp_ioc = 3734 IBDM_GIDINFO2IOCINFO(reprobe_gid, jj); 3735 if (tmp_ioc->ioc_profile.ioc_guid != ioc_guid) 3736 continue; 3737 3738 ioc_matched = 1; 3739 3740 /* 3741 * Explicitly set gl_reprobe_flag to 0 so that 3742 * IBnex is not notified on completion 3743 */ 3744 mutex_enter(&reprobe_gid->gl_mutex); 3745 reprobe_gid->gl_reprobe_flag = 0; 3746 mutex_exit(&reprobe_gid->gl_mutex); 3747 3748 mutex_enter(&ibdm.ibdm_mutex); 3749 ibdm.ibdm_ngid_probes_in_progress++; 3750 mutex_exit(&ibdm.ibdm_mutex); 3751 if (ibdm_send_ioc_profile(reprobe_gid, jj) != 3752 IBDM_SUCCESS) { 3753 IBTF_DPRINTF_L4("ibdm", 3754 "\tprobe_ioc: " 3755 "send_ioc_profile failed " 3756 "for ioc %d", jj); 3757 ibdm_gid_decr_pending(reprobe_gid); 3758 break; 3759 } 3760 mutex_enter(&ibdm.ibdm_mutex); 3761 ibdm_wait_probe_completion(); 3762 mutex_exit(&ibdm.ibdm_mutex); 3763 break; 3764 } 3765 if (ioc_matched == 0) 3766 ibdm_gid_decr_pending(reprobe_gid); 3767 else { 3768 mutex_enter(&ibdm.ibdm_hl_mutex); 3769 break; 3770 } 3771 } else if (new_gid != NULL) { 3772 mutex_exit(&ibdm.ibdm_hl_mutex); 3773 node_gid = node_gid ? node_gid : new_gid; 3774 3775 /* 3776 * New or reinserted GID : Enable notification 3777 * to IBnex 3778 */ 3779 mutex_enter(&node_gid->gl_mutex); 3780 node_gid->gl_reprobe_flag = 1; 3781 mutex_exit(&node_gid->gl_mutex); 3782 3783 ibdm_probe_gid(node_gid); 3784 3785 mutex_enter(&ibdm.ibdm_hl_mutex); 3786 } else { 3787 IBTF_DPRINTF_L2("ibdm", "\tibdm_probe_ioc " 3788 "Invalid state!"); 3789 } 3790 3791 } 3792 mutex_exit(&ibdm.ibdm_hl_mutex); 3793 IBTF_DPRINTF_L4("ibdm", "\tprobe_ioc : End\n"); 3794 } 3795 3796 3797 /* 3798 * ibdm_probe_gid() 3799 * Selectively probes the GID 3800 */ 3801 static void 3802 ibdm_probe_gid(ibdm_dp_gidinfo_t *gid_info) 3803 { 3804 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid:"); 3805 mutex_enter(&gid_info->gl_mutex); 3806 gid_info->gl_pending_cmds++; 3807 gid_info->gl_state = IBDM_GET_CLASSPORTINFO; 3808 mutex_exit(&gid_info->gl_mutex); 3809 if (ibdm_send_classportinfo(gid_info) != IBDM_SUCCESS) { 3810 mutex_enter(&gid_info->gl_mutex); 3811 gid_info->gl_state = IBDM_GID_PROBING_FAILED; 3812 --gid_info->gl_pending_cmds; 3813 mutex_exit(&gid_info->gl_mutex); 3814 ibdm_delete_glhca_list(gid_info); 3815 gid_info = gid_info->gl_next; 3816 return; 3817 } 3818 mutex_enter(&ibdm.ibdm_mutex); 3819 ibdm.ibdm_ngid_probes_in_progress++; 3820 gid_info = gid_info->gl_next; 3821 3822 ibdm_wait_probe_completion(); 3823 mutex_exit(&ibdm.ibdm_mutex); 3824 IBTF_DPRINTF_L4("ibdm", "\tprobe_gid: Wakeup signal received"); 3825 } 3826 3827 3828 /* 3829 * ibdm_create_gid_info() 3830 * Allocates a gid_info structure and initializes 3831 * Returns pointer to the structure on success 3832 * and NULL on failure 3833 */ 3834 static ibdm_dp_gidinfo_t * 3835 ibdm_create_gid_info(ibdm_port_attr_t *port, ib_gid_t sgid, ib_gid_t dgid) 3836 { 3837 uint8_t ii, npaths; 3838 sa_path_record_t *path; 3839 size_t len; 3840 ibdm_pkey_tbl_t *pkey_tbl; 3841 ibdm_dp_gidinfo_t *gid_info = NULL; 3842 int ret; 3843 3844 IBTF_DPRINTF_L4("ibdm", "\tcreate_gid_info: Begin"); 3845 npaths = 1; 3846 3847 /* query for reversible paths */ 3848 if (port->pa_sa_hdl) 3849 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, 3850 sgid, dgid, IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, 3851 &len, &path); 3852 else 3853 return (NULL); 3854 3855 if (ret == IBMF_SUCCESS && path) { 3856 ibdm_dump_path_info(path); 3857 3858 gid_info = kmem_zalloc( 3859 sizeof (ibdm_dp_gidinfo_t), KM_SLEEP); 3860 mutex_init(&gid_info->gl_mutex, NULL, MUTEX_DEFAULT, NULL); 3861 gid_info->gl_dgid_hi = path->DGID.gid_prefix; 3862 gid_info->gl_dgid_lo = path->DGID.gid_guid; 3863 gid_info->gl_sgid_hi = path->SGID.gid_prefix; 3864 gid_info->gl_sgid_lo = path->SGID.gid_guid; 3865 gid_info->gl_p_key = path->P_Key; 3866 gid_info->gl_sa_hdl = port->pa_sa_hdl; 3867 gid_info->gl_ibmf_hdl = port->pa_ibmf_hdl; 3868 gid_info->gl_slid = path->SLID; 3869 gid_info->gl_dlid = path->DLID; 3870 gid_info->gl_transactionID = (++ibdm.ibdm_transactionID) 3871 << IBDM_GID_TRANSACTIONID_SHIFT; 3872 gid_info->gl_min_transactionID = gid_info->gl_transactionID; 3873 gid_info->gl_max_transactionID = (ibdm.ibdm_transactionID +1) 3874 << IBDM_GID_TRANSACTIONID_SHIFT; 3875 3876 gid_info->gl_qp_hdl = IBMF_QP_HANDLE_DEFAULT; 3877 for (ii = 0; ii < port->pa_npkeys; ii++) { 3878 if (port->pa_pkey_tbl == NULL) 3879 break; 3880 3881 pkey_tbl = &port->pa_pkey_tbl[ii]; 3882 if ((gid_info->gl_p_key == pkey_tbl->pt_pkey) && 3883 (pkey_tbl->pt_qp_hdl != NULL)) { 3884 gid_info->gl_qp_hdl = pkey_tbl->pt_qp_hdl; 3885 break; 3886 } 3887 } 3888 kmem_free(path, len); 3889 3890 /* 3891 * QP handle for GID not initialized. No matching Pkey 3892 * was found!! ibdm should *not* hit this case. Flag an 3893 * error and drop the GID if ibdm does encounter this. 3894 */ 3895 if (gid_info->gl_qp_hdl == NULL) { 3896 IBTF_DPRINTF_L2(ibdm_string, 3897 "\tcreate_gid_info: No matching Pkey"); 3898 ibdm_delete_gidinfo(gid_info); 3899 return (NULL); 3900 } 3901 3902 ibdm.ibdm_ngids++; 3903 if (ibdm.ibdm_dp_gidlist_head == NULL) { 3904 ibdm.ibdm_dp_gidlist_head = gid_info; 3905 ibdm.ibdm_dp_gidlist_tail = gid_info; 3906 } else { 3907 ibdm.ibdm_dp_gidlist_tail->gl_next = gid_info; 3908 gid_info->gl_prev = ibdm.ibdm_dp_gidlist_tail; 3909 ibdm.ibdm_dp_gidlist_tail = gid_info; 3910 } 3911 } 3912 3913 return (gid_info); 3914 } 3915 3916 3917 /* 3918 * ibdm_get_node_records 3919 * Sends a SA query to get the NODE record 3920 * Returns pointer to the sa_node_record_t on success 3921 * and NULL on failure 3922 */ 3923 static sa_node_record_t * 3924 ibdm_get_node_records(ibmf_saa_handle_t sa_hdl, size_t *length, ib_guid_t guid) 3925 { 3926 sa_node_record_t req, *resp = NULL; 3927 ibmf_saa_access_args_t args; 3928 int ret; 3929 3930 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: Begin"); 3931 3932 bzero(&req, sizeof (sa_node_record_t)); 3933 req.NodeInfo.NodeGUID = guid; 3934 3935 args.sq_attr_id = SA_NODERECORD_ATTRID; 3936 args.sq_access_type = IBMF_SAA_RETRIEVE; 3937 args.sq_component_mask = SA_NODEINFO_COMPMASK_NODEGUID; 3938 args.sq_template = &req; 3939 args.sq_callback = NULL; 3940 args.sq_callback_arg = NULL; 3941 3942 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3943 if (ret != IBMF_SUCCESS) { 3944 IBTF_DPRINTF_L2("ibdm", "\tget_node_records:" 3945 " SA Retrieve Failed: %d", ret); 3946 return (NULL); 3947 } 3948 if ((resp == NULL) || (*length == 0)) { 3949 IBTF_DPRINTF_L2("ibdm", "\tget_node_records: No records"); 3950 return (NULL); 3951 } 3952 3953 IBTF_DPRINTF_L4("ibdm", "\tget_node_records: NodeGuid %llx " 3954 "PortGUID %llx", resp->NodeInfo.NodeGUID, resp->NodeInfo.PortGUID); 3955 3956 return (resp); 3957 } 3958 3959 3960 /* 3961 * ibdm_get_portinfo() 3962 * Sends a SA query to get the PortInfo record 3963 * Returns pointer to the sa_portinfo_record_t on success 3964 * and NULL on failure 3965 */ 3966 static sa_portinfo_record_t * 3967 ibdm_get_portinfo(ibmf_saa_handle_t sa_hdl, size_t *length, ib_lid_t lid) 3968 { 3969 sa_portinfo_record_t req, *resp = NULL; 3970 ibmf_saa_access_args_t args; 3971 int ret; 3972 3973 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: Begin"); 3974 3975 bzero(&req, sizeof (sa_portinfo_record_t)); 3976 req.EndportLID = lid; 3977 3978 args.sq_attr_id = SA_PORTINFORECORD_ATTRID; 3979 args.sq_access_type = IBMF_SAA_RETRIEVE; 3980 args.sq_component_mask = SA_PORTINFO_COMPMASK_PORTLID; 3981 args.sq_template = &req; 3982 args.sq_callback = NULL; 3983 args.sq_callback_arg = NULL; 3984 3985 ret = ibmf_sa_access(sa_hdl, &args, 0, length, (void **) &resp); 3986 if (ret != IBMF_SUCCESS) { 3987 IBTF_DPRINTF_L2("ibdm", "\tget_portinfo:" 3988 " SA Retrieve Failed: 0x%X", ret); 3989 return (NULL); 3990 } 3991 if ((*length == 0) || (resp == NULL)) 3992 return (NULL); 3993 3994 IBTF_DPRINTF_L4("ibdm", "\tget_portinfo: GidPrefix %llx Cap 0x%x", 3995 resp->PortInfo.GidPrefix, resp->PortInfo.CapabilityMask); 3996 return (resp); 3997 } 3998 3999 4000 /* 4001 * ibdm_ibnex_register_callback 4002 * IB nexus callback routine for HCA attach and detach notification 4003 */ 4004 void 4005 ibdm_ibnex_register_callback(ibdm_callback_t ibnex_dm_callback) 4006 { 4007 IBTF_DPRINTF_L4("ibdm", "\tibnex_register_callbacks"); 4008 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4009 ibdm.ibdm_ibnex_callback = ibnex_dm_callback; 4010 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4011 } 4012 4013 4014 /* 4015 * ibdm_ibnex_unregister_callbacks 4016 */ 4017 void 4018 ibdm_ibnex_unregister_callback() 4019 { 4020 IBTF_DPRINTF_L4("ibdm", "\tibnex_unregister_callbacks"); 4021 mutex_enter(&ibdm.ibdm_ibnex_mutex); 4022 ibdm.ibdm_ibnex_callback = NULL; 4023 mutex_exit(&ibdm.ibdm_ibnex_mutex); 4024 } 4025 4026 4027 /* 4028 * ibdm_ibnex_get_waittime() 4029 * Calculates the wait time based on the last HCA attach time 4030 */ 4031 time_t 4032 ibdm_ibnex_get_waittime(ib_guid_t hca_guid, int *dft_wait) 4033 { 4034 int ii; 4035 time_t temp, wait_time = 0; 4036 ibdm_hca_list_t *hca; 4037 4038 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime hcaguid:%llx" 4039 "\tport settling time %d", hca_guid, *dft_wait); 4040 4041 mutex_enter(&ibdm.ibdm_hl_mutex); 4042 hca = ibdm.ibdm_hca_list_head; 4043 4044 if (hca_guid) { 4045 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4046 if ((hca_guid == hca->hl_hca_guid) && 4047 (hca->hl_nports != hca->hl_nports_active)) { 4048 wait_time = 4049 ddi_get_time() - hca->hl_attach_time; 4050 wait_time = ((wait_time >= *dft_wait) ? 4051 0 : (*dft_wait - wait_time)); 4052 break; 4053 } 4054 hca = hca->hl_next; 4055 } 4056 mutex_exit(&ibdm.ibdm_hl_mutex); 4057 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4058 return (wait_time); 4059 } 4060 4061 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4062 if (hca->hl_nports != hca->hl_nports_active) { 4063 temp = ddi_get_time() - hca->hl_attach_time; 4064 temp = ((temp >= *dft_wait) ? 0 : (*dft_wait - temp)); 4065 wait_time = (temp > wait_time) ? temp : wait_time; 4066 } 4067 } 4068 mutex_exit(&ibdm.ibdm_hl_mutex); 4069 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_waittime %llx", wait_time); 4070 return (wait_time); 4071 } 4072 4073 4074 /* 4075 * ibdm_ibnex_probe_hcaport 4076 * Probes the presence of HCA port (with HCA dip and port number) 4077 * Returns port attributes structure on SUCCESS 4078 */ 4079 ibdm_port_attr_t * 4080 ibdm_ibnex_probe_hcaport(ib_guid_t hca_guid, uint8_t port_num) 4081 { 4082 int ii, jj; 4083 ibdm_hca_list_t *hca_list; 4084 ibdm_port_attr_t *port_attr; 4085 4086 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_hcaport:"); 4087 4088 mutex_enter(&ibdm.ibdm_hl_mutex); 4089 hca_list = ibdm.ibdm_hca_list_head; 4090 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4091 if (hca_list->hl_hca_guid == hca_guid) { 4092 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4093 if (hca_list->hl_port_attr[jj].pa_port_num == 4094 port_num) { 4095 break; 4096 } 4097 } 4098 if (jj != hca_list->hl_nports) 4099 break; 4100 } 4101 hca_list = hca_list->hl_next; 4102 } 4103 if (ii == ibdm.ibdm_hca_count) { 4104 IBTF_DPRINTF_L2("ibdm", "\tibnex_probe_hcaport: not found"); 4105 mutex_exit(&ibdm.ibdm_hl_mutex); 4106 return (NULL); 4107 } 4108 port_attr = (ibdm_port_attr_t *)kmem_zalloc( 4109 sizeof (ibdm_port_attr_t), KM_SLEEP); 4110 bcopy((char *)&hca_list->hl_port_attr[jj], 4111 port_attr, sizeof (ibdm_port_attr_t)); 4112 ibdm_update_port_attr(port_attr); 4113 4114 mutex_exit(&ibdm.ibdm_hl_mutex); 4115 return (port_attr); 4116 } 4117 4118 4119 /* 4120 * ibdm_ibnex_get_port_attrs 4121 * Scan all HCAs for a matching port_guid. 4122 * Returns "port attributes" structure on success. 4123 */ 4124 ibdm_port_attr_t * 4125 ibdm_ibnex_get_port_attrs(ib_guid_t port_guid) 4126 { 4127 int ii, jj; 4128 ibdm_hca_list_t *hca_list; 4129 ibdm_port_attr_t *port_attr; 4130 4131 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_port_attrs:"); 4132 4133 mutex_enter(&ibdm.ibdm_hl_mutex); 4134 hca_list = ibdm.ibdm_hca_list_head; 4135 4136 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4137 for (jj = 0; jj < hca_list->hl_nports; jj++) { 4138 if (hca_list->hl_port_attr[jj].pa_port_guid == 4139 port_guid) { 4140 break; 4141 } 4142 } 4143 if (jj != hca_list->hl_nports) 4144 break; 4145 hca_list = hca_list->hl_next; 4146 } 4147 4148 if (ii == ibdm.ibdm_hca_count) { 4149 IBTF_DPRINTF_L2("ibdm", "\tibnex_get_port_attrs: not found"); 4150 mutex_exit(&ibdm.ibdm_hl_mutex); 4151 return (NULL); 4152 } 4153 4154 port_attr = (ibdm_port_attr_t *)kmem_alloc(sizeof (ibdm_port_attr_t), 4155 KM_SLEEP); 4156 bcopy((char *)&hca_list->hl_port_attr[jj], port_attr, 4157 sizeof (ibdm_port_attr_t)); 4158 ibdm_update_port_attr(port_attr); 4159 4160 mutex_exit(&ibdm.ibdm_hl_mutex); 4161 return (port_attr); 4162 } 4163 4164 4165 /* 4166 * ibdm_ibnex_free_port_attr() 4167 */ 4168 void 4169 ibdm_ibnex_free_port_attr(ibdm_port_attr_t *port_attr) 4170 { 4171 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_port_attr:"); 4172 if (port_attr) { 4173 if (port_attr->pa_pkey_tbl != NULL) { 4174 kmem_free(port_attr->pa_pkey_tbl, 4175 (port_attr->pa_npkeys * sizeof (ibdm_pkey_tbl_t))); 4176 } 4177 kmem_free(port_attr, sizeof (ibdm_port_attr_t)); 4178 } 4179 } 4180 4181 4182 /* 4183 * ibdm_ibnex_get_hca_list() 4184 * Returns portinfo for all the port for all the HCA's 4185 */ 4186 void 4187 ibdm_ibnex_get_hca_list(ibdm_hca_list_t **hca, int *count) 4188 { 4189 ibdm_hca_list_t *head = NULL, *temp, *temp1; 4190 int ii; 4191 4192 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_list:"); 4193 4194 mutex_enter(&ibdm.ibdm_hl_mutex); 4195 temp = ibdm.ibdm_hca_list_head; 4196 for (ii = 0; ii < ibdm.ibdm_hca_count; ii++) { 4197 temp1 = ibdm_dup_hca_attr(temp); 4198 temp1->hl_next = head; 4199 head = temp1; 4200 temp = temp->hl_next; 4201 } 4202 *count = ibdm.ibdm_hca_count; 4203 *hca = head; 4204 mutex_exit(&ibdm.ibdm_hl_mutex); 4205 } 4206 4207 4208 /* 4209 * ibdm_ibnex_get_hca_info_by_guid() 4210 */ 4211 ibdm_hca_list_t * 4212 ibdm_ibnex_get_hca_info_by_guid(ib_guid_t hca_guid) 4213 { 4214 ibdm_hca_list_t *head = NULL, *hca = NULL; 4215 4216 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip"); 4217 4218 mutex_enter(&ibdm.ibdm_hl_mutex); 4219 head = ibdm.ibdm_hca_list_head; 4220 while (head) { 4221 if (head->hl_hca_guid == hca_guid) { 4222 hca = ibdm_dup_hca_attr(head); 4223 hca->hl_next = NULL; 4224 break; 4225 } 4226 head = head->hl_next; 4227 } 4228 mutex_exit(&ibdm.ibdm_hl_mutex); 4229 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_hca_info_by_dip %p", hca); 4230 return (hca); 4231 } 4232 4233 4234 /* 4235 * ibdm_dup_hca_attr() 4236 * Allocate a new HCA attribute strucuture and initialize 4237 * hca attribute structure with the incoming HCA attributes 4238 * returned the allocated hca attributes. 4239 */ 4240 static ibdm_hca_list_t * 4241 ibdm_dup_hca_attr(ibdm_hca_list_t *in_hca) 4242 { 4243 int len; 4244 ibdm_hca_list_t *out_hca; 4245 4246 len = sizeof (ibdm_hca_list_t) + 4247 (in_hca->hl_nports * sizeof (ibdm_port_attr_t)); 4248 IBTF_DPRINTF_L4("ibdm", "\tdup_hca_attr len %d", len); 4249 out_hca = (ibdm_hca_list_t *)kmem_alloc(len, KM_SLEEP); 4250 bcopy((char *)in_hca, 4251 (char *)out_hca, sizeof (ibdm_hca_list_t)); 4252 if (in_hca->hl_nports) { 4253 out_hca->hl_port_attr = (ibdm_port_attr_t *) 4254 ((char *)out_hca + sizeof (ibdm_hca_list_t)); 4255 bcopy((char *)in_hca->hl_port_attr, 4256 (char *)out_hca->hl_port_attr, 4257 (in_hca->hl_nports * sizeof (ibdm_port_attr_t))); 4258 for (len = 0; len < out_hca->hl_nports; len++) 4259 ibdm_update_port_attr(&out_hca->hl_port_attr[len]); 4260 } 4261 return (out_hca); 4262 } 4263 4264 4265 /* 4266 * ibdm_ibnex_free_hca_list() 4267 * Free one/more HCA lists 4268 */ 4269 void 4270 ibdm_ibnex_free_hca_list(ibdm_hca_list_t *hca_list) 4271 { 4272 int ii, len; 4273 ibdm_hca_list_t *temp; 4274 ibdm_port_attr_t *port; 4275 4276 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_hca_list:"); 4277 ASSERT(hca_list); 4278 while (hca_list) { 4279 temp = hca_list; 4280 hca_list = hca_list->hl_next; 4281 for (ii = 0; ii < temp->hl_nports; ii++) { 4282 port = &temp->hl_port_attr[ii]; 4283 len = (port->pa_npkeys * sizeof (ibdm_pkey_tbl_t)); 4284 if (len != 0) 4285 kmem_free(port->pa_pkey_tbl, len); 4286 } 4287 len = sizeof (ibdm_hca_list_t) + (temp->hl_nports * 4288 sizeof (ibdm_port_attr_t)); 4289 kmem_free(temp, len); 4290 } 4291 } 4292 4293 4294 /* 4295 * ibdm_ibnex_probe_iocguid() 4296 * Probes the IOC on the fabric and returns the IOC information 4297 * if present. Otherwise, NULL is returned 4298 */ 4299 /* ARGSUSED */ 4300 ibdm_ioc_info_t * 4301 ibdm_ibnex_probe_ioc(ib_guid_t iou, ib_guid_t ioc_guid, int reprobe_flag) 4302 { 4303 int k; 4304 ibdm_ioc_info_t *ioc_info; 4305 ibdm_dp_gidinfo_t *gid_info; 4306 4307 IBTF_DPRINTF_L4("ibdm", "\tibnex_probe_ioc: (%llX, %llX, %d) Begin", 4308 iou, ioc_guid, reprobe_flag); 4309 /* Check whether we know this already */ 4310 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4311 if (ioc_info == NULL) { 4312 mutex_enter(&ibdm.ibdm_mutex); 4313 while (ibdm.ibdm_busy & IBDM_BUSY) 4314 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4315 ibdm.ibdm_busy |= IBDM_BUSY; 4316 mutex_exit(&ibdm.ibdm_mutex); 4317 ibdm_probe_ioc(iou, ioc_guid, 0); 4318 mutex_enter(&ibdm.ibdm_mutex); 4319 ibdm.ibdm_busy &= ~IBDM_BUSY; 4320 cv_broadcast(&ibdm.ibdm_busy_cv); 4321 mutex_exit(&ibdm.ibdm_mutex); 4322 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4323 } else if (reprobe_flag) { /* Handle Reprobe for the IOC */ 4324 /* Free the ioc_list before reprobe; and cancel any timers */ 4325 mutex_enter(&ibdm.ibdm_mutex); 4326 if (ioc_info->ioc_timeout_id) { 4327 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4328 "ioc_timeout_id = 0x%x", 4329 ioc_info->ioc_timeout_id); 4330 if (untimeout(ioc_info->ioc_timeout_id) == -1) { 4331 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4332 "untimeout ioc_timeout_id failed"); 4333 } 4334 ioc_info->ioc_timeout_id = 0; 4335 } 4336 if (ioc_info->ioc_dc_timeout_id) { 4337 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4338 "ioc_dc_timeout_id = 0x%x", 4339 ioc_info->ioc_dc_timeout_id); 4340 if (untimeout(ioc_info->ioc_dc_timeout_id) == -1) { 4341 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4342 "untimeout ioc_dc_timeout_id failed"); 4343 } 4344 ioc_info->ioc_dc_timeout_id = 0; 4345 } 4346 for (k = 0; k < ioc_info->ioc_profile.ioc_service_entries; k++) 4347 if (ioc_info->ioc_serv[k].se_timeout_id) { 4348 IBTF_DPRINTF_L5("ibdm", "\tprobe_ioc: " 4349 "ioc_info->ioc_serv[k].se_timeout_id = %x", 4350 k, ioc_info->ioc_serv[k].se_timeout_id); 4351 if (untimeout(ioc_info->ioc_serv[k]. 4352 se_timeout_id) == -1) { 4353 IBTF_DPRINTF_L2("ibdm", "\tprobe_ioc: " 4354 "untimeout se_timeout_id %d " 4355 "failed", k); 4356 } 4357 ioc_info->ioc_serv[k].se_timeout_id = 0; 4358 } 4359 mutex_exit(&ibdm.ibdm_mutex); 4360 ibdm_ibnex_free_ioc_list(ioc_info); 4361 4362 mutex_enter(&ibdm.ibdm_mutex); 4363 while (ibdm.ibdm_busy & IBDM_BUSY) 4364 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4365 ibdm.ibdm_busy |= IBDM_BUSY; 4366 mutex_exit(&ibdm.ibdm_mutex); 4367 4368 ibdm_probe_ioc(iou, ioc_guid, 1); 4369 4370 /* 4371 * Skip if gl_reprobe_flag is set, this will be 4372 * a re-inserted / new GID, for which notifications 4373 * have already been send. 4374 */ 4375 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; 4376 gid_info = gid_info->gl_next) { 4377 uint8_t ii, niocs; 4378 ibdm_ioc_info_t *ioc; 4379 4380 if (gid_info->gl_iou == NULL) 4381 continue; 4382 4383 if (gid_info->gl_reprobe_flag) { 4384 gid_info->gl_reprobe_flag = 0; 4385 continue; 4386 } 4387 4388 niocs = gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 4389 for (ii = 0; ii < niocs; ii++) { 4390 ioc = IBDM_GIDINFO2IOCINFO(gid_info, ii); 4391 if (ioc->ioc_profile.ioc_guid == ioc_guid) { 4392 mutex_enter(&ibdm.ibdm_mutex); 4393 ibdm_reprobe_update_port_srv(ioc, 4394 gid_info); 4395 mutex_exit(&ibdm.ibdm_mutex); 4396 } 4397 } 4398 } 4399 mutex_enter(&ibdm.ibdm_mutex); 4400 ibdm.ibdm_busy &= ~IBDM_BUSY; 4401 cv_broadcast(&ibdm.ibdm_busy_cv); 4402 mutex_exit(&ibdm.ibdm_mutex); 4403 4404 ioc_info = ibdm_ibnex_get_ioc_info(ioc_guid); 4405 } 4406 return (ioc_info); 4407 } 4408 4409 4410 /* 4411 * ibdm_ibnex_get_ioc_info() 4412 * Returns pointer to ibdm_ioc_info_t if it finds 4413 * matching record for the ioc_guid, otherwise NULL 4414 * is returned 4415 */ 4416 ibdm_ioc_info_t * 4417 ibdm_ibnex_get_ioc_info(ib_guid_t ioc_guid) 4418 { 4419 int ii; 4420 ibdm_ioc_info_t *ioc = NULL, *tmp = NULL; 4421 ibdm_dp_gidinfo_t *gid_list; 4422 ib_dm_io_unitinfo_t *iou; 4423 4424 IBTF_DPRINTF_L4("ibdm", "\tibnex_get_ioc_info: GUID %llx", ioc_guid); 4425 4426 mutex_enter(&ibdm.ibdm_mutex); 4427 while (ibdm.ibdm_busy & IBDM_BUSY) 4428 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4429 ibdm.ibdm_busy |= IBDM_BUSY; 4430 4431 gid_list = ibdm.ibdm_dp_gidlist_head; 4432 while (gid_list) { 4433 mutex_enter(&gid_list->gl_mutex); 4434 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4435 mutex_exit(&gid_list->gl_mutex); 4436 gid_list = gid_list->gl_next; 4437 continue; 4438 } 4439 if (gid_list->gl_iou == NULL) { 4440 IBTF_DPRINTF_L2("ibdm", 4441 "\tget_ioc_info: No IOU info"); 4442 mutex_exit(&gid_list->gl_mutex); 4443 gid_list = gid_list->gl_next; 4444 continue; 4445 } 4446 iou = &gid_list->gl_iou->iou_info; 4447 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4448 tmp = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4449 if ((tmp->ioc_profile.ioc_guid == ioc_guid) && 4450 (tmp->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS)) { 4451 ioc = ibdm_dup_ioc_info(tmp, gid_list); 4452 mutex_exit(&gid_list->gl_mutex); 4453 ibdm.ibdm_busy &= ~IBDM_BUSY; 4454 cv_broadcast(&ibdm.ibdm_busy_cv); 4455 mutex_exit(&ibdm.ibdm_mutex); 4456 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: End"); 4457 return (ioc); 4458 } 4459 } 4460 if (ii == iou->iou_num_ctrl_slots) 4461 ioc = NULL; 4462 4463 mutex_exit(&gid_list->gl_mutex); 4464 gid_list = gid_list->gl_next; 4465 } 4466 4467 ibdm.ibdm_busy &= ~IBDM_BUSY; 4468 cv_broadcast(&ibdm.ibdm_busy_cv); 4469 mutex_exit(&ibdm.ibdm_mutex); 4470 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_info: failure End"); 4471 return (ioc); 4472 } 4473 4474 4475 /* 4476 * ibdm_ibnex_get_ioc_count() 4477 * Returns number of ibdm_ioc_info_t it finds 4478 */ 4479 int 4480 ibdm_ibnex_get_ioc_count(void) 4481 { 4482 int count = 0, k; 4483 ibdm_ioc_info_t *ioc; 4484 ibdm_dp_gidinfo_t *gid_list; 4485 4486 mutex_enter(&ibdm.ibdm_mutex); 4487 ibdm_sweep_fabric(0); 4488 4489 while (ibdm.ibdm_busy & IBDM_BUSY) 4490 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4491 ibdm.ibdm_busy |= IBDM_BUSY; 4492 4493 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 4494 gid_list = gid_list->gl_next) { 4495 mutex_enter(&gid_list->gl_mutex); 4496 if ((gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) || 4497 (gid_list->gl_iou == NULL)) { 4498 mutex_exit(&gid_list->gl_mutex); 4499 continue; 4500 } 4501 for (k = 0; k < gid_list->gl_iou->iou_info.iou_num_ctrl_slots; 4502 k++) { 4503 ioc = IBDM_GIDINFO2IOCINFO(gid_list, k); 4504 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) 4505 ++count; 4506 } 4507 mutex_exit(&gid_list->gl_mutex); 4508 } 4509 ibdm.ibdm_busy &= ~IBDM_BUSY; 4510 cv_broadcast(&ibdm.ibdm_busy_cv); 4511 mutex_exit(&ibdm.ibdm_mutex); 4512 4513 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_count: count = %d", count); 4514 return (count); 4515 } 4516 4517 4518 /* 4519 * ibdm_ibnex_get_ioc_list() 4520 * Returns information about all the IOCs present on the fabric. 4521 * Reprobes the IOCs and the GID list if list_flag is set to REPROBE_ALL. 4522 * Does not sweep fabric if DONOT_PROBE is set 4523 */ 4524 ibdm_ioc_info_t * 4525 ibdm_ibnex_get_ioc_list(ibdm_ibnex_get_ioclist_mtd_t list_flag) 4526 { 4527 int ii; 4528 ibdm_ioc_info_t *ioc_list = NULL, *tmp, *ioc; 4529 ibdm_dp_gidinfo_t *gid_list; 4530 ib_dm_io_unitinfo_t *iou; 4531 4532 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: Enter"); 4533 4534 mutex_enter(&ibdm.ibdm_mutex); 4535 if (list_flag != IBDM_IBNEX_DONOT_PROBE) 4536 ibdm_sweep_fabric(list_flag == IBDM_IBNEX_REPROBE_ALL); 4537 4538 while (ibdm.ibdm_busy & IBDM_BUSY) 4539 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 4540 ibdm.ibdm_busy |= IBDM_BUSY; 4541 4542 gid_list = ibdm.ibdm_dp_gidlist_head; 4543 while (gid_list) { 4544 mutex_enter(&gid_list->gl_mutex); 4545 if (gid_list->gl_state != IBDM_GID_PROBING_COMPLETE) { 4546 mutex_exit(&gid_list->gl_mutex); 4547 gid_list = gid_list->gl_next; 4548 continue; 4549 } 4550 if (gid_list->gl_iou == NULL) { 4551 IBTF_DPRINTF_L2("ibdm", 4552 "\tget_ioc_list: No IOU info"); 4553 mutex_exit(&gid_list->gl_mutex); 4554 gid_list = gid_list->gl_next; 4555 continue; 4556 } 4557 iou = &gid_list->gl_iou->iou_info; 4558 for (ii = 0; ii < iou->iou_num_ctrl_slots; ii++) { 4559 ioc = IBDM_GIDINFO2IOCINFO(gid_list, ii); 4560 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4561 tmp = ibdm_dup_ioc_info(ioc, gid_list); 4562 tmp->ioc_next = ioc_list; 4563 ioc_list = tmp; 4564 } 4565 } 4566 mutex_exit(&gid_list->gl_mutex); 4567 gid_list = gid_list->gl_next; 4568 } 4569 ibdm.ibdm_busy &= ~IBDM_BUSY; 4570 cv_broadcast(&ibdm.ibdm_busy_cv); 4571 mutex_exit(&ibdm.ibdm_mutex); 4572 4573 IBTF_DPRINTF_L4("ibdm", "\tget_ioc_list: End"); 4574 return (ioc_list); 4575 } 4576 4577 /* 4578 * ibdm_dup_ioc_info() 4579 * Duplicate the IOC information and return the IOC 4580 * information. 4581 */ 4582 static ibdm_ioc_info_t * 4583 ibdm_dup_ioc_info(ibdm_ioc_info_t *in_ioc, ibdm_dp_gidinfo_t *gid_list) 4584 { 4585 ibdm_ioc_info_t *out_ioc; 4586 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*out_ioc)); 4587 4588 out_ioc = kmem_alloc(sizeof (ibdm_ioc_info_t), KM_SLEEP); 4589 bcopy(in_ioc, out_ioc, sizeof (ibdm_ioc_info_t)); 4590 ibdm_update_ioc_port_gidlist(out_ioc, gid_list); 4591 out_ioc->ioc_iou_dc_valid = gid_list->gl_iou->iou_dc_valid; 4592 out_ioc->ioc_iou_diagcode = gid_list->gl_iou->iou_diagcode; 4593 4594 return (out_ioc); 4595 } 4596 4597 4598 /* 4599 * ibdm_free_ioc_list() 4600 * Deallocate memory for IOC list structure 4601 */ 4602 void 4603 ibdm_ibnex_free_ioc_list(ibdm_ioc_info_t *ioc) 4604 { 4605 ibdm_ioc_info_t *temp; 4606 4607 IBTF_DPRINTF_L4("ibdm", "\tibnex_free_ioc_list:"); 4608 while (ioc) { 4609 temp = ioc; 4610 ioc = ioc->ioc_next; 4611 kmem_free(temp->ioc_gid_list, 4612 (sizeof (ibdm_gid_t) * temp->ioc_nportgids)); 4613 if (temp->ioc_hca_list) 4614 ibdm_ibnex_free_hca_list(temp->ioc_hca_list); 4615 kmem_free(temp, sizeof (ibdm_ioc_info_t)); 4616 } 4617 } 4618 4619 4620 /* 4621 * ibdm_ibnex_update_pkey_tbls 4622 * Updates the DM P_Key database. 4623 * NOTE: Two cases are handled here: P_Key being added or removed. 4624 * 4625 * Arguments : NONE 4626 * Return Values : NONE 4627 */ 4628 void 4629 ibdm_ibnex_update_pkey_tbls(void) 4630 { 4631 int h, pp, pidx; 4632 uint_t nports; 4633 uint_t size; 4634 ib_pkey_t new_pkey; 4635 ib_pkey_t *orig_pkey; 4636 ibdm_hca_list_t *hca_list; 4637 ibdm_port_attr_t *port; 4638 ibt_hca_portinfo_t *pinfop; 4639 4640 IBTF_DPRINTF_L4("ibdm", "\tibnex_update_pkey_tbls:"); 4641 4642 mutex_enter(&ibdm.ibdm_hl_mutex); 4643 hca_list = ibdm.ibdm_hca_list_head; 4644 4645 for (h = 0; h < ibdm.ibdm_hca_count; h++) { 4646 4647 /* This updates P_Key Tables for all ports of this HCA */ 4648 (void) ibt_query_hca_ports(hca_list->hl_hca_hdl, 0, &pinfop, 4649 &nports, &size); 4650 4651 /* number of ports shouldn't have changed */ 4652 ASSERT(nports == hca_list->hl_nports); 4653 4654 for (pp = 0; pp < hca_list->hl_nports; pp++) { 4655 port = &hca_list->hl_port_attr[pp]; 4656 4657 /* 4658 * First figure out the P_Keys from IBTL. 4659 * Three things could have happened: 4660 * New P_Keys added 4661 * Existing P_Keys removed 4662 * Both of the above two 4663 * 4664 * Loop through the P_Key Indices and check if a 4665 * give P_Key_Ix matches that of the one seen by 4666 * IBDM. If they match no action is needed. 4667 * 4668 * If they don't match: 4669 * 1. if orig_pkey is invalid and new_pkey is valid 4670 * ---> add new_pkey to DM database 4671 * 2. if orig_pkey is valid and new_pkey is invalid 4672 * ---> remove orig_pkey from DM database 4673 * 3. if orig_pkey and new_pkey are both valid: 4674 * ---> remov orig_pkey from DM database 4675 * ---> add new_pkey to DM database 4676 * 4. if orig_pkey and new_pkey are both invalid: 4677 * ---> do nothing. Updated DM database. 4678 */ 4679 4680 for (pidx = 0; pidx < port->pa_npkeys; pidx++) { 4681 new_pkey = pinfop[pp].p_pkey_tbl[pidx]; 4682 orig_pkey = &port->pa_pkey_tbl[pidx].pt_pkey; 4683 4684 /* keys match - do nothing */ 4685 if (*orig_pkey == new_pkey) 4686 continue; 4687 4688 if (IBDM_INVALID_PKEY(*orig_pkey) && 4689 !IBDM_INVALID_PKEY(new_pkey)) { 4690 /* P_Key was added */ 4691 IBTF_DPRINTF_L5("ibdm", 4692 "\tibnex_update_pkey_tbls: new " 4693 "P_Key added = 0x%x", new_pkey); 4694 *orig_pkey = new_pkey; 4695 ibdm_port_attr_ibmf_init(port, 4696 new_pkey, pp); 4697 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4698 IBDM_INVALID_PKEY(new_pkey)) { 4699 /* P_Key was removed */ 4700 IBTF_DPRINTF_L5("ibdm", 4701 "\tibnex_update_pkey_tbls: P_Key " 4702 "removed = 0x%x", *orig_pkey); 4703 *orig_pkey = new_pkey; 4704 (void) ibdm_port_attr_ibmf_fini(port, 4705 pidx); 4706 } else if (!IBDM_INVALID_PKEY(*orig_pkey) && 4707 !IBDM_INVALID_PKEY(new_pkey)) { 4708 /* P_Key were replaced */ 4709 IBTF_DPRINTF_L5("ibdm", 4710 "\tibnex_update_pkey_tbls: P_Key " 4711 "replaced 0x%x with 0x%x", 4712 *orig_pkey, new_pkey); 4713 (void) ibdm_port_attr_ibmf_fini(port, 4714 pidx); 4715 *orig_pkey = new_pkey; 4716 ibdm_port_attr_ibmf_init(port, 4717 new_pkey, pp); 4718 } else { 4719 /* 4720 * P_Keys are invalid 4721 * set anyway to reflect if 4722 * INVALID_FULL was changed to 4723 * INVALID_LIMITED or vice-versa. 4724 */ 4725 *orig_pkey = new_pkey; 4726 } /* end of else */ 4727 4728 } /* loop of p_key index */ 4729 4730 } /* loop of #ports of HCA */ 4731 4732 ibt_free_portinfo(pinfop, size); 4733 hca_list = hca_list->hl_next; 4734 4735 } /* loop for all HCAs in the system */ 4736 4737 mutex_exit(&ibdm.ibdm_hl_mutex); 4738 } 4739 4740 4741 /* 4742 * ibdm_send_ioc_profile() 4743 * Send IOC Controller Profile request. When the request is completed 4744 * IBMF calls ibdm_process_incoming_mad routine to inform about 4745 * the completion. 4746 */ 4747 static int 4748 ibdm_send_ioc_profile(ibdm_dp_gidinfo_t *gid_info, uint8_t ioc_no) 4749 { 4750 ibmf_msg_t *msg; 4751 ib_mad_hdr_t *hdr; 4752 ibdm_ioc_info_t *ioc_info = &(gid_info->gl_iou->iou_ioc_info[ioc_no]); 4753 ibdm_timeout_cb_args_t *cb_args; 4754 4755 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: " 4756 "gid info 0x%p, ioc_no = %d", gid_info, ioc_no); 4757 4758 /* 4759 * Send command to get IOC profile. 4760 * Allocate a IBMF packet and initialize the packet. 4761 */ 4762 if (ibmf_alloc_msg(gid_info->gl_ibmf_hdl, IBMF_ALLOC_SLEEP, 4763 &msg) != IBMF_SUCCESS) { 4764 IBTF_DPRINTF_L4("ibdm", "\tsend_ioc_profile: pkt alloc fail"); 4765 return (IBDM_FAILURE); 4766 } 4767 4768 ibdm_alloc_send_buffers(msg); 4769 4770 mutex_enter(&gid_info->gl_mutex); 4771 ibdm_bump_transactionID(gid_info); 4772 mutex_exit(&gid_info->gl_mutex); 4773 4774 msg->im_local_addr.ia_local_lid = gid_info->gl_slid; 4775 msg->im_local_addr.ia_remote_lid = gid_info->gl_dlid; 4776 if (gid_info->gl_redirected == B_TRUE) { 4777 if (gid_info->gl_redirect_dlid != 0) { 4778 msg->im_local_addr.ia_remote_lid = 4779 gid_info->gl_redirect_dlid; 4780 } 4781 msg->im_local_addr.ia_remote_qno = gid_info->gl_redirect_QP; 4782 msg->im_local_addr.ia_p_key = gid_info->gl_redirect_pkey; 4783 msg->im_local_addr.ia_q_key = gid_info->gl_redirect_qkey; 4784 } else { 4785 msg->im_local_addr.ia_remote_qno = 1; 4786 msg->im_local_addr.ia_p_key = gid_info->gl_p_key; 4787 msg->im_local_addr.ia_q_key = IB_GSI_QKEY; 4788 } 4789 4790 hdr = IBDM_OUT_IBMFMSG_MADHDR(msg); 4791 hdr->BaseVersion = MAD_CLASS_BASE_VERS_1; 4792 hdr->MgmtClass = MAD_MGMT_CLASS_DEV_MGT; 4793 hdr->ClassVersion = IB_DM_CLASS_VERSION_1; 4794 hdr->R_Method = IB_DM_DEVMGT_METHOD_GET; 4795 hdr->Status = 0; 4796 hdr->TransactionID = h2b64(gid_info->gl_transactionID); 4797 hdr->AttributeID = h2b16(IB_DM_ATTR_IOC_CTRL_PROFILE); 4798 hdr->AttributeModifier = h2b32(ioc_no + 1); 4799 4800 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4801 cb_args = &ioc_info->ioc_cb_args; 4802 cb_args->cb_gid_info = gid_info; 4803 cb_args->cb_retry_count = ibdm_dft_retry_cnt; 4804 cb_args->cb_req_type = IBDM_REQ_TYPE_IOCINFO; 4805 cb_args->cb_ioc_num = ioc_no; 4806 4807 ioc_info->ioc_timeout_id = timeout(ibdm_pkt_timeout_hdlr, 4808 cb_args, IBDM_TIMEOUT_VALUE(ibdm_dft_timeout)); 4809 4810 IBTF_DPRINTF_L5("ibdm", "\tsend_ioc_profile:" 4811 "timeout %x", ioc_info->ioc_timeout_id); 4812 4813 if (ibmf_msg_transport(gid_info->gl_ibmf_hdl, gid_info->gl_qp_hdl, msg, 4814 NULL, ibdm_ibmf_send_cb, cb_args, 0) != IBMF_SUCCESS) { 4815 IBTF_DPRINTF_L2("ibdm", 4816 "\tsend_ioc_profile: msg transport failed"); 4817 ibdm_ibmf_send_cb(gid_info->gl_ibmf_hdl, msg, cb_args); 4818 } 4819 ioc_info->ioc_state = IBDM_IOC_STATE_REPROBE_PROGRESS; 4820 return (IBDM_SUCCESS); 4821 } 4822 4823 4824 /* 4825 * ibdm_port_reachable 4826 * Sends a SA query to get the NODE record for port GUID 4827 * Returns IBDM_SUCCESS if the port GID is reachable 4828 */ 4829 static int 4830 ibdm_port_reachable(ibmf_saa_handle_t sa_hdl, ib_guid_t guid, 4831 ib_guid_t *node_guid) 4832 { 4833 sa_node_record_t req, *resp = NULL; 4834 ibmf_saa_access_args_t args; 4835 int ret; 4836 size_t length; 4837 4838 IBTF_DPRINTF_L4("ibdm", "\tport_reachable: port_guid %llx", 4839 guid); 4840 4841 bzero(&req, sizeof (sa_node_record_t)); 4842 req.NodeInfo.PortGUID = guid; 4843 4844 args.sq_attr_id = SA_NODERECORD_ATTRID; 4845 args.sq_access_type = IBMF_SAA_RETRIEVE; 4846 args.sq_component_mask = SA_NODEINFO_COMPMASK_PORTGUID; 4847 args.sq_template = &req; 4848 args.sq_callback = NULL; 4849 args.sq_callback_arg = NULL; 4850 4851 ret = ibmf_sa_access(sa_hdl, &args, 0, &length, (void **) &resp); 4852 if (ret != IBMF_SUCCESS) { 4853 IBTF_DPRINTF_L2("ibdm", "\tport_reachable:" 4854 " SA Retrieve Failed: %d", ret); 4855 return (IBDM_FAILURE); 4856 } 4857 4858 if ((resp == NULL) || (length == 0)) { 4859 IBTF_DPRINTF_L2("ibdm", "\tport_reachable: No records"); 4860 return (IBDM_FAILURE); 4861 } 4862 4863 if (node_guid != NULL) 4864 *node_guid = resp->NodeInfo.NodeGUID; 4865 4866 kmem_free(resp, length); 4867 4868 return (IBDM_SUCCESS); 4869 } 4870 4871 /* 4872 * Update the gidlist for all affected IOCs when GID becomes 4873 * available/unavailable. 4874 * 4875 * Parameters : 4876 * gidinfo - Incoming / Outgoing GID. 4877 * add_flag - 1 for GID added, 0 for GID removed. 4878 * - (-1) : IOC gid list updated, ioc_list required. 4879 * 4880 * This function gets the GID for the node GUID corresponding to the 4881 * port GID. Gets the IOU info 4882 */ 4883 static ibdm_ioc_info_t * 4884 ibdm_update_ioc_gidlist(ibdm_dp_gidinfo_t *gid_info, int avail_flag) 4885 { 4886 ibdm_dp_gidinfo_t *node_gid = NULL; 4887 uint8_t niocs, ii; 4888 ibdm_ioc_info_t *ioc, *ioc_list = NULL, *tmp; 4889 4890 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist"); 4891 4892 switch (avail_flag) { 4893 case 1 : 4894 node_gid = ibdm_check_dest_nodeguid(gid_info); 4895 break; 4896 case 0 : 4897 node_gid = ibdm_handle_gid_rm(gid_info); 4898 break; 4899 case -1 : 4900 node_gid = gid_info; 4901 break; 4902 default : 4903 break; 4904 } 4905 4906 if (node_gid == NULL) { 4907 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist: " 4908 "No node GID found, port gid 0x%p, avail_flag %d", 4909 gid_info, avail_flag); 4910 return (NULL); 4911 } 4912 4913 mutex_enter(&node_gid->gl_mutex); 4914 if ((node_gid->gl_state != IBDM_GID_PROBING_COMPLETE && 4915 node_gid->gl_state != IBDM_GID_PROBING_SKIPPED) || 4916 node_gid->gl_iou == NULL) { 4917 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist " 4918 "gl_state %x, gl_iou %p", node_gid->gl_state, 4919 node_gid->gl_iou); 4920 mutex_exit(&node_gid->gl_mutex); 4921 return (NULL); 4922 } 4923 4924 niocs = node_gid->gl_iou->iou_info.iou_num_ctrl_slots; 4925 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : niocs %x", 4926 niocs); 4927 for (ii = 0; ii < niocs; ii++) { 4928 ioc = IBDM_GIDINFO2IOCINFO(node_gid, ii); 4929 /* 4930 * Skip IOCs for which probe is not complete or 4931 * reprobe is progress 4932 */ 4933 if (ioc->ioc_state == IBDM_IOC_STATE_PROBE_SUCCESS) { 4934 tmp = ibdm_dup_ioc_info(ioc, node_gid); 4935 tmp->ioc_info_updated.ib_gid_prop_updated = 1; 4936 tmp->ioc_next = ioc_list; 4937 ioc_list = tmp; 4938 } 4939 } 4940 mutex_exit(&node_gid->gl_mutex); 4941 4942 IBTF_DPRINTF_L4("ibdm", "\tupdate_ioc_gidlist : return %p", 4943 ioc_list); 4944 return (ioc_list); 4945 } 4946 4947 /* 4948 * ibdm_saa_event_cb : 4949 * Event handling which does *not* require ibdm_hl_mutex to be 4950 * held are executed in the same thread. This is to prevent 4951 * deadlocks with HCA port down notifications which hold the 4952 * ibdm_hl_mutex. 4953 * 4954 * GID_AVAILABLE event is handled here. A taskq is spawned to 4955 * handle GID_UNAVAILABLE. 4956 * 4957 * A new mutex ibdm_ibnex_mutex has been introduced to protect 4958 * ibnex_callback. This has been done to prevent any possible 4959 * deadlock (described above) while handling GID_AVAILABLE. 4960 * 4961 * IBMF calls the event callback for a HCA port. The SA handle 4962 * for this port would be valid, till the callback returns. 4963 * IBDM calling IBDM using the above SA handle should be valid. 4964 * 4965 * IBDM will additionally check (SA handle != NULL), before 4966 * calling IBMF. 4967 */ 4968 /*ARGSUSED*/ 4969 static void 4970 ibdm_saa_event_cb(ibmf_saa_handle_t ibmf_saa_handle, 4971 ibmf_saa_subnet_event_t ibmf_saa_event, 4972 ibmf_saa_event_details_t *event_details, void *callback_arg) 4973 { 4974 ibdm_saa_event_arg_t *event_arg; 4975 ib_gid_t sgid, dgid; 4976 ibdm_port_attr_t *hca_port; 4977 ibdm_dp_gidinfo_t *gid_info, *node_gid_info = NULL; 4978 ib_guid_t nodeguid; 4979 4980 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 4981 4982 hca_port = (ibdm_port_attr_t *)callback_arg; 4983 4984 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_cb(%x, %x, %x, %x)\n", 4985 ibmf_saa_handle, ibmf_saa_event, event_details, 4986 callback_arg); 4987 #ifdef DEBUG 4988 if (ibdm_ignore_saa_event) 4989 return; 4990 #endif 4991 4992 if (ibmf_saa_event == IBMF_SAA_EVENT_GID_AVAILABLE) { 4993 /* 4994 * Ensure no other probe / sweep fabric is in 4995 * progress. 4996 */ 4997 mutex_enter(&ibdm.ibdm_mutex); 4998 while (ibdm.ibdm_busy & IBDM_BUSY) 4999 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5000 ibdm.ibdm_busy |= IBDM_BUSY; 5001 mutex_exit(&ibdm.ibdm_mutex); 5002 5003 /* 5004 * If we already know about this GID, return. 5005 * GID_AVAILABLE may be reported for multiple HCA 5006 * ports. 5007 */ 5008 if ((ibdm_check_dgid(event_details->ie_gid.gid_guid, 5009 event_details->ie_gid.gid_prefix)) != NULL) { 5010 mutex_enter(&ibdm.ibdm_mutex); 5011 ibdm.ibdm_busy &= ~IBDM_BUSY; 5012 cv_broadcast(&ibdm.ibdm_busy_cv); 5013 mutex_exit(&ibdm.ibdm_mutex); 5014 return; 5015 } 5016 5017 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5018 "Insertion notified", 5019 event_details->ie_gid.gid_prefix, 5020 event_details->ie_gid.gid_guid); 5021 5022 /* This is a new gid, insert it to GID list */ 5023 sgid.gid_prefix = hca_port->pa_sn_prefix; 5024 sgid.gid_guid = hca_port->pa_port_guid; 5025 dgid.gid_prefix = event_details->ie_gid.gid_prefix; 5026 dgid.gid_guid = event_details->ie_gid.gid_guid; 5027 gid_info = ibdm_create_gid_info(hca_port, sgid, dgid); 5028 if (gid_info == NULL) { 5029 IBTF_DPRINTF_L4("ibdm", "\tGID_AVAILABLE: " 5030 "create_gid_info returned NULL"); 5031 mutex_enter(&ibdm.ibdm_mutex); 5032 ibdm.ibdm_busy &= ~IBDM_BUSY; 5033 cv_broadcast(&ibdm.ibdm_busy_cv); 5034 mutex_exit(&ibdm.ibdm_mutex); 5035 return; 5036 } 5037 mutex_enter(&gid_info->gl_mutex); 5038 gid_info->gl_state = IBDM_GID_PROBING_SKIPPED; 5039 mutex_exit(&gid_info->gl_mutex); 5040 5041 /* Get the node GUID */ 5042 if (ibdm_port_reachable(ibmf_saa_handle, dgid.gid_guid, 5043 &nodeguid) != IBDM_SUCCESS) { 5044 /* 5045 * Set the state to PROBE_NOT_DONE for the 5046 * next sweep to probe it 5047 */ 5048 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_taskq: " 5049 "Skipping GID : port GUID not found"); 5050 mutex_enter(&gid_info->gl_mutex); 5051 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5052 mutex_exit(&gid_info->gl_mutex); 5053 mutex_enter(&ibdm.ibdm_mutex); 5054 ibdm.ibdm_busy &= ~IBDM_BUSY; 5055 cv_broadcast(&ibdm.ibdm_busy_cv); 5056 mutex_exit(&ibdm.ibdm_mutex); 5057 return; 5058 } 5059 5060 gid_info->gl_nodeguid = nodeguid; 5061 gid_info->gl_portguid = dgid.gid_guid; 5062 5063 /* 5064 * Get the gid info with the same node GUID. 5065 */ 5066 mutex_enter(&ibdm.ibdm_mutex); 5067 node_gid_info = ibdm.ibdm_dp_gidlist_head; 5068 while (node_gid_info) { 5069 if (node_gid_info->gl_nodeguid == 5070 gid_info->gl_nodeguid && 5071 node_gid_info->gl_iou != NULL) { 5072 break; 5073 } 5074 node_gid_info = node_gid_info->gl_next; 5075 } 5076 mutex_exit(&ibdm.ibdm_mutex); 5077 5078 /* 5079 * Handling a new GID requires filling of gl_hca_list. 5080 * This require ibdm hca_list to be parsed and hence 5081 * holding the ibdm_hl_mutex. Spawning a new thread to 5082 * handle this. 5083 */ 5084 if (node_gid_info == NULL) { 5085 if (taskq_dispatch(system_taskq, 5086 ibdm_saa_handle_new_gid, (void *)gid_info, 5087 TQ_NOSLEEP) == NULL) { 5088 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5089 "new_gid taskq_dispatch failed"); 5090 return; 5091 } 5092 } 5093 5094 mutex_enter(&ibdm.ibdm_mutex); 5095 ibdm.ibdm_busy &= ~IBDM_BUSY; 5096 cv_broadcast(&ibdm.ibdm_busy_cv); 5097 mutex_exit(&ibdm.ibdm_mutex); 5098 return; 5099 } 5100 5101 if (ibmf_saa_event != IBMF_SAA_EVENT_GID_UNAVAILABLE) 5102 return; 5103 5104 event_arg = (ibdm_saa_event_arg_t *)kmem_alloc( 5105 sizeof (ibdm_saa_event_arg_t), KM_SLEEP); 5106 event_arg->ibmf_saa_handle = ibmf_saa_handle; 5107 event_arg->ibmf_saa_event = ibmf_saa_event; 5108 bcopy(event_details, &event_arg->event_details, 5109 sizeof (ibmf_saa_event_details_t)); 5110 event_arg->callback_arg = callback_arg; 5111 5112 if (taskq_dispatch(system_taskq, ibdm_saa_event_taskq, 5113 (void *)event_arg, TQ_NOSLEEP) == NULL) { 5114 IBTF_DPRINTF_L2("ibdm", "\tsaa_event_cb: " 5115 "taskq_dispatch failed"); 5116 ibdm_free_saa_event_arg(event_arg); 5117 return; 5118 } 5119 } 5120 5121 /* 5122 * Handle a new GID discovered by GID_AVAILABLE saa event. 5123 */ 5124 void 5125 ibdm_saa_handle_new_gid(void *arg) 5126 { 5127 ibdm_dp_gidinfo_t *gid_info; 5128 ibdm_hca_list_t *hca_list = NULL; 5129 ibdm_port_attr_t *port = NULL; 5130 ibdm_ioc_info_t *ioc_list = NULL; 5131 5132 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid(%p)", arg); 5133 5134 gid_info = (ibdm_dp_gidinfo_t *)arg; 5135 5136 /* 5137 * Ensure that no other sweep / probe has completed 5138 * probing this gid. 5139 */ 5140 mutex_enter(&gid_info->gl_mutex); 5141 if (gid_info->gl_state != IBDM_GID_PROBE_NOT_DONE) { 5142 mutex_exit(&gid_info->gl_mutex); 5143 return; 5144 } 5145 mutex_exit(&gid_info->gl_mutex); 5146 5147 /* 5148 * Parse HCAs to fill gl_hca_list 5149 */ 5150 mutex_enter(&ibdm.ibdm_hl_mutex); 5151 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5152 ibdm_get_next_port(&hca_list, &port, 1)) { 5153 if (ibdm_port_reachable(port->pa_sa_hdl, 5154 gid_info->gl_portguid, NULL) == 5155 IBDM_SUCCESS) { 5156 ibdm_addto_glhcalist(gid_info, hca_list); 5157 } 5158 } 5159 mutex_exit(&ibdm.ibdm_hl_mutex); 5160 5161 /* 5162 * Ensure no other probe / sweep fabric is in 5163 * progress. 5164 */ 5165 mutex_enter(&ibdm.ibdm_mutex); 5166 while (ibdm.ibdm_busy & IBDM_BUSY) 5167 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5168 ibdm.ibdm_busy |= IBDM_BUSY; 5169 mutex_exit(&ibdm.ibdm_mutex); 5170 5171 /* 5172 * New IOU probe it, to check if new IOCs 5173 */ 5174 IBTF_DPRINTF_L4(ibdm_string, "\tsaa_handle_new_gid: " 5175 "new GID : probing"); 5176 mutex_enter(&ibdm.ibdm_mutex); 5177 ibdm.ibdm_ngid_probes_in_progress++; 5178 mutex_exit(&ibdm.ibdm_mutex); 5179 mutex_enter(&gid_info->gl_mutex); 5180 gid_info->gl_reprobe_flag = 0; 5181 gid_info->gl_state = IBDM_GID_PROBE_NOT_DONE; 5182 mutex_exit(&gid_info->gl_mutex); 5183 ibdm_probe_gid_thread((void *)gid_info); 5184 5185 mutex_enter(&ibdm.ibdm_mutex); 5186 ibdm_wait_probe_completion(); 5187 mutex_exit(&ibdm.ibdm_mutex); 5188 5189 if (gid_info->gl_iou == NULL) { 5190 mutex_enter(&ibdm.ibdm_mutex); 5191 ibdm.ibdm_busy &= ~IBDM_BUSY; 5192 cv_broadcast(&ibdm.ibdm_busy_cv); 5193 mutex_exit(&ibdm.ibdm_mutex); 5194 return; 5195 } 5196 5197 /* 5198 * Update GID list in all IOCs affected by this 5199 */ 5200 ioc_list = ibdm_update_ioc_gidlist(gid_info, 1); 5201 5202 /* 5203 * Pass on the IOCs with updated GIDs to IBnexus 5204 */ 5205 if (ioc_list) { 5206 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5207 if (ibdm.ibdm_ibnex_callback != NULL) { 5208 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5209 IBDM_EVENT_IOC_PROP_UPDATE); 5210 } 5211 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5212 } 5213 5214 mutex_enter(&ibdm.ibdm_mutex); 5215 ibdm.ibdm_busy &= ~IBDM_BUSY; 5216 cv_broadcast(&ibdm.ibdm_busy_cv); 5217 mutex_exit(&ibdm.ibdm_mutex); 5218 } 5219 5220 /* 5221 * ibdm_saa_event_taskq : 5222 * GID_UNAVAILABLE Event handling requires ibdm_hl_mutex to be 5223 * held. The GID_UNAVAILABLE handling is done in a taskq to 5224 * prevent deadlocks with HCA port down notifications which hold 5225 * ibdm_hl_mutex. 5226 */ 5227 void 5228 ibdm_saa_event_taskq(void *arg) 5229 { 5230 ibdm_saa_event_arg_t *event_arg; 5231 ibmf_saa_handle_t ibmf_saa_handle; 5232 ibmf_saa_subnet_event_t ibmf_saa_event; 5233 ibmf_saa_event_details_t *event_details; 5234 void *callback_arg; 5235 5236 ibdm_dp_gidinfo_t *gid_info; 5237 ibdm_port_attr_t *hca_port, *port = NULL; 5238 ibdm_hca_list_t *hca_list = NULL; 5239 int sa_handle_valid = 0; 5240 ibdm_ioc_info_t *ioc_list = NULL; 5241 5242 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*event_arg)); 5243 5244 event_arg = (ibdm_saa_event_arg_t *)arg; 5245 ibmf_saa_handle = event_arg->ibmf_saa_handle; 5246 ibmf_saa_event = event_arg->ibmf_saa_event; 5247 event_details = &event_arg->event_details; 5248 callback_arg = event_arg->callback_arg; 5249 5250 ASSERT(callback_arg != NULL); 5251 ASSERT(ibmf_saa_event == IBMF_SAA_EVENT_GID_UNAVAILABLE); 5252 IBTF_DPRINTF_L4("ibdm", "\tsaa_event_taskq(%x, %x, %x, %x)", 5253 ibmf_saa_handle, ibmf_saa_event, event_details, 5254 callback_arg); 5255 5256 hca_port = (ibdm_port_attr_t *)callback_arg; 5257 5258 /* Check if the port_attr is still valid */ 5259 mutex_enter(&ibdm.ibdm_hl_mutex); 5260 for (ibdm_get_next_port(&hca_list, &port, 0); port; 5261 ibdm_get_next_port(&hca_list, &port, 0)) { 5262 if (port == hca_port && port->pa_port_guid == 5263 hca_port->pa_port_guid) { 5264 if (ibmf_saa_handle == hca_port->pa_sa_hdl) 5265 sa_handle_valid = 1; 5266 break; 5267 } 5268 } 5269 mutex_exit(&ibdm.ibdm_hl_mutex); 5270 if (sa_handle_valid == 0) { 5271 ibdm_free_saa_event_arg(event_arg); 5272 return; 5273 } 5274 5275 if (hca_port && (hca_port->pa_sa_hdl == NULL || 5276 ibmf_saa_handle != hca_port->pa_sa_hdl)) { 5277 ibdm_free_saa_event_arg(event_arg); 5278 return; 5279 } 5280 hca_list = NULL; 5281 port = NULL; 5282 5283 /* 5284 * Check if the GID is visible to other HCA ports. 5285 * Return if so. 5286 */ 5287 mutex_enter(&ibdm.ibdm_hl_mutex); 5288 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5289 ibdm_get_next_port(&hca_list, &port, 1)) { 5290 if (ibdm_port_reachable(port->pa_sa_hdl, 5291 event_details->ie_gid.gid_guid, NULL) == 5292 IBDM_SUCCESS) { 5293 mutex_exit(&ibdm.ibdm_hl_mutex); 5294 ibdm_free_saa_event_arg(event_arg); 5295 return; 5296 } 5297 } 5298 mutex_exit(&ibdm.ibdm_hl_mutex); 5299 5300 /* 5301 * Ensure no other probe / sweep fabric is in 5302 * progress. 5303 */ 5304 mutex_enter(&ibdm.ibdm_mutex); 5305 while (ibdm.ibdm_busy & IBDM_BUSY) 5306 cv_wait(&ibdm.ibdm_busy_cv, &ibdm.ibdm_mutex); 5307 ibdm.ibdm_busy |= IBDM_BUSY; 5308 mutex_exit(&ibdm.ibdm_mutex); 5309 5310 /* 5311 * If this GID is no longer in GID list, return 5312 * GID_UNAVAILABLE may be reported for multiple HCA 5313 * ports. 5314 */ 5315 mutex_enter(&ibdm.ibdm_mutex); 5316 gid_info = ibdm.ibdm_dp_gidlist_head; 5317 while (gid_info) { 5318 if (gid_info->gl_portguid == 5319 event_details->ie_gid.gid_guid) { 5320 break; 5321 } 5322 gid_info = gid_info->gl_next; 5323 } 5324 mutex_exit(&ibdm.ibdm_mutex); 5325 if (gid_info == NULL) { 5326 mutex_enter(&ibdm.ibdm_mutex); 5327 ibdm.ibdm_busy &= ~IBDM_BUSY; 5328 cv_broadcast(&ibdm.ibdm_busy_cv); 5329 mutex_exit(&ibdm.ibdm_mutex); 5330 ibdm_free_saa_event_arg(event_arg); 5331 return; 5332 } 5333 5334 IBTF_DPRINTF_L4("ibdm", "\tGID (prefix %x, guid %llx) " 5335 "Unavailable notification", 5336 event_details->ie_gid.gid_prefix, 5337 event_details->ie_gid.gid_guid); 5338 5339 /* 5340 * Update GID list in all IOCs affected by this 5341 */ 5342 if (gid_info->gl_state == IBDM_GID_PROBING_SKIPPED || 5343 gid_info->gl_state == IBDM_GID_PROBING_COMPLETE) 5344 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5345 5346 /* 5347 * Remove GID from the global GID list 5348 * Handle the case where all port GIDs for an 5349 * IOU have been hot-removed. Check both gid_info 5350 * & ioc_info for checking ngids. 5351 */ 5352 mutex_enter(&ibdm.ibdm_mutex); 5353 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5354 mutex_enter(&gid_info->gl_mutex); 5355 (void) ibdm_free_iou_info(gid_info); 5356 mutex_exit(&gid_info->gl_mutex); 5357 } 5358 if (gid_info->gl_prev != NULL) 5359 gid_info->gl_prev->gl_next = gid_info->gl_next; 5360 if (gid_info->gl_next != NULL) 5361 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5362 5363 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5364 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5365 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5366 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5367 ibdm.ibdm_ngids--; 5368 5369 ibdm.ibdm_busy &= ~IBDM_BUSY; 5370 cv_broadcast(&ibdm.ibdm_busy_cv); 5371 mutex_exit(&ibdm.ibdm_mutex); 5372 5373 mutex_destroy(&gid_info->gl_mutex); 5374 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5375 5376 /* 5377 * Pass on the IOCs with updated GIDs to IBnexus 5378 */ 5379 if (ioc_list) { 5380 IBTF_DPRINTF_L4("ibdm", "\tGID_UNAVAILABLE " 5381 "IOC_PROP_UPDATE for %p\n", ioc_list); 5382 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5383 if (ibdm.ibdm_ibnex_callback != NULL) { 5384 (*ibdm.ibdm_ibnex_callback)((void *) 5385 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5386 } 5387 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5388 } 5389 5390 ibdm_free_saa_event_arg(event_arg); 5391 } 5392 5393 5394 static int 5395 ibdm_cmp_gid_list(ibdm_gid_t *new, ibdm_gid_t *prev) 5396 { 5397 ibdm_gid_t *scan_new, *scan_prev; 5398 int cmp_failed = 0; 5399 5400 ASSERT(new != NULL); 5401 ASSERT(prev != NULL); 5402 5403 /* 5404 * Search for each new gid anywhere in the prev GID list. 5405 * Note that the gid list could have been re-ordered. 5406 */ 5407 for (scan_new = new; scan_new; scan_new = scan_new->gid_next) { 5408 for (scan_prev = prev, cmp_failed = 1; scan_prev; 5409 scan_prev = scan_prev->gid_next) { 5410 if (scan_prev->gid_dgid_hi == scan_new->gid_dgid_hi && 5411 scan_prev->gid_dgid_lo == scan_new->gid_dgid_lo) { 5412 cmp_failed = 0; 5413 break; 5414 } 5415 } 5416 5417 if (cmp_failed) 5418 return (1); 5419 } 5420 return (0); 5421 } 5422 5423 /* 5424 * This is always called in a single thread 5425 * This function updates the gid_list and serv_list of IOC 5426 * The current gid_list is in ioc_info_t(contains only port 5427 * guids for which probe is done) & gidinfo_t(other port gids) 5428 * The gids in both locations are used for comparision. 5429 */ 5430 static void 5431 ibdm_reprobe_update_port_srv(ibdm_ioc_info_t *ioc, ibdm_dp_gidinfo_t *gidinfo) 5432 { 5433 ibdm_gid_t *cur_gid_list; 5434 uint_t cur_nportgids; 5435 5436 ASSERT(MUTEX_HELD(&ibdm.ibdm_mutex)); 5437 5438 ioc->ioc_info_updated.ib_prop_updated = 0; 5439 5440 5441 /* Current GID list in gid_info only */ 5442 cur_gid_list = gidinfo->gl_gid; 5443 cur_nportgids = gidinfo->gl_ngids; 5444 5445 /* 5446 * Service entry names and IDs are not compared currently. 5447 * This may require change. 5448 */ 5449 if (ioc->ioc_prev_serv_cnt != ioc->ioc_profile.ioc_service_entries) 5450 ioc->ioc_info_updated.ib_srv_prop_updated = 1; 5451 5452 if (ioc->ioc_prev_nportgids != cur_nportgids || 5453 ioc->ioc_prev_gid_list == NULL || cur_gid_list == NULL) { 5454 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5455 } else if (ibdm_cmp_gid_list(ioc->ioc_prev_gid_list, cur_gid_list)) { 5456 ioc->ioc_info_updated.ib_gid_prop_updated = 1; 5457 } 5458 5459 /* Zero out previous entries */ 5460 ibdm_free_gid_list(ioc->ioc_prev_gid_list); 5461 if (ioc->ioc_prev_serv) 5462 kmem_free(ioc->ioc_prev_serv, ioc->ioc_prev_serv_cnt * 5463 sizeof (ibdm_srvents_info_t)); 5464 ioc->ioc_prev_serv_cnt = 0; 5465 ioc->ioc_prev_nportgids = 0; 5466 ioc->ioc_prev_serv = NULL; 5467 ioc->ioc_prev_gid_list = NULL; 5468 } 5469 5470 /* 5471 * Handle GID removal. This returns gid_info of an GID for the same 5472 * node GUID, if found. For an GID with IOU information, the same 5473 * gid_info is returned if no gid_info with same node_guid is found. 5474 */ 5475 static ibdm_dp_gidinfo_t * 5476 ibdm_handle_gid_rm(ibdm_dp_gidinfo_t *rm_gid) 5477 { 5478 ibdm_dp_gidinfo_t *gid_list; 5479 5480 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm(0x%p)", rm_gid); 5481 5482 if (rm_gid->gl_iou == NULL) { 5483 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm NO iou"); 5484 /* 5485 * Search for a GID with same node_guid and 5486 * gl_iou != NULL 5487 */ 5488 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5489 gid_list = gid_list->gl_next) { 5490 if (gid_list->gl_iou != NULL && (gid_list->gl_nodeguid 5491 == rm_gid->gl_nodeguid)) 5492 break; 5493 } 5494 5495 if (gid_list) 5496 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5497 5498 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5499 return (gid_list); 5500 } else { 5501 /* 5502 * Search for a GID with same node_guid and 5503 * gl_iou == NULL 5504 */ 5505 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm with iou"); 5506 for (gid_list = ibdm.ibdm_dp_gidlist_head; gid_list; 5507 gid_list = gid_list->gl_next) { 5508 if (gid_list->gl_iou == NULL && (gid_list->gl_nodeguid 5509 == rm_gid->gl_nodeguid)) 5510 break; 5511 } 5512 5513 if (gid_list) { 5514 /* 5515 * Copy the following fields from rm_gid : 5516 * 1. gl_state 5517 * 2. gl_iou 5518 * 3. gl_gid & gl_ngids 5519 * 5520 * Note : Function is synchronized by 5521 * ibdm_busy flag. 5522 * 5523 * Note : Redirect info is initialized if 5524 * any MADs for the GID fail 5525 */ 5526 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm " 5527 "copying info to GID with gl_iou != NULl"); 5528 gid_list->gl_state = rm_gid->gl_state; 5529 gid_list->gl_iou = rm_gid->gl_iou; 5530 gid_list->gl_gid = rm_gid->gl_gid; 5531 gid_list->gl_ngids = rm_gid->gl_ngids; 5532 5533 /* Remove the GID from gl_gid list */ 5534 ibdm_rmfrom_glgid_list(gid_list, rm_gid); 5535 } else { 5536 /* 5537 * Handle a case where all GIDs to the IOU have 5538 * been removed. 5539 */ 5540 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm 0 GID " 5541 "to IOU"); 5542 5543 ibdm_rmfrom_glgid_list(rm_gid, rm_gid); 5544 return (rm_gid); 5545 } 5546 IBTF_DPRINTF_L4("ibdm", "\thandle_gid_rm ret %p", gid_list); 5547 return (gid_list); 5548 } 5549 } 5550 5551 static void 5552 ibdm_rmfrom_glgid_list(ibdm_dp_gidinfo_t *gid_info, 5553 ibdm_dp_gidinfo_t *rm_gid) 5554 { 5555 ibdm_gid_t *tmp, *prev; 5556 5557 IBTF_DPRINTF_L4("ibdm", "\trmfrom_glgid (%p, %p)", 5558 gid_info, rm_gid); 5559 5560 for (tmp = gid_info->gl_gid, prev = NULL; tmp; ) { 5561 if (tmp->gid_dgid_hi == rm_gid->gl_dgid_hi && 5562 tmp->gid_dgid_lo == rm_gid->gl_dgid_lo) { 5563 if (prev == NULL) 5564 gid_info->gl_gid = tmp->gid_next; 5565 else 5566 prev->gid_next = tmp->gid_next; 5567 5568 kmem_free(tmp, sizeof (ibdm_gid_t)); 5569 gid_info->gl_ngids--; 5570 break; 5571 } else { 5572 prev = tmp; 5573 tmp = tmp->gid_next; 5574 } 5575 } 5576 } 5577 5578 static void 5579 ibdm_addto_gidlist(ibdm_gid_t **src_ptr, ibdm_gid_t *dest) 5580 { 5581 ibdm_gid_t *head = NULL, *new, *tail; 5582 5583 /* First copy the destination */ 5584 for (; dest; dest = dest->gid_next) { 5585 new = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5586 new->gid_dgid_hi = dest->gid_dgid_hi; 5587 new->gid_dgid_lo = dest->gid_dgid_lo; 5588 new->gid_next = head; 5589 head = new; 5590 } 5591 5592 /* Insert this to the source */ 5593 if (*src_ptr == NULL) 5594 *src_ptr = head; 5595 else { 5596 for (tail = *src_ptr; tail->gid_next != NULL; 5597 tail = tail->gid_next) 5598 ; 5599 5600 tail->gid_next = head; 5601 } 5602 } 5603 5604 static void 5605 ibdm_free_gid_list(ibdm_gid_t *head) 5606 { 5607 ibdm_gid_t *delete; 5608 5609 for (delete = head; delete; ) { 5610 head = delete->gid_next; 5611 kmem_free(delete, sizeof (ibdm_gid_t)); 5612 delete = head; 5613 } 5614 } 5615 5616 /* 5617 * This function rescans the DM capable GIDs (gl_state is 5618 * GID_PROBE_COMPLETE or IBDM_GID_PROBING_SKIPPED.This 5619 * basically checks if the DM capable GID is reachable. If 5620 * not this is handled the same way as GID_UNAVAILABLE, 5621 * except that notifications are not send to IBnexus. 5622 * 5623 * This function also initializes the ioc_prev_list for 5624 * a particular IOC (when called from probe_ioc, with 5625 * ioc_guidp != NULL) or all IOCs for the gid (called from 5626 * sweep_fabric, ioc_guidp == NULL). 5627 */ 5628 static void 5629 ibdm_rescan_gidlist(ib_guid_t *ioc_guidp) 5630 { 5631 ibdm_dp_gidinfo_t *gid_info, *tmp; 5632 int ii, niocs, found; 5633 ibdm_hca_list_t *hca_list = NULL; 5634 ibdm_port_attr_t *port = NULL; 5635 ibdm_ioc_info_t *ioc_list; 5636 5637 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5638 found = 0; 5639 if (gid_info->gl_state != IBDM_GID_PROBING_SKIPPED && 5640 gid_info->gl_state != IBDM_GID_PROBING_COMPLETE) { 5641 gid_info = gid_info->gl_next; 5642 continue; 5643 } 5644 5645 /* 5646 * Check if the GID is visible to any HCA ports. 5647 * Return if so. 5648 */ 5649 mutex_enter(&ibdm.ibdm_hl_mutex); 5650 for (ibdm_get_next_port(&hca_list, &port, 1); port; 5651 ibdm_get_next_port(&hca_list, &port, 1)) { 5652 if (ibdm_port_reachable(port->pa_sa_hdl, 5653 gid_info->gl_dgid_lo, NULL) == IBDM_SUCCESS) { 5654 found = 1; 5655 break; 5656 } 5657 } 5658 mutex_exit(&ibdm.ibdm_hl_mutex); 5659 5660 if (found) { 5661 if (gid_info->gl_iou == NULL) { 5662 gid_info = gid_info->gl_next; 5663 continue; 5664 } 5665 5666 /* Intialize the ioc_prev_gid_list */ 5667 niocs = 5668 gid_info->gl_iou->iou_info.iou_num_ctrl_slots; 5669 for (ii = 0; ii < niocs; ii++) { 5670 ioc_list = IBDM_GIDINFO2IOCINFO(gid_info, ii); 5671 5672 if (ioc_guidp == NULL || (*ioc_guidp == 5673 ioc_list->ioc_profile.ioc_guid)) { 5674 /* Add info of GIDs in gid_info also */ 5675 ibdm_addto_gidlist( 5676 &ioc_list->ioc_prev_gid_list, 5677 gid_info->gl_gid); 5678 ioc_list->ioc_prev_nportgids = 5679 gid_info->gl_ngids; 5680 } 5681 } 5682 gid_info = gid_info->gl_next; 5683 continue; 5684 } 5685 5686 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5687 "deleted port GUID %llx", 5688 gid_info->gl_dgid_lo); 5689 5690 /* 5691 * Update GID list in all IOCs affected by this 5692 */ 5693 ioc_list = ibdm_update_ioc_gidlist(gid_info, 0); 5694 5695 /* 5696 * Remove GID from the global GID list 5697 * Handle the case where all port GIDs for an 5698 * IOU have been hot-removed. 5699 */ 5700 mutex_enter(&ibdm.ibdm_mutex); 5701 if (gid_info->gl_iou != NULL && gid_info->gl_ngids == 0) { 5702 mutex_enter(&gid_info->gl_mutex); 5703 (void) ibdm_free_iou_info(gid_info); 5704 mutex_exit(&gid_info->gl_mutex); 5705 } 5706 tmp = gid_info->gl_next; 5707 if (gid_info->gl_prev != NULL) 5708 gid_info->gl_prev->gl_next = gid_info->gl_next; 5709 if (gid_info->gl_next != NULL) 5710 gid_info->gl_next->gl_prev = gid_info->gl_prev; 5711 5712 if (gid_info == ibdm.ibdm_dp_gidlist_head) 5713 ibdm.ibdm_dp_gidlist_head = gid_info->gl_next; 5714 if (gid_info == ibdm.ibdm_dp_gidlist_tail) 5715 ibdm.ibdm_dp_gidlist_tail = gid_info->gl_prev; 5716 ibdm.ibdm_ngids--; 5717 5718 mutex_destroy(&gid_info->gl_mutex); 5719 kmem_free(gid_info, sizeof (ibdm_dp_gidinfo_t)); 5720 gid_info = tmp; 5721 5722 mutex_exit(&ibdm.ibdm_mutex); 5723 5724 /* 5725 * Pass on the IOCs with updated GIDs to IBnexus 5726 */ 5727 if (ioc_list) { 5728 IBTF_DPRINTF_L4("ibdm", "\trescan_gidlist " 5729 "IOC_PROP_UPDATE for %p\n", ioc_list); 5730 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5731 if (ibdm.ibdm_ibnex_callback != NULL) { 5732 (*ibdm.ibdm_ibnex_callback)((void *) 5733 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 5734 } 5735 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5736 } 5737 } 5738 } 5739 5740 /* 5741 * This function notifies IBnex of IOCs on this GID. 5742 * Notification is for GIDs with gl_reprobe_flag set. 5743 * The flag is set when IOC probe / fabric sweep 5744 * probes a GID starting from CLASS port info. 5745 * 5746 * IBnexus will have information of a reconnected IOC 5747 * if it had probed it before. If this is a new IOC, 5748 * IBnexus ignores the notification. 5749 * 5750 * This function should be called with no locks held. 5751 */ 5752 static void 5753 ibdm_notify_newgid_iocs(ibdm_dp_gidinfo_t *gid_info) 5754 { 5755 ibdm_ioc_info_t *ioc_list; 5756 5757 if (gid_info->gl_reprobe_flag == 0 || 5758 gid_info->gl_iou == NULL) 5759 return; 5760 5761 ioc_list = ibdm_update_ioc_gidlist(gid_info, -1); 5762 5763 /* 5764 * Pass on the IOCs with updated GIDs to IBnexus 5765 */ 5766 if (ioc_list) { 5767 mutex_enter(&ibdm.ibdm_ibnex_mutex); 5768 if (ibdm.ibdm_ibnex_callback != NULL) { 5769 (*ibdm.ibdm_ibnex_callback)((void *)ioc_list, 5770 IBDM_EVENT_IOC_PROP_UPDATE); 5771 } 5772 mutex_exit(&ibdm.ibdm_ibnex_mutex); 5773 } 5774 } 5775 5776 5777 static void 5778 ibdm_free_saa_event_arg(ibdm_saa_event_arg_t *arg) 5779 { 5780 if (arg != NULL) 5781 kmem_free(arg, sizeof (ibdm_saa_event_arg_t)); 5782 } 5783 5784 /* 5785 * This function parses the list of HCAs and HCA ports 5786 * to return the port_attr of the next HCA port. A port 5787 * connected to IB fabric (port_state active) is returned, 5788 * if connected_flag is set. 5789 */ 5790 static void 5791 ibdm_get_next_port(ibdm_hca_list_t **inp_hcap, 5792 ibdm_port_attr_t **inp_portp, int connect_flag) 5793 { 5794 int ii; 5795 ibdm_port_attr_t *port, *next_port = NULL; 5796 ibdm_port_attr_t *inp_port; 5797 ibdm_hca_list_t *hca_list; 5798 int found = 0; 5799 5800 ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5801 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port(%p, %p, %x)", 5802 inp_hcap, inp_portp, connect_flag); 5803 5804 hca_list = *inp_hcap; 5805 inp_port = *inp_portp; 5806 5807 if (hca_list == NULL) 5808 hca_list = ibdm.ibdm_hca_list_head; 5809 5810 for (; hca_list; hca_list = hca_list->hl_next) { 5811 for (ii = 0; ii < hca_list->hl_nports; ii++) { 5812 port = &hca_list->hl_port_attr[ii]; 5813 5814 /* 5815 * inp_port != NULL; 5816 * Skip till we find the matching port 5817 */ 5818 if (inp_port && !found) { 5819 if (inp_port == port) 5820 found = 1; 5821 continue; 5822 } 5823 5824 if (!connect_flag) { 5825 next_port = port; 5826 break; 5827 } 5828 5829 if (port->pa_sa_hdl == NULL) 5830 ibdm_initialize_port(port); 5831 if (port->pa_sa_hdl == NULL) 5832 (void) ibdm_fini_port(port); 5833 else if (next_port == NULL && 5834 port->pa_sa_hdl != NULL && 5835 port->pa_state == IBT_PORT_ACTIVE) { 5836 next_port = port; 5837 break; 5838 } 5839 } 5840 5841 if (next_port) 5842 break; 5843 } 5844 5845 IBTF_DPRINTF_L4(ibdm_string, "\tget_next_port : " 5846 "returns hca_list %p port %p", hca_list, next_port); 5847 *inp_hcap = hca_list; 5848 *inp_portp = next_port; 5849 } 5850 5851 static void 5852 ibdm_add_to_gl_gid(ibdm_dp_gidinfo_t *nodegid, ibdm_dp_gidinfo_t *addgid) 5853 { 5854 ibdm_gid_t *tmp; 5855 5856 tmp = kmem_zalloc(sizeof (ibdm_gid_t), KM_SLEEP); 5857 tmp->gid_dgid_hi = addgid->gl_dgid_hi; 5858 tmp->gid_dgid_lo = addgid->gl_dgid_lo; 5859 5860 mutex_enter(&nodegid->gl_mutex); 5861 tmp->gid_next = nodegid->gl_gid; 5862 nodegid->gl_gid = tmp; 5863 nodegid->gl_ngids++; 5864 mutex_exit(&nodegid->gl_mutex); 5865 } 5866 5867 static void 5868 ibdm_addto_glhcalist(ibdm_dp_gidinfo_t *gid_info, 5869 ibdm_hca_list_t *hca) 5870 { 5871 ibdm_hca_list_t *head, *prev = NULL, *temp; 5872 5873 IBTF_DPRINTF_L4(ibdm_string, "\taddto_glhcalist(%p, %p) " 5874 ": gl_hca_list %p", gid_info, hca, gid_info->gl_hca_list); 5875 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5876 mutex_enter(&gid_info->gl_mutex); 5877 head = gid_info->gl_hca_list; 5878 if (head == NULL) { 5879 head = ibdm_dup_hca_attr(hca); 5880 head->hl_next = NULL; 5881 gid_info->gl_hca_list = head; 5882 mutex_exit(&gid_info->gl_mutex); 5883 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5884 "gid %p, gl_hca_list %p", gid_info, 5885 gid_info->gl_hca_list); 5886 return; 5887 } 5888 5889 /* Check if already in the list */ 5890 while (head) { 5891 if (head->hl_hca_guid == hca->hl_hca_guid) { 5892 mutex_exit(&gid_info->gl_mutex); 5893 IBTF_DPRINTF_L4(ibdm_string, 5894 "\taddto_glhcalist : gid %x hca %x dup", 5895 gid_info, hca); 5896 return; 5897 } 5898 prev = head; 5899 head = head->hl_next; 5900 } 5901 5902 /* Add this HCA to gl_hca_list */ 5903 temp = ibdm_dup_hca_attr(hca); 5904 temp->hl_next = NULL; 5905 prev->hl_next = temp; 5906 5907 mutex_exit(&gid_info->gl_mutex); 5908 5909 IBTF_DPRINTF_L4(ibdm_string, "\tadd_to_glhcalist: " 5910 "gid %p, gl_hca_list %p", gid_info, gid_info->gl_hca_list); 5911 } 5912 5913 static void 5914 ibdm_delete_glhca_list(ibdm_dp_gidinfo_t *gid_info) 5915 { 5916 ASSERT(!MUTEX_HELD(&gid_info->gl_mutex)); 5917 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5918 5919 mutex_enter(&gid_info->gl_mutex); 5920 if (gid_info->gl_hca_list) 5921 ibdm_ibnex_free_hca_list(gid_info->gl_hca_list); 5922 gid_info->gl_hca_list = NULL; 5923 mutex_exit(&gid_info->gl_mutex); 5924 } 5925 5926 5927 static void 5928 ibdm_reset_all_dgids(ibmf_saa_handle_t port_sa_hdl) 5929 { 5930 IBTF_DPRINTF_L4(ibdm_string, "\treset_all_dgids(%X)", 5931 port_sa_hdl); 5932 5933 ASSERT(!MUTEX_HELD(&ibdm.ibdm_mutex)); 5934 ASSERT(!MUTEX_HELD(&ibdm.ibdm_hl_mutex)); 5935 5936 /* Check : Not busy in another probe / sweep */ 5937 mutex_enter(&ibdm.ibdm_mutex); 5938 if ((ibdm.ibdm_busy & IBDM_BUSY) == 0) { 5939 ibdm_dp_gidinfo_t *gid_info; 5940 5941 ibdm.ibdm_busy |= IBDM_BUSY; 5942 mutex_exit(&ibdm.ibdm_mutex); 5943 5944 /* 5945 * Check if any GID is using the SA & IBMF handle 5946 * of HCA port going down. Reset ibdm_dp_gidinfo_t 5947 * using another HCA port which can reach the GID. 5948 * This is for DM capable GIDs only, no need to do 5949 * this for others 5950 * 5951 * Delete the GID if no alternate HCA port to reach 5952 * it is found. 5953 */ 5954 for (gid_info = ibdm.ibdm_dp_gidlist_head; gid_info; ) { 5955 ibdm_dp_gidinfo_t *tmp; 5956 5957 IBTF_DPRINTF_L4(ibdm_string, "\tevent_hdlr " 5958 "checking gidinfo %p", gid_info); 5959 5960 if (gid_info->gl_nodeguid != 0 && 5961 gid_info->gl_sa_hdl == port_sa_hdl) { 5962 IBTF_DPRINTF_L3(ibdm_string, 5963 "\tevent_hdlr: down HCA port hdl " 5964 "matches gid %p", gid_info); 5965 5966 ibdm_reset_gidinfo(gid_info); 5967 if (gid_info->gl_disconnected) { 5968 IBTF_DPRINTF_L3(ibdm_string, 5969 "\tevent_hdlr: deleting" 5970 " gid %p", gid_info); 5971 tmp = gid_info; 5972 gid_info = gid_info->gl_next; 5973 ibdm_delete_gidinfo(tmp); 5974 } else 5975 gid_info = gid_info->gl_next; 5976 } else 5977 gid_info = gid_info->gl_next; 5978 } 5979 5980 mutex_enter(&ibdm.ibdm_mutex); 5981 ibdm.ibdm_busy &= ~IBDM_BUSY; 5982 cv_signal(&ibdm.ibdm_busy_cv); 5983 } 5984 mutex_exit(&ibdm.ibdm_mutex); 5985 } 5986 5987 static void 5988 ibdm_reset_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 5989 { 5990 ibdm_hca_list_t *hca_list = NULL; 5991 ibdm_port_attr_t *port = NULL; 5992 int gid_reinited = 0; 5993 sa_node_record_t *nr, *tmp; 5994 sa_portinfo_record_t *pi; 5995 size_t nr_len = 0, pi_len = 0; 5996 size_t path_len; 5997 ib_gid_t sgid, dgid; 5998 int ret, ii, nrecords; 5999 sa_path_record_t *path; 6000 uint8_t npaths = 1; 6001 6002 IBTF_DPRINTF_L4(ibdm_string, "\treset_gidinfo(%p)", gidinfo); 6003 6004 /* 6005 * Get list of all the ports reachable from the local known HCA 6006 * ports which are active 6007 */ 6008 mutex_enter(&ibdm.ibdm_hl_mutex); 6009 for (ibdm_get_next_port(&hca_list, &port, 1); port; 6010 ibdm_get_next_port(&hca_list, &port, 1)) { 6011 6012 6013 /* 6014 * Get the path and re-populate the gidinfo. 6015 * Getting the path is the same probe_ioc 6016 * Init the gid info as in ibdm_create_gidinfo() 6017 */ 6018 nr = ibdm_get_node_records(port->pa_sa_hdl, &nr_len, 6019 gidinfo->gl_nodeguid); 6020 if (nr == NULL) { 6021 IBTF_DPRINTF_L4(ibdm_string, 6022 "\treset_gidinfo : no records"); 6023 continue; 6024 } 6025 6026 nrecords = (nr_len / sizeof (sa_node_record_t)); 6027 for (tmp = nr, ii = 0; (ii < nrecords); ii++, tmp++) { 6028 if (tmp->NodeInfo.PortGUID == gidinfo->gl_portguid) 6029 break; 6030 } 6031 6032 if (ii == nrecords) { 6033 IBTF_DPRINTF_L4(ibdm_string, 6034 "\treset_gidinfo : no record for portguid"); 6035 kmem_free(nr, nr_len); 6036 continue; 6037 } 6038 6039 pi = ibdm_get_portinfo(port->pa_sa_hdl, &pi_len, tmp->LID); 6040 if (pi == NULL) { 6041 IBTF_DPRINTF_L4(ibdm_string, 6042 "\treset_gidinfo : no portinfo"); 6043 kmem_free(nr, nr_len); 6044 continue; 6045 } 6046 6047 sgid.gid_prefix = port->pa_sn_prefix; 6048 sgid.gid_guid = port->pa_port_guid; 6049 dgid.gid_prefix = pi->PortInfo.GidPrefix; 6050 dgid.gid_guid = tmp->NodeInfo.PortGUID; 6051 6052 ret = ibmf_saa_gid_to_pathrecords(port->pa_sa_hdl, sgid, dgid, 6053 IBMF_SAA_PKEY_WC, 0, B_TRUE, &npaths, 0, &path_len, &path); 6054 6055 if ((ret != IBMF_SUCCESS) && path) { 6056 IBTF_DPRINTF_L4(ibdm_string, 6057 "\treset_gidinfo : no paths"); 6058 kmem_free(pi, pi_len); 6059 kmem_free(nr, nr_len); 6060 continue; 6061 } 6062 6063 gidinfo->gl_dgid_hi = path->DGID.gid_prefix; 6064 gidinfo->gl_dgid_lo = path->DGID.gid_guid; 6065 gidinfo->gl_sgid_hi = path->SGID.gid_prefix; 6066 gidinfo->gl_sgid_lo = path->SGID.gid_guid; 6067 gidinfo->gl_p_key = path->P_Key; 6068 gidinfo->gl_sa_hdl = port->pa_sa_hdl; 6069 gidinfo->gl_ibmf_hdl = port->pa_ibmf_hdl; 6070 gidinfo->gl_slid = path->SLID; 6071 gidinfo->gl_dlid = path->DLID; 6072 6073 /* Reset redirect info, next MAD will set if redirected */ 6074 gidinfo->gl_redirected = 0; 6075 6076 gid_reinited = 1; 6077 6078 kmem_free(path, path_len); 6079 kmem_free(pi, pi_len); 6080 kmem_free(nr, nr_len); 6081 break; 6082 } 6083 mutex_exit(&ibdm.ibdm_hl_mutex); 6084 6085 if (!gid_reinited) 6086 gidinfo->gl_disconnected = 1; 6087 } 6088 6089 static void 6090 ibdm_delete_gidinfo(ibdm_dp_gidinfo_t *gidinfo) 6091 { 6092 ibdm_ioc_info_t *ioc_list; 6093 int in_gidlist = 0; 6094 6095 /* 6096 * Check if gidinfo has been inserted into the 6097 * ibdm_dp_gidlist_head list. gl_next or gl_prev 6098 * != NULL, if gidinfo is the list. 6099 */ 6100 if (gidinfo->gl_prev != NULL || 6101 gidinfo->gl_next != NULL) 6102 in_gidlist = 1; 6103 6104 ioc_list = ibdm_update_ioc_gidlist(gidinfo, 0); 6105 6106 /* 6107 * Remove GID from the global GID list 6108 * Handle the case where all port GIDs for an 6109 * IOU have been hot-removed. 6110 */ 6111 mutex_enter(&ibdm.ibdm_mutex); 6112 if (gidinfo->gl_iou != NULL && gidinfo->gl_ngids == 0) { 6113 mutex_enter(&gidinfo->gl_mutex); 6114 (void) ibdm_free_iou_info(gidinfo); 6115 mutex_exit(&gidinfo->gl_mutex); 6116 } 6117 6118 if (in_gidlist) { 6119 if (gidinfo->gl_prev != NULL) 6120 gidinfo->gl_prev->gl_next = gidinfo->gl_next; 6121 if (gidinfo->gl_next != NULL) 6122 gidinfo->gl_next->gl_prev = gidinfo->gl_prev; 6123 6124 if (gidinfo == ibdm.ibdm_dp_gidlist_head) 6125 ibdm.ibdm_dp_gidlist_head = gidinfo->gl_next; 6126 if (gidinfo == ibdm.ibdm_dp_gidlist_tail) 6127 ibdm.ibdm_dp_gidlist_tail = gidinfo->gl_prev; 6128 ibdm.ibdm_ngids--; 6129 } 6130 mutex_exit(&ibdm.ibdm_mutex); 6131 6132 mutex_destroy(&gidinfo->gl_mutex); 6133 kmem_free(gidinfo, sizeof (ibdm_dp_gidinfo_t)); 6134 6135 /* 6136 * Pass on the IOCs with updated GIDs to IBnexus 6137 */ 6138 if (ioc_list) { 6139 IBTF_DPRINTF_L4("ibdm", "\tdelete_gidinfo " 6140 "IOC_PROP_UPDATE for %p\n", ioc_list); 6141 mutex_enter(&ibdm.ibdm_ibnex_mutex); 6142 if (ibdm.ibdm_ibnex_callback != NULL) { 6143 (*ibdm.ibdm_ibnex_callback)((void *) 6144 ioc_list, IBDM_EVENT_IOC_PROP_UPDATE); 6145 } 6146 mutex_exit(&ibdm.ibdm_ibnex_mutex); 6147 } 6148 } 6149 6150 6151 static void 6152 ibdm_fill_srv_attr_mod(ib_mad_hdr_t *hdr, ibdm_timeout_cb_args_t *cb_args) 6153 { 6154 uint32_t attr_mod; 6155 6156 attr_mod = (cb_args->cb_ioc_num + 1) << 16; 6157 attr_mod |= cb_args->cb_srvents_start; 6158 attr_mod |= (cb_args->cb_srvents_end) << 8; 6159 hdr->AttributeModifier = h2b32(attr_mod); 6160 } 6161 6162 static void 6163 ibdm_bump_transactionID(ibdm_dp_gidinfo_t *gid_info) 6164 { 6165 ASSERT(MUTEX_HELD(&gid_info->gl_mutex)); 6166 gid_info->gl_transactionID++; 6167 if (gid_info->gl_transactionID == gid_info->gl_max_transactionID) { 6168 IBTF_DPRINTF_L4(ibdm_string, 6169 "\tbump_transactionID(%p), wrapup", gid_info); 6170 gid_info->gl_transactionID = gid_info->gl_min_transactionID; 6171 } 6172 } 6173 6174 /* For debugging purpose only */ 6175 #ifdef DEBUG 6176 void 6177 ibdm_dump_ibmf_msg(ibmf_msg_t *ibmf_msg, int flag) 6178 { 6179 ib_mad_hdr_t *mad_hdr; 6180 6181 IBTF_DPRINTF_L4("ibdm", "\t\t(IBMF_PKT): Local address info"); 6182 IBTF_DPRINTF_L4("ibdm", "\t\t ------------------"); 6183 6184 IBTF_DPRINTF_L4("ibdm", "\tLocal Lid : 0x%x\tRemote Lid : 0x%x" 6185 " Remote Qp : 0x%x", ibmf_msg->im_local_addr.ia_local_lid, 6186 ibmf_msg->im_local_addr.ia_remote_lid, 6187 ibmf_msg->im_local_addr.ia_remote_qno); 6188 IBTF_DPRINTF_L4("ibdm", "\tP_key : 0x%x\tQ_key : 0x%x", 6189 ibmf_msg->im_local_addr.ia_p_key, ibmf_msg->im_local_addr.ia_q_key); 6190 6191 if (flag) 6192 mad_hdr = (ib_mad_hdr_t *)IBDM_OUT_IBMFMSG_MADHDR(ibmf_msg); 6193 else 6194 mad_hdr = IBDM_IN_IBMFMSG_MADHDR(ibmf_msg); 6195 6196 IBTF_DPRINTF_L4("ibdm", "\t\t MAD Header info"); 6197 IBTF_DPRINTF_L4("ibdm", "\t\t ---------------"); 6198 6199 IBTF_DPRINTF_L4("ibdm", "\tBase version : 0x%x" 6200 "\tMgmt Class : 0x%x", mad_hdr->BaseVersion, mad_hdr->MgmtClass); 6201 IBTF_DPRINTF_L4("ibdm", "\tClass version : 0x%x" 6202 "\tR Method : 0x%x", 6203 mad_hdr->ClassVersion, mad_hdr->R_Method); 6204 IBTF_DPRINTF_L4("ibdm", "\tMAD Status : 0x%x" 6205 "\tTransaction ID : 0x%llx", 6206 mad_hdr->Status, mad_hdr->TransactionID); 6207 IBTF_DPRINTF_L4("ibdm", "\t Attribute ID : 0x%x" 6208 "\tAttribute Modified : 0x%lx", 6209 mad_hdr->AttributeID, mad_hdr->AttributeModifier); 6210 } 6211 6212 void 6213 ibdm_dump_path_info(sa_path_record_t *path) 6214 { 6215 IBTF_DPRINTF_L4("ibdm", "\t\t Path information"); 6216 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------"); 6217 6218 IBTF_DPRINTF_L4("ibdm", "\t DGID hi : %llx\tDGID lo : %llx", 6219 path->DGID.gid_prefix, path->DGID.gid_guid); 6220 IBTF_DPRINTF_L4("ibdm", "\t SGID hi : %llx\tSGID lo : %llx", 6221 path->SGID.gid_prefix, path->SGID.gid_guid); 6222 IBTF_DPRINTF_L4("ibdm", "\t SLID : %x\tDlID : %x", 6223 path->SLID, path->DLID); 6224 IBTF_DPRINTF_L4("ibdm", "\t P Key : %x", path->P_Key); 6225 } 6226 6227 6228 void 6229 ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *classportinfo) 6230 { 6231 IBTF_DPRINTF_L4("ibdm", "\t\t CLASSPORT INFO"); 6232 IBTF_DPRINTF_L4("ibdm", "\t\t --------------"); 6233 6234 IBTF_DPRINTF_L4("ibdm", "\t Response Time Value : 0x%x", 6235 ((b2h32(classportinfo->RespTimeValue)) & 0x1F)); 6236 6237 IBTF_DPRINTF_L4("ibdm", "\t Redirected QP : 0x%x", 6238 (b2h32(classportinfo->RedirectQP))); 6239 IBTF_DPRINTF_L4("ibdm", "\t Redirected P KEY : 0x%x", 6240 b2h16(classportinfo->RedirectP_Key)); 6241 IBTF_DPRINTF_L4("ibdm", "\t Redirected Q KEY : 0x%x", 6242 b2h16(classportinfo->RedirectQ_Key)); 6243 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID hi : 0x%x", 6244 b2h64(classportinfo->RedirectGID_hi)); 6245 IBTF_DPRINTF_L4("ibdm", "\t Redirected GID lo : 0x%x", 6246 b2h64(classportinfo->RedirectGID_lo)); 6247 } 6248 6249 6250 void 6251 ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *iou_info) 6252 { 6253 IBTF_DPRINTF_L4("ibdm", "\t\t I/O UnitInfo"); 6254 IBTF_DPRINTF_L4("ibdm", "\t\t ------------"); 6255 6256 IBTF_DPRINTF_L4("ibdm", "\tChange ID : 0x%x", 6257 b2h16(iou_info->iou_changeid)); 6258 IBTF_DPRINTF_L4("ibdm", "\t#of ctrl slots : %d", 6259 iou_info->iou_num_ctrl_slots); 6260 IBTF_DPRINTF_L4("ibdm", "\tIOU flag : 0x%x", 6261 iou_info->iou_flag); 6262 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 0 : 0x%x", 6263 iou_info->iou_ctrl_list[0]); 6264 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 1 : 0x%x", 6265 iou_info->iou_ctrl_list[1]); 6266 IBTF_DPRINTF_L4("ibdm", "\tContrl list byte 2 : 0x%x", 6267 iou_info->iou_ctrl_list[2]); 6268 } 6269 6270 6271 void 6272 ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *ioc) 6273 { 6274 IBTF_DPRINTF_L4("ibdm", "\t\t IOC Controller Profile"); 6275 IBTF_DPRINTF_L4("ibdm", "\t\t ----------------------"); 6276 6277 IBTF_DPRINTF_L4("ibdm", "\tIOC Guid : %llx", ioc->ioc_guid); 6278 IBTF_DPRINTF_L4("ibdm", "\tVendorID : 0x%x", ioc->ioc_vendorid); 6279 IBTF_DPRINTF_L4("ibdm", "\tDevice Id : 0x%x", ioc->ioc_deviceid); 6280 IBTF_DPRINTF_L4("ibdm", "\tDevice Ver : 0x%x", ioc->ioc_device_ver); 6281 IBTF_DPRINTF_L4("ibdm", "\tSubsys ID : 0x%x", ioc->ioc_subsys_id); 6282 IBTF_DPRINTF_L4("ibdm", "\tIO class : 0x%x", ioc->ioc_io_class); 6283 IBTF_DPRINTF_L4("ibdm", "\tIO subclass : 0x%x", ioc->ioc_io_subclass); 6284 IBTF_DPRINTF_L4("ibdm", "\tProtocol : 0x%x", ioc->ioc_protocol); 6285 IBTF_DPRINTF_L4("ibdm", "\tProtocolV : 0x%x", ioc->ioc_protocol_ver); 6286 IBTF_DPRINTF_L4("ibdm", "\tmsg qdepth : %d", ioc->ioc_send_msg_qdepth); 6287 IBTF_DPRINTF_L4("ibdm", "\trdma qdepth : %d", 6288 ioc->ioc_rdma_read_qdepth); 6289 IBTF_DPRINTF_L4("ibdm", "\tsndmsg sz : %d", ioc->ioc_send_msg_sz); 6290 IBTF_DPRINTF_L4("ibdm", "\trdma xfersz : %d", ioc->ioc_rdma_xfer_sz); 6291 IBTF_DPRINTF_L4("ibdm", "\topcal mask : 0x%x", 6292 ioc->ioc_ctrl_opcap_mask); 6293 IBTF_DPRINTF_L4("ibdm", "\tsrventries : %x", ioc->ioc_service_entries); 6294 } 6295 6296 6297 void 6298 ibdm_dump_service_entries(ib_dm_srv_t *srv_ents) 6299 { 6300 IBTF_DPRINTF_L4("ibdm", 6301 "\thandle_srventry_mad: service id : %llx", srv_ents->srv_id); 6302 6303 IBTF_DPRINTF_L4("ibdm", "\thandle_srventry_mad: " 6304 "Service Name : %s", srv_ents->srv_name); 6305 } 6306 6307 int ibdm_allow_sweep_fabric_timestamp = 1; 6308 6309 void 6310 ibdm_dump_sweep_fabric_timestamp(int flag) 6311 { 6312 static hrtime_t x; 6313 if (flag) { 6314 if (ibdm_allow_sweep_fabric_timestamp) { 6315 IBTF_DPRINTF_L4("ibdm", "\tTime taken to complete " 6316 "sweep %lld ms", ((gethrtime() - x)/ 1000000)); 6317 } 6318 x = 0; 6319 } else 6320 x = gethrtime(); 6321 } 6322 #endif 6323