1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IB_MGT_IBDM_IBDM_IMPL_H 28 #define _SYS_IB_MGT_IBDM_IBDM_IMPL_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 /* 33 * ibdm_impl.h 34 * 35 * This file contains definitions of the data structures, macros etc 36 * related to the IBDM module. 37 */ 38 39 #include <sys/ib/mgt/ibdm/ibdm_ibnex.h> 40 #include <sys/ib/ibtl/impl/ibtl_util.h> 41 42 #ifdef __cplusplus 43 extern "C" { 44 #endif 45 46 typedef struct ibdm_mad_classportinfo_s { 47 uint8_t BaseVersion; /* ver. of MAD base format */ 48 uint8_t ClassVersion; /* ver. of MAD class format */ 49 uint16_t CapabilityMask; /* capabilities of this class */ 50 uint32_t RespTimeValue; /* reserved : 27 bits */ 51 /* resptime value : 5 bits */ 52 uint64_t RedirectGID_hi; /* dest gid of redirect msgs */ 53 uint64_t RedirectGID_lo; /* dest gid of redirect msgs */ 54 uint32_t RedirectTC; /* traffic class: 8 bits */ 55 /* SL: 4 bits */ 56 /* Flow label: 20 bits */ 57 ib_lid_t RedirectLID; /* dlid for class services */ 58 ib_pkey_t RedirectP_Key; /* p_key for class services */ 59 uint32_t RedirectQP; /* Reserved: 8 bits */ 60 /* QPN: 24 bits */ 61 ib_qkey_t RedirectQ_Key; /* q_key for class services */ 62 uint64_t TrapGID_hi; /* dest gid of trap msgs */ 63 uint64_t TrapGID_lo; /* dest gid of trap msgs */ 64 uint32_t TrapTC; /* Trap traffic class, etc., */ 65 ib_lid_t TrapLID; /* dlid for traps */ 66 ib_pkey_t TrapP_Key; /* p_key for traps */ 67 uint32_t TrapHL; /* Trap hop limit,etc., */ 68 ib_qkey_t TrapQ_Key; /* q_key for traps */ 69 } ibdm_mad_classportinfo_t; 70 71 /* values for "cb_req_type" */ 72 #define IBDM_REQ_TYPE_INVALID 0x0 73 #define IBDM_REQ_TYPE_CLASSPORTINFO 0x1 74 #define IBDM_REQ_TYPE_IOUINFO 0x2 75 #define IBDM_REQ_TYPE_IOCINFO 0x4 76 #define IBDM_REQ_TYPE_SRVENTS 0x8 77 #define IBDM_REQ_TYPE_IOU_DIAGCODE 0x10 78 #define IBDM_REQ_TYPE_IOC_DIAGCODE 0x20 79 80 typedef struct ibdm_taskq_args_s { 81 ibmf_handle_t tq_ibmf_handle; 82 ibmf_msg_t *tq_ibmf_msg; 83 void *tq_args; 84 } ibdm_taskq_args_t; 85 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibdm_taskq_args_t)) 86 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ib_mad_hdr_t)) 87 _NOTE(SCHEME_PROTECTS_DATA("unique per call", ibmf_msg_t)) 88 89 #define IBDM_GID_PRESENT 0x1 90 #define IBDM_GID_NOT_PRESENT 0x0 91 92 #define IBDM_IBMF_PKT_DUP_RESP 0x1 93 #define IBDM_IBMF_PKT_REUSED 0x2 94 #define IBDM_IBMF_PKT_UNEXP_RESP 0x4 95 96 #define IBDM_MAX_SERV_ENTRIES_PER_REQ 4 97 98 typedef struct ibdm_gid_s { 99 uint64_t gid_dgid_hi; 100 uint64_t gid_dgid_lo; 101 struct ibdm_gid_s *gid_next; 102 } ibdm_gid_t; 103 104 #define IBDM_GID_PROBE_NOT_DONE 0x00 105 #define IBDM_GET_CLASSPORTINFO 0x01 106 #define IBDM_GET_IOUNITINFO 0x02 107 #define IBDM_GET_IOC_DETAILS 0x04 108 #define IBDM_GID_PROBING_COMPLETE 0x08 109 #define IBDM_GID_PROBING_SKIPPED 0x10 110 #define IBDM_GID_PROBING_FAILED 0x20 111 112 /* 113 * The state diagram for the gl_state 114 * 115 * IBDM_GID_PROBE_NOT_DONE --- 1 -> IBDM_GID_GET_CLASSPORTINFO 116 * | | | 117 * | 2 3 118 * | | | 119 * | v v 120 * | IBDM_GID_PROBING_FAILED IBDM_GET_IOUNITINFO 121 * | | 122 * 6 4 123 * | | 124 * v v 125 * IBDM_GID_PROBING_SKIPPLED IBDM_GET_IOC_DETAILS 126 * | 127 * 5 128 * | 129 * v 130 * IBDM_GID_PROBE_COMPLETE 131 * 132 * Initial state : IBDM_GID_PROBE_NOT_DONE 133 * 1 = Port supports DM MAD's and a request to ClassportInfor is sent 134 * 3 = Received ClassPortInfo and sent IOUnitInfo 135 * 4 = Recevied IOUunitInfo and sent IOC profile, diagcodes, and 136 * service entries requests 137 * 5 = Received all the IOC information 138 * 2 = Failed to probe the GID 139 * Port does not support DM MAD's 140 * Port did not respond property 141 * 6 = A different GID for the same port, skip the probe 142 * 143 * Reprobe state transition : 144 * 145 * IBDM_GID_PROBE_COMPLETE 146 * | 147 * 7 148 * | 149 * v 150 * IBDM_GET_IOC_DETAILS 151 * | 152 * 8 153 * | 154 * v 155 * IBDM_GID_PROBE_COMPLETE 156 * 157 * 7 = Reprobe request for one or more IOCs initiated. 158 * 8 = Reprobe done(IOC COntroller Profile & Service entries) 159 */ 160 161 typedef struct ibdm_dp_gidinfo_s { 162 kmutex_t gl_mutex; 163 uint_t gl_state; 164 int gl_reprobe_flag; /* pass this to taskq */ 165 struct ibdm_dp_gidinfo_s *gl_next; 166 struct ibdm_dp_gidinfo_s *gl_prev; 167 ibdm_iou_info_t *gl_iou; 168 int gl_pending_cmds; 169 ibmf_qp_handle_t gl_qp_hdl; 170 uint64_t gl_transactionID; 171 ibdm_timeout_cb_args_t gl_iou_cb_args; 172 ib_lid_t gl_dlid; 173 ib_lid_t gl_slid; 174 uint64_t gl_dgid_hi; 175 uint64_t gl_dgid_lo; 176 uint64_t gl_sgid_hi; 177 uint64_t gl_sgid_lo; 178 ib_guid_t gl_nodeguid; 179 ib_guid_t gl_portguid; 180 ib_pkey_t gl_p_key; 181 boolean_t gl_redirected; 182 uint32_t gl_redirect_dlid; 183 uint32_t gl_redirect_QP; 184 ib_pkey_t gl_redirect_pkey; 185 ib_qkey_t gl_redirect_qkey; 186 uint64_t gl_redirectGID_hi; 187 uint64_t gl_redirectGID_lo; 188 ibmf_handle_t gl_ibmf_hdl; 189 ibmf_saa_handle_t gl_sa_hdl; 190 timeout_id_t gl_timeout_id; 191 ibdm_timeout_cb_args_t gl_cpi_cb_args; 192 uint32_t gl_ngids; 193 ibdm_gid_t *gl_gid; 194 uint32_t gl_resp_timeout; 195 int gl_num_iocs; 196 ibdm_hca_list_t *gl_hca_list; 197 int gl_disconnected; 198 uint64_t gl_min_transactionID; 199 uint64_t gl_max_transactionID; 200 } ibdm_dp_gidinfo_t; 201 _NOTE(MUTEX_PROTECTS_DATA(ibdm_dp_gidinfo_s::gl_mutex, 202 ibdm_dp_gidinfo_s::{gl_state gl_timeout_id gl_pending_cmds})) 203 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_dp_gidinfo_s)) 204 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_dp_gidinfo_s::{gl_ibmf_hdl gl_sa_hdl})) 205 206 /* 207 * The transaction ID for the GID contains of two parts : 208 * 1. Upper 32 bits which is unique for each GID. 209 * 2. Lower 32 bits which is unique for each MAD. 210 * The assumptions are : 211 * 1. At most 2 power 32 DM capable GIDs on the IB fabric 212 * 2. IBDM sends maximum of 2 power 32 MADs to the same DM GID 213 * The limits are sufficient for practical configurations. 214 */ 215 #define IBDM_GID_TRANSACTIONID_SHIFT ((ulong_t)32) 216 #define IBDM_GID_TRANSACTIONID_MASK 0xFFFFFFFF00000000 217 218 typedef struct ibdm_s { 219 /* Protects IBDM's critical data */ 220 kmutex_t ibdm_mutex; 221 uint32_t ibdm_hca_count; 222 kmutex_t ibdm_hl_mutex; 223 kmutex_t ibdm_ibnex_mutex; 224 ibdm_hca_list_t *ibdm_hca_list_head; 225 ibdm_hca_list_t *ibdm_hca_list_tail; 226 227 ibdm_dp_gidinfo_t *ibdm_dp_gidlist_head; 228 ibdm_dp_gidinfo_t *ibdm_dp_gidlist_tail; 229 230 kcondvar_t ibdm_probe_cv; 231 kcondvar_t ibdm_busy_cv; 232 uint32_t ibdm_ngid_probes_in_progress; 233 uint64_t ibdm_transactionID; 234 uint32_t ibdm_ngids; 235 uint32_t ibdm_busy; 236 int ibdm_state; 237 ibt_clnt_hdl_t ibdm_ibt_clnt_hdl; 238 239 /* 240 * These are callback routines registered by the IB nexus driver. 241 * These callbacks are used to inform the IB nexus driver about 242 * the arrival/removal of HCA and IOC's 243 */ 244 ibdm_callback_t ibdm_ibnex_callback; 245 } ibdm_t; 246 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_mutex, ibdm_s::{ibdm_ibt_clnt_hdl 247 ibdm_busy ibdm_state})) 248 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibdm_s::ibdm_ibt_clnt_hdl)) 249 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_hl_mutex, 250 ibdm_s::{ibdm_hca_list_head ibdm_hca_list_tail})) 251 _NOTE(MUTEX_PROTECTS_DATA(ibdm_s::ibdm_ibnex_mutex, 252 ibdm_s::ibdm_ibnex_callback)) 253 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by cv", ibdm_s)) 254 _NOTE(LOCK_ORDER(ibdm_s::ibdm_mutex ibdm_dp_gidinfo_s::gl_mutex)) 255 256 /* valid values for ibdm_state */ 257 #define IBDM_LOCKS_ALLOCED 0x01 /* global locks alloced */ 258 #define IBDM_CVS_ALLOCED 0x02 /* global "cv"s alloced */ 259 #define IBDM_IBT_ATTACHED 0x04 /* ibt_attach() called */ 260 #define IBDM_HCA_ATTACHED 0x08 /* ibdm_handle_hca() called */ 261 262 #define IBDM_8_BIT_MASK 0x00FF 263 #define IBDM_16_BIT_MASK 0xFFFF 264 #define IBDM_RETRY_COUNT 0x2 265 266 #define IBDM_BUSY 0x1 267 #define IBDM_PROBE_IN_PROGRESS 0x2 268 269 #define IBDM_MAD_SIZE 256 270 #define DM_CLASSPORTINFO_SZ 72 271 272 #define IBDM_DFT_TIMEOUT 4 273 #define IBDM_DFT_NRETRIES 3 274 275 #define IBDM_ENABLE_TASKQ_HANDLING 1 276 #define IBDM_DISABLE_TASKQ_HANLDING 0 277 278 typedef struct ibdm_saa_event_arg_s { 279 ibmf_saa_handle_t ibmf_saa_handle; 280 ibmf_saa_subnet_event_t ibmf_saa_event; 281 ibmf_saa_event_details_t event_details; 282 void *callback_arg; 283 } ibdm_saa_event_arg_t; 284 285 #define IBDM_TIMEOUT_VALUE(t) (drv_usectohz(t * 1000000)) 286 287 #define IBDM_OUT_IBMFMSG_MADHDR(msg)\ 288 (msg->im_msgbufs_send.im_bufs_mad_hdr) 289 290 #define IBDM_IN_IBMFMSG_MADHDR(msg)\ 291 (msg->im_msgbufs_recv.im_bufs_mad_hdr) 292 293 #define IBDM_IN_IBMFMSG_STATUS(msg)\ 294 b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->Status) 295 296 #define IBDM_IN_IBMFMSG_ATTR(msg)\ 297 b2h16(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeID) 298 299 #define IBDM_IN_IBMFMSG_ATTRMOD(msg)\ 300 b2h32(msg->im_msgbufs_recv.im_bufs_mad_hdr->AttributeModifier) 301 302 #define IBDM_IN_IBMFMSG2IOU(msg) (ib_dm_io_unitinfo_t *)\ 303 (msg->im_msgbufs_recv.im_bufs_cl_data) 304 305 #define IBDM_IN_IBMFMSG2IOC(msg) (ib_dm_ioc_ctrl_profile_t *)\ 306 (msg->im_msgbufs_recv.im_bufs_cl_data) 307 308 #define IBDM_IN_IBMFMSG2SRVENT(msg) (ib_dm_srv_t *)\ 309 (msg->im_msgbufs_recv.im_bufs_cl_data) 310 311 #define IBDM_IN_IBMFMSG2DIAGCODE(msg) (uint32_t *)\ 312 (msg->im_msgbufs_recv.im_bufs_cl_data) 313 314 #define IBDM_GIDINFO2IOCINFO(gid_info, idx) \ 315 (ibdm_ioc_info_t *)&gid_info->gl_iou->iou_ioc_info[idx]; 316 317 #define IBDM_IS_IOC_NUM_INVALID(ioc_no, gid_info)\ 318 ((ioc_no < 1) || (ioc_no >\ 319 gid_info->gl_iou->iou_info.iou_num_ctrl_slots)) 320 321 #define IBDM_INVALID_PKEY(pkey) \ 322 (((pkey) == IB_PKEY_INVALID_FULL) || \ 323 ((pkey) == IB_PKEY_INVALID_LIMITED)) 324 325 #ifdef DEBUG 326 327 void ibdm_dump_ibmf_msg(ibmf_msg_t *, int); 328 void ibdm_dump_path_info(sa_path_record_t *); 329 void ibdm_dump_classportinfo(ibdm_mad_classportinfo_t *); 330 void ibdm_dump_iounitinfo(ib_dm_io_unitinfo_t *); 331 void ibdm_dump_ioc_profile(ib_dm_ioc_ctrl_profile_t *); 332 void ibdm_dump_service_entries(ib_dm_srv_t *); 333 void ibdm_dump_sweep_fabric_timestamp(int); 334 335 #define ibdm_dump_ibmf_msg(a, b) ibdm_dump_ibmf_msg(a, b) 336 #define ibdm_dump_path_info(a) ibdm_dump_path_info(a) 337 #define ibdm_dump_classportinfo(a) ibdm_dump_classportinfo(a) 338 #define ibdm_dump_iounitinfo(a) ibdm_dump_iounitinfo(a) 339 #define ibdm_dump_ioc_profile(a) ibdm_dump_ioc_profile(a) 340 #define ibdm_dump_service_entries(a) ibdm_dump_service_entries(a) 341 342 #else 343 344 #define ibdm_dump_ibmf_msg(a, b) 345 #define ibdm_dump_path_info(a) 346 #define ibdm_dump_classportinfo(a) 347 #define ibdm_dump_iounitinfo(a) 348 #define ibdm_dump_ioc_profile(a) 349 #define ibdm_dump_service_entries(a) 350 #define ibdm_dump_sweep_fabric_timestamp(a) 351 352 #endif 353 354 #ifdef __cplusplus 355 } 356 #endif 357 358 #endif /* _SYS_IB_MGT_IBDM_IBDM_IMPL_H */ 359