1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 */ 17 18 /* 19 * Controls the management of commands that are issues to and from the HCA 20 * command queue. 21 */ 22 23 #include <mlxcx.h> 24 25 #include <sys/debug.h> 26 #include <sys/sysmacros.h> 27 28 /* 29 * When we start up the command queue, it will undergo some internal 30 * initialization after we set the command queue address. These values allow us 31 * to control how much time we should wait for that to occur. 32 */ 33 clock_t mlxcx_cmd_init_delay = 1000 * 10; /* 10 ms in us */ 34 uint_t mlxcx_cmd_init_trys = 100; /* Wait at most 1s */ 35 36 clock_t mlxcx_cmd_delay = 1000 * 1; /* 1 ms in us */ 37 uint_t mlxcx_cmd_tries = 5000; /* Wait at most 1s */ 38 39 /* 40 * This macro is used to identify that we care about our own function that we're 41 * communicating with. We always use this function. 42 */ 43 #define MLXCX_FUNCTION_SELF (to_be16(0)) 44 45 static const char * 46 mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret) 47 { 48 switch (ret) { 49 case MLXCX_CMD_R_OK: 50 return ("MLXCX_CMD_R_OK"); 51 case MLXCX_CMD_R_INTERNAL_ERR: 52 return ("MLXCX_CMD_R_INTERNAL_ERR"); 53 case MLXCX_CMD_R_BAD_OP: 54 return ("MLXCX_CMD_R_BAD_OP"); 55 case MLXCX_CMD_R_BAD_PARAM: 56 return ("MLXCX_CMD_R_BAD_PARAM"); 57 case MLXCX_CMD_R_BAD_SYS_STATE: 58 return ("MLXCX_CMD_R_BAD_SYS_STATE"); 59 case MLXCX_CMD_R_BAD_RESOURCE: 60 return ("MLXCX_CMD_R_BAD_RESOURCE"); 61 case MLXCX_CMD_R_RESOURCE_BUSY: 62 return ("MLXCX_CMD_R_RESOURCE_BUSY"); 63 case MLXCX_CMD_R_EXCEED_LIM: 64 return ("MLXCX_CMD_R_EXCEED_LIM"); 65 case MLXCX_CMD_R_BAD_RES_STATE: 66 return ("MLXCX_CMD_R_BAD_RES_STATE"); 67 case MLXCX_CMD_R_BAD_INDEX: 68 return ("MLXCX_CMD_R_BAD_INDEX"); 69 case MLXCX_CMD_R_NO_RESOURCES: 70 return ("MLXCX_CMD_R_NO_RESOURCES"); 71 case MLXCX_CMD_R_BAD_INPUT_LEN: 72 return ("MLXCX_CMD_R_BAD_INPUT_LEN"); 73 case MLXCX_CMD_R_BAD_OUTPUT_LEN: 74 return ("MLXCX_CMD_R_BAD_OUTPUT_LEN"); 75 case MLXCX_CMD_R_BAD_RESOURCE_STATE: 76 return ("MLXCX_CMD_R_BAD_RESOURCE_STATE"); 77 case MLXCX_CMD_R_BAD_PKT: 78 return ("MLXCX_CMD_R_BAD_PKT"); 79 case MLXCX_CMD_R_BAD_SIZE: 80 return ("MLXCX_CMD_R_BAD_SIZE"); 81 default: 82 return ("Unknown command"); 83 } 84 } 85 86 static const char * 87 mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op) 88 { 89 switch (op) { 90 case MLXCX_OP_QUERY_HCA_CAP: 91 return ("MLXCX_OP_QUERY_HCA_CAP"); 92 case MLXCX_OP_QUERY_ADAPTER: 93 return ("MLXCX_OP_QUERY_ADAPTER"); 94 case MLXCX_OP_INIT_HCA: 95 return ("MLXCX_OP_INIT_HCA"); 96 case MLXCX_OP_TEARDOWN_HCA: 97 return ("MLXCX_OP_TEARDOWN_HCA"); 98 case MLXCX_OP_ENABLE_HCA: 99 return ("MLXCX_OP_ENABLE_HCA"); 100 case MLXCX_OP_DISABLE_HCA: 101 return ("MLXCX_OP_DISABLE_HCA"); 102 case MLXCX_OP_QUERY_PAGES: 103 return ("MLXCX_OP_QUERY_PAGES"); 104 case MLXCX_OP_MANAGE_PAGES: 105 return ("MLXCX_OP_MANAGE_PAGES"); 106 case MLXCX_OP_SET_HCA_CAP: 107 return ("MLXCX_OP_SET_HCA_CAP"); 108 case MLXCX_OP_QUERY_ISSI: 109 return ("MLXCX_OP_QUERY_ISSI"); 110 case MLXCX_OP_SET_ISSI: 111 return ("MLXCX_OP_SET_ISSI"); 112 case MLXCX_OP_SET_DRIVER_VERSION: 113 return ("MLXCX_OP_SET_DRIVER_VERSION"); 114 case MLXCX_OP_QUERY_OTHER_HCA_CAP: 115 return ("MLXCX_OP_QUERY_OTHER_HCA_CAP"); 116 case MLXCX_OP_MODIFY_OTHER_HCA_CAP: 117 return ("MLXCX_OP_MODIFY_OTHER_HCA_CAP"); 118 case MLXCX_OP_SET_TUNNELED_OPERATIONS: 119 return ("MLXCX_OP_SET_TUNNELED_OPERATIONS"); 120 case MLXCX_OP_CREATE_MKEY: 121 return ("MLXCX_OP_CREATE_MKEY"); 122 case MLXCX_OP_QUERY_MKEY: 123 return ("MLXCX_OP_QUERY_MKEY"); 124 case MLXCX_OP_DESTROY_MKEY: 125 return ("MLXCX_OP_DESTROY_MKEY"); 126 case MLXCX_OP_QUERY_SPECIAL_CONTEXTS: 127 return ("MLXCX_OP_QUERY_SPECIAL_CONTEXTS"); 128 case MLXCX_OP_PAGE_FAULT_RESUME: 129 return ("MLXCX_OP_PAGE_FAULT_RESUME"); 130 case MLXCX_OP_CREATE_EQ: 131 return ("MLXCX_OP_CREATE_EQ"); 132 case MLXCX_OP_DESTROY_EQ: 133 return ("MLXCX_OP_DESTROY_EQ"); 134 case MLXCX_OP_QUERY_EQ: 135 return ("MLXCX_OP_QUERY_EQ"); 136 case MLXCX_OP_GEN_EQE: 137 return ("MLXCX_OP_GEN_EQE"); 138 case MLXCX_OP_CREATE_CQ: 139 return ("MLXCX_OP_CREATE_CQ"); 140 case MLXCX_OP_DESTROY_CQ: 141 return ("MLXCX_OP_DESTROY_CQ"); 142 case MLXCX_OP_QUERY_CQ: 143 return ("MLXCX_OP_QUERY_CQ"); 144 case MLXCX_OP_MODIFY_CQ: 145 return ("MLXCX_OP_MODIFY_CQ"); 146 case MLXCX_OP_CREATE_QP: 147 return ("MLXCX_OP_CREATE_QP"); 148 case MLXCX_OP_DESTROY_QP: 149 return ("MLXCX_OP_DESTROY_QP"); 150 case MLXCX_OP_RST2INIT_QP: 151 return ("MLXCX_OP_RST2INIT_QP"); 152 case MLXCX_OP_INIT2RTR_QP: 153 return ("MLXCX_OP_INIT2RTR_QP"); 154 case MLXCX_OP_RTR2RTS_QP: 155 return ("MLXCX_OP_RTR2RTS_QP"); 156 case MLXCX_OP_RTS2RTS_QP: 157 return ("MLXCX_OP_RTS2RTS_QP"); 158 case MLXCX_OP_SQERR2RTS_QP: 159 return ("MLXCX_OP_SQERR2RTS_QP"); 160 case MLXCX_OP__2ERR_QP: 161 return ("MLXCX_OP__2ERR_QP"); 162 case MLXCX_OP__2RST_QP: 163 return ("MLXCX_OP__2RST_QP"); 164 case MLXCX_OP_QUERY_QP: 165 return ("MLXCX_OP_QUERY_QP"); 166 case MLXCX_OP_SQD_RTS_QP: 167 return ("MLXCX_OP_SQD_RTS_QP"); 168 case MLXCX_OP_INIT2INIT_QP: 169 return ("MLXCX_OP_INIT2INIT_QP"); 170 case MLXCX_OP_CREATE_PSV: 171 return ("MLXCX_OP_CREATE_PSV"); 172 case MLXCX_OP_DESTROY_PSV: 173 return ("MLXCX_OP_DESTROY_PSV"); 174 case MLXCX_OP_CREATE_SRQ: 175 return ("MLXCX_OP_CREATE_SRQ"); 176 case MLXCX_OP_DESTROY_SRQ: 177 return ("MLXCX_OP_DESTROY_SRQ"); 178 case MLXCX_OP_QUERY_SRQ: 179 return ("MLXCX_OP_QUERY_SRQ"); 180 case MLXCX_OP_ARM_RQ: 181 return ("MLXCX_OP_ARM_RQ"); 182 case MLXCX_OP_CREATE_XRC_SRQ: 183 return ("MLXCX_OP_CREATE_XRC_SRQ"); 184 case MLXCX_OP_DESTROY_XRC_SRQ: 185 return ("MLXCX_OP_DESTROY_XRC_SRQ"); 186 case MLXCX_OP_QUERY_XRC_SRQ: 187 return ("MLXCX_OP_QUERY_XRC_SRQ"); 188 case MLXCX_OP_ARM_XRC_SRQ: 189 return ("MLXCX_OP_ARM_XRC_SRQ"); 190 case MLXCX_OP_CREATE_DCT: 191 return ("MLXCX_OP_CREATE_DCT"); 192 case MLXCX_OP_DESTROY_DCT: 193 return ("MLXCX_OP_DESTROY_DCT"); 194 case MLXCX_OP_DRAIN_DCT: 195 return ("MLXCX_OP_DRAIN_DCT"); 196 case MLXCX_OP_QUERY_DCT: 197 return ("MLXCX_OP_QUERY_DCT"); 198 case MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION: 199 return ("MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION"); 200 case MLXCX_OP_CREATE_XRQ: 201 return ("MLXCX_OP_CREATE_XRQ"); 202 case MLXCX_OP_DESTROY_XRQ: 203 return ("MLXCX_OP_DESTROY_XRQ"); 204 case MLXCX_OP_QUERY_XRQ: 205 return ("MLXCX_OP_QUERY_XRQ"); 206 case MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER: 207 return ("MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER"); 208 case MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER: 209 return ("MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER"); 210 case MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER: 211 return ("MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER"); 212 case MLXCX_OP_ATTACH_NVMF_NAMESPACE: 213 return ("MLXCX_OP_ATTACH_NVMF_NAMESPACE"); 214 case MLXCX_OP_DETACH_NVMF_NAMESPACE: 215 return ("MLXCX_OP_DETACH_NVMF_NAMESPACE"); 216 case MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 217 return ("MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY"); 218 case MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY: 219 return ("MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY"); 220 case MLXCX_OP_QUERY_XRQ_ERROR_PARAMS: 221 return ("MLXCX_OP_QUERY_XRQ_ERROR_PARAMS"); 222 case MLXCX_OP_QUERY_VPORT_STATE: 223 return ("MLXCX_OP_QUERY_VPORT_STATE"); 224 case MLXCX_OP_MODIFY_VPORT_STATE: 225 return ("MLXCX_OP_MODIFY_VPORT_STATE"); 226 case MLXCX_OP_QUERY_ESW_VPORT_CONTEXT: 227 return ("MLXCX_OP_QUERY_ESW_VPORT_CONTEXT"); 228 case MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT: 229 return ("MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT"); 230 case MLXCX_OP_QUERY_NIC_VPORT_CONTEXT: 231 return ("MLXCX_OP_QUERY_NIC_VPORT_CONTEXT"); 232 case MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT: 233 return ("MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT"); 234 case MLXCX_OP_QUERY_ROCE_ADDRESS: 235 return ("MLXCX_OP_QUERY_ROCE_ADDRESS"); 236 case MLXCX_OP_SET_ROCE_ADDRESS: 237 return ("MLXCX_OP_SET_ROCE_ADDRESS"); 238 case MLXCX_OP_QUERY_HCA_VPORT_CONTEXT: 239 return ("MLXCX_OP_QUERY_HCA_VPORT_CONTEXT"); 240 case MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT: 241 return ("MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT"); 242 case MLXCX_OP_QUERY_HCA_VPORT_GID: 243 return ("MLXCX_OP_QUERY_HCA_VPORT_GID"); 244 case MLXCX_OP_QUERY_HCA_VPORT_PKEY: 245 return ("MLXCX_OP_QUERY_HCA_VPORT_PKEY"); 246 case MLXCX_OP_QUERY_VPORT_COUNTER: 247 return ("MLXCX_OP_QUERY_VPORT_COUNTER"); 248 case MLXCX_OP_ALLOC_Q_COUNTER: 249 return ("MLXCX_OP_ALLOC_Q_COUNTER"); 250 case MLXCX_OP_DEALLOC_Q_COUNTER: 251 return ("MLXCX_OP_DEALLOC_Q_COUNTER"); 252 case MLXCX_OP_QUERY_Q_COUNTER: 253 return ("MLXCX_OP_QUERY_Q_COUNTER"); 254 case MLXCX_OP_SET_PP_RATE_LIMIT: 255 return ("MLXCX_OP_SET_PP_RATE_LIMIT"); 256 case MLXCX_OP_QUERY_PP_RATE_LIMIT: 257 return ("MLXCX_OP_QUERY_PP_RATE_LIMIT"); 258 case MLXCX_OP_ALLOC_PD: 259 return ("MLXCX_OP_ALLOC_PD"); 260 case MLXCX_OP_DEALLOC_PD: 261 return ("MLXCX_OP_DEALLOC_PD"); 262 case MLXCX_OP_ALLOC_UAR: 263 return ("MLXCX_OP_ALLOC_UAR"); 264 case MLXCX_OP_DEALLOC_UAR: 265 return ("MLXCX_OP_DEALLOC_UAR"); 266 case MLXCX_OP_CONFIG_INT_MODERATION: 267 return ("MLXCX_OP_CONFIG_INT_MODERATION"); 268 case MLXCX_OP_ACCESS_REG: 269 return ("MLXCX_OP_ACCESS_REG"); 270 case MLXCX_OP_ATTACH_TO_MCG: 271 return ("MLXCX_OP_ATTACH_TO_MCG"); 272 case MLXCX_OP_DETACH_FROM_MCG: 273 return ("MLXCX_OP_DETACH_FROM_MCG"); 274 case MLXCX_OP_MAD_IFC: 275 return ("MLXCX_OP_MAD_IFC"); 276 case MLXCX_OP_QUERY_MAD_DEMUX: 277 return ("MLXCX_OP_QUERY_MAD_DEMUX"); 278 case MLXCX_OP_SET_MAD_DEMUX: 279 return ("MLXCX_OP_SET_MAD_DEMUX"); 280 case MLXCX_OP_NOP: 281 return ("MLXCX_OP_NOP"); 282 case MLXCX_OP_ALLOC_XRCD: 283 return ("MLXCX_OP_ALLOC_XRCD"); 284 case MLXCX_OP_DEALLOC_XRCD: 285 return ("MLXCX_OP_DEALLOC_XRCD"); 286 case MLXCX_OP_ALLOC_TRANSPORT_DOMAIN: 287 return ("MLXCX_OP_ALLOC_TRANSPORT_DOMAIN"); 288 case MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN: 289 return ("MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN"); 290 case MLXCX_OP_QUERY_CONG_STATUS: 291 return ("MLXCX_OP_QUERY_CONG_STATUS"); 292 case MLXCX_OP_MODIFY_CONG_STATUS: 293 return ("MLXCX_OP_MODIFY_CONG_STATUS"); 294 case MLXCX_OP_QUERY_CONG_PARAMS: 295 return ("MLXCX_OP_QUERY_CONG_PARAMS"); 296 case MLXCX_OP_MODIFY_CONG_PARAMS: 297 return ("MLXCX_OP_MODIFY_CONG_PARAMS"); 298 case MLXCX_OP_QUERY_CONG_STATISTICS: 299 return ("MLXCX_OP_QUERY_CONG_STATISTICS"); 300 case MLXCX_OP_ADD_VXLAN_UDP_DPORT: 301 return ("MLXCX_OP_ADD_VXLAN_UDP_DPORT"); 302 case MLXCX_OP_DELETE_VXLAN_UDP_DPORT: 303 return ("MLXCX_OP_DELETE_VXLAN_UDP_DPORT"); 304 case MLXCX_OP_SET_L2_TABLE_ENTRY: 305 return ("MLXCX_OP_SET_L2_TABLE_ENTRY"); 306 case MLXCX_OP_QUERY_L2_TABLE_ENTRY: 307 return ("MLXCX_OP_QUERY_L2_TABLE_ENTRY"); 308 case MLXCX_OP_DELETE_L2_TABLE_ENTRY: 309 return ("MLXCX_OP_DELETE_L2_TABLE_ENTRY"); 310 case MLXCX_OP_SET_WOL_ROL: 311 return ("MLXCX_OP_SET_WOL_ROL"); 312 case MLXCX_OP_QUERY_WOL_ROL: 313 return ("MLXCX_OP_QUERY_WOL_ROL"); 314 case MLXCX_OP_CREATE_TIR: 315 return ("MLXCX_OP_CREATE_TIR"); 316 case MLXCX_OP_MODIFY_TIR: 317 return ("MLXCX_OP_MODIFY_TIR"); 318 case MLXCX_OP_DESTROY_TIR: 319 return ("MLXCX_OP_DESTROY_TIR"); 320 case MLXCX_OP_QUERY_TIR: 321 return ("MLXCX_OP_QUERY_TIR"); 322 case MLXCX_OP_CREATE_SQ: 323 return ("MLXCX_OP_CREATE_SQ"); 324 case MLXCX_OP_MODIFY_SQ: 325 return ("MLXCX_OP_MODIFY_SQ"); 326 case MLXCX_OP_DESTROY_SQ: 327 return ("MLXCX_OP_DESTROY_SQ"); 328 case MLXCX_OP_QUERY_SQ: 329 return ("MLXCX_OP_QUERY_SQ"); 330 case MLXCX_OP_CREATE_RQ: 331 return ("MLXCX_OP_CREATE_RQ"); 332 case MLXCX_OP_MODIFY_RQ: 333 return ("MLXCX_OP_MODIFY_RQ"); 334 case MLXCX_OP_DESTROY_RQ: 335 return ("MLXCX_OP_DESTROY_RQ"); 336 case MLXCX_OP_QUERY_RQ: 337 return ("MLXCX_OP_QUERY_RQ"); 338 case MLXCX_OP_CREATE_RMP: 339 return ("MLXCX_OP_CREATE_RMP"); 340 case MLXCX_OP_MODIFY_RMP: 341 return ("MLXCX_OP_MODIFY_RMP"); 342 case MLXCX_OP_DESTROY_RMP: 343 return ("MLXCX_OP_DESTROY_RMP"); 344 case MLXCX_OP_QUERY_RMP: 345 return ("MLXCX_OP_QUERY_RMP"); 346 case MLXCX_OP_CREATE_TIS: 347 return ("MLXCX_OP_CREATE_TIS"); 348 case MLXCX_OP_MODIFY_TIS: 349 return ("MLXCX_OP_MODIFY_TIS"); 350 case MLXCX_OP_DESTROY_TIS: 351 return ("MLXCX_OP_DESTROY_TIS"); 352 case MLXCX_OP_QUERY_TIS: 353 return ("MLXCX_OP_QUERY_TIS"); 354 case MLXCX_OP_CREATE_RQT: 355 return ("MLXCX_OP_CREATE_RQT"); 356 case MLXCX_OP_MODIFY_RQT: 357 return ("MLXCX_OP_MODIFY_RQT"); 358 case MLXCX_OP_DESTROY_RQT: 359 return ("MLXCX_OP_DESTROY_RQT"); 360 case MLXCX_OP_QUERY_RQT: 361 return ("MLXCX_OP_QUERY_RQT"); 362 case MLXCX_OP_SET_FLOW_TABLE_ROOT: 363 return ("MLXCX_OP_SET_FLOW_TABLE_ROOT"); 364 case MLXCX_OP_CREATE_FLOW_TABLE: 365 return ("MLXCX_OP_CREATE_FLOW_TABLE"); 366 case MLXCX_OP_DESTROY_FLOW_TABLE: 367 return ("MLXCX_OP_DESTROY_FLOW_TABLE"); 368 case MLXCX_OP_QUERY_FLOW_TABLE: 369 return ("MLXCX_OP_QUERY_FLOW_TABLE"); 370 case MLXCX_OP_CREATE_FLOW_GROUP: 371 return ("MLXCX_OP_CREATE_FLOW_GROUP"); 372 case MLXCX_OP_DESTROY_FLOW_GROUP: 373 return ("MLXCX_OP_DESTROY_FLOW_GROUP"); 374 case MLXCX_OP_QUERY_FLOW_GROUP: 375 return ("MLXCX_OP_QUERY_FLOW_GROUP"); 376 case MLXCX_OP_SET_FLOW_TABLE_ENTRY: 377 return ("MLXCX_OP_SET_FLOW_TABLE_ENTRY"); 378 case MLXCX_OP_QUERY_FLOW_TABLE_ENTRY: 379 return ("MLXCX_OP_QUERY_FLOW_TABLE_ENTRY"); 380 case MLXCX_OP_DELETE_FLOW_TABLE_ENTRY: 381 return ("MLXCX_OP_DELETE_FLOW_TABLE_ENTRY"); 382 case MLXCX_OP_ALLOC_FLOW_COUNTER: 383 return ("MLXCX_OP_ALLOC_FLOW_COUNTER"); 384 case MLXCX_OP_DEALLOC_FLOW_COUNTER: 385 return ("MLXCX_OP_DEALLOC_FLOW_COUNTER"); 386 case MLXCX_OP_QUERY_FLOW_COUNTER: 387 return ("MLXCX_OP_QUERY_FLOW_COUNTER"); 388 case MLXCX_OP_MODIFY_FLOW_TABLE: 389 return ("MLXCX_OP_MODIFY_FLOW_TABLE"); 390 case MLXCX_OP_ALLOC_ENCAP_HEADER: 391 return ("MLXCX_OP_ALLOC_ENCAP_HEADER"); 392 case MLXCX_OP_DEALLOC_ENCAP_HEADER: 393 return ("MLXCX_OP_DEALLOC_ENCAP_HEADER"); 394 case MLXCX_OP_QUERY_ENCAP_HEADER: 395 return ("MLXCX_OP_QUERY_ENCAP_HEADER"); 396 default: 397 return ("Unknown Opcode"); 398 } 399 } 400 401 const char * 402 mlxcx_port_status_string(mlxcx_port_status_t st) 403 { 404 switch (st) { 405 case MLXCX_PORT_STATUS_UP: 406 return ("UP"); 407 case MLXCX_PORT_STATUS_DOWN: 408 return ("DOWN"); 409 case MLXCX_PORT_STATUS_UP_ONCE: 410 return ("UP_ONCE"); 411 case MLXCX_PORT_STATUS_DISABLED: 412 return ("DISABLED"); 413 default: 414 return ("UNKNOWN"); 415 } 416 } 417 418 void 419 mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) 420 { 421 if (p & MLXCX_PROTO_SGMII) 422 (void) strlcat(buf, "SGMII|", size); 423 if (p & MLXCX_PROTO_1000BASE_KX) 424 (void) strlcat(buf, "1000BASE_KX|", size); 425 if (p & MLXCX_PROTO_10GBASE_CX4) 426 (void) strlcat(buf, "10GBASE_CX4|", size); 427 if (p & MLXCX_PROTO_10GBASE_KX4) 428 (void) strlcat(buf, "10GBASE_KX4|", size); 429 if (p & MLXCX_PROTO_10GBASE_KR) 430 (void) strlcat(buf, "10GBASE_KR|", size); 431 if (p & MLXCX_PROTO_40GBASE_CR4) 432 (void) strlcat(buf, "40GBASE_CR4|", size); 433 if (p & MLXCX_PROTO_40GBASE_KR4) 434 (void) strlcat(buf, "40GBASE_KR4|", size); 435 if (p & MLXCX_PROTO_SGMII_100BASE) 436 (void) strlcat(buf, "SGMII_100BASE|", size); 437 if (p & MLXCX_PROTO_10GBASE_CR) 438 (void) strlcat(buf, "10GBASE_CR|", size); 439 if (p & MLXCX_PROTO_10GBASE_SR) 440 (void) strlcat(buf, "10GBASE_SR|", size); 441 if (p & MLXCX_PROTO_10GBASE_ER_LR) 442 (void) strlcat(buf, "10GBASE_ER_LR|", size); 443 if (p & MLXCX_PROTO_40GBASE_SR4) 444 (void) strlcat(buf, "40GBASE_SR4|", size); 445 if (p & MLXCX_PROTO_40GBASE_LR4_ER4) 446 (void) strlcat(buf, "40GBASE_LR4_ER4|", size); 447 if (p & MLXCX_PROTO_50GBASE_SR2) 448 (void) strlcat(buf, "50GBASE_SR2|", size); 449 if (p & MLXCX_PROTO_100GBASE_CR4) 450 (void) strlcat(buf, "100GBASE_CR4|", size); 451 if (p & MLXCX_PROTO_100GBASE_SR4) 452 (void) strlcat(buf, "100GBASE_SR4|", size); 453 if (p & MLXCX_PROTO_100GBASE_KR4) 454 (void) strlcat(buf, "100GBASE_KR4|", size); 455 if (p & MLXCX_PROTO_25GBASE_CR) 456 (void) strlcat(buf, "25GBASE_CR|", size); 457 if (p & MLXCX_PROTO_25GBASE_KR) 458 (void) strlcat(buf, "25GBASE_KR|", size); 459 if (p & MLXCX_PROTO_25GBASE_SR) 460 (void) strlcat(buf, "25GBASE_SR|", size); 461 if (p & MLXCX_PROTO_50GBASE_CR2) 462 (void) strlcat(buf, "50GBASE_CR2|", size); 463 /* Chop off the trailing '|' */ 464 if (strlen(buf) > 0) 465 buf[strlen(buf) - 1] = '\0'; 466 } 467 468 void 469 mlxcx_cmd_queue_fini(mlxcx_t *mlxp) 470 { 471 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 472 473 if (cmd->mcmd_tokens != NULL) { 474 id_space_destroy(cmd->mcmd_tokens); 475 cmd->mcmd_tokens = NULL; 476 } 477 478 if (cmd->mcmd_taskq != NULL) { 479 ddi_taskq_destroy(cmd->mcmd_taskq); 480 cmd->mcmd_taskq = NULL; 481 } 482 483 cv_destroy(&cmd->mcmd_cv); 484 mutex_destroy(&cmd->mcmd_lock); 485 486 mlxcx_dma_free(&cmd->mcmd_dma); 487 } 488 489 boolean_t 490 mlxcx_cmd_queue_init(mlxcx_t *mlxp) 491 { 492 uint32_t tmp, cmd_low, cmd_high, i; 493 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 494 char buf[32]; 495 char tq_name[TASKQ_NAMELEN]; 496 const ddi_dma_cookie_t *ck; 497 498 ddi_device_acc_attr_t acc; 499 ddi_dma_attr_t attr; 500 501 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FIRMWARE); 502 mlxp->mlx_fw_maj = MLXCX_ISS_FW_MAJOR(tmp); 503 mlxp->mlx_fw_min = MLXCX_ISS_FW_MINOR(tmp); 504 505 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FW_CMD); 506 mlxp->mlx_fw_rev = MLXCX_ISS_FW_REV(tmp); 507 mlxp->mlx_cmd_rev = MLXCX_ISS_CMD_REV(tmp); 508 509 if (mlxp->mlx_cmd_rev != MLXCX_CMD_REVISION) { 510 mlxcx_warn(mlxp, "found unsupported command revision: %u, " 511 "expected %u", mlxp->mlx_cmd_rev, MLXCX_CMD_REVISION); 512 return (B_FALSE); 513 } 514 515 cmd_low = mlxcx_get32(mlxp, MLXCX_ISS_CMD_LOW); 516 cmd->mcmd_size_l2 = MLXCX_ISS_CMDQ_SIZE(cmd_low); 517 cmd->mcmd_stride_l2 = MLXCX_ISS_CMDQ_STRIDE(cmd_low); 518 cmd->mcmd_size = 1U << cmd->mcmd_size_l2; 519 520 if (cmd->mcmd_size > MLXCX_CMD_MAX) { 521 mlxcx_warn(mlxp, "command queue size %u is too " 522 "large. Maximum is %u", cmd->mcmd_size, MLXCX_CMD_MAX); 523 return (B_FALSE); 524 } 525 526 cmd->mcmd_mask = (uint32_t)((1ULL << cmd->mcmd_size) - 1); 527 528 mutex_init(&cmd->mcmd_lock, NULL, MUTEX_DRIVER, NULL); 529 cv_init(&cmd->mcmd_cv, NULL, CV_DRIVER, NULL); 530 531 (void) snprintf(buf, sizeof (buf), "mlxcx_tokens_%d", mlxp->mlx_inst); 532 if ((cmd->mcmd_tokens = id_space_create(buf, 1, UINT8_MAX)) == NULL) { 533 mlxcx_warn(mlxp, "failed to allocate token id space"); 534 mlxcx_cmd_queue_fini(mlxp); 535 return (B_FALSE); 536 } 537 538 (void) snprintf(tq_name, sizeof (tq_name), "cmdq_%d", mlxp->mlx_inst); 539 if ((cmd->mcmd_taskq = ddi_taskq_create(mlxp->mlx_dip, tq_name, 1, 540 TASKQ_DEFAULTPRI, 0)) == NULL) { 541 mlxcx_warn(mlxp, "failed to create command queue task queue"); 542 mlxcx_cmd_queue_fini(mlxp); 543 return (B_FALSE); 544 } 545 546 mlxcx_dma_acc_attr(mlxp, &acc); 547 mlxcx_dma_page_attr(mlxp, &attr); 548 549 if (!mlxcx_dma_alloc(mlxp, &cmd->mcmd_dma, &attr, &acc, B_TRUE, 550 MLXCX_CMD_DMA_PAGE_SIZE, B_TRUE)) { 551 mlxcx_warn(mlxp, "failed to allocate command dma buffer"); 552 mlxcx_cmd_queue_fini(mlxp); 553 return (B_FALSE); 554 } 555 556 ck = mlxcx_dma_cookie_one(&cmd->mcmd_dma); 557 cmd_high = (uint32_t)(ck->dmac_laddress >> 32); 558 cmd_low = (uint32_t)(ck->dmac_laddress & UINT32_MAX); 559 560 mlxcx_put32(mlxp, MLXCX_ISS_CMD_HIGH, cmd_high); 561 mlxcx_put32(mlxp, MLXCX_ISS_CMD_LOW, cmd_low); 562 563 /* 564 * Before this is ready, the initializing bit must become zero. 565 */ 566 for (i = 0; i < mlxcx_cmd_init_trys; i++) { 567 uint32_t init = mlxcx_get32(mlxp, MLXCX_ISS_INIT); 568 569 if (MLXCX_ISS_INITIALIZING(init) == 0) 570 break; 571 delay(drv_usectohz(mlxcx_cmd_init_delay)); 572 } 573 if (i == mlxcx_cmd_init_trys) { 574 mlxcx_warn(mlxp, "timed out initializing command queue"); 575 mlxcx_cmd_queue_fini(mlxp); 576 return (B_FALSE); 577 } 578 579 /* 580 * start in polling mode. 581 */ 582 mlxcx_cmd_eq_disable(mlxp); 583 584 return (B_TRUE); 585 } 586 587 void 588 mlxcx_cmd_eq_enable(mlxcx_t *mlxp) 589 { 590 mlxp->mlx_cmd.mcmd_polled = B_FALSE; 591 } 592 593 void 594 mlxcx_cmd_eq_disable(mlxcx_t *mlxp) 595 { 596 mlxp->mlx_cmd.mcmd_polled = B_TRUE; 597 } 598 599 static void 600 mlxcx_cmd_in_header_init(mlxcx_cmd_t *cmd, mlxcx_cmd_in_t *in, 601 mlxcx_cmd_op_t op, uint16_t mod) 602 { 603 ASSERT3U(op, <=, UINT16_MAX); 604 in->mci_opcode = to_be16(op); 605 in->mci_op_mod = to_be16(mod); 606 cmd->mlcmd_op = op; 607 } 608 609 static boolean_t 610 mlxcx_cmd_mbox_alloc(mlxcx_t *mlxp, list_t *listp, uint8_t nblocks) 611 { 612 uint8_t i; 613 ddi_device_acc_attr_t acc; 614 ddi_dma_attr_t attr; 615 616 mlxcx_dma_acc_attr(mlxp, &acc); 617 mlxcx_dma_page_attr(mlxp, &attr); 618 619 for (i = 0; i < nblocks; i++) { 620 mlxcx_cmd_mbox_t *mbox; 621 622 mbox = kmem_zalloc(sizeof (*mbox), KM_SLEEP); 623 if (!mlxcx_dma_alloc(mlxp, &mbox->mlbox_dma, &attr, &acc, 624 B_TRUE, sizeof (mlxcx_cmd_mailbox_t), B_TRUE)) { 625 mlxcx_warn(mlxp, "failed to allocate mailbox dma " 626 "buffer"); 627 kmem_free(mbox, sizeof (*mbox)); 628 /* 629 * mlxcx_cmd_fini will clean up any mboxes that we 630 * already placed onto listp. 631 */ 632 return (B_FALSE); 633 } 634 mbox->mlbox_data = (void *)mbox->mlbox_dma.mxdb_va; 635 list_insert_tail(listp, mbox); 636 } 637 638 return (B_TRUE); 639 } 640 641 static void 642 mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t *mbox) 643 { 644 mlxcx_dma_free(&mbox->mlbox_dma); 645 kmem_free(mbox, sizeof (mlxcx_cmd_mbox_t)); 646 } 647 648 static void 649 mlxcx_cmd_fini(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 650 { 651 mlxcx_cmd_mbox_t *mbox; 652 653 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_out)) != NULL) { 654 mlxcx_cmd_mbox_free(mbox); 655 } 656 list_destroy(&cmd->mlcmd_mbox_out); 657 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_in)) != NULL) { 658 mlxcx_cmd_mbox_free(mbox); 659 } 660 list_destroy(&cmd->mlcmd_mbox_in); 661 id_free(mlxp->mlx_cmd.mcmd_tokens, cmd->mlcmd_token); 662 cv_destroy(&cmd->mlcmd_cv); 663 mutex_destroy(&cmd->mlcmd_lock); 664 } 665 666 static void 667 mlxcx_cmd_init(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 668 { 669 bzero(cmd, sizeof (*cmd)); 670 mutex_init(&cmd->mlcmd_lock, NULL, MUTEX_DRIVER, NULL); 671 cv_init(&cmd->mlcmd_cv, NULL, CV_DRIVER, NULL); 672 cmd->mlcmd_token = id_alloc(mlxp->mlx_cmd.mcmd_tokens); 673 cmd->mlcmd_poll = mlxp->mlx_cmd.mcmd_polled; 674 list_create(&cmd->mlcmd_mbox_in, sizeof (mlxcx_cmd_mbox_t), 675 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 676 list_create(&cmd->mlcmd_mbox_out, sizeof (mlxcx_cmd_mbox_t), 677 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 678 } 679 680 static void 681 mlxcx_cmd_prep_input(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 682 { 683 uint32_t rem = cmd->mlcmd_inlen; 684 uint8_t i; 685 const void *in = cmd->mlcmd_in; 686 uint32_t copy; 687 mlxcx_cmd_mbox_t *mbox; 688 const ddi_dma_cookie_t *ck; 689 690 copy = MIN(MLXCX_CMD_INLINE_INPUT_LEN, rem); 691 bcopy(in, ent->mce_input, copy); 692 693 rem -= copy; 694 in += copy; 695 696 if (rem == 0) { 697 ent->mce_in_mbox = to_be64(0); 698 VERIFY3U(cmd->mlcmd_nboxes_in, ==, 0); 699 return; 700 } 701 702 mbox = list_head(&cmd->mlcmd_mbox_in); 703 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 704 ent->mce_in_mbox = to_be64(ck->dmac_laddress); 705 for (i = 0; mbox != NULL; 706 mbox = list_next(&cmd->mlcmd_mbox_in, mbox), i++) { 707 mlxcx_cmd_mbox_t *next; 708 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 709 710 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 711 bcopy(in, mp->mlxb_data, copy); 712 rem -= copy; 713 in += copy; 714 715 mp->mlxb_token = cmd->mlcmd_token; 716 mp->mlxb_blockno = to_be32(i); 717 718 next = list_next(&cmd->mlcmd_mbox_in, mbox); 719 if (next == NULL) { 720 mp->mlxb_nextp = to_be64(0); 721 } else { 722 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 723 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 724 } 725 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 726 } 727 VERIFY3U(i, ==, cmd->mlcmd_nboxes_in); 728 VERIFY0(rem); 729 } 730 731 static void 732 mlxcx_cmd_prep_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 733 { 734 uint8_t i; 735 mlxcx_cmd_mbox_t *mbox; 736 const ddi_dma_cookie_t *ck; 737 738 if (cmd->mlcmd_nboxes_out == 0) { 739 ent->mce_out_mbox = to_be64(0); 740 return; 741 } 742 743 mbox = list_head(&cmd->mlcmd_mbox_out); 744 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 745 ent->mce_out_mbox = to_be64(ck->dmac_laddress); 746 for (i = 0, mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 747 mbox = list_next(&cmd->mlcmd_mbox_out, mbox), i++) { 748 mlxcx_cmd_mbox_t *next; 749 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 750 751 mp->mlxb_token = cmd->mlcmd_token; 752 mp->mlxb_blockno = to_be32(i); 753 754 next = list_next(&cmd->mlcmd_mbox_out, mbox); 755 if (next == NULL) { 756 mp->mlxb_nextp = to_be64(0); 757 } else { 758 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 759 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 760 } 761 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 762 } 763 VERIFY3U(i, ==, cmd->mlcmd_nboxes_out); 764 } 765 766 static void 767 mlxcx_cmd_copy_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 768 { 769 void *out = cmd->mlcmd_out; 770 uint32_t rem = cmd->mlcmd_outlen; 771 uint32_t copy; 772 mlxcx_cmd_mbox_t *mbox; 773 774 copy = MIN(rem, MLXCX_CMD_INLINE_OUTPUT_LEN); 775 bcopy(ent->mce_output, out, copy); 776 out += copy; 777 rem -= copy; 778 779 if (rem == 0) { 780 VERIFY0(cmd->mlcmd_nboxes_out); 781 return; 782 } 783 784 for (mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 785 mbox = list_next(&cmd->mlcmd_mbox_out, mbox)) { 786 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORKERNEL); 787 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 788 bcopy(mbox->mlbox_data->mlxb_data, out, copy); 789 out += copy; 790 rem -= copy; 791 } 792 VERIFY0(rem); 793 } 794 795 static uint_t 796 mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t *cmdq) 797 { 798 uint_t slot; 799 800 mutex_enter(&cmdq->mcmd_lock); 801 slot = ddi_ffs(cmdq->mcmd_mask); 802 while (slot == 0) { 803 cv_wait(&cmdq->mcmd_cv, &cmdq->mcmd_lock); 804 slot = ddi_ffs(cmdq->mcmd_mask); 805 } 806 807 cmdq->mcmd_mask &= ~(1U << --slot); 808 809 ASSERT3P(cmdq->mcmd_active[slot], ==, NULL); 810 811 mutex_exit(&cmdq->mcmd_lock); 812 813 return (slot); 814 } 815 816 static void 817 mlxcx_cmd_release_slot(mlxcx_cmd_queue_t *cmdq, uint_t slot) 818 { 819 mutex_enter(&cmdq->mcmd_lock); 820 cmdq->mcmd_mask |= 1U << slot; 821 cv_broadcast(&cmdq->mcmd_cv); 822 mutex_exit(&cmdq->mcmd_lock); 823 } 824 825 static void 826 mlxcx_cmd_done(mlxcx_cmd_t *cmd, uint_t slot) 827 { 828 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 829 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 830 mlxcx_cmd_ent_t *ent; 831 832 /* 833 * Command is done. Save relevant data. Once we broadcast on the CV and 834 * drop the lock, we must not touch it again. 835 */ 836 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 837 838 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 839 (slot << cmdq->mcmd_stride_l2)); 840 841 mutex_enter(&cmd->mlcmd_lock); 842 cmd->mlcmd_status = MLXCX_CMD_STATUS(ent->mce_status); 843 if (cmd->mlcmd_status == 0) 844 mlxcx_cmd_copy_output(ent, cmd); 845 846 cmd->mlcmd_state = MLXCX_CMD_S_DONE; 847 cv_broadcast(&cmd->mlcmd_cv); 848 mutex_exit(&cmd->mlcmd_lock); 849 850 cmdq->mcmd_active[slot] = NULL; 851 mlxcx_cmd_release_slot(cmdq, slot); 852 } 853 854 static void 855 mlxcx_cmd_taskq(void *arg) 856 { 857 mlxcx_cmd_t *cmd = arg; 858 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 859 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 860 mlxcx_cmd_ent_t *ent; 861 uint_t poll, slot; 862 863 ASSERT3S(cmd->mlcmd_op, !=, 0); 864 865 slot = mlxcx_cmd_reserve_slot(cmdq); 866 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 867 (slot << cmdq->mcmd_stride_l2)); 868 869 cmdq->mcmd_active[slot] = cmd; 870 871 /* 872 * Command queue is currently ours as we set busy. 873 */ 874 bzero(ent, sizeof (*ent)); 875 ent->mce_type = MLXCX_CMD_TRANSPORT_PCI; 876 ent->mce_in_length = to_be32(cmd->mlcmd_inlen); 877 ent->mce_out_length = to_be32(cmd->mlcmd_outlen); 878 ent->mce_token = cmd->mlcmd_token; 879 ent->mce_sig = 0; 880 ent->mce_status = MLXCX_CMD_HW_OWNED; 881 mlxcx_cmd_prep_input(ent, cmd); 882 mlxcx_cmd_prep_output(ent, cmd); 883 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORDEV); 884 885 mlxcx_put32(mlxp, MLXCX_ISS_CMD_DOORBELL, 1 << slot); 886 887 if (!cmd->mlcmd_poll) 888 return; 889 890 for (poll = 0; poll < mlxcx_cmd_tries; poll++) { 891 delay(drv_usectohz(mlxcx_cmd_delay)); 892 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 893 if ((ent->mce_status & MLXCX_CMD_HW_OWNED) == 0) 894 break; 895 } 896 897 /* 898 * Command is done (or timed out). Save relevant data. Once we broadcast 899 * on the CV and drop the lock, we must not touch the cmd again. 900 */ 901 902 if (poll == mlxcx_cmd_tries) { 903 mutex_enter(&cmd->mlcmd_lock); 904 cmd->mlcmd_status = MLXCX_CMD_R_TIMEOUT; 905 cmd->mlcmd_state = MLXCX_CMD_S_ERROR; 906 cv_broadcast(&cmd->mlcmd_cv); 907 mutex_exit(&cmd->mlcmd_lock); 908 909 mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_NO_RESPONSE); 910 911 cmdq->mcmd_active[slot] = NULL; 912 mlxcx_cmd_release_slot(cmdq, slot); 913 914 return; 915 } 916 917 mlxcx_cmd_done(cmd, slot); 918 } 919 920 void 921 mlxcx_cmd_completion(mlxcx_t *mlxp, mlxcx_eventq_ent_t *ent) 922 { 923 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 924 mlxcx_evdata_cmd_completion_t *eqe_cmd = &ent->mleqe_cmd_completion; 925 mlxcx_cmd_t *cmd; 926 uint32_t comp_vec = from_be32(eqe_cmd->mled_cmd_completion_vec); 927 uint_t slot; 928 929 DTRACE_PROBE2(cmd_event, mlxcx_t *, mlxp, 930 mlxcx_evdata_cmd_completion_t *, eqe_cmd); 931 932 while ((slot = ddi_ffs(comp_vec)) != 0) { 933 comp_vec &= ~(1U << --slot); 934 935 cmd = cmdq->mcmd_active[slot]; 936 if (cmd->mlcmd_poll) 937 continue; 938 939 mlxcx_cmd_done(cmd, slot); 940 } 941 } 942 943 static boolean_t 944 mlxcx_cmd_send(mlxcx_t *mlxp, mlxcx_cmd_t *cmd, const void *in, uint32_t inlen, 945 void *out, uint32_t outlen) 946 { 947 if (inlen > MLXCX_CMD_INLINE_INPUT_LEN) { 948 uint32_t need = inlen - MLXCX_CMD_INLINE_INPUT_LEN; 949 uint8_t nblocks; 950 951 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 952 mlxcx_warn(mlxp, "requested too many input blocks for " 953 "%u byte input len", inlen); 954 return (B_FALSE); 955 } 956 957 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 958 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_in, nblocks)) { 959 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 960 "input mailbox", nblocks); 961 return (B_FALSE); 962 } 963 cmd->mlcmd_nboxes_in = nblocks; 964 } 965 966 if (outlen > MLXCX_CMD_INLINE_OUTPUT_LEN) { 967 uint32_t need = outlen - MLXCX_CMD_INLINE_OUTPUT_LEN; 968 uint8_t nblocks; 969 970 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 971 mlxcx_warn(mlxp, "requested too many output blocks for " 972 "%u byte output len", outlen); 973 return (B_FALSE); 974 } 975 976 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 977 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_out, 978 nblocks)) { 979 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 980 "output mailbox", nblocks); 981 return (B_FALSE); 982 } 983 cmd->mlcmd_nboxes_out = nblocks; 984 } 985 986 cmd->mlcmd_in = in; 987 cmd->mlcmd_inlen = inlen; 988 cmd->mlcmd_out = out; 989 cmd->mlcmd_outlen = outlen; 990 cmd->mlcmd_mlxp = mlxp; 991 992 /* 993 * Now that all allocations have been done, all that remains is for us 994 * to dispatch the request to process this to the taskq for it to be 995 * processed. 996 */ 997 if (ddi_taskq_dispatch(mlxp->mlx_cmd.mcmd_taskq, mlxcx_cmd_taskq, cmd, 998 DDI_SLEEP) != DDI_SUCCESS) { 999 mlxcx_warn(mlxp, "failed to submit command to taskq"); 1000 return (B_FALSE); 1001 } 1002 1003 return (B_TRUE); 1004 } 1005 1006 static void 1007 mlxcx_cmd_wait(mlxcx_cmd_t *cmd) 1008 { 1009 mutex_enter(&cmd->mlcmd_lock); 1010 while (cmd->mlcmd_state == 0) { 1011 cv_wait(&cmd->mlcmd_cv, &cmd->mlcmd_lock); 1012 } 1013 mutex_exit(&cmd->mlcmd_lock); 1014 } 1015 1016 static boolean_t 1017 mlxcx_cmd_evaluate(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 1018 { 1019 mlxcx_cmd_out_t *out; 1020 1021 if ((cmd->mlcmd_state & MLXCX_CMD_S_ERROR) != 0) { 1022 mlxcx_warn(mlxp, "command %s (0x%x) failed due to an internal " 1023 "driver error", 1024 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1025 cmd->mlcmd_op); 1026 return (B_FALSE); 1027 } 1028 1029 if (cmd->mlcmd_status != 0) { 1030 mlxcx_warn(mlxp, "command %s (0x%x) failed with command queue " 1031 "error 0x%x", 1032 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1033 cmd->mlcmd_op, cmd->mlcmd_status); 1034 return (B_FALSE); 1035 } 1036 1037 out = cmd->mlcmd_out; 1038 if (out->mco_status != MLXCX_CMD_R_OK) { 1039 mlxcx_warn(mlxp, "command %s 0x%x failed with status code %s " 1040 "(0x%x)", mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1041 cmd->mlcmd_op, mlxcx_cmd_response_string(out->mco_status), 1042 out->mco_status); 1043 return (B_FALSE); 1044 } 1045 1046 return (B_TRUE); 1047 } 1048 1049 boolean_t 1050 mlxcx_cmd_disable_hca(mlxcx_t *mlxp) 1051 { 1052 mlxcx_cmd_t cmd; 1053 mlxcx_cmd_disable_hca_in_t in; 1054 mlxcx_cmd_disable_hca_out_t out; 1055 boolean_t ret; 1056 1057 bzero(&in, sizeof (in)); 1058 bzero(&out, sizeof (out)); 1059 1060 mlxcx_cmd_init(mlxp, &cmd); 1061 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_disable_hca_head, 1062 MLXCX_OP_DISABLE_HCA, 0); 1063 in.mlxi_disable_hca_func = MLXCX_FUNCTION_SELF; 1064 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1065 mlxcx_cmd_fini(mlxp, &cmd); 1066 return (B_FALSE); 1067 } 1068 mlxcx_cmd_wait(&cmd); 1069 1070 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1071 mlxcx_cmd_fini(mlxp, &cmd); 1072 return (ret); 1073 } 1074 1075 boolean_t 1076 mlxcx_cmd_enable_hca(mlxcx_t *mlxp) 1077 { 1078 mlxcx_cmd_t cmd; 1079 mlxcx_cmd_enable_hca_in_t in; 1080 mlxcx_cmd_enable_hca_out_t out; 1081 boolean_t ret; 1082 1083 bzero(&in, sizeof (in)); 1084 bzero(&out, sizeof (out)); 1085 1086 mlxcx_cmd_init(mlxp, &cmd); 1087 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_enable_hca_head, 1088 MLXCX_OP_ENABLE_HCA, 0); 1089 in.mlxi_enable_hca_func = MLXCX_FUNCTION_SELF; 1090 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1091 mlxcx_cmd_fini(mlxp, &cmd); 1092 return (B_FALSE); 1093 } 1094 mlxcx_cmd_wait(&cmd); 1095 1096 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1097 mlxcx_cmd_fini(mlxp, &cmd); 1098 return (ret); 1099 } 1100 1101 boolean_t 1102 mlxcx_cmd_query_issi(mlxcx_t *mlxp, uint32_t *issip) 1103 { 1104 mlxcx_cmd_t cmd; 1105 mlxcx_cmd_query_issi_in_t in; 1106 mlxcx_cmd_query_issi_out_t out; 1107 boolean_t ret; 1108 1109 bzero(&in, sizeof (in)); 1110 bzero(&out, sizeof (out)); 1111 1112 mlxcx_cmd_init(mlxp, &cmd); 1113 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_issi_head, 1114 MLXCX_OP_QUERY_ISSI, 0); 1115 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1116 mlxcx_cmd_fini(mlxp, &cmd); 1117 return (B_FALSE); 1118 } 1119 mlxcx_cmd_wait(&cmd); 1120 1121 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1122 if (ret) { 1123 *issip = out.mlxo_supported_issi; 1124 } else if (cmd.mlcmd_status == 0 && 1125 out.mlxo_query_issi_head.mco_status == MLXCX_CMD_R_BAD_OP) { 1126 /* 1127 * The PRM says that if we get a bad operation, that means this 1128 * command isn't supported so it only supports version 1 of the 1129 * ISSI, which means bit zero should be set. 1130 */ 1131 ret = B_TRUE; 1132 *issip = 1; 1133 } 1134 mlxcx_cmd_fini(mlxp, &cmd); 1135 return (ret); 1136 } 1137 1138 boolean_t 1139 mlxcx_cmd_set_issi(mlxcx_t *mlxp, uint16_t issi) 1140 { 1141 mlxcx_cmd_t cmd; 1142 mlxcx_cmd_set_issi_in_t in; 1143 mlxcx_cmd_set_issi_out_t out; 1144 boolean_t ret; 1145 1146 bzero(&in, sizeof (in)); 1147 bzero(&out, sizeof (out)); 1148 1149 mlxcx_cmd_init(mlxp, &cmd); 1150 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_issi_head, 1151 MLXCX_OP_SET_ISSI, 0); 1152 in.mlxi_set_issi_current = to_be16(issi); 1153 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1154 mlxcx_cmd_fini(mlxp, &cmd); 1155 return (B_FALSE); 1156 } 1157 mlxcx_cmd_wait(&cmd); 1158 1159 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1160 mlxcx_cmd_fini(mlxp, &cmd); 1161 return (ret); 1162 } 1163 1164 boolean_t 1165 mlxcx_cmd_query_pages(mlxcx_t *mlxp, uint_t type, int32_t *npages) 1166 { 1167 mlxcx_cmd_t cmd; 1168 mlxcx_cmd_query_pages_in_t in; 1169 mlxcx_cmd_query_pages_out_t out; 1170 boolean_t ret; 1171 1172 switch (type) { 1173 case MLXCX_QUERY_PAGES_OPMOD_BOOT: 1174 case MLXCX_QUERY_PAGES_OPMOD_INIT: 1175 case MLXCX_QUERY_PAGES_OPMOD_REGULAR: 1176 break; 1177 default: 1178 mlxcx_warn(mlxp, "!passed invalid type to query pages: %u", 1179 type); 1180 return (B_FALSE); 1181 } 1182 1183 bzero(&in, sizeof (in)); 1184 bzero(&out, sizeof (out)); 1185 1186 mlxcx_cmd_init(mlxp, &cmd); 1187 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_pages_head, 1188 MLXCX_OP_QUERY_PAGES, type); 1189 in.mlxi_query_pages_func = MLXCX_FUNCTION_SELF; 1190 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1191 mlxcx_cmd_fini(mlxp, &cmd); 1192 return (B_FALSE); 1193 } 1194 mlxcx_cmd_wait(&cmd); 1195 1196 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1197 if (ret) { 1198 *npages = from_be32(out.mlxo_query_pages_npages); 1199 } 1200 mlxcx_cmd_fini(mlxp, &cmd); 1201 1202 return (ret); 1203 } 1204 1205 boolean_t 1206 mlxcx_cmd_give_pages(mlxcx_t *mlxp, uint_t type, int32_t npages, 1207 mlxcx_dev_page_t **pages) 1208 { 1209 mlxcx_cmd_t cmd; 1210 mlxcx_cmd_manage_pages_in_t *in; 1211 mlxcx_cmd_manage_pages_out_t out; 1212 size_t insize, outsize; 1213 boolean_t ret; 1214 uint32_t i; 1215 uint64_t pa; 1216 const ddi_dma_cookie_t *ck; 1217 1218 switch (type) { 1219 case MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL: 1220 if (npages != 0) { 1221 mlxcx_warn(mlxp, "passed non-zero number of pages (%d) " 1222 "but asked to fail page allocation", npages); 1223 return (B_FALSE); 1224 } 1225 break; 1226 case MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES: 1227 ASSERT3S(npages, <=, MLXCX_MANAGE_PAGES_MAX_PAGES); 1228 if (npages <= 0) { 1229 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1230 "to give pages", npages); 1231 return (B_FALSE); 1232 } 1233 break; 1234 default: 1235 mlxcx_warn(mlxp, "!passed invalid type to give pages: %u", 1236 type); 1237 return (B_FALSE); 1238 } 1239 1240 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas) + 1241 npages * sizeof (uint64_t); 1242 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, mlxo_manage_pages_pas); 1243 1244 in = kmem_zalloc(insize, KM_SLEEP); 1245 bzero(&out, sizeof (out)); 1246 1247 mlxcx_cmd_init(mlxp, &cmd); 1248 mlxcx_cmd_in_header_init(&cmd, &in->mlxi_manage_pages_head, 1249 MLXCX_OP_MANAGE_PAGES, type); 1250 in->mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1251 in->mlxi_manage_pages_npages = to_be32(npages); 1252 for (i = 0; i < npages; i++) { 1253 ck = mlxcx_dma_cookie_one(&pages[i]->mxdp_dma); 1254 pa = ck->dmac_laddress; 1255 ASSERT3U(pa & 0xfff, ==, 0); 1256 ASSERT3U(ck->dmac_size, ==, MLXCX_HW_PAGE_SIZE); 1257 in->mlxi_manage_pages_pas[i] = to_be64(pa); 1258 } 1259 1260 if ((ret = mlxcx_cmd_send(mlxp, &cmd, in, insize, &out, outsize))) { 1261 mlxcx_cmd_wait(&cmd); 1262 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1263 } 1264 1265 mlxcx_cmd_fini(mlxp, &cmd); 1266 1267 kmem_free(in, insize); 1268 return (ret); 1269 } 1270 1271 boolean_t 1272 mlxcx_cmd_return_pages(mlxcx_t *mlxp, int32_t nreq, uint64_t *pas, 1273 int32_t *nret) 1274 { 1275 mlxcx_cmd_t cmd; 1276 mlxcx_cmd_manage_pages_in_t in; 1277 mlxcx_cmd_manage_pages_out_t *out; 1278 size_t insize, outsize; 1279 boolean_t ret; 1280 uint32_t i; 1281 1282 if (nreq <= 0) { 1283 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1284 "to return pages", nreq); 1285 return (B_FALSE); 1286 } 1287 1288 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas); 1289 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, 1290 mlxo_manage_pages_pas) + nreq * sizeof (uint64_t); 1291 1292 bzero(&in, sizeof (in)); 1293 out = kmem_alloc(outsize, KM_SLEEP); 1294 1295 mlxcx_cmd_init(mlxp, &cmd); 1296 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_manage_pages_head, 1297 MLXCX_OP_MANAGE_PAGES, MLXCX_MANAGE_PAGES_OPMOD_RETURN_PAGES); 1298 in.mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1299 in.mlxi_manage_pages_npages = to_be32(nreq); 1300 1301 if ((ret = mlxcx_cmd_send(mlxp, &cmd, &in, insize, out, outsize))) { 1302 mlxcx_cmd_wait(&cmd); 1303 1304 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1305 if (ret) { 1306 *nret = from_be32(out->mlxo_manage_pages_npages); 1307 for (i = 0; i < *nret; i++) { 1308 pas[i] = 1309 from_be64(out->mlxo_manage_pages_pas[i]); 1310 } 1311 } 1312 } 1313 1314 mlxcx_cmd_fini(mlxp, &cmd); 1315 1316 kmem_free(out, outsize); 1317 return (ret); 1318 } 1319 1320 boolean_t 1321 mlxcx_cmd_query_hca_cap(mlxcx_t *mlxp, mlxcx_hca_cap_type_t type, 1322 mlxcx_hca_cap_mode_t mode, mlxcx_hca_cap_t *capp) 1323 { 1324 mlxcx_cmd_t cmd; 1325 mlxcx_cmd_query_hca_cap_in_t in; 1326 mlxcx_cmd_query_hca_cap_out_t *out; 1327 boolean_t ret; 1328 uint16_t opmode; 1329 1330 bzero(&in, sizeof (in)); 1331 out = kmem_zalloc(sizeof (mlxcx_cmd_query_hca_cap_out_t), KM_SLEEP); 1332 mlxcx_cmd_init(mlxp, &cmd); 1333 1334 opmode = type << 1 | mode; 1335 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_hca_cap_head, 1336 MLXCX_OP_QUERY_HCA_CAP, opmode); 1337 1338 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), out, sizeof (*out))) { 1339 mlxcx_cmd_fini(mlxp, &cmd); 1340 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1341 return (B_FALSE); 1342 } 1343 mlxcx_cmd_wait(&cmd); 1344 1345 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1346 if (ret) { 1347 capp->mhc_mode = mode; 1348 capp->mhc_type = type; 1349 ASSERT3U(sizeof (out->mlxo_query_hca_cap_data), ==, 1350 sizeof (capp->mhc_bulk)); 1351 bcopy(out->mlxo_query_hca_cap_data, capp->mhc_bulk, 1352 sizeof (capp->mhc_bulk)); 1353 } 1354 mlxcx_cmd_fini(mlxp, &cmd); 1355 1356 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1357 return (B_TRUE); 1358 } 1359 1360 boolean_t 1361 mlxcx_cmd_init_hca(mlxcx_t *mlxp) 1362 { 1363 mlxcx_cmd_t cmd; 1364 mlxcx_cmd_init_hca_in_t in; 1365 mlxcx_cmd_init_hca_out_t out; 1366 boolean_t ret; 1367 1368 bzero(&in, sizeof (in)); 1369 bzero(&out, sizeof (out)); 1370 1371 mlxcx_cmd_init(mlxp, &cmd); 1372 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_init_hca_head, 1373 MLXCX_OP_INIT_HCA, 0); 1374 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1375 mlxcx_cmd_fini(mlxp, &cmd); 1376 return (B_FALSE); 1377 } 1378 mlxcx_cmd_wait(&cmd); 1379 1380 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1381 mlxcx_cmd_fini(mlxp, &cmd); 1382 return (ret); 1383 } 1384 1385 boolean_t 1386 mlxcx_cmd_set_driver_version(mlxcx_t *mlxp, const char *version) 1387 { 1388 mlxcx_cmd_t cmd; 1389 mlxcx_cmd_set_driver_version_in_t in; 1390 mlxcx_cmd_set_driver_version_out_t out; 1391 boolean_t ret; 1392 1393 bzero(&in, sizeof (in)); 1394 bzero(&out, sizeof (out)); 1395 1396 mlxcx_cmd_init(mlxp, &cmd); 1397 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_driver_version_head, 1398 MLXCX_OP_SET_DRIVER_VERSION, 0); 1399 VERIFY3U(strlcpy(in.mlxi_set_driver_version_version, version, 1400 sizeof (in.mlxi_set_driver_version_version)), <=, 1401 sizeof (in.mlxi_set_driver_version_version)); 1402 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1403 mlxcx_cmd_fini(mlxp, &cmd); 1404 return (B_FALSE); 1405 } 1406 mlxcx_cmd_wait(&cmd); 1407 1408 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1409 mlxcx_cmd_fini(mlxp, &cmd); 1410 return (ret); 1411 } 1412 1413 boolean_t 1414 mlxcx_cmd_alloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1415 { 1416 mlxcx_cmd_t cmd; 1417 mlxcx_cmd_alloc_uar_in_t in; 1418 mlxcx_cmd_alloc_uar_out_t out; 1419 boolean_t ret; 1420 size_t i; 1421 1422 bzero(&in, sizeof (in)); 1423 bzero(&out, sizeof (out)); 1424 1425 mlxcx_cmd_init(mlxp, &cmd); 1426 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_uar_head, 1427 MLXCX_OP_ALLOC_UAR, 0); 1428 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1429 mlxcx_cmd_fini(mlxp, &cmd); 1430 return (B_FALSE); 1431 } 1432 mlxcx_cmd_wait(&cmd); 1433 1434 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1435 if (ret) { 1436 mlup->mlu_allocated = B_TRUE; 1437 mlup->mlu_num = from_be24(out.mlxo_alloc_uar_uar); 1438 VERIFY3U(mlup->mlu_num, >, 0); 1439 mlup->mlu_base = mlup->mlu_num * MLXCX_HW_PAGE_SIZE; 1440 1441 for (i = 0; i < MLXCX_BF_PER_UAR; ++i) { 1442 mlup->mlu_bf[i].mbf_even = mlup->mlu_base + 1443 MLXCX_BF_BASE + MLXCX_BF_SIZE * 2 * i; 1444 mlup->mlu_bf[i].mbf_odd = mlup->mlu_bf[i].mbf_even + 1445 MLXCX_BF_SIZE; 1446 } 1447 } 1448 mlxcx_cmd_fini(mlxp, &cmd); 1449 return (ret); 1450 } 1451 1452 boolean_t 1453 mlxcx_cmd_dealloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1454 { 1455 mlxcx_cmd_t cmd; 1456 mlxcx_cmd_dealloc_uar_in_t in; 1457 mlxcx_cmd_dealloc_uar_out_t out; 1458 boolean_t ret; 1459 1460 bzero(&in, sizeof (in)); 1461 bzero(&out, sizeof (out)); 1462 1463 mlxcx_cmd_init(mlxp, &cmd); 1464 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_uar_head, 1465 MLXCX_OP_DEALLOC_UAR, 0); 1466 VERIFY(mlup->mlu_allocated); 1467 in.mlxi_dealloc_uar_uar = to_be24(mlup->mlu_num); 1468 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1469 mlxcx_cmd_fini(mlxp, &cmd); 1470 return (B_FALSE); 1471 } 1472 mlxcx_cmd_wait(&cmd); 1473 1474 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1475 if (ret) { 1476 mlup->mlu_allocated = B_FALSE; 1477 mlup->mlu_num = 0; 1478 } 1479 mlxcx_cmd_fini(mlxp, &cmd); 1480 return (ret); 1481 } 1482 1483 boolean_t 1484 mlxcx_cmd_alloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1485 { 1486 mlxcx_cmd_t cmd; 1487 mlxcx_cmd_alloc_pd_in_t in; 1488 mlxcx_cmd_alloc_pd_out_t out; 1489 boolean_t ret; 1490 1491 bzero(&in, sizeof (in)); 1492 bzero(&out, sizeof (out)); 1493 1494 mlxcx_cmd_init(mlxp, &cmd); 1495 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_pd_head, 1496 MLXCX_OP_ALLOC_PD, 0); 1497 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1498 mlxcx_cmd_fini(mlxp, &cmd); 1499 return (B_FALSE); 1500 } 1501 mlxcx_cmd_wait(&cmd); 1502 1503 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1504 if (ret) { 1505 mlpd->mlpd_allocated = B_TRUE; 1506 mlpd->mlpd_num = from_be24(out.mlxo_alloc_pd_pdn); 1507 } 1508 mlxcx_cmd_fini(mlxp, &cmd); 1509 return (ret); 1510 } 1511 1512 boolean_t 1513 mlxcx_cmd_dealloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1514 { 1515 mlxcx_cmd_t cmd; 1516 mlxcx_cmd_dealloc_pd_in_t in; 1517 mlxcx_cmd_dealloc_pd_out_t out; 1518 boolean_t ret; 1519 1520 bzero(&in, sizeof (in)); 1521 bzero(&out, sizeof (out)); 1522 1523 mlxcx_cmd_init(mlxp, &cmd); 1524 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_pd_head, 1525 MLXCX_OP_DEALLOC_PD, 0); 1526 VERIFY(mlpd->mlpd_allocated); 1527 in.mlxi_dealloc_pd_pdn = to_be24(mlpd->mlpd_num); 1528 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1529 mlxcx_cmd_fini(mlxp, &cmd); 1530 return (B_FALSE); 1531 } 1532 mlxcx_cmd_wait(&cmd); 1533 1534 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1535 if (ret) { 1536 mlpd->mlpd_allocated = B_FALSE; 1537 mlpd->mlpd_num = 0; 1538 } 1539 mlxcx_cmd_fini(mlxp, &cmd); 1540 return (ret); 1541 } 1542 1543 boolean_t 1544 mlxcx_cmd_alloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1545 { 1546 mlxcx_cmd_t cmd; 1547 mlxcx_cmd_alloc_tdom_in_t in; 1548 mlxcx_cmd_alloc_tdom_out_t out; 1549 boolean_t ret; 1550 1551 bzero(&in, sizeof (in)); 1552 bzero(&out, sizeof (out)); 1553 1554 mlxcx_cmd_init(mlxp, &cmd); 1555 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_tdom_head, 1556 MLXCX_OP_ALLOC_TRANSPORT_DOMAIN, 0); 1557 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1558 mlxcx_cmd_fini(mlxp, &cmd); 1559 return (B_FALSE); 1560 } 1561 mlxcx_cmd_wait(&cmd); 1562 1563 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1564 if (ret) { 1565 mltd->mltd_allocated = B_TRUE; 1566 mltd->mltd_num = from_be24(out.mlxo_alloc_tdom_tdomn); 1567 } 1568 mlxcx_cmd_fini(mlxp, &cmd); 1569 return (ret); 1570 } 1571 1572 boolean_t 1573 mlxcx_cmd_dealloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1574 { 1575 mlxcx_cmd_t cmd; 1576 mlxcx_cmd_dealloc_tdom_in_t in; 1577 mlxcx_cmd_dealloc_tdom_out_t out; 1578 boolean_t ret; 1579 1580 bzero(&in, sizeof (in)); 1581 bzero(&out, sizeof (out)); 1582 1583 mlxcx_cmd_init(mlxp, &cmd); 1584 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_tdom_head, 1585 MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN, 0); 1586 VERIFY(mltd->mltd_allocated); 1587 in.mlxi_dealloc_tdom_tdomn = to_be24(mltd->mltd_num); 1588 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1589 mlxcx_cmd_fini(mlxp, &cmd); 1590 return (B_FALSE); 1591 } 1592 mlxcx_cmd_wait(&cmd); 1593 1594 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1595 if (ret) { 1596 mltd->mltd_allocated = B_FALSE; 1597 mltd->mltd_num = 0; 1598 } 1599 mlxcx_cmd_fini(mlxp, &cmd); 1600 return (ret); 1601 } 1602 1603 boolean_t 1604 mlxcx_cmd_teardown_hca(mlxcx_t *mlxp) 1605 { 1606 mlxcx_cmd_t cmd; 1607 mlxcx_cmd_teardown_hca_in_t in; 1608 mlxcx_cmd_teardown_hca_out_t out; 1609 boolean_t ret; 1610 1611 bzero(&in, sizeof (in)); 1612 bzero(&out, sizeof (out)); 1613 1614 mlxcx_cmd_init(mlxp, &cmd); 1615 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_teardown_hca_head, 1616 MLXCX_OP_TEARDOWN_HCA, 0); 1617 in.mlxi_teardown_hca_profile = to_be16(MLXCX_TEARDOWN_HCA_GRACEFUL); 1618 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1619 mlxcx_cmd_fini(mlxp, &cmd); 1620 return (B_FALSE); 1621 } 1622 mlxcx_cmd_wait(&cmd); 1623 1624 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1625 mlxcx_cmd_fini(mlxp, &cmd); 1626 return (ret); 1627 } 1628 1629 boolean_t 1630 mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1631 { 1632 mlxcx_cmd_t cmd; 1633 mlxcx_cmd_query_nic_vport_ctx_in_t in; 1634 mlxcx_cmd_query_nic_vport_ctx_out_t out; 1635 boolean_t ret; 1636 const mlxcx_nic_vport_ctx_t *ctx; 1637 1638 bzero(&in, sizeof (in)); 1639 bzero(&out, sizeof (out)); 1640 1641 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1642 mlxcx_cmd_init(mlxp, &cmd); 1643 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_nic_vport_ctx_head, 1644 MLXCX_OP_QUERY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 1645 1646 in.mlxi_query_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 1647 1648 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1649 mlxcx_cmd_fini(mlxp, &cmd); 1650 return (B_FALSE); 1651 } 1652 mlxcx_cmd_wait(&cmd); 1653 1654 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1655 if (ret) { 1656 ctx = &out.mlxo_query_nic_vport_ctx_context; 1657 mlp->mlp_guid = from_be64(ctx->mlnvc_port_guid); 1658 mlp->mlp_mtu = from_be16(ctx->mlnvc_mtu); 1659 bcopy(ctx->mlnvc_permanent_address, mlp->mlp_mac_address, 1660 sizeof (mlp->mlp_mac_address)); 1661 mlp->mlp_wqe_min_inline = get_bits64(ctx->mlnvc_flags, 1662 MLXCX_VPORT_CTX_MIN_WQE_INLINE); 1663 } 1664 mlxcx_cmd_fini(mlxp, &cmd); 1665 return (ret); 1666 } 1667 1668 static const char * 1669 mlxcx_reg_name(mlxcx_register_id_t rid) 1670 { 1671 switch (rid) { 1672 case MLXCX_REG_PMTU: 1673 return ("PMTU"); 1674 case MLXCX_REG_PAOS: 1675 return ("PAOS"); 1676 case MLXCX_REG_PTYS: 1677 return ("PTYS"); 1678 case MLXCX_REG_MSGI: 1679 return ("MSGI"); 1680 case MLXCX_REG_PMAOS: 1681 return ("PMAOS"); 1682 case MLXCX_REG_MLCR: 1683 return ("MLCR"); 1684 case MLXCX_REG_MCIA: 1685 return ("MCIA"); 1686 case MLXCX_REG_PPCNT: 1687 return ("PPCNT"); 1688 case MLXCX_REG_PPLM: 1689 return ("PPLM"); 1690 default: 1691 return ("???"); 1692 } 1693 } 1694 1695 boolean_t 1696 mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod, 1697 mlxcx_register_id_t rid, mlxcx_register_data_t *data) 1698 { 1699 mlxcx_cmd_t cmd; 1700 mlxcx_cmd_access_register_in_t in; 1701 mlxcx_cmd_access_register_out_t out; 1702 boolean_t ret; 1703 size_t dsize, insize, outsize; 1704 1705 bzero(&in, sizeof (in)); 1706 bzero(&out, sizeof (out)); 1707 1708 mlxcx_cmd_init(mlxp, &cmd); 1709 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_access_register_head, 1710 MLXCX_OP_ACCESS_REG, opmod); 1711 1712 in.mlxi_access_register_register_id = to_be16(rid); 1713 1714 switch (rid) { 1715 case MLXCX_REG_PMTU: 1716 dsize = sizeof (mlxcx_reg_pmtu_t); 1717 break; 1718 case MLXCX_REG_PAOS: 1719 dsize = sizeof (mlxcx_reg_paos_t); 1720 break; 1721 case MLXCX_REG_PTYS: 1722 dsize = sizeof (mlxcx_reg_ptys_t); 1723 break; 1724 case MLXCX_REG_MLCR: 1725 dsize = sizeof (mlxcx_reg_mlcr_t); 1726 break; 1727 case MLXCX_REG_PMAOS: 1728 dsize = sizeof (mlxcx_reg_pmaos_t); 1729 break; 1730 case MLXCX_REG_MCIA: 1731 dsize = sizeof (mlxcx_reg_mcia_t); 1732 break; 1733 case MLXCX_REG_PPCNT: 1734 dsize = sizeof (mlxcx_reg_ppcnt_t); 1735 break; 1736 case MLXCX_REG_PPLM: 1737 dsize = sizeof (mlxcx_reg_pplm_t); 1738 break; 1739 default: 1740 dsize = 0; 1741 VERIFY(0); 1742 return (B_FALSE); 1743 } 1744 insize = dsize + offsetof(mlxcx_cmd_access_register_in_t, 1745 mlxi_access_register_data); 1746 outsize = dsize + offsetof(mlxcx_cmd_access_register_out_t, 1747 mlxo_access_register_data); 1748 1749 bcopy(data, &in.mlxi_access_register_data, dsize); 1750 1751 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, outsize)) { 1752 mlxcx_cmd_fini(mlxp, &cmd); 1753 return (B_FALSE); 1754 } 1755 mlxcx_cmd_wait(&cmd); 1756 1757 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1758 if (ret) { 1759 bcopy(&out.mlxo_access_register_data, data, dsize); 1760 } else { 1761 mlxcx_warn(mlxp, "failed OP_ACCESS_REG was for register " 1762 "%04x (%s)", rid, mlxcx_reg_name(rid)); 1763 } 1764 mlxcx_cmd_fini(mlxp, &cmd); 1765 return (ret); 1766 } 1767 1768 boolean_t 1769 mlxcx_cmd_query_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1770 { 1771 mlxcx_register_data_t data; 1772 boolean_t ret; 1773 1774 /* 1775 * Since we modify the port here we require that the caller is holding 1776 * the port mutex. 1777 */ 1778 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1779 bzero(&data, sizeof (data)); 1780 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1781 1782 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1783 MLXCX_REG_PMTU, &data); 1784 1785 if (ret) { 1786 mlp->mlp_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_admin_mtu); 1787 mlp->mlp_max_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_max_mtu); 1788 } 1789 1790 return (ret); 1791 } 1792 1793 boolean_t 1794 mlxcx_cmd_query_module_status(mlxcx_t *mlxp, uint_t id, 1795 mlxcx_module_status_t *pstatus, mlxcx_module_error_type_t *perr) 1796 { 1797 mlxcx_register_data_t data; 1798 boolean_t ret; 1799 1800 bzero(&data, sizeof (data)); 1801 ASSERT3U(id, <, 0xff); 1802 data.mlrd_pmaos.mlrd_pmaos_module = (uint8_t)id; 1803 1804 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1805 MLXCX_REG_PMAOS, &data); 1806 1807 if (ret) { 1808 if (pstatus != NULL) 1809 *pstatus = data.mlrd_pmaos.mlrd_pmaos_oper_status; 1810 if (perr != NULL) 1811 *perr = data.mlrd_pmaos.mlrd_pmaos_error_type; 1812 } 1813 1814 return (ret); 1815 } 1816 1817 boolean_t 1818 mlxcx_cmd_set_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1819 { 1820 mlxcx_register_data_t data; 1821 boolean_t ret; 1822 1823 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1824 bzero(&data, sizeof (data)); 1825 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1826 data.mlrd_pmtu.mlrd_pmtu_admin_mtu = to_be16(mlp->mlp_mtu); 1827 1828 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1829 MLXCX_REG_PMTU, &data); 1830 1831 return (ret); 1832 } 1833 1834 boolean_t 1835 mlxcx_cmd_set_port_led(mlxcx_t *mlxp, mlxcx_port_t *mlp, uint16_t sec) 1836 { 1837 mlxcx_register_data_t data; 1838 boolean_t ret; 1839 1840 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1841 bzero(&data, sizeof (data)); 1842 data.mlrd_mlcr.mlrd_mlcr_local_port = mlp->mlp_num + 1; 1843 set_bits8(&data.mlrd_mlcr.mlrd_mlcr_flags, MLXCX_MLCR_LED_TYPE, 1844 MLXCX_LED_TYPE_PORT); 1845 data.mlrd_mlcr.mlrd_mlcr_beacon_duration = to_be16(sec); 1846 1847 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1848 MLXCX_REG_MLCR, &data); 1849 1850 return (ret); 1851 } 1852 1853 boolean_t 1854 mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1855 { 1856 mlxcx_register_data_t data; 1857 boolean_t ret; 1858 1859 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1860 bzero(&data, sizeof (data)); 1861 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1862 1863 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1864 MLXCX_REG_PAOS, &data); 1865 1866 if (ret) { 1867 mlp->mlp_admin_status = data.mlrd_paos.mlrd_paos_admin_status; 1868 mlp->mlp_oper_status = data.mlrd_paos.mlrd_paos_oper_status; 1869 } 1870 1871 return (ret); 1872 } 1873 1874 boolean_t 1875 mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1876 mlxcx_port_status_t status) 1877 { 1878 mlxcx_register_data_t data; 1879 boolean_t ret; 1880 1881 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1882 bzero(&data, sizeof (data)); 1883 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1884 data.mlrd_paos.mlrd_paos_admin_status = status; 1885 set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN); 1886 1887 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1888 MLXCX_REG_PAOS, &data); 1889 1890 return (ret); 1891 } 1892 1893 boolean_t 1894 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1895 { 1896 mlxcx_register_data_t data; 1897 boolean_t ret; 1898 1899 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1900 bzero(&data, sizeof (data)); 1901 data.mlrd_ptys.mlrd_ptys_local_port = mlp->mlp_num + 1; 1902 set_bit8(&data.mlrd_ptys.mlrd_ptys_proto_mask, 1903 MLXCX_PTYS_PROTO_MASK_ETH); 1904 1905 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1906 MLXCX_REG_PTYS, &data); 1907 1908 if (ret) { 1909 if (get_bit8(data.mlrd_ptys.mlrd_ptys_autoneg_flags, 1910 MLXCX_AUTONEG_DISABLE)) { 1911 mlp->mlp_autoneg = B_FALSE; 1912 } else { 1913 mlp->mlp_autoneg = B_TRUE; 1914 } 1915 mlp->mlp_max_proto = 1916 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_cap); 1917 mlp->mlp_admin_proto = 1918 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin); 1919 mlp->mlp_oper_proto = 1920 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper); 1921 } 1922 1923 return (ret); 1924 } 1925 1926 boolean_t 1927 mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1928 { 1929 mlxcx_register_data_t data; 1930 boolean_t ret; 1931 1932 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1933 bzero(&data, sizeof (data)); 1934 data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1; 1935 1936 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1937 MLXCX_REG_PPLM, &data); 1938 1939 if (ret) { 1940 mlp->mlp_fec_active = 1941 from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active); 1942 } 1943 1944 return (ret); 1945 } 1946 1947 boolean_t 1948 mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1949 mlxcx_pplm_fec_caps_t fec) 1950 { 1951 mlxcx_register_data_t data_in, data_out; 1952 mlxcx_pplm_fec_caps_t caps; 1953 mlxcx_reg_pplm_t *pplm_in, *pplm_out; 1954 boolean_t ret; 1955 1956 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1957 bzero(&data_in, sizeof (data_in)); 1958 pplm_in = &data_in.mlrd_pplm; 1959 pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1; 1960 1961 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1962 MLXCX_REG_PPLM, &data_in); 1963 1964 if (!ret) 1965 return (B_FALSE); 1966 1967 bzero(&data_out, sizeof (data_out)); 1968 pplm_out = &data_out.mlrd_pplm; 1969 pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1; 1970 1971 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1972 MLXCX_PPLM_CAP_56G); 1973 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1974 MLXCX_PPLM_CAP_56G, fec & caps); 1975 1976 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1977 MLXCX_PPLM_CAP_100G); 1978 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1979 MLXCX_PPLM_CAP_100G, fec & caps); 1980 1981 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1982 MLXCX_PPLM_CAP_50G); 1983 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1984 MLXCX_PPLM_CAP_50G, fec & caps); 1985 1986 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1987 MLXCX_PPLM_CAP_25G); 1988 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1989 MLXCX_PPLM_CAP_25G, fec & caps); 1990 1991 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1992 MLXCX_PPLM_CAP_10_40G); 1993 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1994 MLXCX_PPLM_CAP_10_40G, fec & caps); 1995 1996 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1997 MLXCX_REG_PPLM, &data_out); 1998 1999 return (ret); 2000 } 2001 2002 boolean_t 2003 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp, 2004 mlxcx_modify_nic_vport_ctx_fields_t fields) 2005 { 2006 mlxcx_cmd_t cmd; 2007 mlxcx_cmd_modify_nic_vport_ctx_in_t in; 2008 mlxcx_cmd_modify_nic_vport_ctx_out_t out; 2009 boolean_t ret; 2010 mlxcx_nic_vport_ctx_t *ctx; 2011 2012 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2013 bzero(&in, sizeof (in)); 2014 bzero(&out, sizeof (out)); 2015 2016 mlxcx_cmd_init(mlxp, &cmd); 2017 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_nic_vport_ctx_head, 2018 MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 2019 2020 in.mlxi_modify_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 2021 in.mlxi_modify_nic_vport_ctx_field_select = to_be32(fields); 2022 2023 ctx = &in.mlxi_modify_nic_vport_ctx_context; 2024 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2025 set_bit16(&ctx->mlnvc_promisc_list_type, 2026 MLXCX_VPORT_PROMISC_ALL); 2027 } 2028 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_MTU) { 2029 ctx->mlnvc_mtu = to_be16(mlp->mlp_mtu); 2030 } 2031 2032 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2033 mlxcx_cmd_fini(mlxp, &cmd); 2034 return (B_FALSE); 2035 } 2036 mlxcx_cmd_wait(&cmd); 2037 2038 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2039 if (ret) { 2040 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2041 mlp->mlp_flags |= MLXCX_PORT_VPORT_PROMISC; 2042 } 2043 } 2044 mlxcx_cmd_fini(mlxp, &cmd); 2045 return (ret); 2046 } 2047 2048 boolean_t 2049 mlxcx_cmd_create_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2050 { 2051 mlxcx_cmd_t cmd; 2052 mlxcx_cmd_create_eq_in_t in; 2053 mlxcx_cmd_create_eq_out_t out; 2054 boolean_t ret; 2055 mlxcx_eventq_ctx_t *ctx; 2056 size_t rem, insize; 2057 const ddi_dma_cookie_t *c; 2058 uint64_t pa, npages; 2059 2060 bzero(&in, sizeof (in)); 2061 bzero(&out, sizeof (out)); 2062 2063 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2064 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2065 VERIFY0(mleq->mleq_state & MLXCX_EQ_CREATED); 2066 2067 mlxcx_cmd_init(mlxp, &cmd); 2068 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_eq_head, 2069 MLXCX_OP_CREATE_EQ, 0); 2070 2071 ctx = &in.mlxi_create_eq_context; 2072 ctx->mleqc_uar_page = to_be24(mleq->mleq_uar->mlu_num); 2073 ctx->mleqc_log_eq_size = mleq->mleq_entshift; 2074 ctx->mleqc_intr = mleq->mleq_intr_index; 2075 2076 in.mlxi_create_eq_event_bitmask = to_be64(mleq->mleq_events); 2077 2078 npages = 0; 2079 c = NULL; 2080 while ((c = mlxcx_dma_cookie_iter(&mleq->mleq_dma, c)) != NULL) { 2081 pa = c->dmac_laddress; 2082 rem = c->dmac_size; 2083 while (rem > 0) { 2084 ASSERT3U(pa & 0xfff, ==, 0); 2085 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2086 in.mlxi_create_eq_pas[npages++] = to_be64(pa); 2087 rem -= MLXCX_HW_PAGE_SIZE; 2088 pa += MLXCX_HW_PAGE_SIZE; 2089 } 2090 } 2091 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2092 2093 insize = offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) + 2094 sizeof (uint64_t) * npages; 2095 2096 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2097 mlxcx_cmd_fini(mlxp, &cmd); 2098 return (B_FALSE); 2099 } 2100 mlxcx_cmd_wait(&cmd); 2101 2102 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2103 if (ret) { 2104 mleq->mleq_state |= MLXCX_EQ_CREATED; 2105 mleq->mleq_num = out.mlxo_create_eq_eqn; 2106 } 2107 mlxcx_cmd_fini(mlxp, &cmd); 2108 return (ret); 2109 } 2110 2111 boolean_t 2112 mlxcx_cmd_query_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq, 2113 mlxcx_eventq_ctx_t *ctxp) 2114 { 2115 mlxcx_cmd_t cmd; 2116 mlxcx_cmd_query_eq_in_t in; 2117 mlxcx_cmd_query_eq_out_t out; 2118 boolean_t ret; 2119 2120 bzero(&in, sizeof (in)); 2121 bzero(&out, sizeof (out)); 2122 2123 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2124 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2125 2126 mlxcx_cmd_init(mlxp, &cmd); 2127 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_eq_head, 2128 MLXCX_OP_QUERY_EQ, 0); 2129 2130 in.mlxi_query_eq_eqn = mleq->mleq_num; 2131 2132 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2133 mlxcx_cmd_fini(mlxp, &cmd); 2134 return (B_FALSE); 2135 } 2136 mlxcx_cmd_wait(&cmd); 2137 2138 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2139 if (ret) { 2140 bcopy(&out.mlxo_query_eq_context, ctxp, 2141 sizeof (mlxcx_eventq_ctx_t)); 2142 } 2143 mlxcx_cmd_fini(mlxp, &cmd); 2144 return (ret); 2145 } 2146 2147 boolean_t 2148 mlxcx_cmd_destroy_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2149 { 2150 mlxcx_cmd_t cmd; 2151 mlxcx_cmd_destroy_eq_in_t in; 2152 mlxcx_cmd_destroy_eq_out_t out; 2153 boolean_t ret; 2154 2155 bzero(&in, sizeof (in)); 2156 bzero(&out, sizeof (out)); 2157 2158 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2159 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2160 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2161 2162 mlxcx_cmd_init(mlxp, &cmd); 2163 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_eq_head, 2164 MLXCX_OP_DESTROY_EQ, 0); 2165 2166 in.mlxi_destroy_eq_eqn = mleq->mleq_num; 2167 2168 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2169 mlxcx_cmd_fini(mlxp, &cmd); 2170 return (B_FALSE); 2171 } 2172 mlxcx_cmd_wait(&cmd); 2173 2174 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2175 if (ret) { 2176 mleq->mleq_state |= MLXCX_EQ_DESTROYED; 2177 } 2178 mlxcx_cmd_fini(mlxp, &cmd); 2179 return (ret); 2180 } 2181 2182 boolean_t 2183 mlxcx_cmd_query_special_ctxs(mlxcx_t *mlxp) 2184 { 2185 mlxcx_cmd_t cmd; 2186 mlxcx_cmd_query_special_ctxs_in_t in; 2187 mlxcx_cmd_query_special_ctxs_out_t out; 2188 boolean_t ret; 2189 2190 bzero(&in, sizeof (in)); 2191 bzero(&out, sizeof (out)); 2192 2193 mlxcx_cmd_init(mlxp, &cmd); 2194 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_special_ctxs_head, 2195 MLXCX_OP_QUERY_SPECIAL_CONTEXTS, 0); 2196 2197 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2198 mlxcx_cmd_fini(mlxp, &cmd); 2199 return (B_FALSE); 2200 } 2201 mlxcx_cmd_wait(&cmd); 2202 2203 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2204 if (ret) { 2205 mlxp->mlx_rsvd_lkey = from_be32( 2206 out.mlxo_query_special_ctxs_resd_lkey); 2207 } 2208 mlxcx_cmd_fini(mlxp, &cmd); 2209 return (ret); 2210 } 2211 2212 boolean_t 2213 mlxcx_cmd_create_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2214 { 2215 mlxcx_cmd_t cmd; 2216 mlxcx_cmd_create_cq_in_t in; 2217 mlxcx_cmd_create_cq_out_t out; 2218 boolean_t ret; 2219 mlxcx_completionq_ctx_t *ctx; 2220 size_t rem, insize; 2221 const ddi_dma_cookie_t *c; 2222 uint64_t pa, npages; 2223 2224 bzero(&in, sizeof (in)); 2225 bzero(&out, sizeof (out)); 2226 2227 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2228 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2229 VERIFY0(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2230 2231 mlxcx_cmd_init(mlxp, &cmd); 2232 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_cq_head, 2233 MLXCX_OP_CREATE_CQ, 0); 2234 2235 ctx = &in.mlxi_create_cq_context; 2236 ctx->mlcqc_uar_page = to_be24(mlcq->mlcq_uar->mlu_num); 2237 ctx->mlcqc_log_cq_size = mlcq->mlcq_entshift; 2238 ctx->mlcqc_eqn = mlcq->mlcq_eq->mleq_num; 2239 ctx->mlcqc_cq_period = to_be16(mlcq->mlcq_cqemod_period_usec); 2240 ctx->mlcqc_cq_max_count = to_be16(mlcq->mlcq_cqemod_count); 2241 2242 c = mlxcx_dma_cookie_one(&mlcq->mlcq_doorbell_dma); 2243 ctx->mlcqc_dbr_addr = to_be64(c->dmac_laddress); 2244 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_completionq_doorbell_t)); 2245 2246 npages = 0; 2247 c = NULL; 2248 while ((c = mlxcx_dma_cookie_iter(&mlcq->mlcq_dma, c)) != NULL) { 2249 pa = c->dmac_laddress; 2250 rem = c->dmac_size; 2251 while (rem > 0) { 2252 ASSERT3U(pa & 0xfff, ==, 0); 2253 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2254 in.mlxi_create_cq_pas[npages++] = to_be64(pa); 2255 rem -= MLXCX_HW_PAGE_SIZE; 2256 pa += MLXCX_HW_PAGE_SIZE; 2257 } 2258 } 2259 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2260 2261 insize = offsetof(mlxcx_cmd_create_cq_in_t, mlxi_create_cq_pas) + 2262 sizeof (uint64_t) * npages; 2263 2264 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2265 mlxcx_cmd_fini(mlxp, &cmd); 2266 return (B_FALSE); 2267 } 2268 mlxcx_cmd_wait(&cmd); 2269 2270 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2271 if (ret) { 2272 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_CREATED); 2273 mlcq->mlcq_num = from_be24(out.mlxo_create_cq_cqn); 2274 } 2275 mlxcx_cmd_fini(mlxp, &cmd); 2276 return (ret); 2277 } 2278 2279 boolean_t 2280 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2281 mlxcx_rq_ctx_t *ctxp) 2282 { 2283 mlxcx_cmd_t cmd; 2284 mlxcx_cmd_query_rq_in_t in; 2285 mlxcx_cmd_query_rq_out_t out; 2286 boolean_t ret; 2287 2288 bzero(&in, sizeof (in)); 2289 bzero(&out, sizeof (out)); 2290 2291 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2292 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2293 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2294 2295 mlxcx_cmd_init(mlxp, &cmd); 2296 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_rq_head, 2297 MLXCX_OP_QUERY_RQ, 0); 2298 2299 in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num); 2300 2301 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2302 mlxcx_cmd_fini(mlxp, &cmd); 2303 return (B_FALSE); 2304 } 2305 mlxcx_cmd_wait(&cmd); 2306 2307 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2308 if (ret) { 2309 bcopy(&out.mlxo_query_rq_context, ctxp, 2310 sizeof (mlxcx_rq_ctx_t)); 2311 } 2312 mlxcx_cmd_fini(mlxp, &cmd); 2313 return (ret); 2314 } 2315 2316 boolean_t 2317 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2318 mlxcx_sq_ctx_t *ctxp) 2319 { 2320 mlxcx_cmd_t cmd; 2321 mlxcx_cmd_query_sq_in_t in; 2322 mlxcx_cmd_query_sq_out_t out; 2323 boolean_t ret; 2324 2325 bzero(&in, sizeof (in)); 2326 bzero(&out, sizeof (out)); 2327 2328 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2329 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2330 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 2331 2332 mlxcx_cmd_init(mlxp, &cmd); 2333 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_sq_head, 2334 MLXCX_OP_QUERY_SQ, 0); 2335 2336 in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num); 2337 2338 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2339 mlxcx_cmd_fini(mlxp, &cmd); 2340 return (B_FALSE); 2341 } 2342 mlxcx_cmd_wait(&cmd); 2343 2344 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2345 if (ret) { 2346 bcopy(&out.mlxo_query_sq_context, ctxp, 2347 sizeof (mlxcx_sq_ctx_t)); 2348 } 2349 mlxcx_cmd_fini(mlxp, &cmd); 2350 return (ret); 2351 } 2352 2353 boolean_t 2354 mlxcx_cmd_query_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, 2355 mlxcx_completionq_ctx_t *ctxp) 2356 { 2357 mlxcx_cmd_t cmd; 2358 mlxcx_cmd_query_cq_in_t in; 2359 mlxcx_cmd_query_cq_out_t out; 2360 boolean_t ret; 2361 2362 bzero(&in, sizeof (in)); 2363 bzero(&out, sizeof (out)); 2364 2365 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2366 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2367 2368 mlxcx_cmd_init(mlxp, &cmd); 2369 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_cq_head, 2370 MLXCX_OP_QUERY_CQ, 0); 2371 2372 in.mlxi_query_cq_cqn = to_be24(mlcq->mlcq_num); 2373 2374 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2375 mlxcx_cmd_fini(mlxp, &cmd); 2376 return (B_FALSE); 2377 } 2378 mlxcx_cmd_wait(&cmd); 2379 2380 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2381 if (ret) { 2382 bcopy(&out.mlxo_query_cq_context, ctxp, 2383 sizeof (mlxcx_completionq_ctx_t)); 2384 } 2385 mlxcx_cmd_fini(mlxp, &cmd); 2386 return (ret); 2387 } 2388 2389 boolean_t 2390 mlxcx_cmd_destroy_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2391 { 2392 mlxcx_cmd_t cmd; 2393 mlxcx_cmd_destroy_cq_in_t in; 2394 mlxcx_cmd_destroy_cq_out_t out; 2395 boolean_t ret; 2396 2397 bzero(&in, sizeof (in)); 2398 bzero(&out, sizeof (out)); 2399 2400 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2401 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2402 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2403 2404 mlxcx_cmd_init(mlxp, &cmd); 2405 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_cq_head, 2406 MLXCX_OP_DESTROY_CQ, 0); 2407 2408 in.mlxi_destroy_cq_cqn = to_be24(mlcq->mlcq_num); 2409 2410 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2411 mlxcx_cmd_fini(mlxp, &cmd); 2412 return (B_FALSE); 2413 } 2414 mlxcx_cmd_wait(&cmd); 2415 2416 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2417 if (ret) { 2418 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_DESTROYED); 2419 } 2420 mlxcx_cmd_fini(mlxp, &cmd); 2421 return (ret); 2422 } 2423 2424 boolean_t 2425 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2426 { 2427 mlxcx_cmd_t cmd; 2428 mlxcx_cmd_create_rq_in_t in; 2429 mlxcx_cmd_create_rq_out_t out; 2430 boolean_t ret; 2431 mlxcx_rq_ctx_t *ctx; 2432 size_t rem, insize; 2433 const ddi_dma_cookie_t *c; 2434 uint64_t pa, npages; 2435 2436 bzero(&in, sizeof (in)); 2437 bzero(&out, sizeof (out)); 2438 2439 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2440 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2441 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2442 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2443 2444 mlxcx_cmd_init(mlxp, &cmd); 2445 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rq_head, 2446 MLXCX_OP_CREATE_RQ, 0); 2447 2448 ctx = &in.mlxi_create_rq_context; 2449 2450 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_RLKEY); 2451 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_FLUSH_IN_ERROR); 2452 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_VLAN_STRIP_DISABLE); 2453 ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 2454 2455 set_bits32(&ctx->mlrqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 2456 MLXCX_WORKQ_TYPE_CYCLIC); 2457 ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 2458 ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 2459 ctx->mlrqc_wq.mlwqc_log_wq_stride = MLXCX_RECVQ_STRIDE_SHIFT; 2460 2461 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 2462 ctx->mlrqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 2463 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 2464 2465 npages = 0; 2466 c = NULL; 2467 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 2468 pa = c->dmac_laddress; 2469 rem = c->dmac_size; 2470 while (rem > 0) { 2471 ASSERT3U(pa & 0xfff, ==, 0); 2472 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2473 ctx->mlrqc_wq.mlwqc_pas[npages++] = to_be64(pa); 2474 rem -= MLXCX_HW_PAGE_SIZE; 2475 pa += MLXCX_HW_PAGE_SIZE; 2476 } 2477 } 2478 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 2479 2480 insize = offsetof(mlxcx_cmd_create_rq_in_t, mlxi_create_rq_context) + 2481 offsetof(mlxcx_rq_ctx_t, mlrqc_wq) + 2482 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 2483 sizeof (uint64_t) * npages; 2484 2485 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2486 mlxcx_cmd_fini(mlxp, &cmd); 2487 return (B_FALSE); 2488 } 2489 mlxcx_cmd_wait(&cmd); 2490 2491 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2492 if (ret) { 2493 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 2494 mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn); 2495 } 2496 mlxcx_cmd_fini(mlxp, &cmd); 2497 return (ret); 2498 } 2499 2500 boolean_t 2501 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2502 { 2503 mlxcx_cmd_t cmd; 2504 mlxcx_cmd_modify_rq_in_t in; 2505 mlxcx_cmd_modify_rq_out_t out; 2506 boolean_t ret; 2507 ddi_fm_error_t err; 2508 2509 bzero(&in, sizeof (in)); 2510 bzero(&out, sizeof (out)); 2511 2512 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2513 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2514 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2515 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2516 2517 /* 2518 * Before starting the queue, we have to be sure that it is 2519 * empty and the doorbell and counters are set to 0. 2520 */ 2521 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 2522 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 2523 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 2524 2525 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 2526 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 2527 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 2528 DDI_FME_VERSION); 2529 if (err.fme_status != DDI_FM_OK) 2530 return (B_FALSE); 2531 mlwq->mlwq_pc = 0; 2532 2533 mlxcx_cmd_init(mlxp, &cmd); 2534 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2535 MLXCX_OP_MODIFY_RQ, 0); 2536 2537 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2538 2539 /* From state */ 2540 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2541 MLXCX_RQ_STATE_RST); 2542 /* To state */ 2543 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2544 MLXCX_RQ_STATE_RDY); 2545 2546 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2547 mlxcx_cmd_fini(mlxp, &cmd); 2548 return (B_FALSE); 2549 } 2550 mlxcx_cmd_wait(&cmd); 2551 2552 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2553 if (ret) { 2554 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 2555 } 2556 mlxcx_cmd_fini(mlxp, &cmd); 2557 return (ret); 2558 } 2559 2560 boolean_t 2561 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2562 { 2563 mlxcx_cmd_t cmd; 2564 mlxcx_cmd_modify_rq_in_t in; 2565 mlxcx_cmd_modify_rq_out_t out; 2566 boolean_t ret; 2567 2568 bzero(&in, sizeof (in)); 2569 bzero(&out, sizeof (out)); 2570 2571 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2572 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2573 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2574 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2575 2576 mlxcx_cmd_init(mlxp, &cmd); 2577 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2578 MLXCX_OP_MODIFY_RQ, 0); 2579 2580 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2581 2582 /* From state */ 2583 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2584 MLXCX_RQ_STATE_RDY); 2585 /* To state */ 2586 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2587 MLXCX_RQ_STATE_RST); 2588 2589 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2590 mlxcx_cmd_fini(mlxp, &cmd); 2591 return (B_FALSE); 2592 } 2593 mlxcx_cmd_wait(&cmd); 2594 2595 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2596 if (ret) { 2597 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 2598 } 2599 mlxcx_cmd_fini(mlxp, &cmd); 2600 return (ret); 2601 } 2602 2603 boolean_t 2604 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2605 { 2606 mlxcx_cmd_t cmd; 2607 mlxcx_cmd_destroy_rq_in_t in; 2608 mlxcx_cmd_destroy_rq_out_t out; 2609 boolean_t ret; 2610 2611 bzero(&in, sizeof (in)); 2612 bzero(&out, sizeof (out)); 2613 2614 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2615 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2616 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2617 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2618 2619 mlxcx_cmd_init(mlxp, &cmd); 2620 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rq_head, 2621 MLXCX_OP_DESTROY_RQ, 0); 2622 2623 in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num); 2624 2625 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2626 mlxcx_cmd_fini(mlxp, &cmd); 2627 return (B_FALSE); 2628 } 2629 mlxcx_cmd_wait(&cmd); 2630 2631 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2632 if (ret) { 2633 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 2634 } 2635 mlxcx_cmd_fini(mlxp, &cmd); 2636 return (ret); 2637 } 2638 2639 boolean_t 2640 mlxcx_cmd_create_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2641 { 2642 mlxcx_cmd_t cmd; 2643 mlxcx_cmd_create_tir_in_t in; 2644 mlxcx_cmd_create_tir_out_t out; 2645 mlxcx_tir_ctx_t *ctx; 2646 boolean_t ret; 2647 2648 bzero(&in, sizeof (in)); 2649 bzero(&out, sizeof (out)); 2650 2651 VERIFY0(mltir->mltir_state & MLXCX_TIR_CREATED); 2652 2653 mlxcx_cmd_init(mlxp, &cmd); 2654 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tir_head, 2655 MLXCX_OP_CREATE_TIR, 0); 2656 2657 ctx = &in.mlxi_create_tir_context; 2658 ctx->mltirc_transport_domain = to_be24(mltir->mltir_tdom->mltd_num); 2659 set_bits8(&ctx->mltirc_disp_type, MLXCX_TIR_CTX_DISP_TYPE, 2660 mltir->mltir_type); 2661 switch (mltir->mltir_type) { 2662 case MLXCX_TIR_INDIRECT: 2663 VERIFY(mltir->mltir_rqtable != NULL); 2664 VERIFY(mltir->mltir_rqtable->mlrqt_state & MLXCX_RQT_CREATED); 2665 ctx->mltirc_indirect_table = 2666 to_be24(mltir->mltir_rqtable->mlrqt_num); 2667 set_bits8(&ctx->mltirc_hash_lb, MLXCX_TIR_RX_HASH_FN, 2668 mltir->mltir_hash_fn); 2669 bcopy(mltir->mltir_toeplitz_key, 2670 ctx->mltirc_rx_hash_toeplitz_key, 2671 sizeof (ctx->mltirc_rx_hash_toeplitz_key)); 2672 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2673 MLXCX_RX_HASH_L3_TYPE, mltir->mltir_l3_type); 2674 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2675 MLXCX_RX_HASH_L4_TYPE, mltir->mltir_l4_type); 2676 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2677 MLXCX_RX_HASH_FIELDS, mltir->mltir_hash_fields); 2678 break; 2679 case MLXCX_TIR_DIRECT: 2680 VERIFY(mltir->mltir_rq != NULL); 2681 VERIFY(mltir->mltir_rq->mlwq_state & MLXCX_WQ_CREATED); 2682 ctx->mltirc_inline_rqn = to_be24(mltir->mltir_rq->mlwq_num); 2683 break; 2684 default: 2685 VERIFY(0); 2686 } 2687 2688 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2689 mlxcx_cmd_fini(mlxp, &cmd); 2690 return (B_FALSE); 2691 } 2692 mlxcx_cmd_wait(&cmd); 2693 2694 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2695 if (ret) { 2696 mltir->mltir_state |= MLXCX_TIR_CREATED; 2697 mltir->mltir_num = from_be24(out.mlxo_create_tir_tirn); 2698 } 2699 mlxcx_cmd_fini(mlxp, &cmd); 2700 return (ret); 2701 } 2702 2703 boolean_t 2704 mlxcx_cmd_destroy_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2705 { 2706 mlxcx_cmd_t cmd; 2707 mlxcx_cmd_destroy_tir_in_t in; 2708 mlxcx_cmd_destroy_tir_out_t out; 2709 boolean_t ret; 2710 2711 bzero(&in, sizeof (in)); 2712 bzero(&out, sizeof (out)); 2713 2714 VERIFY(mltir->mltir_state & MLXCX_TIR_CREATED); 2715 VERIFY0(mltir->mltir_state & MLXCX_TIR_DESTROYED); 2716 2717 mlxcx_cmd_init(mlxp, &cmd); 2718 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tir_head, 2719 MLXCX_OP_DESTROY_TIR, 0); 2720 2721 in.mlxi_destroy_tir_tirn = to_be24(mltir->mltir_num); 2722 2723 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2724 mlxcx_cmd_fini(mlxp, &cmd); 2725 return (B_FALSE); 2726 } 2727 mlxcx_cmd_wait(&cmd); 2728 2729 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2730 if (ret) { 2731 mltir->mltir_state |= MLXCX_TIR_DESTROYED; 2732 } 2733 mlxcx_cmd_fini(mlxp, &cmd); 2734 return (ret); 2735 } 2736 2737 boolean_t 2738 mlxcx_cmd_create_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2739 { 2740 mlxcx_cmd_t cmd; 2741 mlxcx_cmd_create_tis_in_t in; 2742 mlxcx_cmd_create_tis_out_t out; 2743 mlxcx_tis_ctx_t *ctx; 2744 boolean_t ret; 2745 2746 bzero(&in, sizeof (in)); 2747 bzero(&out, sizeof (out)); 2748 2749 VERIFY0(mltis->mltis_state & MLXCX_TIS_CREATED); 2750 2751 mlxcx_cmd_init(mlxp, &cmd); 2752 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tis_head, 2753 MLXCX_OP_CREATE_TIS, 0); 2754 2755 ctx = &in.mlxi_create_tis_context; 2756 ctx->mltisc_transport_domain = to_be24(mltis->mltis_tdom->mltd_num); 2757 2758 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2759 mlxcx_cmd_fini(mlxp, &cmd); 2760 return (B_FALSE); 2761 } 2762 mlxcx_cmd_wait(&cmd); 2763 2764 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2765 if (ret) { 2766 mltis->mltis_state |= MLXCX_TIS_CREATED; 2767 mltis->mltis_num = from_be24(out.mlxo_create_tis_tisn); 2768 } 2769 mlxcx_cmd_fini(mlxp, &cmd); 2770 return (ret); 2771 } 2772 2773 boolean_t 2774 mlxcx_cmd_destroy_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2775 { 2776 mlxcx_cmd_t cmd; 2777 mlxcx_cmd_destroy_tis_in_t in; 2778 mlxcx_cmd_destroy_tis_out_t out; 2779 boolean_t ret; 2780 2781 bzero(&in, sizeof (in)); 2782 bzero(&out, sizeof (out)); 2783 2784 VERIFY(mltis->mltis_state & MLXCX_TIR_CREATED); 2785 VERIFY0(mltis->mltis_state & MLXCX_TIR_DESTROYED); 2786 2787 mlxcx_cmd_init(mlxp, &cmd); 2788 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tis_head, 2789 MLXCX_OP_DESTROY_TIS, 0); 2790 2791 in.mlxi_destroy_tis_tisn = to_be24(mltis->mltis_num); 2792 2793 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2794 mlxcx_cmd_fini(mlxp, &cmd); 2795 return (B_FALSE); 2796 } 2797 mlxcx_cmd_wait(&cmd); 2798 2799 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2800 if (ret) { 2801 mltis->mltis_state |= MLXCX_TIS_DESTROYED; 2802 } 2803 mlxcx_cmd_fini(mlxp, &cmd); 2804 return (ret); 2805 } 2806 2807 boolean_t 2808 mlxcx_cmd_create_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2809 { 2810 mlxcx_cmd_t cmd; 2811 mlxcx_cmd_create_flow_table_in_t in; 2812 mlxcx_cmd_create_flow_table_out_t out; 2813 mlxcx_flow_table_ctx_t *ctx; 2814 boolean_t ret; 2815 2816 bzero(&in, sizeof (in)); 2817 bzero(&out, sizeof (out)); 2818 2819 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2820 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2821 2822 mlxcx_cmd_init(mlxp, &cmd); 2823 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_table_head, 2824 MLXCX_OP_CREATE_FLOW_TABLE, 0); 2825 2826 in.mlxi_create_flow_table_vport_number = 2827 to_be16(mlft->mlft_port->mlp_num); 2828 in.mlxi_create_flow_table_table_type = mlft->mlft_type; 2829 ctx = &in.mlxi_create_flow_table_context; 2830 ctx->mlftc_log_size = mlft->mlft_entshift; 2831 ctx->mlftc_level = mlft->mlft_level; 2832 2833 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2834 mlxcx_cmd_fini(mlxp, &cmd); 2835 return (B_FALSE); 2836 } 2837 mlxcx_cmd_wait(&cmd); 2838 2839 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2840 if (ret) { 2841 mlft->mlft_num = from_be24(out.mlxo_create_flow_table_table_id); 2842 mlft->mlft_state |= MLXCX_FLOW_TABLE_CREATED; 2843 } 2844 mlxcx_cmd_fini(mlxp, &cmd); 2845 return (ret); 2846 } 2847 2848 boolean_t 2849 mlxcx_cmd_destroy_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2850 { 2851 mlxcx_cmd_t cmd; 2852 mlxcx_cmd_destroy_flow_table_in_t in; 2853 mlxcx_cmd_destroy_flow_table_out_t out; 2854 boolean_t ret; 2855 2856 bzero(&in, sizeof (in)); 2857 bzero(&out, sizeof (out)); 2858 2859 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2860 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2861 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2862 2863 mlxcx_cmd_init(mlxp, &cmd); 2864 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_table_head, 2865 MLXCX_OP_DESTROY_FLOW_TABLE, 0); 2866 2867 in.mlxi_destroy_flow_table_vport_number = 2868 to_be16(mlft->mlft_port->mlp_num); 2869 in.mlxi_destroy_flow_table_table_type = mlft->mlft_type; 2870 in.mlxi_destroy_flow_table_table_id = to_be24(mlft->mlft_num); 2871 2872 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2873 mlxcx_cmd_fini(mlxp, &cmd); 2874 return (B_FALSE); 2875 } 2876 mlxcx_cmd_wait(&cmd); 2877 2878 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2879 if (ret) { 2880 mlft->mlft_state |= MLXCX_FLOW_TABLE_DESTROYED; 2881 } 2882 mlxcx_cmd_fini(mlxp, &cmd); 2883 return (ret); 2884 } 2885 2886 boolean_t 2887 mlxcx_cmd_set_flow_table_root(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2888 { 2889 mlxcx_cmd_t cmd; 2890 mlxcx_cmd_set_flow_table_root_in_t in; 2891 mlxcx_cmd_set_flow_table_root_out_t out; 2892 boolean_t ret; 2893 2894 bzero(&in, sizeof (in)); 2895 bzero(&out, sizeof (out)); 2896 2897 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2898 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2899 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2900 2901 mlxcx_cmd_init(mlxp, &cmd); 2902 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_root_head, 2903 MLXCX_OP_SET_FLOW_TABLE_ROOT, 0); 2904 2905 in.mlxi_set_flow_table_root_vport_number = 2906 to_be16(mlft->mlft_port->mlp_num); 2907 in.mlxi_set_flow_table_root_table_type = mlft->mlft_type; 2908 in.mlxi_set_flow_table_root_table_id = to_be24(mlft->mlft_num); 2909 2910 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2911 mlxcx_cmd_fini(mlxp, &cmd); 2912 return (B_FALSE); 2913 } 2914 mlxcx_cmd_wait(&cmd); 2915 2916 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2917 if (ret) { 2918 mlft->mlft_state |= MLXCX_FLOW_TABLE_ROOT; 2919 } 2920 mlxcx_cmd_fini(mlxp, &cmd); 2921 return (ret); 2922 } 2923 2924 boolean_t 2925 mlxcx_cmd_create_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 2926 { 2927 mlxcx_cmd_t cmd; 2928 mlxcx_cmd_create_flow_group_in_t in; 2929 mlxcx_cmd_create_flow_group_out_t out; 2930 boolean_t ret; 2931 const mlxcx_flow_table_t *mlft; 2932 mlxcx_flow_header_match_t *hdrs; 2933 mlxcx_flow_params_match_t *params; 2934 2935 bzero(&in, sizeof (in)); 2936 bzero(&out, sizeof (out)); 2937 2938 mlft = mlfg->mlfg_table; 2939 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2940 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2941 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2942 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 2943 2944 mlxcx_cmd_init(mlxp, &cmd); 2945 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_group_head, 2946 MLXCX_OP_CREATE_FLOW_GROUP, 0); 2947 2948 in.mlxi_create_flow_group_vport_number = 2949 to_be16(mlft->mlft_port->mlp_num); 2950 in.mlxi_create_flow_group_table_type = mlft->mlft_type; 2951 in.mlxi_create_flow_group_table_id = to_be24(mlft->mlft_num); 2952 in.mlxi_create_flow_group_start_flow_index = 2953 to_be32(mlfg->mlfg_start_idx); 2954 in.mlxi_create_flow_group_end_flow_index = 2955 to_be32(mlfg->mlfg_start_idx + (mlfg->mlfg_size - 1)); 2956 2957 hdrs = &in.mlxi_create_flow_group_match_criteria.mlfm_outer_headers; 2958 params = &in.mlxi_create_flow_group_match_criteria.mlfm_misc_parameters; 2959 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 2960 in.mlxi_create_flow_group_match_criteria_en |= 2961 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2962 (void) memset(&hdrs->mlfh_smac, 0xff, sizeof (hdrs->mlfh_smac)); 2963 } 2964 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 2965 in.mlxi_create_flow_group_match_criteria_en |= 2966 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2967 (void) memset(&hdrs->mlfh_dmac, 0xff, sizeof (hdrs->mlfh_dmac)); 2968 } 2969 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 2970 in.mlxi_create_flow_group_match_criteria_en |= 2971 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2972 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_CVLAN_TAG); 2973 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_SVLAN_TAG); 2974 } 2975 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 2976 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 2977 set_bits16(&hdrs->mlfh_first_vid_flags, 2978 MLXCX_FLOW_HDR_FIRST_VID, UINT16_MAX); 2979 } 2980 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 2981 in.mlxi_create_flow_group_match_criteria_en |= 2982 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2983 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 2984 UINT32_MAX); 2985 } 2986 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 2987 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 2988 (void) memset(&hdrs->mlfh_src_ip, 0xff, 2989 sizeof (hdrs->mlfh_src_ip)); 2990 } 2991 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 2992 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 2993 (void) memset(&hdrs->mlfh_src_ip, 0xff, 2994 sizeof (hdrs->mlfh_dst_ip)); 2995 } 2996 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 2997 in.mlxi_create_flow_group_match_criteria_en |= 2998 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2999 hdrs->mlfh_ip_protocol = UINT8_MAX; 3000 } 3001 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3002 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3003 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3004 sizeof (hdrs->mlfh_src_ip)); 3005 } 3006 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3007 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3008 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3009 sizeof (hdrs->mlfh_dst_ip)); 3010 } 3011 3012 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3013 in.mlxi_create_flow_group_match_criteria_en |= 3014 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3015 params->mlfp_source_sqn = to_be24(UINT32_MAX); 3016 } 3017 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3018 in.mlxi_create_flow_group_match_criteria_en |= 3019 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3020 params->mlfp_vxlan_vni = to_be24(UINT32_MAX); 3021 } 3022 3023 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3024 mlxcx_cmd_fini(mlxp, &cmd); 3025 return (B_FALSE); 3026 } 3027 mlxcx_cmd_wait(&cmd); 3028 3029 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3030 if (ret) { 3031 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_CREATED; 3032 mlfg->mlfg_num = from_be24(out.mlxo_create_flow_group_group_id); 3033 } 3034 mlxcx_cmd_fini(mlxp, &cmd); 3035 return (ret); 3036 } 3037 3038 boolean_t 3039 mlxcx_cmd_destroy_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 3040 { 3041 mlxcx_cmd_t cmd; 3042 mlxcx_cmd_destroy_flow_group_in_t in; 3043 mlxcx_cmd_destroy_flow_group_out_t out; 3044 boolean_t ret; 3045 const mlxcx_flow_table_t *mlft; 3046 3047 bzero(&in, sizeof (in)); 3048 bzero(&out, sizeof (out)); 3049 3050 mlft = mlfg->mlfg_table; 3051 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3052 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3053 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3054 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3055 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3056 3057 mlxcx_cmd_init(mlxp, &cmd); 3058 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_group_head, 3059 MLXCX_OP_DESTROY_FLOW_GROUP, 0); 3060 3061 in.mlxi_destroy_flow_group_vport_number = 3062 to_be16(mlft->mlft_port->mlp_num); 3063 in.mlxi_destroy_flow_group_table_type = mlft->mlft_type; 3064 in.mlxi_destroy_flow_group_table_id = to_be24(mlft->mlft_num); 3065 in.mlxi_destroy_flow_group_group_id = to_be32(mlfg->mlfg_num); 3066 3067 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3068 mlxcx_cmd_fini(mlxp, &cmd); 3069 return (B_FALSE); 3070 } 3071 mlxcx_cmd_wait(&cmd); 3072 3073 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3074 if (ret) { 3075 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_DESTROYED; 3076 } 3077 mlxcx_cmd_fini(mlxp, &cmd); 3078 return (ret); 3079 } 3080 3081 boolean_t 3082 mlxcx_cmd_set_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3083 { 3084 mlxcx_cmd_t cmd; 3085 mlxcx_cmd_set_flow_table_entry_in_t in; 3086 mlxcx_cmd_set_flow_table_entry_out_t out; 3087 boolean_t ret; 3088 size_t insize; 3089 mlxcx_flow_entry_ctx_t *ctx; 3090 const mlxcx_flow_table_t *mlft; 3091 mlxcx_flow_group_t *mlfg; 3092 mlxcx_flow_dest_t *d; 3093 uint_t i; 3094 mlxcx_flow_header_match_t *hdrs; 3095 mlxcx_flow_params_match_t *params; 3096 mlxcx_cmd_set_flow_table_entry_opmod_t opmod; 3097 3098 bzero(&in, sizeof (in)); 3099 bzero(&out, sizeof (out)); 3100 3101 mlft = mlfe->mlfe_table; 3102 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3103 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3104 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3105 3106 mlfg = mlfe->mlfe_group; 3107 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3108 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3109 3110 opmod = MLXCX_CMD_FLOW_ENTRY_SET_NEW; 3111 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3112 ASSERT(mlfe->mlfe_state & MLXCX_FLOW_ENTRY_DIRTY); 3113 opmod = MLXCX_CMD_FLOW_ENTRY_MODIFY; 3114 } 3115 3116 mlxcx_cmd_init(mlxp, &cmd); 3117 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_entry_head, 3118 MLXCX_OP_SET_FLOW_TABLE_ENTRY, opmod); 3119 3120 in.mlxi_set_flow_table_entry_vport_number = 3121 to_be16(mlft->mlft_port->mlp_num); 3122 in.mlxi_set_flow_table_entry_table_type = mlft->mlft_type; 3123 in.mlxi_set_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3124 in.mlxi_set_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3125 3126 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3127 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3128 MLXCX_CMD_FLOW_ENTRY_SET_ACTION); 3129 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3130 MLXCX_CMD_FLOW_ENTRY_SET_DESTINATION); 3131 } 3132 3133 ctx = &in.mlxi_set_flow_table_entry_context; 3134 ctx->mlfec_group_id = to_be32(mlfg->mlfg_num); 3135 3136 insize = offsetof(mlxcx_cmd_set_flow_table_entry_in_t, 3137 mlxi_set_flow_table_entry_context) + 3138 offsetof(mlxcx_flow_entry_ctx_t, mlfec_destination); 3139 3140 ctx->mlfec_action = to_be16(mlfe->mlfe_action); 3141 3142 switch (mlfe->mlfe_action) { 3143 case MLXCX_FLOW_ACTION_ALLOW: 3144 case MLXCX_FLOW_ACTION_DROP: 3145 break; 3146 case MLXCX_FLOW_ACTION_FORWARD: 3147 ASSERT3U(mlfe->mlfe_ndest, <=, MLXCX_FLOW_MAX_DESTINATIONS); 3148 ASSERT3U(mlfe->mlfe_ndest, <=, 3149 mlxp->mlx_caps->mlc_max_rx_fe_dest); 3150 ctx->mlfec_destination_list_size = to_be24(mlfe->mlfe_ndest); 3151 for (i = 0; i < mlfe->mlfe_ndest; ++i) { 3152 insize += sizeof (mlxcx_flow_dest_t); 3153 d = &ctx->mlfec_destination[i]; 3154 if (mlfe->mlfe_dest[i].mlfed_tir != NULL) { 3155 d->mlfd_destination_type = MLXCX_FLOW_DEST_TIR; 3156 d->mlfd_destination_id = to_be24( 3157 mlfe->mlfe_dest[i].mlfed_tir->mltir_num); 3158 } else if (mlfe->mlfe_dest[i].mlfed_flow != NULL) { 3159 d->mlfd_destination_type = 3160 MLXCX_FLOW_DEST_FLOW_TABLE; 3161 d->mlfd_destination_id = to_be24( 3162 mlfe->mlfe_dest[i].mlfed_flow->mlft_num); 3163 } else { 3164 /* Invalid flow entry destination */ 3165 VERIFY(0); 3166 } 3167 } 3168 break; 3169 case MLXCX_FLOW_ACTION_COUNT: 3170 /* We don't support count actions yet. */ 3171 VERIFY(0); 3172 break; 3173 case MLXCX_FLOW_ACTION_ENCAP: 3174 case MLXCX_FLOW_ACTION_DECAP: 3175 /* We don't support encap/decap actions yet. */ 3176 VERIFY(0); 3177 break; 3178 } 3179 3180 hdrs = &ctx->mlfec_match_value.mlfm_outer_headers; 3181 params = &ctx->mlfec_match_value.mlfm_misc_parameters; 3182 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 3183 bcopy(mlfe->mlfe_smac, hdrs->mlfh_smac, 3184 sizeof (hdrs->mlfh_smac)); 3185 } 3186 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 3187 bcopy(mlfe->mlfe_dmac, hdrs->mlfh_dmac, 3188 sizeof (hdrs->mlfh_dmac)); 3189 } 3190 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 3191 switch (mlfe->mlfe_vlan_type) { 3192 case MLXCX_VLAN_TYPE_CVLAN: 3193 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3194 MLXCX_FLOW_HDR_CVLAN_TAG); 3195 break; 3196 case MLXCX_VLAN_TYPE_SVLAN: 3197 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3198 MLXCX_FLOW_HDR_SVLAN_TAG); 3199 break; 3200 default: 3201 break; 3202 } 3203 } 3204 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 3205 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 3206 set_bits16(&hdrs->mlfh_first_vid_flags, 3207 MLXCX_FLOW_HDR_FIRST_VID, mlfe->mlfe_vid); 3208 } 3209 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 3210 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 3211 mlfe->mlfe_ip_version); 3212 } 3213 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3214 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3215 bcopy(mlfe->mlfe_srcip, hdrs->mlfh_src_ip, 3216 sizeof (hdrs->mlfh_src_ip)); 3217 } 3218 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3219 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3220 bcopy(mlfe->mlfe_dstip, hdrs->mlfh_src_ip, 3221 sizeof (hdrs->mlfh_dst_ip)); 3222 } 3223 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3224 hdrs->mlfh_ip_protocol = mlfe->mlfe_ip_proto; 3225 } 3226 3227 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3228 params->mlfp_source_sqn = to_be24(mlfe->mlfe_sqn); 3229 } 3230 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3231 params->mlfp_vxlan_vni = to_be24(mlfe->mlfe_vxlan_vni); 3232 } 3233 3234 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3235 mlxcx_cmd_fini(mlxp, &cmd); 3236 return (B_FALSE); 3237 } 3238 mlxcx_cmd_wait(&cmd); 3239 3240 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3241 if (ret) { 3242 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_CREATED; 3243 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY; 3244 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_BUSY; 3245 } 3246 mlxcx_cmd_fini(mlxp, &cmd); 3247 return (ret); 3248 } 3249 3250 boolean_t 3251 mlxcx_cmd_delete_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3252 { 3253 mlxcx_cmd_t cmd; 3254 mlxcx_cmd_delete_flow_table_entry_in_t in; 3255 mlxcx_cmd_delete_flow_table_entry_out_t out; 3256 boolean_t ret; 3257 const mlxcx_flow_table_t *mlft; 3258 3259 bzero(&in, sizeof (in)); 3260 bzero(&out, sizeof (out)); 3261 3262 mlft = mlfe->mlfe_table; 3263 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3264 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3265 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3266 3267 mlxcx_cmd_init(mlxp, &cmd); 3268 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_delete_flow_table_entry_head, 3269 MLXCX_OP_DELETE_FLOW_TABLE_ENTRY, 0); 3270 3271 in.mlxi_delete_flow_table_entry_vport_number = 3272 to_be16(mlft->mlft_port->mlp_num); 3273 in.mlxi_delete_flow_table_entry_table_type = mlft->mlft_type; 3274 in.mlxi_delete_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3275 in.mlxi_delete_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3276 3277 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3278 mlxcx_cmd_fini(mlxp, &cmd); 3279 return (B_FALSE); 3280 } 3281 mlxcx_cmd_wait(&cmd); 3282 3283 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3284 if (ret) { 3285 /* 3286 * Note that flow entries have a different lifecycle to most 3287 * other things we create -- we have to be able to re-use them 3288 * after they have been deleted, since they exist at a fixed 3289 * position in their flow table. 3290 * 3291 * So we clear the CREATED bit here for them to let us call 3292 * create_flow_table_entry() on the same entry again later. 3293 */ 3294 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_CREATED; 3295 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_DELETED; 3296 } 3297 mlxcx_cmd_fini(mlxp, &cmd); 3298 return (ret); 3299 } 3300 3301 boolean_t 3302 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3303 { 3304 mlxcx_cmd_t cmd; 3305 mlxcx_cmd_create_sq_in_t in; 3306 mlxcx_cmd_create_sq_out_t out; 3307 boolean_t ret; 3308 mlxcx_sq_ctx_t *ctx; 3309 size_t rem, insize; 3310 const ddi_dma_cookie_t *c; 3311 uint64_t pa, npages; 3312 3313 bzero(&in, sizeof (in)); 3314 bzero(&out, sizeof (out)); 3315 3316 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3317 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 3318 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3319 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3320 3321 mlxcx_cmd_init(mlxp, &cmd); 3322 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_sq_head, 3323 MLXCX_OP_CREATE_SQ, 0); 3324 3325 ctx = &in.mlxi_create_sq_context; 3326 3327 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_RLKEY); 3328 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_FLUSH_IN_ERROR); 3329 set_bits32(&ctx->mlsqc_flags, MLXCX_SQ_MIN_WQE_INLINE, 3330 mlwq->mlwq_inline_mode); 3331 ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 3332 3333 VERIFY(mlwq->mlwq_tis != NULL); 3334 ctx->mlsqc_tis_lst_sz = to_be16(1); 3335 ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num); 3336 3337 set_bits32(&ctx->mlsqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 3338 MLXCX_WORKQ_TYPE_CYCLIC); 3339 ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 3340 ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num); 3341 ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 3342 ctx->mlsqc_wq.mlwqc_log_wq_stride = MLXCX_SENDQ_STRIDE_SHIFT; 3343 3344 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 3345 ctx->mlsqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 3346 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 3347 3348 npages = 0; 3349 c = NULL; 3350 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 3351 pa = c->dmac_laddress; 3352 rem = c->dmac_size; 3353 while (rem > 0) { 3354 ASSERT3U(pa & 0xfff, ==, 0); 3355 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 3356 ctx->mlsqc_wq.mlwqc_pas[npages++] = to_be64(pa); 3357 rem -= MLXCX_HW_PAGE_SIZE; 3358 pa += MLXCX_HW_PAGE_SIZE; 3359 } 3360 } 3361 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 3362 3363 insize = offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) + 3364 offsetof(mlxcx_sq_ctx_t, mlsqc_wq) + 3365 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 3366 sizeof (uint64_t) * npages; 3367 3368 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3369 mlxcx_cmd_fini(mlxp, &cmd); 3370 return (B_FALSE); 3371 } 3372 mlxcx_cmd_wait(&cmd); 3373 3374 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3375 if (ret) { 3376 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 3377 mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn); 3378 } 3379 mlxcx_cmd_fini(mlxp, &cmd); 3380 return (ret); 3381 } 3382 3383 boolean_t 3384 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3385 { 3386 mlxcx_cmd_t cmd; 3387 mlxcx_cmd_modify_sq_in_t in; 3388 mlxcx_cmd_modify_sq_out_t out; 3389 boolean_t ret; 3390 ddi_fm_error_t err; 3391 3392 bzero(&in, sizeof (in)); 3393 bzero(&out, sizeof (out)); 3394 3395 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3396 ASSERT(mlwq->mlwq_cq != NULL); 3397 3398 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3399 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3400 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3401 3402 /* 3403 * Before starting the queue, we have to be sure that it is 3404 * empty and the doorbell and counters are set to 0. 3405 */ 3406 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 3407 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 3408 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 3409 3410 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 3411 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 3412 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 3413 DDI_FME_VERSION); 3414 if (err.fme_status != DDI_FM_OK) 3415 return (B_FALSE); 3416 mlwq->mlwq_pc = 0; 3417 3418 mlxcx_cmd_init(mlxp, &cmd); 3419 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3420 MLXCX_OP_MODIFY_SQ, 0); 3421 3422 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3423 3424 /* From state */ 3425 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3426 MLXCX_SQ_STATE_RST); 3427 /* To state */ 3428 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3429 MLXCX_SQ_STATE_RDY); 3430 3431 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3432 mlxcx_cmd_fini(mlxp, &cmd); 3433 return (B_FALSE); 3434 } 3435 mlxcx_cmd_wait(&cmd); 3436 3437 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3438 if (ret) { 3439 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 3440 } 3441 mlxcx_cmd_fini(mlxp, &cmd); 3442 return (ret); 3443 } 3444 3445 boolean_t 3446 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3447 { 3448 mlxcx_cmd_t cmd; 3449 mlxcx_cmd_modify_sq_in_t in; 3450 mlxcx_cmd_modify_sq_out_t out; 3451 boolean_t ret; 3452 3453 bzero(&in, sizeof (in)); 3454 bzero(&out, sizeof (out)); 3455 3456 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3457 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3458 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3459 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3460 3461 mlxcx_cmd_init(mlxp, &cmd); 3462 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3463 MLXCX_OP_MODIFY_SQ, 0); 3464 3465 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3466 3467 /* From state */ 3468 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3469 MLXCX_SQ_STATE_RDY); 3470 /* To state */ 3471 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3472 MLXCX_SQ_STATE_RST); 3473 3474 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3475 mlxcx_cmd_fini(mlxp, &cmd); 3476 return (B_FALSE); 3477 } 3478 mlxcx_cmd_wait(&cmd); 3479 3480 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3481 if (ret) { 3482 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 3483 } 3484 mlxcx_cmd_fini(mlxp, &cmd); 3485 return (ret); 3486 } 3487 3488 boolean_t 3489 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3490 { 3491 mlxcx_cmd_t cmd; 3492 mlxcx_cmd_destroy_sq_in_t in; 3493 mlxcx_cmd_destroy_sq_out_t out; 3494 boolean_t ret; 3495 3496 bzero(&in, sizeof (in)); 3497 bzero(&out, sizeof (out)); 3498 3499 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3500 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3501 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3502 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3503 3504 mlxcx_cmd_init(mlxp, &cmd); 3505 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_sq_head, 3506 MLXCX_OP_DESTROY_SQ, 0); 3507 3508 in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num); 3509 3510 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3511 mlxcx_cmd_fini(mlxp, &cmd); 3512 return (B_FALSE); 3513 } 3514 mlxcx_cmd_wait(&cmd); 3515 3516 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3517 if (ret) { 3518 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 3519 } 3520 mlxcx_cmd_fini(mlxp, &cmd); 3521 return (ret); 3522 } 3523 3524 boolean_t 3525 mlxcx_cmd_create_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3526 { 3527 mlxcx_cmd_t cmd; 3528 mlxcx_cmd_create_rqt_in_t in; 3529 mlxcx_cmd_create_rqt_out_t out; 3530 mlxcx_rqtable_ctx_t *ctx; 3531 boolean_t ret; 3532 uint_t i; 3533 3534 bzero(&in, sizeof (in)); 3535 bzero(&out, sizeof (out)); 3536 3537 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3538 3539 mlxcx_cmd_init(mlxp, &cmd); 3540 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rqt_head, 3541 MLXCX_OP_CREATE_RQT, 0); 3542 3543 ctx = &in.mlxi_create_rqt_context; 3544 ASSERT3U(mlrqt->mlrqt_max, <=, MLXCX_RQT_MAX_RQ_REFS); 3545 ASSERT3U(mlrqt->mlrqt_max, <=, mlxp->mlx_caps->mlc_max_rqt_size); 3546 ctx->mlrqtc_max_size = to_be16(mlrqt->mlrqt_max); 3547 ctx->mlrqtc_actual_size = to_be16(mlrqt->mlrqt_used); 3548 for (i = 0; i < mlrqt->mlrqt_used; ++i) { 3549 ctx->mlrqtc_rqref[i].mlrqtr_rqn = to_be24( 3550 mlrqt->mlrqt_rq[i]->mlwq_num); 3551 } 3552 3553 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3554 mlxcx_cmd_fini(mlxp, &cmd); 3555 return (B_FALSE); 3556 } 3557 mlxcx_cmd_wait(&cmd); 3558 3559 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3560 if (ret) { 3561 mlrqt->mlrqt_num = from_be24(out.mlxo_create_rqt_rqtn); 3562 mlrqt->mlrqt_state |= MLXCX_RQT_CREATED; 3563 mlrqt->mlrqt_state &= ~MLXCX_RQT_DIRTY; 3564 } 3565 mlxcx_cmd_fini(mlxp, &cmd); 3566 return (ret); 3567 } 3568 3569 boolean_t 3570 mlxcx_cmd_destroy_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3571 { 3572 mlxcx_cmd_t cmd; 3573 mlxcx_cmd_destroy_rqt_in_t in; 3574 mlxcx_cmd_destroy_rqt_out_t out; 3575 boolean_t ret; 3576 3577 bzero(&in, sizeof (in)); 3578 bzero(&out, sizeof (out)); 3579 3580 VERIFY(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3581 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_DESTROYED); 3582 3583 mlxcx_cmd_init(mlxp, &cmd); 3584 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rqt_head, 3585 MLXCX_OP_DESTROY_RQT, 0); 3586 3587 in.mlxi_destroy_rqt_rqtn = to_be24(mlrqt->mlrqt_num); 3588 3589 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3590 mlxcx_cmd_fini(mlxp, &cmd); 3591 return (B_FALSE); 3592 } 3593 mlxcx_cmd_wait(&cmd); 3594 3595 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3596 if (ret) { 3597 mlrqt->mlrqt_state |= MLXCX_RQT_DESTROYED; 3598 } 3599 mlxcx_cmd_fini(mlxp, &cmd); 3600 return (ret); 3601 } 3602 3603 boolean_t 3604 mlxcx_cmd_set_int_mod(mlxcx_t *mlxp, uint_t intr, uint_t min_delay) 3605 { 3606 mlxcx_cmd_t cmd; 3607 mlxcx_cmd_config_int_mod_in_t in; 3608 mlxcx_cmd_config_int_mod_out_t out; 3609 boolean_t ret; 3610 3611 bzero(&in, sizeof (in)); 3612 bzero(&out, sizeof (out)); 3613 3614 mlxcx_cmd_init(mlxp, &cmd); 3615 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_config_int_mod_head, 3616 MLXCX_OP_CONFIG_INT_MODERATION, MLXCX_CMD_CONFIG_INT_MOD_WRITE); 3617 3618 in.mlxi_config_int_mod_int_vector = to_be16(intr); 3619 in.mlxi_config_int_mod_min_delay = to_be16(min_delay); 3620 3621 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3622 mlxcx_cmd_fini(mlxp, &cmd); 3623 return (B_FALSE); 3624 } 3625 mlxcx_cmd_wait(&cmd); 3626 3627 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3628 mlxcx_cmd_fini(mlxp, &cmd); 3629 return (ret); 3630 } 3631 3632 /* 3633 * CTASSERTs here are for the structs in mlxcx_reg.h, to check they match 3634 * against offsets from the PRM. 3635 * 3636 * They're not in the header file, to avoid them being used by multiple .c 3637 * files. 3638 */ 3639 3640 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_unknown_data) == 0x20); 3641 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_signature) == 0x3c + 2); 3642 CTASSERT(sizeof (mlxcx_eventq_ent_t) == 64); 3643 3644 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_byte_cnt) == 0x2C); 3645 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_wqe_opcode) == 0x38); 3646 3647 CTASSERT(sizeof (mlxcx_completionq_error_ent_t) == 3648 sizeof (mlxcx_completionq_ent_t)); 3649 CTASSERT(sizeof (mlxcx_wqe_control_seg_t) == (1 << 4)); 3650 3651 CTASSERT(offsetof(mlxcx_wqe_eth_seg_t, mles_inline_headers) == 0x0e); 3652 CTASSERT(sizeof (mlxcx_wqe_eth_seg_t) == (1 << 5)); 3653 3654 CTASSERT(sizeof (mlxcx_wqe_data_seg_t) == (1 << 4)); 3655 3656 CTASSERT(sizeof (mlxcx_sendq_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3657 3658 CTASSERT(sizeof (mlxcx_sendq_bf_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3659 3660 CTASSERT(sizeof (mlxcx_sendq_extra_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3661 3662 CTASSERT(sizeof (mlxcx_recvq_ent_t) == (1 << MLXCX_RECVQ_STRIDE_SHIFT)); 3663 3664 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_dbr_addr) == 0x10); 3665 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_pas) == 0xc0); 3666 3667 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_cqn) == 0x09); 3668 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_wq) == 0x30); 3669 3670 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_cqn) == 0x09); 3671 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_lst_sz) == 0x20); 3672 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_num) == 0x2d); 3673 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_wq) == 0x30); 3674 3675 CTASSERT(sizeof (mlxcx_tis_ctx_t) == 0xa0); 3676 CTASSERT(offsetof(mlxcx_tis_ctx_t, mltisc_transport_domain) == 0x25); 3677 3678 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_max_size) == 0x16); 3679 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_rqref) == 0xF0); 3680 3681 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_event_bitmask) == 3682 0x58); 3683 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) == 0x110); 3684 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_context) == 0x10); 3685 3686 CTASSERT(offsetof(mlxcx_cmd_create_tir_in_t, mlxi_create_tir_context) == 0x20); 3687 3688 CTASSERT(offsetof(mlxcx_cmd_create_tis_in_t, mlxi_create_tis_context) == 0x20); 3689 3690 CTASSERT(offsetof(mlxcx_cmd_query_special_ctxs_out_t, 3691 mlxo_query_special_ctxs_resd_lkey) == 0x0c); 3692 3693 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_context) == 0x10); 3694 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_pas) == 0x110); 3695 3696 CTASSERT(offsetof(mlxcx_cmd_query_rq_out_t, mlxo_query_rq_context) == 0x20); 3697 3698 CTASSERT(offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) == 0x20); 3699 3700 CTASSERT(offsetof(mlxcx_cmd_modify_sq_in_t, mlxi_modify_sq_context) == 0x20); 3701 3702 CTASSERT(offsetof(mlxcx_cmd_query_sq_out_t, mlxo_query_sq_context) == 0x20); 3703 3704 CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20); 3705 3706 CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C); 3707 3708 CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64); 3709 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c); 3710 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18); 3711 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30); 3712 3713 CTASSERT(offsetof(mlxcx_reg_mcia_t, mlrd_mcia_data) == 0x10); 3714 3715 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3716 mlppc_ieee_802_3_in_range_len_err) == 0x50); 3717 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3718 mlppc_ieee_802_3_pause_tx) == 0x90); 3719 3720 CTASSERT(sizeof (mlxcx_reg_ppcnt_t) == 256); 3721 CTASSERT(offsetof(mlxcx_reg_ppcnt_t, mlrd_ppcnt_data) == 0x08); 3722 3723 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3724 mlxi_access_register_argument) == 0x0C); 3725 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3726 mlxi_access_register_data) == 0x10); 3727 3728 CTASSERT(offsetof(mlxcx_cmd_access_register_out_t, 3729 mlxo_access_register_data) == 0x10); 3730