1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 */ 17 18 /* 19 * Mellanox Connect-X 4/5/6 driver. 20 * 21 * More details in mlxcx.c 22 */ 23 24 #ifndef _MLXCX_H 25 #define _MLXCX_H 26 27 /* 28 * mlxcx(7D) defintions 29 */ 30 31 #include <sys/ddi.h> 32 #include <sys/sunddi.h> 33 #include <sys/ddifm.h> 34 #include <sys/id_space.h> 35 #include <sys/list.h> 36 #include <sys/taskq_impl.h> 37 #include <sys/stddef.h> 38 #include <sys/stream.h> 39 #include <sys/strsun.h> 40 #include <sys/mac_provider.h> 41 #include <sys/mac_ether.h> 42 #include <sys/cpuvar.h> 43 #include <sys/ethernet.h> 44 45 #include <inet/ip.h> 46 #include <inet/ip6.h> 47 48 #include <sys/ddifm.h> 49 #include <sys/fm/protocol.h> 50 #include <sys/fm/util.h> 51 #include <sys/fm/io/ddi.h> 52 53 #include <mlxcx_reg.h> 54 55 #ifdef __cplusplus 56 extern "C" { 57 #endif 58 59 /* 60 * Get access to the first PCI BAR. 61 */ 62 #define MLXCX_REG_NUMBER 1 63 64 /* 65 * The command queue is supposed to be a page, which is 4k. 66 */ 67 #define MLXCX_CMD_DMA_PAGE_SIZE 4096 68 69 /* 70 * Queues can allocate in units of this much memory. 71 */ 72 #define MLXCX_QUEUE_DMA_PAGE_SIZE 4096 73 74 /* 75 * We advertise two sizes of groups to MAC -- a certain number of "large" 76 * groups (including the default group, which is sized to at least ncpus) 77 * followed by a certain number of "small" groups. 78 * 79 * This allows us to have a larger amount of classification resources available 80 * for zones/VMs without resorting to software classification. 81 */ 82 #define MLXCX_RX_NGROUPS_LARGE_DFLT 2 83 #define MLXCX_RX_NRINGS_PER_LARGE_GROUP_DFLT 16 84 #define MLXCX_RX_NGROUPS_SMALL_DFLT 256 85 #define MLXCX_RX_NRINGS_PER_SMALL_GROUP_DFLT 4 86 87 #define MLXCX_TX_NGROUPS_DFLT 1 88 #define MLXCX_TX_NRINGS_PER_GROUP_DFLT 64 89 90 /* 91 * Queues will be sized to (1 << *Q_SIZE_SHIFT) entries long. 92 */ 93 #define MLXCX_EQ_SIZE_SHIFT_DFLT 9 94 95 /* 96 * The CQ, SQ and RQ sizes can effect throughput on higher speed interfaces. 97 * EQ less so, as it only takes a single EQ entry to indicate there are 98 * multiple completions on the CQ. 99 * 100 * Particularly on the Rx side, the RQ (and corresponding CQ) would run 101 * low on available entries. A symptom of this is the refill taskq running 102 * frequently. A larger RQ (and CQ) alleviates this, and as there is a 103 * close relationship between SQ and CQ size, the SQ is increased too. 104 */ 105 #define MLXCX_CQ_SIZE_SHIFT_DFLT 10 106 #define MLXCX_CQ_SIZE_SHIFT_25G 12 107 108 /* 109 * Default to making SQs bigger than RQs for 9k MTU, since most packets will 110 * spill over into more than one slot. RQ WQEs are always 1 slot. 111 */ 112 #define MLXCX_SQ_SIZE_SHIFT_DFLT 11 113 #define MLXCX_SQ_SIZE_SHIFT_25G 13 114 115 #define MLXCX_RQ_SIZE_SHIFT_DFLT 10 116 #define MLXCX_RQ_SIZE_SHIFT_25G 12 117 118 #define MLXCX_CQ_HWM_GAP 16 119 #define MLXCX_CQ_LWM_GAP 24 120 121 #define MLXCX_WQ_HWM_GAP MLXCX_CQ_HWM_GAP 122 #define MLXCX_WQ_LWM_GAP MLXCX_CQ_LWM_GAP 123 124 #define MLXCX_RQ_REFILL_STEP 64 125 126 /* 127 * CQ event moderation 128 */ 129 #define MLXCX_CQEMOD_PERIOD_USEC_DFLT 50 130 #define MLXCX_CQEMOD_COUNT_DFLT \ 131 (8 * ((1 << MLXCX_CQ_SIZE_SHIFT_DFLT) / 10)) 132 133 /* 134 * EQ interrupt moderation 135 */ 136 #define MLXCX_INTRMOD_PERIOD_USEC_DFLT 10 137 138 /* Size of root flow tables */ 139 #define MLXCX_FTBL_ROOT_SIZE_SHIFT_DFLT 12 140 141 /* Size of 2nd level flow tables for VLAN filtering */ 142 #define MLXCX_FTBL_VLAN_SIZE_SHIFT_DFLT 4 143 144 /* 145 * How big does an mblk have to be before we dma_bind() it instead of 146 * bcopying? 147 */ 148 #define MLXCX_TX_BIND_THRESHOLD_DFLT 2048 149 150 /* 151 * How often to check the status of completion queues for overflow and 152 * other problems. 153 */ 154 #define MLXCX_WQ_CHECK_INTERVAL_SEC_DFLT 300 155 #define MLXCX_CQ_CHECK_INTERVAL_SEC_DFLT 300 156 #define MLXCX_EQ_CHECK_INTERVAL_SEC_DFLT 30 157 158 /* 159 * After this many packets, the packets received so far are passed to 160 * the mac layer. 161 */ 162 #define MLXCX_RX_PER_CQ_DEFAULT 256 163 #define MLXCX_RX_PER_CQ_MIN 16 164 #define MLXCX_RX_PER_CQ_MAX 4096 165 166 #define MLXCX_DOORBELL_TRIES_DFLT 3 167 extern uint_t mlxcx_doorbell_tries; 168 169 #define MLXCX_STUCK_INTR_COUNT_DFLT 128 170 extern uint_t mlxcx_stuck_intr_count; 171 172 #define MLXCX_BUF_BIND_MAX_ATTEMTPS 50 173 174 #define MLXCX_MTU_OFFSET \ 175 (sizeof (struct ether_vlan_header) + ETHERFCSL) 176 177 /* 178 * This is the current version of the command structure that the driver expects 179 * to be found in the ISS. 180 */ 181 #define MLXCX_CMD_REVISION 5 182 183 #ifdef DEBUG 184 #define MLXCX_DMA_SYNC(dma, flag) VERIFY0(ddi_dma_sync( \ 185 (dma).mxdb_dma_handle, 0, 0, \ 186 (flag))) 187 #else 188 #define MLXCX_DMA_SYNC(dma, flag) (void) ddi_dma_sync( \ 189 (dma).mxdb_dma_handle, 0, 0, \ 190 (flag)) 191 #endif 192 193 #define MLXCX_FM_SERVICE_MLXCX "mlxcx" 194 195 /* 196 * This macro defines the expected value of the 'Interface Step Sequence ID' 197 * (issi) which represents the version of the start up and tear down sequence. 198 * We must check that hardware supports this and tell it which version we're 199 * using as well. 200 */ 201 #define MLXCX_CURRENT_ISSI 1 202 203 /* 204 * This is the size of a page that the hardware expects from us when 205 * manipulating pages. 206 */ 207 #define MLXCX_HW_PAGE_SIZE 4096 208 209 /* 210 * This is a special lkey value used to terminate a list of scatter pointers. 211 */ 212 #define MLXCX_NULL_LKEY 0x100 213 214 /* 215 * The max function id we support in manage pages requests. 216 * At the moment we only support/expect func 0 from manage pages, but 217 * structures and code are in place to support any number. 218 */ 219 #define MLXCX_FUNC_ID_MAX 0 220 221 /* 222 * Forwards 223 */ 224 struct mlxcx; 225 typedef struct mlxcx mlxcx_t; 226 typedef struct mlxcx_cmd mlxcx_cmd_t; 227 typedef struct mlxcx_port mlxcx_port_t; 228 229 typedef struct { 230 mlxcx_t *mlp_mlx; 231 int32_t mlp_npages; 232 uint16_t mlp_func; 233 } mlxcx_pages_request_t; 234 235 typedef struct mlxcx_async_param { 236 mlxcx_t *mla_mlx; 237 taskq_ent_t mla_tqe; 238 boolean_t mla_pending; 239 kmutex_t mla_mtx; 240 241 /* 242 * Parameters specific to the function dispatched. 243 */ 244 union { 245 void *mla_arg; 246 mlxcx_pages_request_t mla_pages; 247 mlxcx_port_t *mla_port; 248 }; 249 } mlxcx_async_param_t; 250 251 typedef enum { 252 MLXCX_DMABUF_HDL_ALLOC = 1 << 0, 253 MLXCX_DMABUF_MEM_ALLOC = 1 << 1, 254 MLXCX_DMABUF_BOUND = 1 << 2, 255 MLXCX_DMABUF_FOREIGN = 1 << 3, 256 } mlxcx_dma_buffer_flags_t; 257 258 typedef struct mlxcx_dma_buffer { 259 mlxcx_dma_buffer_flags_t mxdb_flags; 260 caddr_t mxdb_va; /* Buffer VA */ 261 size_t mxdb_len; /* Buffer logical len */ 262 ddi_acc_handle_t mxdb_acc_handle; 263 ddi_dma_handle_t mxdb_dma_handle; 264 uint_t mxdb_ncookies; 265 } mlxcx_dma_buffer_t; 266 267 typedef struct mlxcx_dev_page { 268 list_node_t mxdp_list; 269 avl_node_t mxdp_tree; 270 uintptr_t mxdp_pa; 271 mlxcx_dma_buffer_t mxdp_dma; 272 } mlxcx_dev_page_t; 273 274 /* 275 * Data structure to keep track of all information related to the command queue. 276 */ 277 typedef enum { 278 MLXCX_CMD_QUEUE_S_IDLE = 1, 279 MLXCX_CMD_QUEUE_S_BUSY, 280 MLXCX_CMD_QUEUE_S_BROKEN 281 } mlxcx_cmd_queue_status_t; 282 283 typedef struct mlxcx_cmd_queue { 284 kmutex_t mcmd_lock; 285 kcondvar_t mcmd_cv; 286 mlxcx_dma_buffer_t mcmd_dma; 287 288 boolean_t mcmd_polled; 289 290 uint8_t mcmd_size_l2; 291 uint8_t mcmd_stride_l2; 292 uint_t mcmd_size; 293 /* 294 * The mask has a bit for each command slot, there are a maximum 295 * of 32 slots. When the bit is set in the mask, it indicates 296 * the slot is available. 297 */ 298 uint32_t mcmd_mask; 299 300 mlxcx_cmd_t *mcmd_active[MLXCX_CMD_MAX]; 301 302 ddi_taskq_t *mcmd_taskq; 303 id_space_t *mcmd_tokens; 304 } mlxcx_cmd_queue_t; 305 306 typedef struct mlxcd_cmd_mbox { 307 list_node_t mlbox_node; 308 mlxcx_dma_buffer_t mlbox_dma; 309 mlxcx_cmd_mailbox_t *mlbox_data; 310 } mlxcx_cmd_mbox_t; 311 312 typedef enum { 313 MLXCX_EQ_ALLOC = 1 << 0, /* dma mem alloc'd, size set */ 314 MLXCX_EQ_CREATED = 1 << 1, /* CREATE_EQ sent to hw */ 315 MLXCX_EQ_DESTROYED = 1 << 2, /* DESTROY_EQ sent to hw */ 316 MLXCX_EQ_ARMED = 1 << 3, /* Armed through the UAR */ 317 MLXCX_EQ_POLLING = 1 << 4, /* Currently being polled */ 318 MLXCX_EQ_INTR_ENABLED = 1 << 5, /* ddi_intr_enable()'d */ 319 MLXCX_EQ_INTR_ACTIVE = 1 << 6, /* 'rupt handler running */ 320 MLXCX_EQ_INTR_QUIESCE = 1 << 7, /* 'rupt handler to quiesce */ 321 } mlxcx_eventq_state_t; 322 323 typedef struct mlxcx_bf { 324 kmutex_t mbf_mtx; 325 uint_t mbf_cnt; 326 uint_t mbf_even; 327 uint_t mbf_odd; 328 } mlxcx_bf_t; 329 330 typedef struct mlxcx_uar { 331 boolean_t mlu_allocated; 332 uint_t mlu_num; 333 uint_t mlu_base; 334 335 volatile uint_t mlu_bfcnt; 336 mlxcx_bf_t mlu_bf[MLXCX_BF_PER_UAR]; 337 } mlxcx_uar_t; 338 339 typedef struct mlxcx_pd { 340 boolean_t mlpd_allocated; 341 uint32_t mlpd_num; 342 } mlxcx_pd_t; 343 344 typedef struct mlxcx_tdom { 345 boolean_t mltd_allocated; 346 uint32_t mltd_num; 347 } mlxcx_tdom_t; 348 349 typedef enum { 350 MLXCX_PORT_VPORT_PROMISC = 1 << 0, 351 } mlxcx_port_flags_t; 352 353 typedef struct mlxcx_flow_table mlxcx_flow_table_t; 354 typedef struct mlxcx_flow_group mlxcx_flow_group_t; 355 356 typedef struct { 357 uint64_t mlps_rx_drops; 358 } mlxcx_port_stats_t; 359 360 typedef enum { 361 MLXCX_PORT_INIT = 1 << 0 362 } mlxcx_port_init_t; 363 364 struct mlxcx_port { 365 kmutex_t mlp_mtx; 366 mlxcx_port_init_t mlp_init; 367 mlxcx_t *mlp_mlx; 368 /* 369 * The mlp_num we have here starts at zero (it's an index), but the 370 * numbering we have to use for register access starts at 1. We 371 * currently write mlp_num into the other_vport fields in mlxcx_cmd.c 372 * (where 0 is a magic number meaning "my vport") so if we ever add 373 * support for virtualisation features and deal with more than one 374 * vport, we will probably have to change this. 375 */ 376 uint_t mlp_num; 377 mlxcx_port_flags_t mlp_flags; 378 uint64_t mlp_guid; 379 uint8_t mlp_mac_address[ETHERADDRL]; 380 381 uint_t mlp_mtu; 382 uint_t mlp_max_mtu; 383 384 mlxcx_port_status_t mlp_admin_status; 385 mlxcx_port_status_t mlp_oper_status; 386 387 boolean_t mlp_autoneg; 388 mlxcx_eth_proto_t mlp_max_proto; 389 mlxcx_eth_proto_t mlp_admin_proto; 390 mlxcx_eth_proto_t mlp_oper_proto; 391 mlxcx_pplm_fec_active_t mlp_fec_active; 392 link_fec_t mlp_fec_requested; 393 394 mlxcx_eth_inline_mode_t mlp_wqe_min_inline; 395 396 /* Root flow tables */ 397 mlxcx_flow_table_t *mlp_rx_flow; 398 mlxcx_flow_table_t *mlp_tx_flow; 399 400 mlxcx_flow_group_t *mlp_promisc; 401 mlxcx_flow_group_t *mlp_bcast; 402 mlxcx_flow_group_t *mlp_umcast; 403 404 avl_tree_t mlp_dmac_fe; 405 406 mlxcx_port_stats_t mlp_stats; 407 408 mlxcx_module_status_t mlp_last_modstate; 409 mlxcx_module_error_type_t mlp_last_moderr; 410 411 mlxcx_async_param_t mlx_port_event; 412 }; 413 414 typedef enum { 415 MLXCX_EQ_TYPE_ANY, 416 MLXCX_EQ_TYPE_RX, 417 MLXCX_EQ_TYPE_TX 418 } mlxcx_eventq_type_t; 419 420 /* 421 * mlxcx_event_queue_t is a representation of an event queue (EQ). 422 * There is a 1-1 tie in between an EQ and an interrupt vector, and 423 * knowledge of that effects how some members of the struct are used 424 * and modified. 425 * 426 * Most of the struct members are immmutable except for during set up and 427 * teardown, for those it is safe to access them without a mutex once 428 * the driver is initialized. 429 * 430 * Members which are not immutable and are protected by mleq_mtx are: 431 * * mleq_state - EQ state. Changes during transitions between 432 * polling modes. 433 * * mleq_cq - an AVL tree of completions queues using this EQ. 434 * 435 * Another member which is not immutable is mleq_cc. This is the EQ 436 * consumer counter, it *must* only be incremented in the EQ's interrupt 437 * context. It is also fed back to the hardware during re-arming of 438 * the EQ, again this *must* only happen in the EQ's interrupt context. 439 * 440 * There are a couple of struct members (mleq_check_disarm_cc and 441 * mleq_check_disarm_cnt) which are used to help monitor the health 442 * and consistency of the EQ. They are only used and modified during health 443 * monitoring, which is both infrequent and single threaded, consequently 444 * no mutex guards are needed. 445 * 446 * Care is taken not to use the mleq_mtx when possible, both to avoid 447 * contention in what is "hot" code and avoid breaking requirements 448 * of mac(9E). 449 */ 450 typedef struct mlxcx_event_queue { 451 kmutex_t mleq_mtx; 452 kcondvar_t mleq_cv; 453 mlxcx_t *mleq_mlx; 454 mlxcx_eventq_state_t mleq_state; 455 mlxcx_eventq_type_t mleq_type; 456 457 mlxcx_dma_buffer_t mleq_dma; 458 459 size_t mleq_entshift; 460 size_t mleq_nents; 461 mlxcx_eventq_ent_t *mleq_ent; 462 uint32_t mleq_cc; /* consumer counter */ 463 uint32_t mleq_cc_armed; 464 465 uint32_t mleq_events; 466 467 uint32_t mleq_badintrs; 468 469 /* Hardware eq number */ 470 uint_t mleq_num; 471 /* Index into the mlxcx_t's interrupts array */ 472 uint_t mleq_intr_index; 473 474 /* UAR region that has this EQ's doorbell in it */ 475 mlxcx_uar_t *mleq_uar; 476 477 /* Tree of CQn => mlxcx_completion_queue_t */ 478 avl_tree_t mleq_cqs; 479 480 uint32_t mleq_check_disarm_cc; 481 uint_t mleq_check_disarm_cnt; 482 } mlxcx_event_queue_t; 483 484 typedef enum { 485 MLXCX_TIS_CREATED = 1 << 0, 486 MLXCX_TIS_DESTROYED = 1 << 1, 487 } mlxcx_tis_state_t; 488 489 typedef struct mlxcx_tis { 490 mlxcx_tis_state_t mltis_state; 491 list_node_t mltis_entry; 492 uint_t mltis_num; 493 mlxcx_tdom_t *mltis_tdom; 494 } mlxcx_tis_t; 495 496 typedef enum { 497 MLXCX_BUFFER_INIT, 498 MLXCX_BUFFER_FREE, 499 MLXCX_BUFFER_ON_WQ, 500 MLXCX_BUFFER_ON_LOAN, 501 MLXCX_BUFFER_ON_CHAIN, 502 } mlxcx_buffer_state_t; 503 504 typedef enum { 505 MLXCX_SHARD_READY, 506 MLXCX_SHARD_DRAINING, 507 } mlxcx_shard_state_t; 508 509 typedef struct mlxcx_buf_shard { 510 mlxcx_shard_state_t mlbs_state; 511 list_node_t mlbs_entry; 512 kmutex_t mlbs_mtx; 513 list_t mlbs_busy; 514 list_t mlbs_free; 515 list_t mlbs_loaned; 516 kcondvar_t mlbs_free_nonempty; 517 } mlxcx_buf_shard_t; 518 519 typedef struct mlxcx_buffer { 520 mlxcx_buf_shard_t *mlb_shard; 521 list_node_t mlb_entry; 522 list_node_t mlb_cq_entry; 523 524 struct mlxcx_buffer *mlb_tx_head; /* head of tx chain */ 525 list_t mlb_tx_chain; 526 list_node_t mlb_tx_chain_entry; 527 528 boolean_t mlb_foreign; 529 size_t mlb_used; 530 mblk_t *mlb_tx_mp; 531 532 /* 533 * The number of work queue basic blocks this buf uses. 534 */ 535 uint_t mlb_wqebbs; 536 537 mlxcx_t *mlb_mlx; 538 mlxcx_buffer_state_t mlb_state; 539 uint_t mlb_wqe_index; 540 mlxcx_dma_buffer_t mlb_dma; 541 mblk_t *mlb_mp; 542 frtn_t mlb_frtn; 543 } mlxcx_buffer_t; 544 545 typedef enum { 546 MLXCX_CQ_ALLOC = 1 << 0, 547 MLXCX_CQ_CREATED = 1 << 1, 548 MLXCX_CQ_DESTROYED = 1 << 2, 549 MLXCX_CQ_EQAVL = 1 << 3, 550 MLXCX_CQ_BLOCKED_MAC = 1 << 4, 551 MLXCX_CQ_TEARDOWN = 1 << 5, 552 MLXCX_CQ_POLLING = 1 << 6, 553 MLXCX_CQ_ARMED = 1 << 7, 554 } mlxcx_completionq_state_t; 555 556 typedef struct mlxcx_work_queue mlxcx_work_queue_t; 557 558 typedef struct mlxcx_completion_queue { 559 kmutex_t mlcq_mtx; 560 kmutex_t mlcq_arm_mtx; 561 mlxcx_t *mlcq_mlx; 562 mlxcx_completionq_state_t mlcq_state; 563 564 mlxcx_port_stats_t *mlcq_stats; 565 566 list_node_t mlcq_entry; 567 avl_node_t mlcq_eq_entry; 568 569 uint_t mlcq_num; 570 571 mlxcx_work_queue_t *mlcq_wq; 572 mlxcx_event_queue_t *mlcq_eq; 573 574 /* UAR region that has this CQ's UAR doorbell in it */ 575 mlxcx_uar_t *mlcq_uar; 576 577 mlxcx_dma_buffer_t mlcq_dma; 578 579 size_t mlcq_entshift; 580 size_t mlcq_nents; 581 mlxcx_completionq_ent_t *mlcq_ent; 582 uint32_t mlcq_cc; /* consumer counter */ 583 uint32_t mlcq_cc_armed; /* cc at last arm */ 584 uint32_t mlcq_ec; /* event counter */ 585 uint32_t mlcq_ec_armed; /* ec at last arm */ 586 587 mlxcx_dma_buffer_t mlcq_doorbell_dma; 588 mlxcx_completionq_doorbell_t *mlcq_doorbell; 589 590 uint64_t mlcq_bufcnt; 591 size_t mlcq_bufhwm; 592 size_t mlcq_buflwm; 593 list_t mlcq_buffers; 594 kmutex_t mlcq_bufbmtx; 595 list_t mlcq_buffers_b; 596 597 uint_t mlcq_check_disarm_cnt; 598 uint64_t mlcq_check_disarm_cc; 599 600 uint_t mlcq_cqemod_period_usec; 601 uint_t mlcq_cqemod_count; 602 603 mac_ring_handle_t mlcq_mac_hdl; 604 uint64_t mlcq_mac_gen; 605 606 boolean_t mlcq_fm_repd_qstate; 607 } mlxcx_completion_queue_t; 608 609 typedef enum { 610 MLXCX_WQ_ALLOC = 1 << 0, 611 MLXCX_WQ_CREATED = 1 << 1, 612 MLXCX_WQ_STARTED = 1 << 2, 613 MLXCX_WQ_DESTROYED = 1 << 3, 614 MLXCX_WQ_TEARDOWN = 1 << 4, 615 MLXCX_WQ_BUFFERS = 1 << 5, 616 MLXCX_WQ_REFILLING = 1 << 6, 617 MLXCX_WQ_BLOCKED_MAC = 1 << 7 618 } mlxcx_workq_state_t; 619 620 typedef enum { 621 MLXCX_WQ_TYPE_SENDQ = 1, 622 MLXCX_WQ_TYPE_RECVQ 623 } mlxcx_workq_type_t; 624 625 typedef struct mlxcx_ring_group mlxcx_ring_group_t; 626 627 struct mlxcx_work_queue { 628 kmutex_t mlwq_mtx; 629 mlxcx_t *mlwq_mlx; 630 mlxcx_workq_type_t mlwq_type; 631 mlxcx_workq_state_t mlwq_state; 632 633 list_node_t mlwq_entry; 634 list_node_t mlwq_group_entry; 635 636 mlxcx_ring_group_t *mlwq_group; 637 638 uint_t mlwq_num; 639 640 mlxcx_completion_queue_t *mlwq_cq; 641 mlxcx_pd_t *mlwq_pd; 642 643 /* Required for send queues */ 644 mlxcx_tis_t *mlwq_tis; 645 646 /* UAR region that has this WQ's blueflame buffers in it */ 647 mlxcx_uar_t *mlwq_uar; 648 649 mlxcx_dma_buffer_t mlwq_dma; 650 651 mlxcx_eth_inline_mode_t mlwq_inline_mode; 652 size_t mlwq_entshift; 653 size_t mlwq_nents; 654 /* Discriminate based on mwq_type */ 655 union { 656 mlxcx_sendq_ent_t *mlwq_send_ent; 657 mlxcx_sendq_extra_ent_t *mlwq_send_extra_ent; 658 mlxcx_recvq_ent_t *mlwq_recv_ent; 659 mlxcx_sendq_bf_t *mlwq_bf_ent; 660 }; 661 uint64_t mlwq_pc; /* producer counter */ 662 663 uint64_t mlwq_wqebb_used; 664 size_t mlwq_bufhwm; 665 size_t mlwq_buflwm; 666 667 mlxcx_dma_buffer_t mlwq_doorbell_dma; 668 mlxcx_workq_doorbell_t *mlwq_doorbell; 669 670 mlxcx_buf_shard_t *mlwq_bufs; 671 mlxcx_buf_shard_t *mlwq_foreign_bufs; 672 673 taskq_ent_t mlwq_tqe; 674 675 boolean_t mlwq_fm_repd_qstate; 676 }; 677 678 #define MLXCX_RQT_MAX_SIZE 64 679 680 typedef enum { 681 MLXCX_RQT_CREATED = 1 << 0, 682 MLXCX_RQT_DESTROYED = 1 << 1, 683 MLXCX_RQT_DIRTY = 1 << 2, 684 } mlxcx_rqtable_state_t; 685 686 typedef struct mlxcx_rqtable { 687 mlxcx_rqtable_state_t mlrqt_state; 688 list_node_t mlrqt_entry; 689 uint_t mlrqt_num; 690 691 size_t mlrqt_max; 692 size_t mlrqt_used; 693 694 size_t mlrqt_rq_size; 695 mlxcx_work_queue_t **mlrqt_rq; 696 } mlxcx_rqtable_t; 697 698 typedef enum { 699 MLXCX_TIR_CREATED = 1 << 0, 700 MLXCX_TIR_DESTROYED = 1 << 1, 701 } mlxcx_tir_state_t; 702 703 typedef struct mlxcx_tir { 704 mlxcx_tir_state_t mltir_state; 705 list_node_t mltir_entry; 706 uint_t mltir_num; 707 mlxcx_tdom_t *mltir_tdom; 708 mlxcx_tir_type_t mltir_type; 709 union { 710 mlxcx_rqtable_t *mltir_rqtable; 711 mlxcx_work_queue_t *mltir_rq; 712 }; 713 mlxcx_tir_hash_fn_t mltir_hash_fn; 714 uint8_t mltir_toeplitz_key[40]; 715 mlxcx_tir_rx_hash_l3_type_t mltir_l3_type; 716 mlxcx_tir_rx_hash_l4_type_t mltir_l4_type; 717 mlxcx_tir_rx_hash_fields_t mltir_hash_fields; 718 } mlxcx_tir_t; 719 720 typedef enum { 721 MLXCX_FLOW_GROUP_CREATED = 1 << 0, 722 MLXCX_FLOW_GROUP_BUSY = 1 << 1, 723 MLXCX_FLOW_GROUP_DESTROYED = 1 << 2, 724 } mlxcx_flow_group_state_t; 725 726 typedef enum { 727 MLXCX_FLOW_MATCH_SMAC = 1 << 0, 728 MLXCX_FLOW_MATCH_DMAC = 1 << 1, 729 MLXCX_FLOW_MATCH_VLAN = 1 << 2, 730 MLXCX_FLOW_MATCH_VID = 1 << 3, 731 MLXCX_FLOW_MATCH_IP_VER = 1 << 4, 732 MLXCX_FLOW_MATCH_SRCIP = 1 << 5, 733 MLXCX_FLOW_MATCH_DSTIP = 1 << 6, 734 MLXCX_FLOW_MATCH_IP_PROTO = 1 << 7, 735 MLXCX_FLOW_MATCH_SQN = 1 << 8, 736 MLXCX_FLOW_MATCH_VXLAN = 1 << 9, 737 } mlxcx_flow_mask_t; 738 739 struct mlxcx_flow_group { 740 list_node_t mlfg_entry; 741 list_node_t mlfg_role_entry; 742 mlxcx_flow_group_state_t mlfg_state; 743 mlxcx_flow_table_t *mlfg_table; 744 uint_t mlfg_num; 745 size_t mlfg_start_idx; 746 size_t mlfg_size; 747 size_t mlfg_avail; 748 list_t mlfg_entries; 749 mlxcx_flow_mask_t mlfg_mask; 750 }; 751 752 typedef enum { 753 MLXCX_FLOW_ENTRY_RESERVED = 1 << 0, 754 MLXCX_FLOW_ENTRY_CREATED = 1 << 1, 755 MLXCX_FLOW_ENTRY_DELETED = 1 << 2, 756 MLXCX_FLOW_ENTRY_DIRTY = 1 << 3, 757 } mlxcx_flow_entry_state_t; 758 759 typedef struct { 760 mlxcx_tir_t *mlfed_tir; 761 mlxcx_flow_table_t *mlfed_flow; 762 } mlxcx_flow_entry_dest_t; 763 764 typedef struct mlxcx_flow_entry { 765 list_node_t mlfe_group_entry; 766 avl_node_t mlfe_dmac_entry; 767 mlxcx_flow_entry_state_t mlfe_state; 768 mlxcx_flow_table_t *mlfe_table; 769 mlxcx_flow_group_t *mlfe_group; 770 uint_t mlfe_index; 771 772 mlxcx_flow_action_t mlfe_action; 773 774 /* Criteria for match */ 775 uint8_t mlfe_smac[ETHERADDRL]; 776 uint8_t mlfe_dmac[ETHERADDRL]; 777 778 mlxcx_vlan_type_t mlfe_vlan_type; 779 uint16_t mlfe_vid; 780 781 uint_t mlfe_ip_version; 782 uint8_t mlfe_srcip[IPV6_ADDR_LEN]; 783 uint8_t mlfe_dstip[IPV6_ADDR_LEN]; 784 785 uint_t mlfe_ip_proto; 786 uint16_t mlfe_sport; 787 uint16_t mlfe_dport; 788 789 uint32_t mlfe_sqn; 790 uint32_t mlfe_vxlan_vni; 791 792 /* Destinations */ 793 size_t mlfe_ndest; 794 mlxcx_flow_entry_dest_t mlfe_dest[MLXCX_FLOW_MAX_DESTINATIONS]; 795 796 /* 797 * mlxcx_group_mac_ts joining this entry to N ring groups 798 * only used by FEs on the root rx flow table 799 */ 800 list_t mlfe_ring_groups; 801 } mlxcx_flow_entry_t; 802 803 typedef enum { 804 MLXCX_FLOW_TABLE_CREATED = 1 << 0, 805 MLXCX_FLOW_TABLE_DESTROYED = 1 << 1, 806 MLXCX_FLOW_TABLE_ROOT = 1 << 2 807 } mlxcx_flow_table_state_t; 808 809 struct mlxcx_flow_table { 810 kmutex_t mlft_mtx; 811 mlxcx_flow_table_state_t mlft_state; 812 uint_t mlft_level; 813 uint_t mlft_num; 814 mlxcx_flow_table_type_t mlft_type; 815 816 mlxcx_port_t *mlft_port; 817 818 size_t mlft_entshift; 819 size_t mlft_nents; 820 821 size_t mlft_entsize; 822 mlxcx_flow_entry_t *mlft_ent; 823 824 /* First entry not yet claimed by a group */ 825 size_t mlft_next_ent; 826 827 list_t mlft_groups; 828 }; 829 830 typedef enum { 831 MLXCX_GROUP_RX, 832 MLXCX_GROUP_TX 833 } mlxcx_group_type_t; 834 835 typedef enum { 836 MLXCX_GROUP_INIT = 1 << 0, 837 MLXCX_GROUP_WQS = 1 << 1, 838 MLXCX_GROUP_TIRTIS = 1 << 2, 839 MLXCX_GROUP_FLOWS = 1 << 3, 840 MLXCX_GROUP_RUNNING = 1 << 4, 841 MLXCX_GROUP_RQT = 1 << 5, 842 } mlxcx_group_state_t; 843 844 #define MLXCX_RX_HASH_FT_SIZE_SHIFT 4 845 846 typedef enum { 847 MLXCX_TIR_ROLE_IPv4 = 0, 848 MLXCX_TIR_ROLE_IPv6, 849 MLXCX_TIR_ROLE_TCPv4, 850 MLXCX_TIR_ROLE_TCPv6, 851 MLXCX_TIR_ROLE_UDPv4, 852 MLXCX_TIR_ROLE_UDPv6, 853 MLXCX_TIR_ROLE_OTHER, 854 855 MLXCX_TIRS_PER_GROUP 856 } mlxcx_tir_role_t; 857 858 typedef struct { 859 avl_node_t mlgm_group_entry; 860 list_node_t mlgm_fe_entry; 861 mlxcx_ring_group_t *mlgm_group; 862 uint8_t mlgm_mac[6]; 863 mlxcx_flow_entry_t *mlgm_fe; 864 } mlxcx_group_mac_t; 865 866 typedef struct { 867 list_node_t mlgv_entry; 868 boolean_t mlgv_tagged; 869 uint16_t mlgv_vid; 870 mlxcx_flow_entry_t *mlgv_fe; 871 } mlxcx_group_vlan_t; 872 873 struct mlxcx_ring_group { 874 kmutex_t mlg_mtx; 875 mlxcx_t *mlg_mlx; 876 mlxcx_group_state_t mlg_state; 877 mlxcx_group_type_t mlg_type; 878 879 mac_group_handle_t mlg_mac_hdl; 880 881 union { 882 mlxcx_tis_t mlg_tis; 883 mlxcx_tir_t mlg_tir[MLXCX_TIRS_PER_GROUP]; 884 }; 885 mlxcx_port_t *mlg_port; 886 887 size_t mlg_nwqs; 888 size_t mlg_wqs_size; 889 mlxcx_work_queue_t *mlg_wqs; 890 891 mlxcx_rqtable_t *mlg_rqt; 892 893 /* 894 * Flow table for matching VLAN IDs 895 */ 896 mlxcx_flow_table_t *mlg_rx_vlan_ft; 897 mlxcx_flow_group_t *mlg_rx_vlan_fg; 898 mlxcx_flow_group_t *mlg_rx_vlan_def_fg; 899 mlxcx_flow_group_t *mlg_rx_vlan_promisc_fg; 900 list_t mlg_rx_vlans; 901 902 taskq_t *mlg_refill_tq; 903 904 /* 905 * Flow table for separating out by protocol before hashing 906 */ 907 mlxcx_flow_table_t *mlg_rx_hash_ft; 908 909 /* 910 * Links to flow entries on the root flow table which are pointing to 911 * our rx_vlan_ft. 912 */ 913 avl_tree_t mlg_rx_macs; 914 }; 915 916 typedef enum mlxcx_cmd_state { 917 MLXCX_CMD_S_DONE = 1 << 0, 918 MLXCX_CMD_S_ERROR = 1 << 1 919 } mlxcx_cmd_state_t; 920 921 struct mlxcx_cmd { 922 struct mlxcx *mlcmd_mlxp; 923 kmutex_t mlcmd_lock; 924 kcondvar_t mlcmd_cv; 925 926 boolean_t mlcmd_poll; 927 uint8_t mlcmd_token; 928 mlxcx_cmd_op_t mlcmd_op; 929 930 /* 931 * Command data and extended mailboxes for responses. 932 */ 933 const void *mlcmd_in; 934 uint32_t mlcmd_inlen; 935 void *mlcmd_out; 936 uint32_t mlcmd_outlen; 937 list_t mlcmd_mbox_in; 938 uint8_t mlcmd_nboxes_in; 939 list_t mlcmd_mbox_out; 940 uint8_t mlcmd_nboxes_out; 941 /* 942 * Status information. 943 */ 944 mlxcx_cmd_state_t mlcmd_state; 945 uint8_t mlcmd_status; 946 }; 947 948 /* 949 * Our view of capabilities. 950 */ 951 typedef struct mlxcx_hca_cap { 952 mlxcx_hca_cap_mode_t mhc_mode; 953 mlxcx_hca_cap_type_t mhc_type; 954 union { 955 uint8_t mhc_bulk[MLXCX_HCA_CAP_SIZE]; 956 mlxcx_hca_cap_general_caps_t mhc_general; 957 mlxcx_hca_cap_eth_caps_t mhc_eth; 958 mlxcx_hca_cap_flow_caps_t mhc_flow; 959 }; 960 } mlxcx_hca_cap_t; 961 962 typedef struct { 963 /* Cooked values */ 964 boolean_t mlc_checksum; 965 boolean_t mlc_lso; 966 boolean_t mlc_vxlan; 967 size_t mlc_max_lso_size; 968 size_t mlc_max_rqt_size; 969 970 size_t mlc_max_rx_ft_shift; 971 size_t mlc_max_rx_fe_dest; 972 size_t mlc_max_rx_flows; 973 size_t mlc_max_rx_ft; 974 975 size_t mlc_max_tir; 976 977 /* Raw caps data */ 978 mlxcx_hca_cap_t mlc_hca_cur; 979 mlxcx_hca_cap_t mlc_hca_max; 980 mlxcx_hca_cap_t mlc_ether_cur; 981 mlxcx_hca_cap_t mlc_ether_max; 982 mlxcx_hca_cap_t mlc_nic_flow_cur; 983 mlxcx_hca_cap_t mlc_nic_flow_max; 984 } mlxcx_caps_t; 985 986 typedef struct { 987 uint_t mldp_eq_size_shift; 988 uint_t mldp_cq_size_shift; 989 uint_t mldp_cq_size_shift_default; 990 uint_t mldp_rq_size_shift; 991 uint_t mldp_rq_size_shift_default; 992 uint_t mldp_sq_size_shift; 993 uint_t mldp_sq_size_shift_default; 994 uint_t mldp_cqemod_period_usec; 995 uint_t mldp_cqemod_count; 996 uint_t mldp_intrmod_period_usec; 997 uint_t mldp_rx_ngroups_large; 998 uint_t mldp_rx_ngroups_small; 999 uint_t mldp_rx_nrings_per_large_group; 1000 uint_t mldp_rx_nrings_per_small_group; 1001 uint_t mldp_rx_per_cq; 1002 uint_t mldp_tx_ngroups; 1003 uint_t mldp_tx_nrings_per_group; 1004 uint_t mldp_ftbl_root_size_shift; 1005 size_t mldp_tx_bind_threshold; 1006 uint_t mldp_ftbl_vlan_size_shift; 1007 uint64_t mldp_eq_check_interval_sec; 1008 uint64_t mldp_cq_check_interval_sec; 1009 uint64_t mldp_wq_check_interval_sec; 1010 } mlxcx_drv_props_t; 1011 1012 typedef enum { 1013 MLXCX_ATTACH_FM = 1 << 0, 1014 MLXCX_ATTACH_PCI_CONFIG = 1 << 1, 1015 MLXCX_ATTACH_REGS = 1 << 2, 1016 MLXCX_ATTACH_CMD = 1 << 3, 1017 MLXCX_ATTACH_ENABLE_HCA = 1 << 4, 1018 MLXCX_ATTACH_PAGE_LIST = 1 << 5, 1019 MLXCX_ATTACH_INIT_HCA = 1 << 6, 1020 MLXCX_ATTACH_UAR_PD_TD = 1 << 7, 1021 MLXCX_ATTACH_INTRS = 1 << 8, 1022 MLXCX_ATTACH_PORTS = 1 << 9, 1023 MLXCX_ATTACH_MAC_HDL = 1 << 10, 1024 MLXCX_ATTACH_CQS = 1 << 11, 1025 MLXCX_ATTACH_WQS = 1 << 12, 1026 MLXCX_ATTACH_GROUPS = 1 << 13, 1027 MLXCX_ATTACH_BUFS = 1 << 14, 1028 MLXCX_ATTACH_CAPS = 1 << 15, 1029 MLXCX_ATTACH_CHKTIMERS = 1 << 16, 1030 MLXCX_ATTACH_ASYNC_TQ = 1 << 17, 1031 } mlxcx_attach_progress_t; 1032 1033 struct mlxcx { 1034 /* entry on the mlxcx_glist */ 1035 list_node_t mlx_gentry; 1036 1037 dev_info_t *mlx_dip; 1038 int mlx_inst; 1039 mlxcx_attach_progress_t mlx_attach; 1040 1041 mlxcx_drv_props_t mlx_props; 1042 1043 /* 1044 * Misc. data 1045 */ 1046 uint16_t mlx_fw_maj; 1047 uint16_t mlx_fw_min; 1048 uint16_t mlx_fw_rev; 1049 uint16_t mlx_cmd_rev; 1050 1051 /* 1052 * Various capabilities of hardware. 1053 */ 1054 mlxcx_caps_t *mlx_caps; 1055 1056 uint_t mlx_max_sdu; 1057 uint_t mlx_sdu; 1058 1059 /* 1060 * FM State 1061 */ 1062 int mlx_fm_caps; 1063 1064 /* 1065 * PCI Data 1066 */ 1067 ddi_acc_handle_t mlx_cfg_handle; 1068 ddi_acc_handle_t mlx_regs_handle; 1069 caddr_t mlx_regs_base; 1070 1071 /* 1072 * MAC handle 1073 */ 1074 mac_handle_t mlx_mac_hdl; 1075 1076 /* 1077 * Main command queue for issuing general FW control commands. 1078 */ 1079 mlxcx_cmd_queue_t mlx_cmd; 1080 1081 /* 1082 * Interrupts 1083 */ 1084 uint_t mlx_intr_pri; 1085 uint_t mlx_async_intr_pri; 1086 uint_t mlx_intr_type; /* always MSI-X */ 1087 int mlx_intr_count; 1088 size_t mlx_intr_size; /* allocation size */ 1089 int mlx_intr_cq0; 1090 ddi_intr_handle_t *mlx_intr_handles; 1091 1092 /* 1093 * Basic firmware resources which we use for a variety of things. 1094 * The UAR is a reference to a page where CQ and EQ doorbells are 1095 * located. It also holds all the BlueFlame stuff (which we don't 1096 * use). 1097 */ 1098 mlxcx_uar_t mlx_uar; 1099 /* 1100 * The PD (Protection Domain) and TDOM (Transport Domain) are opaque 1101 * entities to us (they're Infiniband constructs we don't actually care 1102 * about) -- we just allocate them and shove their ID numbers in 1103 * whenever we're asked for one. 1104 * 1105 * The "reserved" LKEY is what we should put in queue entries that 1106 * have references to memory to indicate that they're using linear 1107 * addresses (comes from the QUERY_SPECIAL_CONTEXTS cmd). 1108 */ 1109 mlxcx_pd_t mlx_pd; 1110 mlxcx_tdom_t mlx_tdom; 1111 uint_t mlx_rsvd_lkey; 1112 1113 /* 1114 * Our event queues. These are 1:1 with interrupts. 1115 */ 1116 size_t mlx_eqs_size; /* allocation size */ 1117 mlxcx_event_queue_t *mlx_eqs; 1118 1119 /* 1120 * Page list. These represent the set of 4k pages we've given to 1121 * hardware. 1122 * 1123 * We can add to this list at the request of hardware from interrupt 1124 * context (the PAGE_REQUEST event), so it's protected by pagemtx. 1125 */ 1126 kmutex_t mlx_pagemtx; 1127 uint_t mlx_npages; 1128 avl_tree_t mlx_pages; 1129 1130 mlxcx_async_param_t mlx_npages_req[MLXCX_FUNC_ID_MAX + 1]; 1131 1132 /* 1133 * Taskq for processing asynchronous events which may issue 1134 * commands to the HCA. 1135 */ 1136 taskq_t *mlx_async_tq; 1137 1138 /* 1139 * Port state 1140 */ 1141 uint_t mlx_nports; 1142 size_t mlx_ports_size; 1143 mlxcx_port_t *mlx_ports; 1144 1145 /* 1146 * Completion queues (CQs). These are also indexed off the 1147 * event_queue_ts that they each report to. 1148 */ 1149 list_t mlx_cqs; 1150 1151 uint_t mlx_next_eq; 1152 1153 /* 1154 * Work queues (WQs). 1155 */ 1156 list_t mlx_wqs; 1157 1158 /* 1159 * Ring groups 1160 */ 1161 size_t mlx_rx_ngroups; 1162 size_t mlx_rx_groups_size; 1163 mlxcx_ring_group_t *mlx_rx_groups; 1164 1165 size_t mlx_tx_ngroups; 1166 size_t mlx_tx_groups_size; 1167 mlxcx_ring_group_t *mlx_tx_groups; 1168 1169 kmem_cache_t *mlx_bufs_cache; 1170 list_t mlx_buf_shards; 1171 1172 ddi_periodic_t mlx_eq_checktimer; 1173 ddi_periodic_t mlx_cq_checktimer; 1174 ddi_periodic_t mlx_wq_checktimer; 1175 }; 1176 1177 /* 1178 * Register access 1179 */ 1180 extern uint16_t mlxcx_get16(mlxcx_t *, uintptr_t); 1181 extern uint32_t mlxcx_get32(mlxcx_t *, uintptr_t); 1182 extern uint64_t mlxcx_get64(mlxcx_t *, uintptr_t); 1183 1184 extern void mlxcx_put32(mlxcx_t *, uintptr_t, uint32_t); 1185 extern void mlxcx_put64(mlxcx_t *, uintptr_t, uint64_t); 1186 1187 extern void mlxcx_uar_put32(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint32_t); 1188 extern void mlxcx_uar_put64(mlxcx_t *, mlxcx_uar_t *, uintptr_t, uint64_t); 1189 1190 /* 1191 * Logging functions. 1192 */ 1193 extern void mlxcx_warn(mlxcx_t *, const char *, ...); 1194 extern void mlxcx_note(mlxcx_t *, const char *, ...); 1195 extern void mlxcx_panic(mlxcx_t *, const char *, ...); 1196 1197 extern void mlxcx_fm_ereport(mlxcx_t *, const char *); 1198 1199 extern void mlxcx_check_sq(mlxcx_t *, mlxcx_work_queue_t *); 1200 extern void mlxcx_check_rq(mlxcx_t *, mlxcx_work_queue_t *); 1201 1202 /* 1203 * DMA Functions 1204 */ 1205 extern void mlxcx_dma_free(mlxcx_dma_buffer_t *); 1206 extern boolean_t mlxcx_dma_alloc(mlxcx_t *, mlxcx_dma_buffer_t *, 1207 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, size_t, boolean_t); 1208 extern boolean_t mlxcx_dma_init(mlxcx_t *, mlxcx_dma_buffer_t *, 1209 ddi_dma_attr_t *, boolean_t); 1210 extern boolean_t mlxcx_dma_bind_mblk(mlxcx_t *, mlxcx_dma_buffer_t *, 1211 const mblk_t *, size_t, boolean_t); 1212 extern boolean_t mlxcx_dma_alloc_offset(mlxcx_t *, mlxcx_dma_buffer_t *, 1213 ddi_dma_attr_t *, ddi_device_acc_attr_t *, boolean_t, 1214 size_t, size_t, boolean_t); 1215 extern void mlxcx_dma_unbind(mlxcx_t *, mlxcx_dma_buffer_t *); 1216 extern void mlxcx_dma_acc_attr(mlxcx_t *, ddi_device_acc_attr_t *); 1217 extern void mlxcx_dma_page_attr(mlxcx_t *, ddi_dma_attr_t *); 1218 extern void mlxcx_dma_queue_attr(mlxcx_t *, ddi_dma_attr_t *); 1219 extern void mlxcx_dma_qdbell_attr(mlxcx_t *, ddi_dma_attr_t *); 1220 extern void mlxcx_dma_buf_attr(mlxcx_t *, ddi_dma_attr_t *); 1221 1222 extern boolean_t mlxcx_give_pages(mlxcx_t *, int32_t, int32_t *); 1223 1224 static inline const ddi_dma_cookie_t * 1225 mlxcx_dma_cookie_iter(const mlxcx_dma_buffer_t *db, 1226 const ddi_dma_cookie_t *prev) 1227 { 1228 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1229 return (ddi_dma_cookie_iter(db->mxdb_dma_handle, prev)); 1230 } 1231 1232 static inline const ddi_dma_cookie_t * 1233 mlxcx_dma_cookie_one(const mlxcx_dma_buffer_t *db) 1234 { 1235 ASSERT(db->mxdb_flags & MLXCX_DMABUF_BOUND); 1236 return (ddi_dma_cookie_one(db->mxdb_dma_handle)); 1237 } 1238 1239 /* 1240 * From mlxcx_intr.c 1241 */ 1242 extern boolean_t mlxcx_intr_setup(mlxcx_t *); 1243 extern void mlxcx_intr_disable(mlxcx_t *); 1244 extern void mlxcx_intr_teardown(mlxcx_t *); 1245 extern void mlxcx_arm_eq(mlxcx_t *, mlxcx_event_queue_t *); 1246 extern void mlxcx_arm_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1247 extern void mlxcx_update_cqci(mlxcx_t *, mlxcx_completion_queue_t *); 1248 1249 extern mblk_t *mlxcx_rx_poll(mlxcx_t *, mlxcx_completion_queue_t *, size_t); 1250 1251 /* 1252 * From mlxcx_gld.c 1253 */ 1254 extern boolean_t mlxcx_register_mac(mlxcx_t *); 1255 1256 /* 1257 * From mlxcx_ring.c 1258 */ 1259 extern boolean_t mlxcx_wq_alloc_dma(mlxcx_t *, mlxcx_work_queue_t *); 1260 extern void mlxcx_wq_rele_dma(mlxcx_t *, mlxcx_work_queue_t *); 1261 1262 extern boolean_t mlxcx_buf_create(mlxcx_t *, mlxcx_buf_shard_t *, 1263 mlxcx_buffer_t **); 1264 extern boolean_t mlxcx_buf_create_foreign(mlxcx_t *, mlxcx_buf_shard_t *, 1265 mlxcx_buffer_t **); 1266 extern mlxcx_buffer_t *mlxcx_buf_take(mlxcx_t *, mlxcx_work_queue_t *); 1267 extern size_t mlxcx_buf_take_n(mlxcx_t *, mlxcx_work_queue_t *, 1268 mlxcx_buffer_t **, size_t); 1269 extern boolean_t mlxcx_buf_loan(mlxcx_t *, mlxcx_buffer_t *); 1270 extern void mlxcx_buf_return(mlxcx_t *, mlxcx_buffer_t *); 1271 extern void mlxcx_buf_return_chain(mlxcx_t *, mlxcx_buffer_t *, boolean_t); 1272 extern void mlxcx_buf_destroy(mlxcx_t *, mlxcx_buffer_t *); 1273 extern void mlxcx_shard_ready(mlxcx_buf_shard_t *); 1274 extern void mlxcx_shard_draining(mlxcx_buf_shard_t *); 1275 1276 extern uint_t mlxcx_buf_bind_or_copy(mlxcx_t *, mlxcx_work_queue_t *, 1277 mblk_t *, size_t, mlxcx_buffer_t **); 1278 1279 extern boolean_t mlxcx_rx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1280 extern boolean_t mlxcx_tx_group_setup(mlxcx_t *, mlxcx_ring_group_t *); 1281 1282 extern boolean_t mlxcx_rx_group_start(mlxcx_t *, mlxcx_ring_group_t *); 1283 extern boolean_t mlxcx_tx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1284 mlxcx_work_queue_t *); 1285 extern boolean_t mlxcx_rx_ring_start(mlxcx_t *, mlxcx_ring_group_t *, 1286 mlxcx_work_queue_t *); 1287 1288 extern boolean_t mlxcx_rq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1289 mlxcx_buffer_t *); 1290 extern boolean_t mlxcx_rq_add_buffers(mlxcx_t *, mlxcx_work_queue_t *, 1291 mlxcx_buffer_t **, size_t); 1292 extern boolean_t mlxcx_sq_add_buffer(mlxcx_t *, mlxcx_work_queue_t *, 1293 uint8_t *, size_t, uint32_t, mlxcx_buffer_t *); 1294 extern boolean_t mlxcx_sq_add_nop(mlxcx_t *, mlxcx_work_queue_t *); 1295 extern void mlxcx_rq_refill(mlxcx_t *, mlxcx_work_queue_t *); 1296 1297 extern void mlxcx_teardown_groups(mlxcx_t *); 1298 extern void mlxcx_wq_teardown(mlxcx_t *, mlxcx_work_queue_t *); 1299 extern void mlxcx_cq_teardown(mlxcx_t *, mlxcx_completion_queue_t *); 1300 extern void mlxcx_teardown_rx_group(mlxcx_t *, mlxcx_ring_group_t *); 1301 extern void mlxcx_teardown_tx_group(mlxcx_t *, mlxcx_ring_group_t *); 1302 1303 extern void mlxcx_tx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1304 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1305 extern mblk_t *mlxcx_rx_completion(mlxcx_t *, mlxcx_completion_queue_t *, 1306 mlxcx_completionq_ent_t *, mlxcx_buffer_t *); 1307 1308 extern mlxcx_buf_shard_t *mlxcx_mlbs_create(mlxcx_t *); 1309 1310 /* 1311 * Flow mgmt 1312 */ 1313 extern boolean_t mlxcx_add_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1314 mlxcx_ring_group_t *, const uint8_t *); 1315 extern boolean_t mlxcx_remove_umcast_entry(mlxcx_t *, mlxcx_port_t *, 1316 mlxcx_ring_group_t *, const uint8_t *); 1317 extern void mlxcx_remove_all_umcast_entries(mlxcx_t *, mlxcx_port_t *, 1318 mlxcx_ring_group_t *); 1319 extern boolean_t mlxcx_setup_flow_group(mlxcx_t *, mlxcx_flow_table_t *, 1320 mlxcx_flow_group_t *); 1321 extern void mlxcx_teardown_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1322 1323 extern void mlxcx_remove_all_vlan_entries(mlxcx_t *, mlxcx_ring_group_t *); 1324 extern boolean_t mlxcx_remove_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1325 boolean_t, uint16_t); 1326 extern boolean_t mlxcx_add_vlan_entry(mlxcx_t *, mlxcx_ring_group_t *, 1327 boolean_t, uint16_t); 1328 1329 /* 1330 * Command functions 1331 */ 1332 extern boolean_t mlxcx_cmd_queue_init(mlxcx_t *); 1333 extern void mlxcx_cmd_queue_fini(mlxcx_t *); 1334 1335 extern void mlxcx_cmd_completion(mlxcx_t *, mlxcx_eventq_ent_t *); 1336 extern void mlxcx_cmd_eq_enable(mlxcx_t *); 1337 extern void mlxcx_cmd_eq_disable(mlxcx_t *); 1338 1339 extern boolean_t mlxcx_cmd_enable_hca(mlxcx_t *); 1340 extern boolean_t mlxcx_cmd_disable_hca(mlxcx_t *); 1341 1342 extern boolean_t mlxcx_cmd_query_issi(mlxcx_t *, uint_t *); 1343 extern boolean_t mlxcx_cmd_set_issi(mlxcx_t *, uint16_t); 1344 1345 extern boolean_t mlxcx_cmd_query_pages(mlxcx_t *, uint_t, int32_t *); 1346 extern boolean_t mlxcx_cmd_give_pages(mlxcx_t *, uint_t, int32_t, 1347 mlxcx_dev_page_t **); 1348 extern boolean_t mlxcx_cmd_return_pages(mlxcx_t *, int32_t, uint64_t *, 1349 int32_t *); 1350 1351 extern boolean_t mlxcx_cmd_query_hca_cap(mlxcx_t *, mlxcx_hca_cap_type_t, 1352 mlxcx_hca_cap_mode_t, mlxcx_hca_cap_t *); 1353 1354 extern boolean_t mlxcx_cmd_set_driver_version(mlxcx_t *, const char *); 1355 1356 extern boolean_t mlxcx_cmd_init_hca(mlxcx_t *); 1357 extern boolean_t mlxcx_cmd_teardown_hca(mlxcx_t *); 1358 1359 extern boolean_t mlxcx_cmd_alloc_uar(mlxcx_t *, mlxcx_uar_t *); 1360 extern boolean_t mlxcx_cmd_dealloc_uar(mlxcx_t *, mlxcx_uar_t *); 1361 1362 extern boolean_t mlxcx_cmd_alloc_pd(mlxcx_t *, mlxcx_pd_t *); 1363 extern boolean_t mlxcx_cmd_dealloc_pd(mlxcx_t *, mlxcx_pd_t *); 1364 1365 extern boolean_t mlxcx_cmd_alloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1366 extern boolean_t mlxcx_cmd_dealloc_tdom(mlxcx_t *, mlxcx_tdom_t *); 1367 1368 extern boolean_t mlxcx_cmd_create_eq(mlxcx_t *, mlxcx_event_queue_t *); 1369 extern boolean_t mlxcx_cmd_destroy_eq(mlxcx_t *, mlxcx_event_queue_t *); 1370 extern boolean_t mlxcx_cmd_query_eq(mlxcx_t *, mlxcx_event_queue_t *, 1371 mlxcx_eventq_ctx_t *); 1372 1373 extern boolean_t mlxcx_cmd_create_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1374 extern boolean_t mlxcx_cmd_destroy_cq(mlxcx_t *, mlxcx_completion_queue_t *); 1375 extern boolean_t mlxcx_cmd_query_cq(mlxcx_t *, mlxcx_completion_queue_t *, 1376 mlxcx_completionq_ctx_t *); 1377 1378 extern boolean_t mlxcx_cmd_create_rq(mlxcx_t *, mlxcx_work_queue_t *); 1379 extern boolean_t mlxcx_cmd_start_rq(mlxcx_t *, mlxcx_work_queue_t *); 1380 extern boolean_t mlxcx_cmd_stop_rq(mlxcx_t *, mlxcx_work_queue_t *); 1381 extern boolean_t mlxcx_cmd_destroy_rq(mlxcx_t *, mlxcx_work_queue_t *); 1382 extern boolean_t mlxcx_cmd_query_rq(mlxcx_t *, mlxcx_work_queue_t *, 1383 mlxcx_rq_ctx_t *); 1384 1385 extern boolean_t mlxcx_cmd_create_tir(mlxcx_t *, mlxcx_tir_t *); 1386 extern boolean_t mlxcx_cmd_destroy_tir(mlxcx_t *, mlxcx_tir_t *); 1387 1388 extern boolean_t mlxcx_cmd_create_sq(mlxcx_t *, mlxcx_work_queue_t *); 1389 extern boolean_t mlxcx_cmd_start_sq(mlxcx_t *, mlxcx_work_queue_t *); 1390 extern boolean_t mlxcx_cmd_stop_sq(mlxcx_t *, mlxcx_work_queue_t *); 1391 extern boolean_t mlxcx_cmd_destroy_sq(mlxcx_t *, mlxcx_work_queue_t *); 1392 extern boolean_t mlxcx_cmd_query_sq(mlxcx_t *, mlxcx_work_queue_t *, 1393 mlxcx_sq_ctx_t *); 1394 1395 extern boolean_t mlxcx_cmd_create_tis(mlxcx_t *, mlxcx_tis_t *); 1396 extern boolean_t mlxcx_cmd_destroy_tis(mlxcx_t *, mlxcx_tis_t *); 1397 1398 extern boolean_t mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *); 1399 extern boolean_t mlxcx_cmd_query_special_ctxs(mlxcx_t *); 1400 1401 extern boolean_t mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *, mlxcx_port_t *, 1402 mlxcx_modify_nic_vport_ctx_fields_t); 1403 1404 extern boolean_t mlxcx_cmd_create_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1405 extern boolean_t mlxcx_cmd_destroy_flow_table(mlxcx_t *, mlxcx_flow_table_t *); 1406 extern boolean_t mlxcx_cmd_set_flow_table_root(mlxcx_t *, mlxcx_flow_table_t *); 1407 1408 extern boolean_t mlxcx_cmd_create_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1409 extern boolean_t mlxcx_cmd_set_flow_table_entry(mlxcx_t *, 1410 mlxcx_flow_entry_t *); 1411 extern boolean_t mlxcx_cmd_delete_flow_table_entry(mlxcx_t *, 1412 mlxcx_flow_entry_t *); 1413 extern boolean_t mlxcx_cmd_destroy_flow_group(mlxcx_t *, mlxcx_flow_group_t *); 1414 1415 extern boolean_t mlxcx_cmd_access_register(mlxcx_t *, mlxcx_cmd_reg_opmod_t, 1416 mlxcx_register_id_t, mlxcx_register_data_t *); 1417 extern boolean_t mlxcx_cmd_query_port_mtu(mlxcx_t *, mlxcx_port_t *); 1418 extern boolean_t mlxcx_cmd_query_port_status(mlxcx_t *, mlxcx_port_t *); 1419 extern boolean_t mlxcx_cmd_modify_port_status(mlxcx_t *, mlxcx_port_t *, 1420 mlxcx_port_status_t); 1421 extern boolean_t mlxcx_cmd_query_port_speed(mlxcx_t *, mlxcx_port_t *); 1422 extern boolean_t mlxcx_cmd_query_port_fec(mlxcx_t *, mlxcx_port_t *); 1423 extern boolean_t mlxcx_cmd_modify_port_fec(mlxcx_t *, mlxcx_port_t *, 1424 mlxcx_pplm_fec_caps_t); 1425 1426 extern boolean_t mlxcx_cmd_set_port_mtu(mlxcx_t *, mlxcx_port_t *); 1427 1428 extern boolean_t mlxcx_cmd_create_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1429 extern boolean_t mlxcx_cmd_destroy_rqt(mlxcx_t *, mlxcx_rqtable_t *); 1430 1431 extern boolean_t mlxcx_cmd_set_int_mod(mlxcx_t *, uint_t, uint_t); 1432 1433 extern boolean_t mlxcx_cmd_query_module_status(mlxcx_t *, uint_t, 1434 mlxcx_module_status_t *, mlxcx_module_error_type_t *); 1435 extern boolean_t mlxcx_cmd_set_port_led(mlxcx_t *, mlxcx_port_t *, uint16_t); 1436 1437 /* Comparator for avl_ts */ 1438 extern int mlxcx_cq_compare(const void *, const void *); 1439 extern int mlxcx_dmac_fe_compare(const void *, const void *); 1440 extern int mlxcx_grmac_compare(const void *, const void *); 1441 extern int mlxcx_page_compare(const void *, const void *); 1442 1443 extern void mlxcx_update_link_state(mlxcx_t *, mlxcx_port_t *); 1444 1445 extern void mlxcx_eth_proto_to_string(mlxcx_eth_proto_t, char *, size_t); 1446 extern const char *mlxcx_port_status_string(mlxcx_port_status_t); 1447 1448 extern const char *mlxcx_event_name(mlxcx_event_t); 1449 1450 #ifdef __cplusplus 1451 } 1452 #endif 1453 1454 #endif /* _MLXCX_H */ 1455