1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved. 14 * Copyright 2019 Joyent, Inc. 15 * Copyright 2017 Tegile Systems, Inc. All rights reserved. 16 * Copyright 2020 RackTop Systems, Inc. 17 */ 18 19 /* 20 * i40e - Intel 10/40 Gb Ethernet driver 21 * 22 * The i40e driver is the main software device driver for the Intel 40 Gb family 23 * of devices. Note that these devices come in many flavors with both 40 GbE 24 * ports and 10 GbE ports. This device is the successor to the 82599 family of 25 * devices (ixgbe). 26 * 27 * Unlike previous generations of Intel 1 GbE and 10 GbE devices, the 40 GbE 28 * devices defined in the XL710 controller (previously known as Fortville) are a 29 * rather different beast and have a small switch embedded inside of them. In 30 * addition, the way that most of the programming is done has been overhauled. 31 * As opposed to just using PCIe memory mapped registers, it also has an 32 * administrative queue which is used to communicate with firmware running on 33 * the chip. 34 * 35 * Each physical function in the hardware shows up as a device that this driver 36 * will bind to. The hardware splits many resources evenly across all of the 37 * physical functions present on the device, while other resources are instead 38 * shared across the entire card and its up to the device driver to 39 * intelligently partition them. 40 * 41 * ------------ 42 * Organization 43 * ------------ 44 * 45 * This driver is made up of several files which have their own theory 46 * statements spread across them. We'll touch on the high level purpose of each 47 * file here, and then we'll get into more discussion on how the device is 48 * generally modelled with respect to the interfaces in illumos. 49 * 50 * i40e_gld.c: This file contains all of the bindings to MAC and the networking 51 * stack. 52 * 53 * i40e_intr.c: This file contains all of the interrupt service routines and 54 * contains logic to enable and disable interrupts on the hardware. 55 * It also contains the logic to map hardware resources such as the 56 * rings to and from interrupts and controls their ability to fire. 57 * 58 * There is a big theory statement on interrupts present there. 59 * 60 * i40e_main.c: The file that you're currently in. It interfaces with the 61 * traditional OS DDI interfaces and is in charge of configuring 62 * the device. 63 * 64 * i40e_osdep.[ch]: These files contain interfaces and definitions needed to 65 * work with Intel's common code for the device. 66 * 67 * i40e_stats.c: This file contains the general work and logic around our 68 * kstats. A theory statement on their organization and use of the 69 * hardware exists there. 70 * 71 * i40e_sw.h: This header file contains all of the primary structure definitions 72 * and constants that are used across the entire driver. 73 * 74 * i40e_transceiver.c: This file contains all of the logic for sending and 75 * receiving data. It contains all of the ring and DMA 76 * allocation logic, as well as, the actual interfaces to 77 * send and receive data. 78 * 79 * A big theory statement on ring management, descriptors, 80 * and how it ties into the OS is present there. 81 * 82 * -------------- 83 * General Design 84 * -------------- 85 * 86 * Before we go too far into the general way we've laid out data structures and 87 * the like, it's worth taking some time to explain how the hardware is 88 * organized. This organization informs a lot of how we do things at this time 89 * in the driver. 90 * 91 * Each physical device consists of a number of one or more ports, which are 92 * considered physical functions in the PCI sense and thus each get enumerated 93 * by the system, resulting in an instance being created and attached to. While 94 * there are many resources that are unique to each physical function eg. 95 * instance of the device, there are many that are shared across all of them. 96 * Several resources have an amount reserved for each Virtual Station Interface 97 * (VSI) and then a static pool of resources, available for all functions on the 98 * card. 99 * 100 * The most important resource in hardware are its transmit and receive queue 101 * pairs (i40e_trqpair_t). These should be thought of as rings in GLDv3 102 * parlance. There are a set number of these on each device; however, they are 103 * statically partitioned among all of the different physical functions. 104 * 105 * 'Fortville' (the code name for this device family) is basically a switch. To 106 * map MAC addresses and other things to queues, we end up having to create 107 * Virtual Station Interfaces (VSIs) and establish forwarding rules that direct 108 * traffic to a queue. A VSI owns a collection of queues and has a series of 109 * forwarding rules that point to it. One way to think of this is to treat it 110 * like MAC does a VNIC. When MAC refers to a group, a collection of rings and 111 * classification resources, that is a VSI in i40e. 112 * 113 * The sets of VSIs is shared across the entire device, though there may be some 114 * amount that are reserved to each PF. Because the GLDv3 does not let us change 115 * the number of groups dynamically, we instead statically divide this amount 116 * evenly between all the functions that exist. In addition, we have the same 117 * problem with the mac address forwarding rules. There are a static number that 118 * exist shared across all the functions. 119 * 120 * To handle both of these resources, what we end up doing is going through and 121 * determining which functions belong to the same device. Nominally one might do 122 * this by having a nexus driver; however, a prime requirement for a nexus 123 * driver is identifying the various children and activating them. While it is 124 * possible to get this information from NVRAM, we would end up duplicating a 125 * lot of the PCI enumeration logic. Really, at the end of the day, the device 126 * doesn't give us the traditional identification properties we want from a 127 * nexus driver. 128 * 129 * Instead, we rely on some properties that are guaranteed to be unique. While 130 * it might be tempting to leverage the PBA or serial number of the device from 131 * NVRAM, there is nothing that says that two devices can't be mis-programmed to 132 * have the same values in NVRAM. Instead, we uniquely identify a group of 133 * functions based on their parent in the /devices tree, their PCI bus and PCI 134 * function identifiers. Using either on their own may not be sufficient. 135 * 136 * For each unique PCI device that we encounter, we'll create a i40e_device_t. 137 * From there, because we don't have a good way to tell the GLDv3 about sharing 138 * resources between everything, we'll end up just dividing the resources 139 * evenly between all of the functions. Longer term, if we don't have to declare 140 * to the GLDv3 that these resources are shared, then we'll maintain a pool and 141 * have each PF allocate from the pool in the device, thus if only two of four 142 * ports are being used, for example, then all of the resources can still be 143 * used. 144 * 145 * ------------------------------------------- 146 * Transmit and Receive Queue Pair Allocations 147 * ------------------------------------------- 148 * 149 * NVRAM ends up assigning each PF its own share of the transmit and receive LAN 150 * queue pairs, we have no way of modifying it, only observing it. From there, 151 * it's up to us to map these queues to VSIs and VFs. Since we don't support any 152 * VFs at this time, we only focus on assignments to VSIs. 153 * 154 * At the moment, we used a static mapping of transmit/receive queue pairs to a 155 * given VSI (eg. rings to a group). Though in the fullness of time, we want to 156 * make this something which is fully dynamic and take advantage of documented, 157 * but not yet available functionality for adding filters based on VXLAN and 158 * other encapsulation technologies. 159 * 160 * ------------------------------------- 161 * Broadcast, Multicast, and Promiscuous 162 * ------------------------------------- 163 * 164 * As part of the GLDv3, we need to make sure that we can handle receiving 165 * broadcast and multicast traffic. As well as enabling promiscuous mode when 166 * requested. GLDv3 requires that all broadcast and multicast traffic be 167 * retrieved by the default group, eg. the first one. This is the same thing as 168 * the default VSI. 169 * 170 * To receieve broadcast traffic, we enable it through the admin queue, rather 171 * than use one of our filters for it. For multicast traffic, we reserve a 172 * certain number of the hash filters and assign them to a given PF. When we 173 * exceed those, we then switch to using promiscuous mode for multicast traffic. 174 * 175 * More specifically, once we exceed the number of filters (indicated because 176 * the i40e_t`i40e_resources.ifr_nmcastfilt == 177 * i40e_t`i40e_resources.ifr_nmcastfilt_used), we then instead need to toggle 178 * promiscuous mode. If promiscuous mode is toggled then we keep track of the 179 * number of MACs added to it by incrementing i40e_t`i40e_mcast_promisc_count. 180 * That will stay enabled until that count reaches zero indicating that we have 181 * only added multicast addresses that we have a corresponding entry for. 182 * 183 * Because MAC itself wants to toggle promiscuous mode, which includes both 184 * unicast and multicast traffic, we go through and keep track of that 185 * ourselves. That is maintained through the use of the i40e_t`i40e_promisc_on 186 * member. 187 * 188 * -------------- 189 * VSI Management 190 * -------------- 191 * 192 * The PFs share 384 VSIs. The firmware creates one VSI per PF by default. 193 * During chip start we retrieve the SEID of this VSI and assign it as the 194 * default VSI for our VEB (one VEB per PF). We then add additional VSIs to 195 * the VEB up to the determined number of rx groups: i40e_t`i40e_num_rx_groups. 196 * We currently cap this number to I40E_GROUP_MAX to a) make sure all PFs can 197 * allocate the same number of VSIs, and b) to keep the interrupt multiplexing 198 * under control. In the future, when we improve the interrupt allocation, we 199 * may want to revisit this cap to make better use of the available VSIs. The 200 * VSI allocation and configuration can be found in i40e_chip_start(). 201 * 202 * ---------------- 203 * Structure Layout 204 * ---------------- 205 * 206 * The following images relates the core data structures together. The primary 207 * structure in the system is the i40e_t. It itself contains multiple rings, 208 * i40e_trqpair_t's which contain the various transmit and receive data. The 209 * receive data is stored outside of the i40e_trqpair_t and instead in the 210 * i40e_rx_data_t. The i40e_t has a corresponding i40e_device_t which keeps 211 * track of per-physical device state. Finally, for every active descriptor, 212 * there is a corresponding control block, which is where the 213 * i40e_rx_control_block_t and the i40e_tx_control_block_t come from. 214 * 215 * +-----------------------+ +-----------------------+ 216 * | Global i40e_t list | | Global Device list | 217 * | | +--| | 218 * | i40e_glist | | | i40e_dlist | 219 * +-----------------------+ | +-----------------------+ 220 * | v 221 * | +------------------------+ +-----------------------+ 222 * | | Device-wide Structure |----->| Device-wide Structure |--> ... 223 * | | i40e_device_t | | i40e_device_t | 224 * | | | +-----------------------+ 225 * | | dev_info_t * ------+--> Parent in devices tree. 226 * | | uint_t ------+--> PCI bus number 227 * | | uint_t ------+--> PCI device number 228 * | | uint_t ------+--> Number of functions 229 * | | i40e_switch_rsrcs_t ---+--> Captured total switch resources 230 * | | list_t ------+-------------+ 231 * | +------------------------+ | 232 * | ^ | 233 * | +--------+ | 234 * | | v 235 * | +---------------------------+ | +-------------------+ 236 * +->| GLDv3 Device, per PF |-----|-->| GLDv3 Device (PF) |--> ... 237 * | i40e_t | | | i40e_t | 238 * | **Primary Structure** | | +-------------------+ 239 * | | | 240 * | i40e_device_t * --+-----+ 241 * | i40e_state_t --+---> Device State 242 * | i40e_hw_t --+---> Intel common code structure 243 * | mac_handle_t --+---> GLDv3 handle to MAC 244 * | ddi_periodic_t --+---> Link activity timer 245 * | i40e_vsi_t * --+---> Array of VSIs 246 * | i40e_func_rsrc_t --+---> Available hardware resources 247 * | i40e_switch_rsrc_t * --+---> Switch resource snapshot 248 * | i40e_sdu --+---> Current MTU 249 * | i40e_frame_max --+---> Current HW frame size 250 * | i40e_uaddr_t * --+---> Array of assigned unicast MACs 251 * | i40e_maddr_t * --+---> Array of assigned multicast MACs 252 * | i40e_mcast_promisccount --+---> Active multicast state 253 * | i40e_promisc_on --+---> Current promiscuous mode state 254 * | uint_t --+---> Number of transmit/receive pairs 255 * | i40e_rx_group_t * --+---> Array of Rx groups 256 * | kstat_t * --+---> PF kstats 257 * | i40e_pf_stats_t --+---> PF kstat backing data 258 * | i40e_trqpair_t * --+---------+ 259 * +---------------------------+ | 260 * | 261 * v 262 * +-------------------------------+ +-----------------------------+ 263 * | Transmit/Receive Queue Pair |-------| Transmit/Receive Queue Pair |->... 264 * | i40e_trqpair_t | | i40e_trqpair_t | 265 * + Ring Data Structure | +-----------------------------+ 266 * | | 267 * | mac_ring_handle_t +--> MAC RX ring handle 268 * | mac_ring_handle_t +--> MAC TX ring handle 269 * | i40e_rxq_stat_t --+--> RX Queue stats 270 * | i40e_txq_stat_t --+--> TX Queue stats 271 * | uint32_t (tx ring size) +--> TX Ring Size 272 * | uint32_t (tx free list size) +--> TX Free List Size 273 * | i40e_dma_buffer_t --------+--> TX Descriptor ring DMA 274 * | i40e_tx_desc_t * --------+--> TX descriptor ring 275 * | volatile unt32_t * +--> TX Write back head 276 * | uint32_t -------+--> TX ring head 277 * | uint32_t -------+--> TX ring tail 278 * | uint32_t -------+--> Num TX desc free 279 * | i40e_tx_control_block_t * --+--> TX control block array ---+ 280 * | i40e_tx_control_block_t ** --+--> TCB work list ----+ 281 * | i40e_tx_control_block_t ** --+--> TCB free list ---+ 282 * | uint32_t -------+--> Free TCB count | 283 * | i40e_rx_data_t * -------+--+ v 284 * +-------------------------------+ | +---------------------------+ 285 * | | Per-TX Frame Metadata | 286 * | | i40e_tx_control_block_t | 287 * +--------------------+ | | 288 * | mblk to transmit <--+--- mblk_t * | 289 * | type of transmit <--+--- i40e_tx_type_t | 290 * | TX DMA handle <--+--- ddi_dma_handle_t | 291 * v TX DMA buffer <--+--- i40e_dma_buffer_t | 292 * +------------------------------+ +---------------------------+ 293 * | Core Receive Data | 294 * | i40e_rx_data_t | 295 * | | 296 * | i40e_dma_buffer_t --+--> RX descriptor DMA Data 297 * | i40e_rx_desc_t --+--> RX descriptor ring 298 * | uint32_t --+--> Next free desc. 299 * | i40e_rx_control_block_t * --+--> RX Control Block Array ---+ 300 * | i40e_rx_control_block_t ** --+--> RCB work list ---+ 301 * | i40e_rx_control_block_t ** --+--> RCB free list ---+ 302 * +------------------------------+ | 303 * ^ | 304 * | +---------------------------+ | 305 * | | Per-RX Frame Metadata |<---------------+ 306 * | | i40e_rx_control_block_t | 307 * | | | 308 * | | mblk_t * ----+--> Received mblk_t data 309 * | | uint32_t ----+--> Reference count 310 * | | i40e_dma_buffer_t ----+--> Receive data DMA info 311 * | | frtn_t ----+--> mblk free function info 312 * +-----+-- i40e_rx_data_t * | 313 * +---------------------------+ 314 * 315 * ------------- 316 * Lock Ordering 317 * ------------- 318 * 319 * In order to ensure that we don't deadlock, the following represents the 320 * lock order being used. When grabbing locks, follow the following order. Lower 321 * numbers are more important. Thus, the i40e_glock which is number 0, must be 322 * taken before any other locks in the driver. On the other hand, the 323 * i40e_t`i40e_stat_lock, has the highest number because it's the least 324 * important lock. Note, that just because one lock is higher than another does 325 * not mean that all intermediary locks are required. 326 * 327 * 0) i40e_glock 328 * 1) i40e_t`i40e_general_lock 329 * 330 * 2) i40e_trqpair_t`itrq_rx_lock 331 * 3) i40e_trqpair_t`itrq_tx_lock 332 * 4) i40e_t`i40e_rx_pending_lock 333 * 5) i40e_trqpair_t`itrq_tcb_lock 334 * 335 * 6) i40e_t`i40e_stat_lock 336 * 337 * Rules and expectations: 338 * 339 * 1) A thread holding locks belong to one PF should not hold locks belonging to 340 * a second. If for some reason this becomes necessary, locks should be grabbed 341 * based on the list order in the i40e_device_t, which implies that the 342 * i40e_glock is held. 343 * 344 * 2) When grabbing locks between multiple transmit and receive queues, the 345 * locks for the lowest number transmit/receive queue should be grabbed first. 346 * 347 * 3) When grabbing both the transmit and receive lock for a given queue, always 348 * grab i40e_trqpair_t`itrq_rx_lock before the i40e_trqpair_t`itrq_tx_lock. 349 * 350 * 4) The following pairs of locks are not expected to be held at the same time: 351 * 352 * o i40e_t`i40e_rx_pending_lock and i40e_trqpair_t`itrq_tcb_lock 353 * 354 * ----------- 355 * Future Work 356 * ----------- 357 * 358 * At the moment the i40e_t driver is rather bare bones, allowing us to start 359 * getting data flowing and folks using it while we develop additional features. 360 * While bugs have been filed to cover this future work, the following gives an 361 * overview of expected work: 362 * 363 * o DMA binding and breaking up the locking in ring recycling. 364 * o Enhanced detection of device errors 365 * o Participation in IRM 366 * o FMA device reset 367 * o Stall detection, temperature error detection, etc. 368 * o More dynamic resource pools 369 */ 370 371 #include "i40e_sw.h" 372 373 static char i40e_ident[] = "Intel 10/40Gb Ethernet v1.0.3"; 374 375 /* 376 * The i40e_glock primarily protects the lists below and the i40e_device_t 377 * structures. 378 */ 379 static kmutex_t i40e_glock; 380 static list_t i40e_glist; 381 static list_t i40e_dlist; 382 383 /* 384 * Access attributes for register mapping. 385 */ 386 static ddi_device_acc_attr_t i40e_regs_acc_attr = { 387 DDI_DEVICE_ATTR_V1, 388 DDI_STRUCTURE_LE_ACC, 389 DDI_STRICTORDER_ACC, 390 DDI_FLAGERR_ACC 391 }; 392 393 /* 394 * Logging function for this driver. 395 */ 396 static void 397 i40e_dev_err(i40e_t *i40e, int level, boolean_t console, const char *fmt, 398 va_list ap) 399 { 400 char buf[1024]; 401 402 (void) vsnprintf(buf, sizeof (buf), fmt, ap); 403 404 if (i40e == NULL) { 405 cmn_err(level, (console) ? "%s: %s" : "!%s: %s", 406 I40E_MODULE_NAME, buf); 407 } else { 408 dev_err(i40e->i40e_dip, level, (console) ? "%s" : "!%s", 409 buf); 410 } 411 } 412 413 /* 414 * Because there's the stupid trailing-comma problem with the C preprocessor 415 * and variable arguments, I need to instantiate these. Pardon the redundant 416 * code. 417 */ 418 /*PRINTFLIKE2*/ 419 void 420 i40e_error(i40e_t *i40e, const char *fmt, ...) 421 { 422 va_list ap; 423 424 va_start(ap, fmt); 425 i40e_dev_err(i40e, CE_WARN, B_FALSE, fmt, ap); 426 va_end(ap); 427 } 428 429 /*PRINTFLIKE2*/ 430 void 431 i40e_log(i40e_t *i40e, const char *fmt, ...) 432 { 433 va_list ap; 434 435 va_start(ap, fmt); 436 i40e_dev_err(i40e, CE_NOTE, B_FALSE, fmt, ap); 437 va_end(ap); 438 } 439 440 /*PRINTFLIKE2*/ 441 void 442 i40e_notice(i40e_t *i40e, const char *fmt, ...) 443 { 444 va_list ap; 445 446 va_start(ap, fmt); 447 i40e_dev_err(i40e, CE_NOTE, B_TRUE, fmt, ap); 448 va_end(ap); 449 } 450 451 /* 452 * Various parts of the driver need to know if the controller is from the X722 453 * family, which has a few additional capabilities and different programming 454 * means. We don't consider virtual functions as part of this as they are quite 455 * different and will require substantially more work. 456 */ 457 static boolean_t 458 i40e_is_x722(i40e_t *i40e) 459 { 460 return (i40e->i40e_hw_space.mac.type == I40E_MAC_X722); 461 } 462 463 static void 464 i40e_device_rele(i40e_t *i40e) 465 { 466 i40e_device_t *idp = i40e->i40e_device; 467 468 if (idp == NULL) 469 return; 470 471 mutex_enter(&i40e_glock); 472 VERIFY(idp->id_nreg > 0); 473 list_remove(&idp->id_i40e_list, i40e); 474 idp->id_nreg--; 475 if (idp->id_nreg == 0) { 476 list_remove(&i40e_dlist, idp); 477 list_destroy(&idp->id_i40e_list); 478 kmem_free(idp->id_rsrcs, sizeof (i40e_switch_rsrc_t) * 479 idp->id_rsrcs_alloc); 480 kmem_free(idp, sizeof (i40e_device_t)); 481 } 482 i40e->i40e_device = NULL; 483 mutex_exit(&i40e_glock); 484 } 485 486 static i40e_device_t * 487 i40e_device_find(i40e_t *i40e, dev_info_t *parent, uint_t bus, uint_t device) 488 { 489 i40e_device_t *idp; 490 mutex_enter(&i40e_glock); 491 for (idp = list_head(&i40e_dlist); idp != NULL; 492 idp = list_next(&i40e_dlist, idp)) { 493 if (idp->id_parent == parent && idp->id_pci_bus == bus && 494 idp->id_pci_device == device) { 495 break; 496 } 497 } 498 499 if (idp != NULL) { 500 VERIFY(idp->id_nreg < idp->id_nfuncs); 501 idp->id_nreg++; 502 } else { 503 i40e_hw_t *hw = &i40e->i40e_hw_space; 504 ASSERT(hw->num_ports > 0); 505 ASSERT(hw->num_partitions > 0); 506 507 /* 508 * The Intel common code doesn't exactly keep the number of PCI 509 * functions. But it calculates it during discovery of 510 * partitions and ports. So what we do is undo the calculation 511 * that it does originally, as functions are evenly spread 512 * across ports in the rare case of partitions. 513 */ 514 idp = kmem_alloc(sizeof (i40e_device_t), KM_SLEEP); 515 idp->id_parent = parent; 516 idp->id_pci_bus = bus; 517 idp->id_pci_device = device; 518 idp->id_nfuncs = hw->num_ports * hw->num_partitions; 519 idp->id_nreg = 1; 520 idp->id_rsrcs_alloc = i40e->i40e_switch_rsrc_alloc; 521 idp->id_rsrcs_act = i40e->i40e_switch_rsrc_actual; 522 idp->id_rsrcs = kmem_alloc(sizeof (i40e_switch_rsrc_t) * 523 idp->id_rsrcs_alloc, KM_SLEEP); 524 bcopy(i40e->i40e_switch_rsrcs, idp->id_rsrcs, 525 sizeof (i40e_switch_rsrc_t) * idp->id_rsrcs_alloc); 526 list_create(&idp->id_i40e_list, sizeof (i40e_t), 527 offsetof(i40e_t, i40e_dlink)); 528 529 list_insert_tail(&i40e_dlist, idp); 530 } 531 532 list_insert_tail(&idp->id_i40e_list, i40e); 533 mutex_exit(&i40e_glock); 534 535 return (idp); 536 } 537 538 static void 539 i40e_link_state_set(i40e_t *i40e, link_state_t state) 540 { 541 if (i40e->i40e_link_state == state) 542 return; 543 544 i40e->i40e_link_state = state; 545 mac_link_update(i40e->i40e_mac_hdl, i40e->i40e_link_state); 546 } 547 548 /* 549 * This is a basic link check routine. Mostly we're using this just to see 550 * if we can get any accurate information about the state of the link being 551 * up or down, as well as updating the link state, speed, etc. information. 552 */ 553 void 554 i40e_link_check(i40e_t *i40e) 555 { 556 i40e_hw_t *hw = &i40e->i40e_hw_space; 557 boolean_t ls; 558 int ret; 559 560 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 561 562 hw->phy.get_link_info = B_TRUE; 563 if ((ret = i40e_get_link_status(hw, &ls)) != I40E_SUCCESS) { 564 i40e->i40e_s_link_status_errs++; 565 i40e->i40e_s_link_status_lasterr = ret; 566 return; 567 } 568 569 /* 570 * Firmware abstracts all of the mac and phy information for us, so we 571 * can use i40e_get_link_status to determine the current state. 572 */ 573 if (ls == B_TRUE) { 574 enum i40e_aq_link_speed speed; 575 576 speed = i40e_get_link_speed(hw); 577 578 /* 579 * Translate from an i40e value to a value in Mbits/s. 580 */ 581 switch (speed) { 582 case I40E_LINK_SPEED_100MB: 583 i40e->i40e_link_speed = 100; 584 break; 585 case I40E_LINK_SPEED_1GB: 586 i40e->i40e_link_speed = 1000; 587 break; 588 case I40E_LINK_SPEED_10GB: 589 i40e->i40e_link_speed = 10000; 590 break; 591 case I40E_LINK_SPEED_20GB: 592 i40e->i40e_link_speed = 20000; 593 break; 594 case I40E_LINK_SPEED_40GB: 595 i40e->i40e_link_speed = 40000; 596 break; 597 case I40E_LINK_SPEED_25GB: 598 i40e->i40e_link_speed = 25000; 599 break; 600 default: 601 i40e->i40e_link_speed = 0; 602 break; 603 } 604 605 /* 606 * At this time, hardware does not support half-duplex 607 * operation, hence why we don't ask the hardware about our 608 * current speed. 609 */ 610 i40e->i40e_link_duplex = LINK_DUPLEX_FULL; 611 i40e_link_state_set(i40e, LINK_STATE_UP); 612 } else { 613 i40e->i40e_link_speed = 0; 614 i40e->i40e_link_duplex = 0; 615 i40e_link_state_set(i40e, LINK_STATE_DOWN); 616 } 617 } 618 619 static void 620 i40e_rem_intrs(i40e_t *i40e) 621 { 622 int i, rc; 623 624 for (i = 0; i < i40e->i40e_intr_count; i++) { 625 rc = ddi_intr_free(i40e->i40e_intr_handles[i]); 626 if (rc != DDI_SUCCESS) { 627 i40e_log(i40e, "failed to free interrupt %d: %d", 628 i, rc); 629 } 630 } 631 632 kmem_free(i40e->i40e_intr_handles, i40e->i40e_intr_size); 633 i40e->i40e_intr_handles = NULL; 634 } 635 636 static void 637 i40e_rem_intr_handlers(i40e_t *i40e) 638 { 639 int i, rc; 640 641 for (i = 0; i < i40e->i40e_intr_count; i++) { 642 rc = ddi_intr_remove_handler(i40e->i40e_intr_handles[i]); 643 if (rc != DDI_SUCCESS) { 644 i40e_log(i40e, "failed to remove interrupt %d: %d", 645 i, rc); 646 } 647 } 648 } 649 650 /* 651 * illumos Fault Management Architecture (FMA) support. 652 */ 653 654 int 655 i40e_check_acc_handle(ddi_acc_handle_t handle) 656 { 657 ddi_fm_error_t de; 658 659 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 660 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 661 return (de.fme_status); 662 } 663 664 int 665 i40e_check_dma_handle(ddi_dma_handle_t handle) 666 { 667 ddi_fm_error_t de; 668 669 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 670 return (de.fme_status); 671 } 672 673 /* 674 * Fault service error handling callback function. 675 */ 676 /* ARGSUSED */ 677 static int 678 i40e_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 679 { 680 pci_ereport_post(dip, err, NULL); 681 return (err->fme_status); 682 } 683 684 static void 685 i40e_fm_init(i40e_t *i40e) 686 { 687 ddi_iblock_cookie_t iblk; 688 689 i40e->i40e_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY, 690 i40e->i40e_dip, DDI_PROP_DONTPASS, "fm_capable", 691 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 692 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 693 694 if (i40e->i40e_fm_capabilities < 0) { 695 i40e->i40e_fm_capabilities = 0; 696 } else if (i40e->i40e_fm_capabilities > 0xf) { 697 i40e->i40e_fm_capabilities = DDI_FM_EREPORT_CAPABLE | 698 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | 699 DDI_FM_ERRCB_CAPABLE; 700 } 701 702 /* 703 * Only register with IO Fault Services if we have some capability 704 */ 705 if (i40e->i40e_fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 706 i40e_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 707 } else { 708 i40e_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 709 } 710 711 if (i40e->i40e_fm_capabilities) { 712 ddi_fm_init(i40e->i40e_dip, &i40e->i40e_fm_capabilities, &iblk); 713 714 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 715 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 716 pci_ereport_setup(i40e->i40e_dip); 717 } 718 719 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 720 ddi_fm_handler_register(i40e->i40e_dip, 721 i40e_fm_error_cb, (void*)i40e); 722 } 723 } 724 725 if (i40e->i40e_fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 726 i40e_init_dma_attrs(i40e, B_TRUE); 727 } else { 728 i40e_init_dma_attrs(i40e, B_FALSE); 729 } 730 } 731 732 static void 733 i40e_fm_fini(i40e_t *i40e) 734 { 735 if (i40e->i40e_fm_capabilities) { 736 737 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 738 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 739 pci_ereport_teardown(i40e->i40e_dip); 740 741 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 742 ddi_fm_handler_unregister(i40e->i40e_dip); 743 744 ddi_fm_fini(i40e->i40e_dip); 745 } 746 } 747 748 void 749 i40e_fm_ereport(i40e_t *i40e, char *detail) 750 { 751 uint64_t ena; 752 char buf[FM_MAX_CLASS]; 753 754 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 755 ena = fm_ena_generate(0, FM_ENA_FMT1); 756 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities)) { 757 ddi_fm_ereport_post(i40e->i40e_dip, buf, ena, DDI_NOSLEEP, 758 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 759 } 760 } 761 762 /* 763 * Here we're trying to set the SEID of the default VSI. In general, 764 * when we come through and look at this shortly after attach, we 765 * expect there to only be a single element present, which is the 766 * default VSI. Importantly, each PF seems to not see any other 767 * devices, in part because of the simple switch mode that we're 768 * using. If for some reason, we see more artifacts, we'll need to 769 * revisit what we're doing here. 770 */ 771 static boolean_t 772 i40e_set_def_vsi_seid(i40e_t *i40e) 773 { 774 i40e_hw_t *hw = &i40e->i40e_hw_space; 775 struct i40e_aqc_get_switch_config_resp *sw_config; 776 uint8_t aq_buf[I40E_AQ_LARGE_BUF]; 777 uint16_t next = 0; 778 int rc; 779 780 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 781 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 782 rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next, 783 NULL); 784 if (rc != I40E_SUCCESS) { 785 i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d", 786 rc, hw->aq.asq_last_status); 787 return (B_FALSE); 788 } 789 790 if (LE_16(sw_config->header.num_reported) != 1) { 791 i40e_error(i40e, "encountered multiple (%d) switching units " 792 "during attach, not proceeding", 793 LE_16(sw_config->header.num_reported)); 794 return (B_FALSE); 795 } 796 797 I40E_DEF_VSI_SEID(i40e) = sw_config->element[0].seid; 798 return (B_TRUE); 799 } 800 801 /* 802 * Get the SEID of the uplink MAC. 803 */ 804 static int 805 i40e_get_mac_seid(i40e_t *i40e) 806 { 807 i40e_hw_t *hw = &i40e->i40e_hw_space; 808 struct i40e_aqc_get_switch_config_resp *sw_config; 809 uint8_t aq_buf[I40E_AQ_LARGE_BUF]; 810 uint16_t next = 0; 811 int rc; 812 813 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 814 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 815 rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next, 816 NULL); 817 if (rc != I40E_SUCCESS) { 818 i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d", 819 rc, hw->aq.asq_last_status); 820 return (-1); 821 } 822 823 return (LE_16(sw_config->element[0].uplink_seid)); 824 } 825 826 /* 827 * We need to fill the i40e_hw_t structure with the capabilities of this PF. We 828 * must also provide the memory for it; however, we don't need to keep it around 829 * to the call to the common code. It takes it and parses it into an internal 830 * structure. 831 */ 832 static boolean_t 833 i40e_get_hw_capabilities(i40e_t *i40e, i40e_hw_t *hw) 834 { 835 struct i40e_aqc_list_capabilities_element_resp *buf; 836 int rc; 837 size_t len; 838 uint16_t needed; 839 int nelems = I40E_HW_CAP_DEFAULT; 840 841 len = nelems * sizeof (*buf); 842 843 for (;;) { 844 ASSERT(len > 0); 845 buf = kmem_alloc(len, KM_SLEEP); 846 rc = i40e_aq_discover_capabilities(hw, buf, len, 847 &needed, i40e_aqc_opc_list_func_capabilities, NULL); 848 kmem_free(buf, len); 849 850 if (hw->aq.asq_last_status == I40E_AQ_RC_ENOMEM && 851 nelems == I40E_HW_CAP_DEFAULT) { 852 if (nelems == needed) { 853 i40e_error(i40e, "Capability discovery failed " 854 "due to byzantine common code"); 855 return (B_FALSE); 856 } 857 len = needed; 858 continue; 859 } else if (rc != I40E_SUCCESS || 860 hw->aq.asq_last_status != I40E_AQ_RC_OK) { 861 i40e_error(i40e, "Capability discovery failed: %d", rc); 862 return (B_FALSE); 863 } 864 865 break; 866 } 867 868 return (B_TRUE); 869 } 870 871 /* 872 * Obtain the switch's capabilities as seen by this PF and keep it around for 873 * our later use. 874 */ 875 static boolean_t 876 i40e_get_switch_resources(i40e_t *i40e) 877 { 878 i40e_hw_t *hw = &i40e->i40e_hw_space; 879 uint8_t cnt = 2; 880 uint8_t act; 881 size_t size; 882 i40e_switch_rsrc_t *buf; 883 884 for (;;) { 885 enum i40e_status_code ret; 886 size = cnt * sizeof (i40e_switch_rsrc_t); 887 ASSERT(size > 0); 888 if (size > UINT16_MAX) 889 return (B_FALSE); 890 buf = kmem_alloc(size, KM_SLEEP); 891 892 ret = i40e_aq_get_switch_resource_alloc(hw, &act, buf, 893 cnt, NULL); 894 if (ret == I40E_ERR_ADMIN_QUEUE_ERROR && 895 hw->aq.asq_last_status == I40E_AQ_RC_EINVAL) { 896 kmem_free(buf, size); 897 cnt += I40E_SWITCH_CAP_DEFAULT; 898 continue; 899 } else if (ret != I40E_SUCCESS) { 900 kmem_free(buf, size); 901 i40e_error(i40e, 902 "failed to retrieve switch statistics: %d", ret); 903 return (B_FALSE); 904 } 905 906 break; 907 } 908 909 i40e->i40e_switch_rsrc_alloc = cnt; 910 i40e->i40e_switch_rsrc_actual = act; 911 i40e->i40e_switch_rsrcs = buf; 912 913 return (B_TRUE); 914 } 915 916 static void 917 i40e_cleanup_resources(i40e_t *i40e) 918 { 919 if (i40e->i40e_uaddrs != NULL) { 920 kmem_free(i40e->i40e_uaddrs, sizeof (i40e_uaddr_t) * 921 i40e->i40e_resources.ifr_nmacfilt); 922 i40e->i40e_uaddrs = NULL; 923 } 924 925 if (i40e->i40e_maddrs != NULL) { 926 kmem_free(i40e->i40e_maddrs, sizeof (i40e_maddr_t) * 927 i40e->i40e_resources.ifr_nmcastfilt); 928 i40e->i40e_maddrs = NULL; 929 } 930 931 if (i40e->i40e_switch_rsrcs != NULL) { 932 size_t sz = sizeof (i40e_switch_rsrc_t) * 933 i40e->i40e_switch_rsrc_alloc; 934 ASSERT(sz > 0); 935 kmem_free(i40e->i40e_switch_rsrcs, sz); 936 i40e->i40e_switch_rsrcs = NULL; 937 } 938 939 if (i40e->i40e_device != NULL) 940 i40e_device_rele(i40e); 941 } 942 943 static boolean_t 944 i40e_get_available_resources(i40e_t *i40e) 945 { 946 dev_info_t *parent; 947 uint16_t bus, device, func; 948 uint_t nregs; 949 int *regs, i; 950 i40e_device_t *idp; 951 i40e_hw_t *hw = &i40e->i40e_hw_space; 952 953 parent = ddi_get_parent(i40e->i40e_dip); 954 955 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, i40e->i40e_dip, 0, "reg", 956 ®s, &nregs) != DDI_PROP_SUCCESS) { 957 return (B_FALSE); 958 } 959 960 if (nregs < 1) { 961 ddi_prop_free(regs); 962 return (B_FALSE); 963 } 964 965 bus = PCI_REG_BUS_G(regs[0]); 966 device = PCI_REG_DEV_G(regs[0]); 967 func = PCI_REG_FUNC_G(regs[0]); 968 ddi_prop_free(regs); 969 970 i40e->i40e_hw_space.bus.func = func; 971 i40e->i40e_hw_space.bus.device = device; 972 973 if (i40e_get_switch_resources(i40e) == B_FALSE) { 974 return (B_FALSE); 975 } 976 977 /* 978 * To calculate the total amount of a resource we have available, we 979 * need to add how many our i40e_t thinks it has guaranteed, if any, and 980 * then we need to go through and divide the number of available on the 981 * device, which was snapshotted before anyone should have allocated 982 * anything, and use that to derive how many are available from the 983 * pool. Longer term, we may want to turn this into something that's 984 * more of a pool-like resource that everything can share (though that 985 * may require some more assistance from MAC). 986 * 987 * Though for transmit and receive queue pairs, we just have to ask 988 * firmware instead. 989 */ 990 idp = i40e_device_find(i40e, parent, bus, device); 991 i40e->i40e_device = idp; 992 i40e->i40e_resources.ifr_nvsis = 0; 993 i40e->i40e_resources.ifr_nvsis_used = 0; 994 i40e->i40e_resources.ifr_nmacfilt = 0; 995 i40e->i40e_resources.ifr_nmacfilt_used = 0; 996 i40e->i40e_resources.ifr_nmcastfilt = 0; 997 i40e->i40e_resources.ifr_nmcastfilt_used = 0; 998 999 for (i = 0; i < i40e->i40e_switch_rsrc_actual; i++) { 1000 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 1001 1002 switch (srp->resource_type) { 1003 case I40E_AQ_RESOURCE_TYPE_VSI: 1004 i40e->i40e_resources.ifr_nvsis += 1005 LE_16(srp->guaranteed); 1006 i40e->i40e_resources.ifr_nvsis_used = LE_16(srp->used); 1007 break; 1008 case I40E_AQ_RESOURCE_TYPE_MACADDR: 1009 i40e->i40e_resources.ifr_nmacfilt += 1010 LE_16(srp->guaranteed); 1011 i40e->i40e_resources.ifr_nmacfilt_used = 1012 LE_16(srp->used); 1013 break; 1014 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 1015 i40e->i40e_resources.ifr_nmcastfilt += 1016 LE_16(srp->guaranteed); 1017 i40e->i40e_resources.ifr_nmcastfilt_used = 1018 LE_16(srp->used); 1019 break; 1020 default: 1021 break; 1022 } 1023 } 1024 1025 for (i = 0; i < idp->id_rsrcs_act; i++) { 1026 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 1027 switch (srp->resource_type) { 1028 case I40E_AQ_RESOURCE_TYPE_VSI: 1029 i40e->i40e_resources.ifr_nvsis += 1030 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1031 break; 1032 case I40E_AQ_RESOURCE_TYPE_MACADDR: 1033 i40e->i40e_resources.ifr_nmacfilt += 1034 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1035 break; 1036 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 1037 i40e->i40e_resources.ifr_nmcastfilt += 1038 LE_16(srp->total_unalloced) / idp->id_nfuncs; 1039 default: 1040 break; 1041 } 1042 } 1043 1044 i40e->i40e_resources.ifr_nrx_queue = hw->func_caps.num_rx_qp; 1045 i40e->i40e_resources.ifr_ntx_queue = hw->func_caps.num_tx_qp; 1046 1047 i40e->i40e_uaddrs = kmem_zalloc(sizeof (i40e_uaddr_t) * 1048 i40e->i40e_resources.ifr_nmacfilt, KM_SLEEP); 1049 i40e->i40e_maddrs = kmem_zalloc(sizeof (i40e_maddr_t) * 1050 i40e->i40e_resources.ifr_nmcastfilt, KM_SLEEP); 1051 1052 /* 1053 * Initialize these as multicast addresses to indicate it's invalid for 1054 * sanity purposes. Think of it like 0xdeadbeef. 1055 */ 1056 for (i = 0; i < i40e->i40e_resources.ifr_nmacfilt; i++) 1057 i40e->i40e_uaddrs[i].iua_mac[0] = 0x01; 1058 1059 return (B_TRUE); 1060 } 1061 1062 static boolean_t 1063 i40e_enable_interrupts(i40e_t *i40e) 1064 { 1065 int i, rc; 1066 1067 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1068 rc = ddi_intr_block_enable(i40e->i40e_intr_handles, 1069 i40e->i40e_intr_count); 1070 if (rc != DDI_SUCCESS) { 1071 i40e_error(i40e, "Interrupt block-enable failed: %d", 1072 rc); 1073 return (B_FALSE); 1074 } 1075 } else { 1076 for (i = 0; i < i40e->i40e_intr_count; i++) { 1077 rc = ddi_intr_enable(i40e->i40e_intr_handles[i]); 1078 if (rc != DDI_SUCCESS) { 1079 i40e_error(i40e, 1080 "Failed to enable interrupt %d: %d", i, rc); 1081 while (--i >= 0) { 1082 (void) ddi_intr_disable( 1083 i40e->i40e_intr_handles[i]); 1084 } 1085 return (B_FALSE); 1086 } 1087 } 1088 } 1089 1090 return (B_TRUE); 1091 } 1092 1093 static boolean_t 1094 i40e_disable_interrupts(i40e_t *i40e) 1095 { 1096 int i, rc; 1097 1098 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1099 rc = ddi_intr_block_disable(i40e->i40e_intr_handles, 1100 i40e->i40e_intr_count); 1101 if (rc != DDI_SUCCESS) { 1102 i40e_error(i40e, 1103 "Interrupt block-disabled failed: %d", rc); 1104 return (B_FALSE); 1105 } 1106 } else { 1107 for (i = 0; i < i40e->i40e_intr_count; i++) { 1108 rc = ddi_intr_disable(i40e->i40e_intr_handles[i]); 1109 if (rc != DDI_SUCCESS) { 1110 i40e_error(i40e, 1111 "Failed to disable interrupt %d: %d", 1112 i, rc); 1113 return (B_FALSE); 1114 } 1115 } 1116 } 1117 1118 return (B_TRUE); 1119 } 1120 1121 /* 1122 * Free receive & transmit rings. 1123 */ 1124 static void 1125 i40e_free_trqpairs(i40e_t *i40e) 1126 { 1127 i40e_trqpair_t *itrq; 1128 1129 if (i40e->i40e_rx_groups != NULL) { 1130 kmem_free(i40e->i40e_rx_groups, 1131 sizeof (i40e_rx_group_t) * i40e->i40e_num_rx_groups); 1132 i40e->i40e_rx_groups = NULL; 1133 } 1134 1135 if (i40e->i40e_trqpairs != NULL) { 1136 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1137 itrq = &i40e->i40e_trqpairs[i]; 1138 mutex_destroy(&itrq->itrq_rx_lock); 1139 mutex_destroy(&itrq->itrq_tx_lock); 1140 mutex_destroy(&itrq->itrq_tcb_lock); 1141 1142 /* 1143 * Should have already been cleaned up by start/stop, 1144 * etc. 1145 */ 1146 ASSERT(itrq->itrq_txkstat == NULL); 1147 ASSERT(itrq->itrq_rxkstat == NULL); 1148 } 1149 1150 kmem_free(i40e->i40e_trqpairs, 1151 sizeof (i40e_trqpair_t) * i40e->i40e_num_trqpairs); 1152 i40e->i40e_trqpairs = NULL; 1153 } 1154 1155 cv_destroy(&i40e->i40e_rx_pending_cv); 1156 mutex_destroy(&i40e->i40e_rx_pending_lock); 1157 mutex_destroy(&i40e->i40e_general_lock); 1158 } 1159 1160 /* 1161 * Allocate transmit and receive rings, as well as other data structures that we 1162 * need. 1163 */ 1164 static boolean_t 1165 i40e_alloc_trqpairs(i40e_t *i40e) 1166 { 1167 void *mutexpri = DDI_INTR_PRI(i40e->i40e_intr_pri); 1168 1169 /* 1170 * Now that we have the priority for the interrupts, initialize 1171 * all relevant locks. 1172 */ 1173 mutex_init(&i40e->i40e_general_lock, NULL, MUTEX_DRIVER, mutexpri); 1174 mutex_init(&i40e->i40e_rx_pending_lock, NULL, MUTEX_DRIVER, mutexpri); 1175 cv_init(&i40e->i40e_rx_pending_cv, NULL, CV_DRIVER, NULL); 1176 1177 i40e->i40e_trqpairs = kmem_zalloc(sizeof (i40e_trqpair_t) * 1178 i40e->i40e_num_trqpairs, KM_SLEEP); 1179 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1180 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 1181 1182 itrq->itrq_i40e = i40e; 1183 mutex_init(&itrq->itrq_rx_lock, NULL, MUTEX_DRIVER, mutexpri); 1184 mutex_init(&itrq->itrq_tx_lock, NULL, MUTEX_DRIVER, mutexpri); 1185 mutex_init(&itrq->itrq_tcb_lock, NULL, MUTEX_DRIVER, mutexpri); 1186 itrq->itrq_index = i; 1187 } 1188 1189 i40e->i40e_rx_groups = kmem_zalloc(sizeof (i40e_rx_group_t) * 1190 i40e->i40e_num_rx_groups, KM_SLEEP); 1191 1192 for (uint_t i = 0; i < i40e->i40e_num_rx_groups; i++) { 1193 i40e_rx_group_t *rxg = &i40e->i40e_rx_groups[i]; 1194 1195 rxg->irg_index = i; 1196 rxg->irg_i40e = i40e; 1197 } 1198 1199 return (B_TRUE); 1200 } 1201 1202 1203 1204 /* 1205 * Unless a .conf file already overrode i40e_t structure values, they will 1206 * be 0, and need to be set in conjunction with the now-available HW report. 1207 */ 1208 /* ARGSUSED */ 1209 static void 1210 i40e_hw_to_instance(i40e_t *i40e, i40e_hw_t *hw) 1211 { 1212 if (i40e->i40e_num_trqpairs_per_vsi == 0) { 1213 if (i40e_is_x722(i40e)) { 1214 i40e->i40e_num_trqpairs_per_vsi = 1215 I40E_722_MAX_TC_QUEUES; 1216 } else { 1217 i40e->i40e_num_trqpairs_per_vsi = 1218 I40E_710_MAX_TC_QUEUES; 1219 } 1220 } 1221 1222 if (i40e->i40e_num_rx_groups == 0) { 1223 i40e->i40e_num_rx_groups = I40E_GROUP_MAX; 1224 } 1225 } 1226 1227 /* 1228 * Free any resources required by, or setup by, the Intel common code. 1229 */ 1230 static void 1231 i40e_common_code_fini(i40e_t *i40e) 1232 { 1233 i40e_hw_t *hw = &i40e->i40e_hw_space; 1234 int rc; 1235 1236 rc = i40e_shutdown_lan_hmc(hw); 1237 if (rc != I40E_SUCCESS) 1238 i40e_error(i40e, "failed to shutdown LAN hmc: %d", rc); 1239 1240 rc = i40e_shutdown_adminq(hw); 1241 if (rc != I40E_SUCCESS) 1242 i40e_error(i40e, "failed to shutdown admin queue: %d", rc); 1243 } 1244 1245 /* 1246 * Initialize and call Intel common-code routines, includes some setup 1247 * the common code expects from the driver. Also prints on failure, so 1248 * the caller doesn't have to. 1249 */ 1250 static boolean_t 1251 i40e_common_code_init(i40e_t *i40e, i40e_hw_t *hw) 1252 { 1253 int rc; 1254 1255 i40e_clear_hw(hw); 1256 rc = i40e_pf_reset(hw); 1257 if (rc != 0) { 1258 i40e_error(i40e, "failed to reset hardware: %d", rc); 1259 i40e_fm_ereport(i40e, DDI_FM_DEVICE_NO_RESPONSE); 1260 return (B_FALSE); 1261 } 1262 1263 rc = i40e_init_shared_code(hw); 1264 if (rc != 0) { 1265 i40e_error(i40e, "failed to initialize i40e core: %d", rc); 1266 return (B_FALSE); 1267 } 1268 1269 hw->aq.num_arq_entries = I40E_DEF_ADMINQ_SIZE; 1270 hw->aq.num_asq_entries = I40E_DEF_ADMINQ_SIZE; 1271 hw->aq.arq_buf_size = I40E_ADMINQ_BUFSZ; 1272 hw->aq.asq_buf_size = I40E_ADMINQ_BUFSZ; 1273 1274 rc = i40e_init_adminq(hw); 1275 if (rc != 0) { 1276 i40e_error(i40e, "failed to initialize firmware admin queue: " 1277 "%d, potential firmware version mismatch", rc); 1278 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 1279 return (B_FALSE); 1280 } 1281 1282 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 1283 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) { 1284 i40e_log(i40e, "The driver for the device detected a newer " 1285 "version of the NVM image (%d.%d) than expected (%d.%d).\n" 1286 "Please install the most recent version of the network " 1287 "driver.\n", hw->aq.api_maj_ver, hw->aq.api_min_ver, 1288 I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR); 1289 } else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 1290 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) { 1291 i40e_log(i40e, "The driver for the device detected an older" 1292 " version of the NVM image (%d.%d) than expected (%d.%d)." 1293 "\nPlease update the NVM image.\n", 1294 hw->aq.api_maj_ver, hw->aq.api_min_ver, 1295 I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR - 1); 1296 } 1297 1298 i40e_clear_pxe_mode(hw); 1299 1300 /* 1301 * We need to call this so that the common code can discover 1302 * capabilities of the hardware, which it uses throughout the rest. 1303 */ 1304 if (!i40e_get_hw_capabilities(i40e, hw)) { 1305 i40e_error(i40e, "failed to obtain hardware capabilities"); 1306 return (B_FALSE); 1307 } 1308 1309 if (i40e_get_available_resources(i40e) == B_FALSE) { 1310 i40e_error(i40e, "failed to obtain hardware resources"); 1311 return (B_FALSE); 1312 } 1313 1314 i40e_hw_to_instance(i40e, hw); 1315 1316 rc = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 1317 hw->func_caps.num_rx_qp, 0, 0); 1318 if (rc != 0) { 1319 i40e_error(i40e, "failed to initialize hardware memory cache: " 1320 "%d", rc); 1321 return (B_FALSE); 1322 } 1323 1324 rc = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 1325 if (rc != 0) { 1326 i40e_error(i40e, "failed to configure hardware memory cache: " 1327 "%d", rc); 1328 return (B_FALSE); 1329 } 1330 1331 (void) i40e_aq_stop_lldp(hw, TRUE, NULL); 1332 1333 rc = i40e_get_mac_addr(hw, hw->mac.addr); 1334 if (rc != I40E_SUCCESS) { 1335 i40e_error(i40e, "failed to retrieve hardware mac address: %d", 1336 rc); 1337 return (B_FALSE); 1338 } 1339 1340 rc = i40e_validate_mac_addr(hw->mac.addr); 1341 if (rc != 0) { 1342 i40e_error(i40e, "failed to validate internal mac address: " 1343 "%d", rc); 1344 return (B_FALSE); 1345 } 1346 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 1347 if ((rc = i40e_get_port_mac_addr(hw, hw->mac.port_addr)) != 1348 I40E_SUCCESS) { 1349 i40e_error(i40e, "failed to retrieve port mac address: %d", 1350 rc); 1351 return (B_FALSE); 1352 } 1353 1354 /* 1355 * We need to obtain the Default Virtual Station SEID (VSI) 1356 * before we can perform other operations on the device. 1357 */ 1358 if (!i40e_set_def_vsi_seid(i40e)) { 1359 i40e_error(i40e, "failed to obtain Default VSI SEID"); 1360 return (B_FALSE); 1361 } 1362 1363 return (B_TRUE); 1364 } 1365 1366 static void 1367 i40e_unconfigure(dev_info_t *devinfo, i40e_t *i40e) 1368 { 1369 int rc; 1370 1371 if (i40e->i40e_attach_progress & I40E_ATTACH_ENABLE_INTR) 1372 (void) i40e_disable_interrupts(i40e); 1373 1374 if ((i40e->i40e_attach_progress & I40E_ATTACH_LINK_TIMER) && 1375 i40e->i40e_periodic_id != 0) { 1376 ddi_periodic_delete(i40e->i40e_periodic_id); 1377 i40e->i40e_periodic_id = 0; 1378 } 1379 1380 if (i40e->i40e_attach_progress & I40E_ATTACH_UFM_INIT) 1381 ddi_ufm_fini(i40e->i40e_ufmh); 1382 1383 if (i40e->i40e_attach_progress & I40E_ATTACH_MAC) { 1384 rc = mac_unregister(i40e->i40e_mac_hdl); 1385 if (rc != 0) { 1386 i40e_error(i40e, "failed to unregister from mac: %d", 1387 rc); 1388 } 1389 } 1390 1391 if (i40e->i40e_attach_progress & I40E_ATTACH_STATS) { 1392 i40e_stats_fini(i40e); 1393 } 1394 1395 if (i40e->i40e_attach_progress & I40E_ATTACH_ADD_INTR) 1396 i40e_rem_intr_handlers(i40e); 1397 1398 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_RINGSLOCKS) 1399 i40e_free_trqpairs(i40e); 1400 1401 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_INTR) 1402 i40e_rem_intrs(i40e); 1403 1404 if (i40e->i40e_attach_progress & I40E_ATTACH_COMMON_CODE) 1405 i40e_common_code_fini(i40e); 1406 1407 i40e_cleanup_resources(i40e); 1408 1409 if (i40e->i40e_attach_progress & I40E_ATTACH_PROPS) 1410 (void) ddi_prop_remove_all(devinfo); 1411 1412 if (i40e->i40e_attach_progress & I40E_ATTACH_REGS_MAP && 1413 i40e->i40e_osdep_space.ios_reg_handle != NULL) { 1414 ddi_regs_map_free(&i40e->i40e_osdep_space.ios_reg_handle); 1415 i40e->i40e_osdep_space.ios_reg_handle = NULL; 1416 } 1417 1418 if ((i40e->i40e_attach_progress & I40E_ATTACH_PCI_CONFIG) && 1419 i40e->i40e_osdep_space.ios_cfg_handle != NULL) { 1420 pci_config_teardown(&i40e->i40e_osdep_space.ios_cfg_handle); 1421 i40e->i40e_osdep_space.ios_cfg_handle = NULL; 1422 } 1423 1424 if (i40e->i40e_attach_progress & I40E_ATTACH_FM_INIT) 1425 i40e_fm_fini(i40e); 1426 1427 kmem_free(i40e->i40e_aqbuf, I40E_ADMINQ_BUFSZ); 1428 kmem_free(i40e, sizeof (i40e_t)); 1429 1430 ddi_set_driver_private(devinfo, NULL); 1431 } 1432 1433 static boolean_t 1434 i40e_final_init(i40e_t *i40e) 1435 { 1436 i40e_hw_t *hw = &i40e->i40e_hw_space; 1437 struct i40e_osdep *osdep = OS_DEP(hw); 1438 uint8_t pbanum[I40E_PBANUM_STRLEN]; 1439 enum i40e_status_code irc; 1440 char buf[I40E_DDI_PROP_LEN]; 1441 1442 pbanum[0] = '\0'; 1443 irc = i40e_read_pba_string(hw, pbanum, sizeof (pbanum)); 1444 if (irc != I40E_SUCCESS) { 1445 i40e_log(i40e, "failed to read PBA string: %d", irc); 1446 } else { 1447 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1448 "printed-board-assembly", (char *)pbanum); 1449 } 1450 1451 #ifdef DEBUG 1452 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.fw_maj_ver, 1453 hw->aq.fw_min_ver) < sizeof (buf)); 1454 ASSERT(snprintf(NULL, 0, "%x", hw->aq.fw_build) < sizeof (buf)); 1455 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.api_maj_ver, 1456 hw->aq.api_min_ver) < sizeof (buf)); 1457 #endif 1458 1459 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.fw_maj_ver, 1460 hw->aq.fw_min_ver); 1461 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1462 "firmware-version", buf); 1463 (void) snprintf(buf, sizeof (buf), "%x", hw->aq.fw_build); 1464 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1465 "firmware-build", buf); 1466 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.api_maj_ver, 1467 hw->aq.api_min_ver); 1468 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1469 "api-version", buf); 1470 1471 if (!i40e_set_hw_bus_info(hw)) 1472 return (B_FALSE); 1473 1474 if (i40e_check_acc_handle(osdep->ios_reg_handle) != DDI_FM_OK) { 1475 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 1476 return (B_FALSE); 1477 } 1478 1479 return (B_TRUE); 1480 } 1481 1482 static void 1483 i40e_identify_hardware(i40e_t *i40e) 1484 { 1485 i40e_hw_t *hw = &i40e->i40e_hw_space; 1486 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1487 1488 hw->vendor_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_VENID); 1489 hw->device_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_DEVID); 1490 hw->revision_id = pci_config_get8(osdep->ios_cfg_handle, 1491 PCI_CONF_REVID); 1492 hw->subsystem_device_id = 1493 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBSYSID); 1494 hw->subsystem_vendor_id = 1495 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBVENID); 1496 1497 /* 1498 * Note that we set the hardware's bus information later on, in 1499 * i40e_get_available_resources(). The common code doesn't seem to 1500 * require that it be set in any ways, it seems to be mostly for 1501 * book-keeping. 1502 */ 1503 } 1504 1505 static boolean_t 1506 i40e_regs_map(i40e_t *i40e) 1507 { 1508 dev_info_t *devinfo = i40e->i40e_dip; 1509 i40e_hw_t *hw = &i40e->i40e_hw_space; 1510 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1511 off_t memsize; 1512 int ret; 1513 1514 if (ddi_dev_regsize(devinfo, I40E_ADAPTER_REGSET, &memsize) != 1515 DDI_SUCCESS) { 1516 i40e_error(i40e, "Used invalid register set to map PCIe regs"); 1517 return (B_FALSE); 1518 } 1519 1520 if ((ret = ddi_regs_map_setup(devinfo, I40E_ADAPTER_REGSET, 1521 (caddr_t *)&hw->hw_addr, 0, memsize, &i40e_regs_acc_attr, 1522 &osdep->ios_reg_handle)) != DDI_SUCCESS) { 1523 i40e_error(i40e, "failed to map device registers: %d", ret); 1524 return (B_FALSE); 1525 } 1526 1527 osdep->ios_reg_size = memsize; 1528 return (B_TRUE); 1529 } 1530 1531 /* 1532 * Update parameters required when a new MTU has been configured. Calculate the 1533 * maximum frame size, as well as, size our DMA buffers which we size in 1534 * increments of 1K. 1535 */ 1536 void 1537 i40e_update_mtu(i40e_t *i40e) 1538 { 1539 uint32_t rx, tx; 1540 1541 i40e->i40e_frame_max = i40e->i40e_sdu + 1542 sizeof (struct ether_vlan_header) + ETHERFCSL; 1543 1544 rx = i40e->i40e_frame_max + I40E_BUF_IPHDR_ALIGNMENT; 1545 i40e->i40e_rx_buf_size = ((rx >> 10) + 1546 ((rx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1547 1548 tx = i40e->i40e_frame_max; 1549 i40e->i40e_tx_buf_size = ((tx >> 10) + 1550 ((tx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1551 } 1552 1553 static int 1554 i40e_get_prop(i40e_t *i40e, char *prop, int min, int max, int def) 1555 { 1556 int val; 1557 1558 val = ddi_prop_get_int(DDI_DEV_T_ANY, i40e->i40e_dip, DDI_PROP_DONTPASS, 1559 prop, def); 1560 if (val > max) 1561 val = max; 1562 if (val < min) 1563 val = min; 1564 return (val); 1565 } 1566 1567 static void 1568 i40e_init_properties(i40e_t *i40e) 1569 { 1570 i40e->i40e_sdu = i40e_get_prop(i40e, "default_mtu", 1571 I40E_MIN_MTU, I40E_MAX_MTU, I40E_DEF_MTU); 1572 1573 i40e->i40e_intr_force = i40e_get_prop(i40e, "intr_force", 1574 I40E_INTR_NONE, I40E_INTR_LEGACY, I40E_INTR_NONE); 1575 1576 i40e->i40e_mr_enable = i40e_get_prop(i40e, "mr_enable", 1577 B_FALSE, B_TRUE, B_TRUE); 1578 1579 i40e->i40e_tx_ring_size = i40e_get_prop(i40e, "tx_ring_size", 1580 I40E_MIN_TX_RING_SIZE, I40E_MAX_TX_RING_SIZE, 1581 I40E_DEF_TX_RING_SIZE); 1582 if ((i40e->i40e_tx_ring_size % I40E_DESC_ALIGN) != 0) { 1583 i40e->i40e_tx_ring_size = P2ROUNDUP(i40e->i40e_tx_ring_size, 1584 I40E_DESC_ALIGN); 1585 } 1586 1587 i40e->i40e_tx_block_thresh = i40e_get_prop(i40e, "tx_resched_threshold", 1588 I40E_MIN_TX_BLOCK_THRESH, 1589 i40e->i40e_tx_ring_size - I40E_TX_MAX_COOKIE, 1590 I40E_DEF_TX_BLOCK_THRESH); 1591 1592 i40e->i40e_rx_ring_size = i40e_get_prop(i40e, "rx_ring_size", 1593 I40E_MIN_RX_RING_SIZE, I40E_MAX_RX_RING_SIZE, 1594 I40E_DEF_RX_RING_SIZE); 1595 if ((i40e->i40e_rx_ring_size % I40E_DESC_ALIGN) != 0) { 1596 i40e->i40e_rx_ring_size = P2ROUNDUP(i40e->i40e_rx_ring_size, 1597 I40E_DESC_ALIGN); 1598 } 1599 1600 i40e->i40e_rx_limit_per_intr = i40e_get_prop(i40e, "rx_limit_per_intr", 1601 I40E_MIN_RX_LIMIT_PER_INTR, I40E_MAX_RX_LIMIT_PER_INTR, 1602 I40E_DEF_RX_LIMIT_PER_INTR); 1603 1604 i40e->i40e_tx_hcksum_enable = i40e_get_prop(i40e, "tx_hcksum_enable", 1605 B_FALSE, B_TRUE, B_TRUE); 1606 1607 i40e->i40e_tx_lso_enable = i40e_get_prop(i40e, "tx_lso_enable", 1608 B_FALSE, B_TRUE, B_TRUE); 1609 1610 i40e->i40e_rx_hcksum_enable = i40e_get_prop(i40e, "rx_hcksum_enable", 1611 B_FALSE, B_TRUE, B_TRUE); 1612 1613 i40e->i40e_rx_dma_min = i40e_get_prop(i40e, "rx_dma_threshold", 1614 I40E_MIN_RX_DMA_THRESH, I40E_MAX_RX_DMA_THRESH, 1615 I40E_DEF_RX_DMA_THRESH); 1616 1617 i40e->i40e_tx_dma_min = i40e_get_prop(i40e, "tx_dma_threshold", 1618 I40E_MIN_TX_DMA_THRESH, I40E_MAX_TX_DMA_THRESH, 1619 I40E_DEF_TX_DMA_THRESH); 1620 1621 i40e->i40e_tx_itr = i40e_get_prop(i40e, "tx_intr_throttle", 1622 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_TX_ITR); 1623 1624 i40e->i40e_rx_itr = i40e_get_prop(i40e, "rx_intr_throttle", 1625 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_RX_ITR); 1626 1627 i40e->i40e_other_itr = i40e_get_prop(i40e, "other_intr_throttle", 1628 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_OTHER_ITR); 1629 1630 if (!i40e->i40e_mr_enable) { 1631 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1632 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1633 } 1634 1635 i40e_update_mtu(i40e); 1636 } 1637 1638 /* 1639 * There are a few constraints on interrupts that we're currently imposing, some 1640 * of which are restrictions from hardware. For a fuller treatment, see 1641 * i40e_intr.c. 1642 * 1643 * Currently, to use MSI-X we require two interrupts be available though in 1644 * theory we should participate in IRM and happily use more interrupts. 1645 * 1646 * Hardware only supports a single MSI being programmed and therefore if we 1647 * don't have MSI-X interrupts available at this time, then we ratchet down the 1648 * number of rings and groups available. Obviously, we only bother with a single 1649 * fixed interrupt. 1650 */ 1651 static boolean_t 1652 i40e_alloc_intr_handles(i40e_t *i40e, dev_info_t *devinfo, int intr_type) 1653 { 1654 i40e_hw_t *hw = &i40e->i40e_hw_space; 1655 ddi_acc_handle_t rh = i40e->i40e_osdep_space.ios_reg_handle; 1656 int request, count, actual, rc, min; 1657 uint32_t reg; 1658 1659 switch (intr_type) { 1660 case DDI_INTR_TYPE_FIXED: 1661 case DDI_INTR_TYPE_MSI: 1662 request = 1; 1663 min = 1; 1664 break; 1665 case DDI_INTR_TYPE_MSIX: 1666 min = 2; 1667 if (!i40e->i40e_mr_enable) { 1668 request = 2; 1669 break; 1670 } 1671 reg = I40E_READ_REG(hw, I40E_GLPCI_CNF2); 1672 /* 1673 * Should this read fail, we will drop back to using 1674 * MSI or fixed interrupts. 1675 */ 1676 if (i40e_check_acc_handle(rh) != DDI_FM_OK) { 1677 ddi_fm_service_impact(i40e->i40e_dip, 1678 DDI_SERVICE_DEGRADED); 1679 return (B_FALSE); 1680 } 1681 request = (reg & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >> 1682 I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT; 1683 request++; /* the register value is n - 1 */ 1684 break; 1685 default: 1686 panic("bad interrupt type passed to i40e_alloc_intr_handles: " 1687 "%d", intr_type); 1688 } 1689 1690 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 1691 if (rc != DDI_SUCCESS || count < min) { 1692 i40e_log(i40e, "Get interrupt number failed, " 1693 "returned %d, count %d", rc, count); 1694 return (B_FALSE); 1695 } 1696 1697 rc = ddi_intr_get_navail(devinfo, intr_type, &count); 1698 if (rc != DDI_SUCCESS || count < min) { 1699 i40e_log(i40e, "Get AVAILABLE interrupt number failed, " 1700 "returned %d, count %d", rc, count); 1701 return (B_FALSE); 1702 } 1703 1704 actual = 0; 1705 i40e->i40e_intr_count = 0; 1706 i40e->i40e_intr_count_max = 0; 1707 i40e->i40e_intr_count_min = 0; 1708 1709 i40e->i40e_intr_size = request * sizeof (ddi_intr_handle_t); 1710 ASSERT(i40e->i40e_intr_size != 0); 1711 i40e->i40e_intr_handles = kmem_alloc(i40e->i40e_intr_size, KM_SLEEP); 1712 1713 rc = ddi_intr_alloc(devinfo, i40e->i40e_intr_handles, intr_type, 0, 1714 min(request, count), &actual, DDI_INTR_ALLOC_NORMAL); 1715 if (rc != DDI_SUCCESS) { 1716 i40e_log(i40e, "Interrupt allocation failed with %d.", rc); 1717 goto alloc_handle_fail; 1718 } 1719 1720 i40e->i40e_intr_count = actual; 1721 i40e->i40e_intr_count_max = request; 1722 i40e->i40e_intr_count_min = min; 1723 1724 if (actual < min) { 1725 i40e_log(i40e, "actual (%d) is less than minimum (%d).", 1726 actual, min); 1727 goto alloc_handle_fail; 1728 } 1729 1730 /* 1731 * Record the priority and capabilities for our first vector. Once 1732 * we have it, that's our priority until detach time. Even if we 1733 * eventually participate in IRM, our priority shouldn't change. 1734 */ 1735 rc = ddi_intr_get_pri(i40e->i40e_intr_handles[0], &i40e->i40e_intr_pri); 1736 if (rc != DDI_SUCCESS) { 1737 i40e_log(i40e, 1738 "Getting interrupt priority failed with %d.", rc); 1739 goto alloc_handle_fail; 1740 } 1741 1742 rc = ddi_intr_get_cap(i40e->i40e_intr_handles[0], &i40e->i40e_intr_cap); 1743 if (rc != DDI_SUCCESS) { 1744 i40e_log(i40e, 1745 "Getting interrupt capabilities failed with %d.", rc); 1746 goto alloc_handle_fail; 1747 } 1748 1749 i40e->i40e_intr_type = intr_type; 1750 return (B_TRUE); 1751 1752 alloc_handle_fail: 1753 1754 i40e_rem_intrs(i40e); 1755 return (B_FALSE); 1756 } 1757 1758 static boolean_t 1759 i40e_alloc_intrs(i40e_t *i40e, dev_info_t *devinfo) 1760 { 1761 i40e_hw_t *hw = &i40e->i40e_hw_space; 1762 int intr_types, rc; 1763 uint_t max_trqpairs; 1764 1765 if (i40e_is_x722(i40e)) { 1766 max_trqpairs = I40E_722_MAX_TC_QUEUES; 1767 } else { 1768 max_trqpairs = I40E_710_MAX_TC_QUEUES; 1769 } 1770 1771 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 1772 if (rc != DDI_SUCCESS) { 1773 i40e_error(i40e, "failed to get supported interrupt types: %d", 1774 rc); 1775 return (B_FALSE); 1776 } 1777 1778 i40e->i40e_intr_type = 0; 1779 i40e->i40e_num_rx_groups = I40E_GROUP_MAX; 1780 1781 /* 1782 * We need to determine the number of queue pairs per traffic 1783 * class. We only have one traffic class (TC0), so we'll base 1784 * this off the number of interrupts provided. Furthermore, 1785 * since we only use one traffic class, the number of queues 1786 * per traffic class and per VSI are the same. 1787 */ 1788 if ((intr_types & DDI_INTR_TYPE_MSIX) && 1789 (i40e->i40e_intr_force <= I40E_INTR_MSIX) && 1790 (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSIX))) { 1791 uint32_t n, qp_cap, num_trqpairs; 1792 1793 /* 1794 * While we want the number of queue pairs to match 1795 * the number of interrupts, we must keep stay in 1796 * bounds of the maximum number of queues per traffic 1797 * class. We subtract one from i40e_intr_count to 1798 * account for interrupt zero; which is currently 1799 * restricted to admin queue commands and other 1800 * interrupt causes. 1801 */ 1802 n = MIN(i40e->i40e_intr_count - 1, max_trqpairs); 1803 ASSERT3U(n, >, 0); 1804 1805 /* 1806 * Round up to the nearest power of two to ensure that 1807 * the QBASE aligns with the TC size which must be 1808 * programmed as a power of two. See the queue mapping 1809 * description in section 7.4.9.5.5.1. 1810 * 1811 * If i40e_intr_count - 1 is not a power of two then 1812 * some queue pairs on the same VSI will have to share 1813 * an interrupt. 1814 * 1815 * We may want to revisit this logic in a future where 1816 * we have more interrupts and more VSIs. Otherwise, 1817 * each VSI will use as many interrupts as possible. 1818 * Using more QPs per VSI means better RSS for each 1819 * group, but at the same time may require more 1820 * sharing of interrupts across VSIs. This may be a 1821 * good candidate for a .conf tunable. 1822 */ 1823 n = 0x1 << ddi_fls(n); 1824 i40e->i40e_num_trqpairs_per_vsi = n; 1825 1826 /* 1827 * Make sure the number of tx/rx qpairs does not exceed 1828 * the device's capabilities. 1829 */ 1830 ASSERT3U(i40e->i40e_num_rx_groups, >, 0); 1831 qp_cap = MIN(hw->func_caps.num_rx_qp, hw->func_caps.num_tx_qp); 1832 num_trqpairs = i40e->i40e_num_trqpairs_per_vsi * 1833 i40e->i40e_num_rx_groups; 1834 if (num_trqpairs > qp_cap) { 1835 i40e->i40e_num_rx_groups = MAX(1, qp_cap / 1836 i40e->i40e_num_trqpairs_per_vsi); 1837 num_trqpairs = i40e->i40e_num_trqpairs_per_vsi * 1838 i40e->i40e_num_rx_groups; 1839 i40e_log(i40e, "Rx groups restricted to %u", 1840 i40e->i40e_num_rx_groups); 1841 } 1842 ASSERT3U(num_trqpairs, >, 0); 1843 i40e->i40e_num_trqpairs = num_trqpairs; 1844 return (B_TRUE); 1845 } 1846 1847 /* 1848 * We only use multiple transmit/receive pairs when MSI-X interrupts are 1849 * available due to the fact that the device basically only supports a 1850 * single MSI interrupt. 1851 */ 1852 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1853 i40e->i40e_num_trqpairs_per_vsi = i40e->i40e_num_trqpairs; 1854 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1855 1856 if ((intr_types & DDI_INTR_TYPE_MSI) && 1857 (i40e->i40e_intr_force <= I40E_INTR_MSI)) { 1858 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSI)) 1859 return (B_TRUE); 1860 } 1861 1862 if (intr_types & DDI_INTR_TYPE_FIXED) { 1863 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_FIXED)) 1864 return (B_TRUE); 1865 } 1866 1867 return (B_FALSE); 1868 } 1869 1870 /* 1871 * Map different interrupts to MSI-X vectors. 1872 */ 1873 static boolean_t 1874 i40e_map_intrs_to_vectors(i40e_t *i40e) 1875 { 1876 if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 1877 return (B_TRUE); 1878 } 1879 1880 /* 1881 * Each queue pair is mapped to a single interrupt, so 1882 * transmit and receive interrupts for a given queue share the 1883 * same vector. Vector zero is reserved for the admin queue. 1884 */ 1885 for (uint_t i = 0; i < i40e->i40e_num_trqpairs; i++) { 1886 uint_t vector = i % (i40e->i40e_intr_count - 1); 1887 1888 i40e->i40e_trqpairs[i].itrq_rx_intrvec = vector + 1; 1889 i40e->i40e_trqpairs[i].itrq_tx_intrvec = vector + 1; 1890 } 1891 1892 return (B_TRUE); 1893 } 1894 1895 static boolean_t 1896 i40e_add_intr_handlers(i40e_t *i40e) 1897 { 1898 int rc, vector; 1899 1900 switch (i40e->i40e_intr_type) { 1901 case DDI_INTR_TYPE_MSIX: 1902 for (vector = 0; vector < i40e->i40e_intr_count; vector++) { 1903 rc = ddi_intr_add_handler( 1904 i40e->i40e_intr_handles[vector], 1905 (ddi_intr_handler_t *)i40e_intr_msix, i40e, 1906 (void *)(uintptr_t)vector); 1907 if (rc != DDI_SUCCESS) { 1908 i40e_log(i40e, "Add interrupt handler (MSI-X) " 1909 "failed: return %d, vector %d", rc, vector); 1910 for (vector--; vector >= 0; vector--) { 1911 (void) ddi_intr_remove_handler( 1912 i40e->i40e_intr_handles[vector]); 1913 } 1914 return (B_FALSE); 1915 } 1916 } 1917 break; 1918 case DDI_INTR_TYPE_MSI: 1919 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1920 (ddi_intr_handler_t *)i40e_intr_msi, i40e, NULL); 1921 if (rc != DDI_SUCCESS) { 1922 i40e_log(i40e, "Add interrupt handler (MSI) failed: " 1923 "return %d", rc); 1924 return (B_FALSE); 1925 } 1926 break; 1927 case DDI_INTR_TYPE_FIXED: 1928 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1929 (ddi_intr_handler_t *)i40e_intr_legacy, i40e, NULL); 1930 if (rc != DDI_SUCCESS) { 1931 i40e_log(i40e, "Add interrupt handler (legacy) failed:" 1932 " return %d", rc); 1933 return (B_FALSE); 1934 } 1935 break; 1936 default: 1937 /* Cast to pacify lint */ 1938 panic("i40e_intr_type %p contains an unknown type: %d", 1939 (void *)i40e, i40e->i40e_intr_type); 1940 } 1941 1942 return (B_TRUE); 1943 } 1944 1945 /* 1946 * Perform periodic checks. Longer term, we should be thinking about additional 1947 * things here: 1948 * 1949 * o Stall Detection 1950 * o Temperature sensor detection 1951 * o Device resetting 1952 * o Statistics updating to avoid wraparound 1953 */ 1954 static void 1955 i40e_timer(void *arg) 1956 { 1957 i40e_t *i40e = arg; 1958 1959 mutex_enter(&i40e->i40e_general_lock); 1960 i40e_link_check(i40e); 1961 mutex_exit(&i40e->i40e_general_lock); 1962 } 1963 1964 /* 1965 * Get the hardware state, and scribble away anything that needs scribbling. 1966 */ 1967 static void 1968 i40e_get_hw_state(i40e_t *i40e, i40e_hw_t *hw) 1969 { 1970 int rc; 1971 1972 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 1973 1974 (void) i40e_aq_get_link_info(hw, TRUE, NULL, NULL); 1975 i40e_link_check(i40e); 1976 1977 /* 1978 * Try and determine our PHY. Note that we may have to retry to and 1979 * delay to detect fiber correctly. 1980 */ 1981 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, &i40e->i40e_phy, 1982 NULL); 1983 if (rc == I40E_ERR_UNKNOWN_PHY) { 1984 i40e_msec_delay(200); 1985 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, 1986 &i40e->i40e_phy, NULL); 1987 } 1988 1989 if (rc != I40E_SUCCESS) { 1990 if (rc == I40E_ERR_UNKNOWN_PHY) { 1991 i40e_error(i40e, "encountered unknown PHY type, " 1992 "not attaching."); 1993 } else { 1994 i40e_error(i40e, "error getting physical capabilities: " 1995 "%d, %d", rc, hw->aq.asq_last_status); 1996 } 1997 } 1998 1999 rc = i40e_update_link_info(hw); 2000 if (rc != I40E_SUCCESS) { 2001 i40e_error(i40e, "failed to update link information: %d", rc); 2002 } 2003 2004 /* 2005 * In general, we don't want to mask off (as in stop from being a cause) 2006 * any of the interrupts that the phy might be able to generate. 2007 */ 2008 rc = i40e_aq_set_phy_int_mask(hw, 0, NULL); 2009 if (rc != I40E_SUCCESS) { 2010 i40e_error(i40e, "failed to update phy link mask: %d", rc); 2011 } 2012 } 2013 2014 /* 2015 * Go through and re-initialize any existing filters that we may have set up for 2016 * this device. Note that we would only expect them to exist if hardware had 2017 * already been initialized and we had just reset it. While we're not 2018 * implementing this yet, we're keeping this around for when we add reset 2019 * capabilities, so this isn't forgotten. 2020 */ 2021 /* ARGSUSED */ 2022 static void 2023 i40e_init_macaddrs(i40e_t *i40e, i40e_hw_t *hw) 2024 { 2025 } 2026 2027 /* 2028 * Set the properties which have common values across all the VSIs. 2029 * Consult the "Add VSI" command section (7.4.9.5.5.1) for a 2030 * complete description of these properties. 2031 */ 2032 static void 2033 i40e_set_shared_vsi_props(i40e_t *i40e, 2034 struct i40e_aqc_vsi_properties_data *info, uint_t vsi_idx) 2035 { 2036 uint_t tc_queues; 2037 uint16_t vsi_qp_base; 2038 2039 /* 2040 * It's important that we use bitwise-OR here; callers to this 2041 * function might enable other sections before calling this 2042 * function. 2043 */ 2044 info->valid_sections |= LE_16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID | 2045 I40E_AQ_VSI_PROP_VLAN_VALID); 2046 2047 /* 2048 * Calculate the starting QP index for this VSI. This base is 2049 * relative to the PF queue space; so a value of 0 for PF#1 2050 * represents the absolute index PFLAN_QALLOC_FIRSTQ for PF#1. 2051 */ 2052 vsi_qp_base = vsi_idx * i40e->i40e_num_trqpairs_per_vsi; 2053 info->mapping_flags = LE_16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2054 info->queue_mapping[0] = 2055 LE_16((vsi_qp_base << I40E_AQ_VSI_QUEUE_SHIFT) & 2056 I40E_AQ_VSI_QUEUE_MASK); 2057 2058 /* 2059 * tc_queues determines the size of the traffic class, where 2060 * the size is 2^^tc_queues to a maximum of 64 for the X710 2061 * and 128 for the X722. 2062 * 2063 * Some examples: 2064 * i40e_num_trqpairs_per_vsi == 1 => tc_queues = 0, 2^^0 = 1. 2065 * i40e_num_trqpairs_per_vsi == 7 => tc_queues = 3, 2^^3 = 8. 2066 * i40e_num_trqpairs_per_vsi == 8 => tc_queues = 3, 2^^3 = 8. 2067 * i40e_num_trqpairs_per_vsi == 9 => tc_queues = 4, 2^^4 = 16. 2068 * i40e_num_trqpairs_per_vsi == 17 => tc_queues = 5, 2^^5 = 32. 2069 * i40e_num_trqpairs_per_vsi == 64 => tc_queues = 6, 2^^6 = 64. 2070 */ 2071 tc_queues = ddi_fls(i40e->i40e_num_trqpairs_per_vsi - 1); 2072 2073 /* 2074 * The TC queue mapping is in relation to the VSI queue space. 2075 * Since we are only using one traffic class (TC0) we always 2076 * start at queue offset 0. 2077 */ 2078 info->tc_mapping[0] = 2079 LE_16(((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) & 2080 I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | 2081 ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) & 2082 I40E_AQ_VSI_TC_QUE_NUMBER_MASK)); 2083 2084 /* 2085 * I40E_AQ_VSI_PVLAN_MODE_ALL ("VLAN driver insertion mode") 2086 * 2087 * Allow tagged and untagged packets to be sent to this 2088 * VSI from the host. 2089 * 2090 * I40E_AQ_VSI_PVLAN_EMOD_NOTHING ("VLAN and UP expose mode") 2091 * 2092 * Leave the tag on the frame and place no VLAN 2093 * information in the descriptor. We want this mode 2094 * because our MAC layer will take care of the VLAN tag, 2095 * if there is one. 2096 */ 2097 info->port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2098 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2099 } 2100 2101 /* 2102 * Delete the VSI at this index, if one exists. We assume there is no 2103 * action we can take if this command fails but to log the failure. 2104 */ 2105 static void 2106 i40e_delete_vsi(i40e_t *i40e, uint_t idx) 2107 { 2108 i40e_hw_t *hw = &i40e->i40e_hw_space; 2109 uint16_t seid = i40e->i40e_vsis[idx].iv_seid; 2110 2111 if (seid != 0) { 2112 int rc; 2113 2114 rc = i40e_aq_delete_element(hw, seid, NULL); 2115 2116 if (rc != I40E_SUCCESS) { 2117 i40e_error(i40e, "Failed to delete VSI %d: %d", 2118 rc, hw->aq.asq_last_status); 2119 } 2120 2121 i40e->i40e_vsis[idx].iv_seid = 0; 2122 } 2123 } 2124 2125 /* 2126 * Add a new VSI. 2127 */ 2128 static boolean_t 2129 i40e_add_vsi(i40e_t *i40e, i40e_hw_t *hw, uint_t idx) 2130 { 2131 struct i40e_vsi_context ctx; 2132 i40e_rx_group_t *rxg; 2133 int rc; 2134 2135 /* 2136 * The default VSI is created by the controller. This function 2137 * creates new, non-defualt VSIs only. 2138 */ 2139 ASSERT3U(idx, !=, 0); 2140 2141 bzero(&ctx, sizeof (struct i40e_vsi_context)); 2142 ctx.uplink_seid = i40e->i40e_veb_seid; 2143 ctx.pf_num = hw->pf_id; 2144 ctx.flags = I40E_AQ_VSI_TYPE_PF; 2145 ctx.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 2146 i40e_set_shared_vsi_props(i40e, &ctx.info, idx); 2147 2148 rc = i40e_aq_add_vsi(hw, &ctx, NULL); 2149 if (rc != I40E_SUCCESS) { 2150 i40e_error(i40e, "i40e_aq_add_vsi() failed %d: %d", rc, 2151 hw->aq.asq_last_status); 2152 return (B_FALSE); 2153 } 2154 2155 rxg = &i40e->i40e_rx_groups[idx]; 2156 rxg->irg_vsi_seid = ctx.seid; 2157 i40e->i40e_vsis[idx].iv_number = ctx.vsi_number; 2158 i40e->i40e_vsis[idx].iv_seid = ctx.seid; 2159 i40e->i40e_vsis[idx].iv_stats_id = LE_16(ctx.info.stat_counter_idx); 2160 2161 if (i40e_stat_vsi_init(i40e, idx) == B_FALSE) 2162 return (B_FALSE); 2163 2164 return (B_TRUE); 2165 } 2166 2167 /* 2168 * Configure the hardware for the Default Virtual Station Interface (VSI). 2169 */ 2170 static boolean_t 2171 i40e_config_def_vsi(i40e_t *i40e, i40e_hw_t *hw) 2172 { 2173 struct i40e_vsi_context ctx; 2174 i40e_rx_group_t *def_rxg; 2175 int err; 2176 struct i40e_aqc_remove_macvlan_element_data filt; 2177 2178 bzero(&ctx, sizeof (struct i40e_vsi_context)); 2179 ctx.seid = I40E_DEF_VSI_SEID(i40e); 2180 ctx.pf_num = hw->pf_id; 2181 err = i40e_aq_get_vsi_params(hw, &ctx, NULL); 2182 if (err != I40E_SUCCESS) { 2183 i40e_error(i40e, "get VSI params failed with %d", err); 2184 return (B_FALSE); 2185 } 2186 2187 ctx.info.valid_sections = 0; 2188 i40e->i40e_vsis[0].iv_number = ctx.vsi_number; 2189 i40e->i40e_vsis[0].iv_stats_id = LE_16(ctx.info.stat_counter_idx); 2190 if (i40e_stat_vsi_init(i40e, 0) == B_FALSE) 2191 return (B_FALSE); 2192 2193 i40e_set_shared_vsi_props(i40e, &ctx.info, I40E_DEF_VSI_IDX); 2194 2195 err = i40e_aq_update_vsi_params(hw, &ctx, NULL); 2196 if (err != I40E_SUCCESS) { 2197 i40e_error(i40e, "Update VSI params failed with %d", err); 2198 return (B_FALSE); 2199 } 2200 2201 def_rxg = &i40e->i40e_rx_groups[0]; 2202 def_rxg->irg_vsi_seid = I40E_DEF_VSI_SEID(i40e); 2203 2204 /* 2205 * We have seen three different behaviors in regards to the 2206 * Default VSI and its implicit L2 MAC+VLAN filter. 2207 * 2208 * 1. It has an implicit filter for the factory MAC address 2209 * and this filter counts against 'ifr_nmacfilt_used'. 2210 * 2211 * 2. It has an implicit filter for the factory MAC address 2212 * and this filter DOES NOT count against 'ifr_nmacfilt_used'. 2213 * 2214 * 3. It DOES NOT have an implicit filter. 2215 * 2216 * All three of these cases are accounted for below. If we 2217 * fail to remove the L2 filter (ENOENT) then we assume there 2218 * wasn't one. Otherwise, if we successfully remove the 2219 * filter, we make sure to update the 'ifr_nmacfilt_used' 2220 * count accordingly. 2221 * 2222 * We remove this filter to prevent duplicate delivery of 2223 * packets destined for the primary MAC address as DLS will 2224 * create the same filter on a non-default VSI for the primary 2225 * MAC client. 2226 * 2227 * If you change the following code please test it across as 2228 * many X700 series controllers and firmware revisions as you 2229 * can. 2230 */ 2231 bzero(&filt, sizeof (filt)); 2232 bcopy(hw->mac.port_addr, filt.mac_addr, ETHERADDRL); 2233 filt.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2234 filt.vlan_tag = 0; 2235 2236 ASSERT3U(i40e->i40e_resources.ifr_nmacfilt_used, <=, 1); 2237 i40e_log(i40e, "Num L2 filters: %u", 2238 i40e->i40e_resources.ifr_nmacfilt_used); 2239 2240 err = i40e_aq_remove_macvlan(hw, I40E_DEF_VSI_SEID(i40e), &filt, 1, 2241 NULL); 2242 if (err == I40E_SUCCESS) { 2243 i40e_log(i40e, 2244 "Removed L2 filter from Default VSI with SEID %u", 2245 I40E_DEF_VSI_SEID(i40e)); 2246 } else if (hw->aq.asq_last_status == ENOENT) { 2247 i40e_log(i40e, 2248 "No L2 filter for Default VSI with SEID %u", 2249 I40E_DEF_VSI_SEID(i40e)); 2250 } else { 2251 i40e_error(i40e, "Failed to remove L2 filter from" 2252 " Default VSI with SEID %u: %d (%d)", 2253 I40E_DEF_VSI_SEID(i40e), err, hw->aq.asq_last_status); 2254 2255 return (B_FALSE); 2256 } 2257 2258 /* 2259 * As mentioned above, the controller created an implicit L2 2260 * filter for the primary MAC. We want to remove both the 2261 * filter and decrement the filter count. However, not all 2262 * controllers count this implicit filter against the total 2263 * MAC filter count. So here we are making sure it is either 2264 * one or zero. If it is one, then we know it is for the 2265 * implicit filter and we should decrement since we just 2266 * removed the filter above. If it is zero then we know the 2267 * controller that does not count the implicit filter, and it 2268 * was enough to just remove it; we leave the count alone. 2269 * But if it is neither, then we have never seen a controller 2270 * like this before and we should fail to attach. 2271 * 2272 * It is unfortunate that this code must exist but the 2273 * behavior of this implicit L2 filter and its corresponding 2274 * count were dicovered through empirical testing. The 2275 * programming manuals hint at this filter but do not 2276 * explicitly call out the exact behavior. 2277 */ 2278 if (i40e->i40e_resources.ifr_nmacfilt_used == 1) { 2279 i40e->i40e_resources.ifr_nmacfilt_used--; 2280 } else { 2281 if (i40e->i40e_resources.ifr_nmacfilt_used != 0) { 2282 i40e_error(i40e, "Unexpected L2 filter count: %u" 2283 " (expected 0)", 2284 i40e->i40e_resources.ifr_nmacfilt_used); 2285 return (B_FALSE); 2286 } 2287 } 2288 2289 return (B_TRUE); 2290 } 2291 2292 static boolean_t 2293 i40e_config_rss_key_x722(i40e_t *i40e, i40e_hw_t *hw) 2294 { 2295 for (uint_t i = 0; i < i40e->i40e_num_rx_groups; i++) { 2296 uint32_t seed[I40E_PFQF_HKEY_MAX_INDEX + 1]; 2297 struct i40e_aqc_get_set_rss_key_data key; 2298 const char *u8seed; 2299 enum i40e_status_code status; 2300 uint16_t vsi_number = i40e->i40e_vsis[i].iv_number; 2301 2302 (void) random_get_pseudo_bytes((uint8_t *)seed, sizeof (seed)); 2303 u8seed = (char *)seed; 2304 2305 CTASSERT(sizeof (key) >= (sizeof (key.standard_rss_key) + 2306 sizeof (key.extended_hash_key))); 2307 2308 bcopy(u8seed, key.standard_rss_key, 2309 sizeof (key.standard_rss_key)); 2310 bcopy(&u8seed[sizeof (key.standard_rss_key)], 2311 key.extended_hash_key, sizeof (key.extended_hash_key)); 2312 2313 ASSERT3U(vsi_number, !=, 0); 2314 status = i40e_aq_set_rss_key(hw, vsi_number, &key); 2315 2316 if (status != I40E_SUCCESS) { 2317 i40e_error(i40e, "failed to set RSS key for VSI %u: %d", 2318 vsi_number, status); 2319 return (B_FALSE); 2320 } 2321 } 2322 2323 return (B_TRUE); 2324 } 2325 2326 /* 2327 * Configure the RSS key. For the X710 controller family, this is set on a 2328 * per-PF basis via registers. For the X722, this is done on a per-VSI basis 2329 * through the admin queue. 2330 */ 2331 static boolean_t 2332 i40e_config_rss_key(i40e_t *i40e, i40e_hw_t *hw) 2333 { 2334 if (i40e_is_x722(i40e)) { 2335 if (!i40e_config_rss_key_x722(i40e, hw)) 2336 return (B_FALSE); 2337 } else { 2338 uint32_t seed[I40E_PFQF_HKEY_MAX_INDEX + 1]; 2339 2340 (void) random_get_pseudo_bytes((uint8_t *)seed, sizeof (seed)); 2341 for (uint_t i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 2342 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), seed[i]); 2343 } 2344 2345 return (B_TRUE); 2346 } 2347 2348 /* 2349 * Populate the LUT. The size of each entry in the LUT depends on the controller 2350 * family, with the X722 using a known 7-bit width. On the X710 controller, this 2351 * is programmed through its control registers where as on the X722 this is 2352 * configured through the admin queue. Also of note, the X722 allows the LUT to 2353 * be set on a per-PF or VSI basis. At this time we use the PF setting. If we 2354 * decide to use the per-VSI LUT in the future, then we will need to modify the 2355 * i40e_add_vsi() function to set the RSS LUT bits in the queueing section. 2356 * 2357 * We populate the LUT in a round robin fashion with the rx queue indices from 0 2358 * to i40e_num_trqpairs_per_vsi - 1. 2359 */ 2360 static boolean_t 2361 i40e_config_rss_hlut(i40e_t *i40e, i40e_hw_t *hw) 2362 { 2363 uint32_t *hlut; 2364 uint8_t lut_mask; 2365 uint_t i; 2366 boolean_t ret = B_FALSE; 2367 2368 /* 2369 * We always configure the PF with a table size of 512 bytes in 2370 * i40e_chip_start(). 2371 */ 2372 hlut = kmem_alloc(I40E_HLUT_TABLE_SIZE, KM_NOSLEEP); 2373 if (hlut == NULL) { 2374 i40e_error(i40e, "i40e_config_rss() buffer allocation failed"); 2375 return (B_FALSE); 2376 } 2377 2378 /* 2379 * The width of the X722 is apparently defined to be 7 bits, regardless 2380 * of the capability. 2381 */ 2382 if (i40e_is_x722(i40e)) { 2383 lut_mask = (1 << 7) - 1; 2384 } else { 2385 lut_mask = (1 << hw->func_caps.rss_table_entry_width) - 1; 2386 } 2387 2388 for (i = 0; i < I40E_HLUT_TABLE_SIZE; i++) { 2389 ((uint8_t *)hlut)[i] = 2390 (i % i40e->i40e_num_trqpairs_per_vsi) & lut_mask; 2391 } 2392 2393 if (i40e_is_x722(i40e)) { 2394 enum i40e_status_code status; 2395 2396 status = i40e_aq_set_rss_lut(hw, 0, B_TRUE, (uint8_t *)hlut, 2397 I40E_HLUT_TABLE_SIZE); 2398 2399 if (status != I40E_SUCCESS) { 2400 i40e_error(i40e, "failed to set RSS LUT %d: %d", 2401 status, hw->aq.asq_last_status); 2402 goto out; 2403 } 2404 } else { 2405 for (i = 0; i < I40E_HLUT_TABLE_SIZE >> 2; i++) { 2406 I40E_WRITE_REG(hw, I40E_PFQF_HLUT(i), hlut[i]); 2407 } 2408 } 2409 ret = B_TRUE; 2410 out: 2411 kmem_free(hlut, I40E_HLUT_TABLE_SIZE); 2412 return (ret); 2413 } 2414 2415 /* 2416 * Set up RSS. 2417 * 1. Seed the hash key. 2418 * 2. Enable PCTYPEs for the hash filter. 2419 * 3. Populate the LUT. 2420 */ 2421 static boolean_t 2422 i40e_config_rss(i40e_t *i40e, i40e_hw_t *hw) 2423 { 2424 uint64_t hena; 2425 2426 /* 2427 * 1. Seed the hash key 2428 */ 2429 if (!i40e_config_rss_key(i40e, hw)) 2430 return (B_FALSE); 2431 2432 /* 2433 * 2. Configure PCTYPES 2434 */ 2435 hena = (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | 2436 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | 2437 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | 2438 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | 2439 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4) | 2440 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | 2441 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | 2442 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | 2443 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | 2444 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6) | 2445 (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD); 2446 2447 /* 2448 * Add additional types supported by the X722 controller. 2449 */ 2450 if (i40e_is_x722(i40e)) { 2451 hena |= (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | 2452 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | 2453 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | 2454 (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | 2455 (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | 2456 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); 2457 } 2458 2459 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (uint32_t)hena); 2460 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (uint32_t)(hena >> 32)); 2461 2462 /* 2463 * 3. Populate LUT 2464 */ 2465 return (i40e_config_rss_hlut(i40e, hw)); 2466 } 2467 2468 /* 2469 * Wrapper to kick the chipset on. 2470 */ 2471 static boolean_t 2472 i40e_chip_start(i40e_t *i40e) 2473 { 2474 i40e_hw_t *hw = &i40e->i40e_hw_space; 2475 struct i40e_filter_control_settings filter; 2476 int rc; 2477 uint8_t err; 2478 2479 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || 2480 (hw->aq.fw_maj_ver < 4)) { 2481 i40e_msec_delay(75); 2482 if (i40e_aq_set_link_restart_an(hw, TRUE, NULL) != 2483 I40E_SUCCESS) { 2484 i40e_error(i40e, "failed to restart link: admin queue " 2485 "error: %d", hw->aq.asq_last_status); 2486 return (B_FALSE); 2487 } 2488 } 2489 2490 /* Determine hardware state */ 2491 i40e_get_hw_state(i40e, hw); 2492 2493 /* For now, we always disable Ethernet Flow Control. */ 2494 hw->fc.requested_mode = I40E_FC_NONE; 2495 rc = i40e_set_fc(hw, &err, B_TRUE); 2496 if (rc != I40E_SUCCESS) { 2497 i40e_error(i40e, "Setting flow control failed, returned %d" 2498 " with error: 0x%x", rc, err); 2499 return (B_FALSE); 2500 } 2501 2502 /* Initialize mac addresses. */ 2503 i40e_init_macaddrs(i40e, hw); 2504 2505 /* 2506 * Set up the filter control. If the hash lut size is changed from 2507 * I40E_HASH_LUT_SIZE_512 then I40E_HLUT_TABLE_SIZE and 2508 * i40e_config_rss_hlut() will need to be updated. 2509 */ 2510 bzero(&filter, sizeof (filter)); 2511 filter.enable_ethtype = TRUE; 2512 filter.enable_macvlan = TRUE; 2513 filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; 2514 2515 rc = i40e_set_filter_control(hw, &filter); 2516 if (rc != I40E_SUCCESS) { 2517 i40e_error(i40e, "i40e_set_filter_control() returned %d", rc); 2518 return (B_FALSE); 2519 } 2520 2521 i40e_intr_chip_init(i40e); 2522 2523 rc = i40e_get_mac_seid(i40e); 2524 if (rc == -1) { 2525 i40e_error(i40e, "failed to obtain MAC Uplink SEID"); 2526 return (B_FALSE); 2527 } 2528 i40e->i40e_mac_seid = (uint16_t)rc; 2529 2530 /* 2531 * Create a VEB in order to support multiple VSIs. Each VSI 2532 * functions as a MAC group. This call sets the PF's MAC as 2533 * the uplink port and the PF's default VSI as the default 2534 * downlink port. 2535 */ 2536 rc = i40e_aq_add_veb(hw, i40e->i40e_mac_seid, I40E_DEF_VSI_SEID(i40e), 2537 0x1, B_TRUE, &i40e->i40e_veb_seid, B_FALSE, NULL); 2538 if (rc != I40E_SUCCESS) { 2539 i40e_error(i40e, "i40e_aq_add_veb() failed %d: %d", rc, 2540 hw->aq.asq_last_status); 2541 return (B_FALSE); 2542 } 2543 2544 if (!i40e_config_def_vsi(i40e, hw)) 2545 return (B_FALSE); 2546 2547 for (uint_t i = 1; i < i40e->i40e_num_rx_groups; i++) { 2548 if (!i40e_add_vsi(i40e, hw, i)) 2549 return (B_FALSE); 2550 } 2551 2552 if (!i40e_config_rss(i40e, hw)) 2553 return (B_FALSE); 2554 2555 i40e_flush(hw); 2556 2557 return (B_TRUE); 2558 } 2559 2560 /* 2561 * Take care of tearing down the rx ring. See 8.3.3.1.2 for more information. 2562 */ 2563 static void 2564 i40e_shutdown_rx_rings(i40e_t *i40e) 2565 { 2566 int i; 2567 uint32_t reg; 2568 2569 i40e_hw_t *hw = &i40e->i40e_hw_space; 2570 2571 /* 2572 * Step 1. The interrupt linked list (see i40e_intr.c for more 2573 * information) should have already been cleared before calling this 2574 * function. 2575 */ 2576 #ifdef DEBUG 2577 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 2578 for (i = 1; i < i40e->i40e_intr_count; i++) { 2579 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i - 1)); 2580 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2581 } 2582 } else { 2583 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLST0); 2584 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2585 } 2586 2587 #endif /* DEBUG */ 2588 2589 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2590 /* 2591 * Step 1. Request the queue by clearing QENA_REQ. It may not be 2592 * set due to unwinding from failures and a partially enabled 2593 * ring set. 2594 */ 2595 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2596 if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK)) 2597 continue; 2598 VERIFY((reg & I40E_QRX_ENA_QENA_REQ_MASK) == 2599 I40E_QRX_ENA_QENA_REQ_MASK); 2600 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 2601 I40E_WRITE_REG(hw, I40E_QRX_ENA(i), reg); 2602 } 2603 2604 /* 2605 * Step 2. Wait for the disable to take, by having QENA_STAT in the FPM 2606 * be cleared. Note that we could still receive data in the queue during 2607 * this time. We don't actually wait for this now and instead defer this 2608 * to i40e_shutdown_rings_wait(), after we've interleaved disabling the 2609 * TX queues as well. 2610 */ 2611 } 2612 2613 static void 2614 i40e_shutdown_tx_rings(i40e_t *i40e) 2615 { 2616 int i; 2617 uint32_t reg; 2618 2619 i40e_hw_t *hw = &i40e->i40e_hw_space; 2620 2621 /* 2622 * Step 1. The interrupt linked list should already have been cleared. 2623 */ 2624 #ifdef DEBUG 2625 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 2626 for (i = 1; i < i40e->i40e_intr_count; i++) { 2627 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i - 1)); 2628 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2629 } 2630 } else { 2631 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLST0); 2632 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2633 2634 } 2635 #endif /* DEBUG */ 2636 2637 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2638 /* 2639 * Step 2. Set the SET_QDIS flag for every queue. 2640 */ 2641 i40e_pre_tx_queue_cfg(hw, i, B_FALSE); 2642 } 2643 2644 /* 2645 * Step 3. Wait at least 400 usec (can be done once for all queues). 2646 */ 2647 drv_usecwait(500); 2648 2649 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2650 /* 2651 * Step 4. Clear the QENA_REQ flag which tells hardware to 2652 * quiesce. If QENA_REQ is not already set then that means that 2653 * we likely already tried to disable this queue. 2654 */ 2655 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2656 if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK)) 2657 continue; 2658 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 2659 I40E_WRITE_REG(hw, I40E_QTX_ENA(i), reg); 2660 } 2661 2662 /* 2663 * Step 5. Wait for all drains to finish. This will be done by the 2664 * hardware removing the QENA_STAT flag from the queue. Rather than 2665 * waiting here, we interleave it with all the others in 2666 * i40e_shutdown_rings_wait(). 2667 */ 2668 } 2669 2670 /* 2671 * Wait for all the rings to be shut down. e.g. Steps 2 and 5 from the above 2672 * functions. 2673 */ 2674 static boolean_t 2675 i40e_shutdown_rings_wait(i40e_t *i40e) 2676 { 2677 int i, try; 2678 i40e_hw_t *hw = &i40e->i40e_hw_space; 2679 2680 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2681 uint32_t reg; 2682 2683 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2684 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2685 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2686 break; 2687 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2688 } 2689 2690 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) != 0) { 2691 i40e_error(i40e, "timed out disabling rx queue %d", 2692 i); 2693 return (B_FALSE); 2694 } 2695 2696 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2697 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2698 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2699 break; 2700 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2701 } 2702 2703 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) != 0) { 2704 i40e_error(i40e, "timed out disabling tx queue %d", 2705 i); 2706 return (B_FALSE); 2707 } 2708 } 2709 2710 return (B_TRUE); 2711 } 2712 2713 static boolean_t 2714 i40e_shutdown_rings(i40e_t *i40e) 2715 { 2716 i40e_shutdown_rx_rings(i40e); 2717 i40e_shutdown_tx_rings(i40e); 2718 return (i40e_shutdown_rings_wait(i40e)); 2719 } 2720 2721 static void 2722 i40e_setup_rx_descs(i40e_trqpair_t *itrq) 2723 { 2724 int i; 2725 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2726 2727 for (i = 0; i < rxd->rxd_ring_size; i++) { 2728 i40e_rx_control_block_t *rcb; 2729 i40e_rx_desc_t *rdesc; 2730 2731 rcb = rxd->rxd_work_list[i]; 2732 rdesc = &rxd->rxd_desc_ring[i]; 2733 2734 rdesc->read.pkt_addr = 2735 CPU_TO_LE64((uintptr_t)rcb->rcb_dma.dmab_dma_address); 2736 rdesc->read.hdr_addr = 0; 2737 } 2738 } 2739 2740 static boolean_t 2741 i40e_setup_rx_hmc(i40e_trqpair_t *itrq) 2742 { 2743 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2744 i40e_t *i40e = itrq->itrq_i40e; 2745 i40e_hw_t *hw = &i40e->i40e_hw_space; 2746 2747 struct i40e_hmc_obj_rxq rctx; 2748 int err; 2749 2750 bzero(&rctx, sizeof (struct i40e_hmc_obj_rxq)); 2751 rctx.base = rxd->rxd_desc_area.dmab_dma_address / 2752 I40E_HMC_RX_CTX_UNIT; 2753 rctx.qlen = rxd->rxd_ring_size; 2754 VERIFY(i40e->i40e_rx_buf_size >= I40E_HMC_RX_DBUFF_MIN); 2755 VERIFY(i40e->i40e_rx_buf_size <= I40E_HMC_RX_DBUFF_MAX); 2756 rctx.dbuff = i40e->i40e_rx_buf_size >> I40E_RXQ_CTX_DBUFF_SHIFT; 2757 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 2758 rctx.dtype = I40E_HMC_RX_DTYPE_NOSPLIT; 2759 rctx.dsize = I40E_HMC_RX_DSIZE_32BYTE; 2760 rctx.crcstrip = I40E_HMC_RX_CRCSTRIP_ENABLE; 2761 rctx.fc_ena = I40E_HMC_RX_FC_DISABLE; 2762 rctx.l2tsel = I40E_HMC_RX_L2TAGORDER; 2763 rctx.hsplit_0 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2764 rctx.hsplit_1 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2765 rctx.showiv = I40E_HMC_RX_INVLAN_DONTSTRIP; 2766 rctx.rxmax = i40e->i40e_frame_max; 2767 rctx.tphrdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2768 rctx.tphwdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2769 rctx.tphdata_ena = I40E_HMC_RX_TPH_DISABLE; 2770 rctx.tphhead_ena = I40E_HMC_RX_TPH_DISABLE; 2771 rctx.lrxqthresh = I40E_HMC_RX_LOWRXQ_NOINTR; 2772 2773 /* 2774 * This must be set to 0x1, see Table 8-12 in section 8.3.3.2.2. 2775 */ 2776 rctx.prefena = I40E_HMC_RX_PREFENA; 2777 2778 err = i40e_clear_lan_rx_queue_context(hw, itrq->itrq_index); 2779 if (err != I40E_SUCCESS) { 2780 i40e_error(i40e, "failed to clear rx queue %d context: %d", 2781 itrq->itrq_index, err); 2782 return (B_FALSE); 2783 } 2784 2785 err = i40e_set_lan_rx_queue_context(hw, itrq->itrq_index, &rctx); 2786 if (err != I40E_SUCCESS) { 2787 i40e_error(i40e, "failed to set rx queue %d context: %d", 2788 itrq->itrq_index, err); 2789 return (B_FALSE); 2790 } 2791 2792 return (B_TRUE); 2793 } 2794 2795 /* 2796 * Take care of setting up the descriptor rings and actually programming the 2797 * device. See 8.3.3.1.1 for the full list of steps we need to do to enable the 2798 * rx rings. 2799 */ 2800 static boolean_t 2801 i40e_setup_rx_rings(i40e_t *i40e) 2802 { 2803 int i; 2804 i40e_hw_t *hw = &i40e->i40e_hw_space; 2805 2806 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2807 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 2808 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2809 uint32_t reg; 2810 2811 /* 2812 * Step 1. Program all receive ring descriptors. 2813 */ 2814 i40e_setup_rx_descs(itrq); 2815 2816 /* 2817 * Step 2. Program the queue's FPM/HMC context. 2818 */ 2819 if (i40e_setup_rx_hmc(itrq) == B_FALSE) 2820 return (B_FALSE); 2821 2822 /* 2823 * Step 3. Clear the queue's tail pointer and set it to the end 2824 * of the space. 2825 */ 2826 I40E_WRITE_REG(hw, I40E_QRX_TAIL(i), 0); 2827 I40E_WRITE_REG(hw, I40E_QRX_TAIL(i), rxd->rxd_ring_size - 1); 2828 2829 /* 2830 * Step 4. Enable the queue via the QENA_REQ. 2831 */ 2832 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2833 VERIFY0(reg & (I40E_QRX_ENA_QENA_REQ_MASK | 2834 I40E_QRX_ENA_QENA_STAT_MASK)); 2835 reg |= I40E_QRX_ENA_QENA_REQ_MASK; 2836 I40E_WRITE_REG(hw, I40E_QRX_ENA(i), reg); 2837 } 2838 2839 /* 2840 * Note, we wait for every queue to be enabled before we start checking. 2841 * This will hopefully cause most queues to be enabled at this point. 2842 */ 2843 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2844 uint32_t j, reg; 2845 2846 /* 2847 * Step 5. Verify that QENA_STAT has been set. It's promised 2848 * that this should occur within about 10 us, but like other 2849 * systems, we give the card a bit more time. 2850 */ 2851 for (j = 0; j < I40E_RING_WAIT_NTRIES; j++) { 2852 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2853 2854 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) 2855 break; 2856 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2857 } 2858 2859 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { 2860 i40e_error(i40e, "failed to enable rx queue %d, timed " 2861 "out.", i); 2862 return (B_FALSE); 2863 } 2864 } 2865 2866 return (B_TRUE); 2867 } 2868 2869 static boolean_t 2870 i40e_setup_tx_hmc(i40e_trqpair_t *itrq) 2871 { 2872 i40e_t *i40e = itrq->itrq_i40e; 2873 i40e_hw_t *hw = &i40e->i40e_hw_space; 2874 2875 struct i40e_hmc_obj_txq tctx; 2876 struct i40e_vsi_context context; 2877 int err; 2878 2879 bzero(&tctx, sizeof (struct i40e_hmc_obj_txq)); 2880 tctx.new_context = I40E_HMC_TX_NEW_CONTEXT; 2881 tctx.base = itrq->itrq_desc_area.dmab_dma_address / 2882 I40E_HMC_TX_CTX_UNIT; 2883 tctx.fc_ena = I40E_HMC_TX_FC_DISABLE; 2884 tctx.timesync_ena = I40E_HMC_TX_TS_DISABLE; 2885 tctx.fd_ena = I40E_HMC_TX_FD_DISABLE; 2886 tctx.alt_vlan_ena = I40E_HMC_TX_ALT_VLAN_DISABLE; 2887 tctx.head_wb_ena = I40E_HMC_TX_WB_ENABLE; 2888 tctx.qlen = itrq->itrq_tx_ring_size; 2889 tctx.tphrdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2890 tctx.tphrpacket_ena = I40E_HMC_TX_TPH_DISABLE; 2891 tctx.tphwdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2892 tctx.head_wb_addr = itrq->itrq_desc_area.dmab_dma_address + 2893 sizeof (i40e_tx_desc_t) * itrq->itrq_tx_ring_size; 2894 2895 /* 2896 * This field isn't actually documented, like crc, but it suggests that 2897 * it should be zeroed. We leave both of these here because of that for 2898 * now. We should check with Intel on why these are here even. 2899 */ 2900 tctx.crc = 0; 2901 tctx.rdylist_act = 0; 2902 2903 /* 2904 * We're supposed to assign the rdylist field with the value of the 2905 * traffic class index for the first device. We query the VSI parameters 2906 * again to get what the handle is. Note that every queue is always 2907 * assigned to traffic class zero, because we don't actually use them. 2908 */ 2909 bzero(&context, sizeof (struct i40e_vsi_context)); 2910 context.seid = I40E_DEF_VSI_SEID(i40e); 2911 context.pf_num = hw->pf_id; 2912 err = i40e_aq_get_vsi_params(hw, &context, NULL); 2913 if (err != I40E_SUCCESS) { 2914 i40e_error(i40e, "get VSI params failed with %d", err); 2915 return (B_FALSE); 2916 } 2917 tctx.rdylist = LE_16(context.info.qs_handle[0]); 2918 2919 err = i40e_clear_lan_tx_queue_context(hw, itrq->itrq_index); 2920 if (err != I40E_SUCCESS) { 2921 i40e_error(i40e, "failed to clear tx queue %d context: %d", 2922 itrq->itrq_index, err); 2923 return (B_FALSE); 2924 } 2925 2926 err = i40e_set_lan_tx_queue_context(hw, itrq->itrq_index, &tctx); 2927 if (err != I40E_SUCCESS) { 2928 i40e_error(i40e, "failed to set tx queue %d context: %d", 2929 itrq->itrq_index, err); 2930 return (B_FALSE); 2931 } 2932 2933 return (B_TRUE); 2934 } 2935 2936 /* 2937 * Take care of setting up the descriptor rings and actually programming the 2938 * device. See 8.4.3.1.1 for what we need to do here. 2939 */ 2940 static boolean_t 2941 i40e_setup_tx_rings(i40e_t *i40e) 2942 { 2943 int i; 2944 i40e_hw_t *hw = &i40e->i40e_hw_space; 2945 2946 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2947 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 2948 uint32_t reg; 2949 2950 /* 2951 * Step 1. Clear the queue disable flag and verify that the 2952 * index is set correctly. 2953 */ 2954 i40e_pre_tx_queue_cfg(hw, i, B_TRUE); 2955 2956 /* 2957 * Step 2. Prepare the queue's FPM/HMC context. 2958 */ 2959 if (i40e_setup_tx_hmc(itrq) == B_FALSE) 2960 return (B_FALSE); 2961 2962 /* 2963 * Step 3. Verify that it's clear that this PF owns this queue. 2964 */ 2965 reg = I40E_QTX_CTL_PF_QUEUE; 2966 reg |= (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2967 I40E_QTX_CTL_PF_INDX_MASK; 2968 I40E_WRITE_REG(hw, I40E_QTX_CTL(itrq->itrq_index), reg); 2969 i40e_flush(hw); 2970 2971 /* 2972 * Step 4. Set the QENA_REQ flag. 2973 */ 2974 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2975 VERIFY0(reg & (I40E_QTX_ENA_QENA_REQ_MASK | 2976 I40E_QTX_ENA_QENA_STAT_MASK)); 2977 reg |= I40E_QTX_ENA_QENA_REQ_MASK; 2978 I40E_WRITE_REG(hw, I40E_QTX_ENA(i), reg); 2979 } 2980 2981 /* 2982 * Note, we wait for every queue to be enabled before we start checking. 2983 * This will hopefully cause most queues to be enabled at this point. 2984 */ 2985 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2986 uint32_t j, reg; 2987 2988 /* 2989 * Step 5. Verify that QENA_STAT has been set. It's promised 2990 * that this should occur within about 10 us, but like BSD, 2991 * we'll try for up to 100 ms for this queue. 2992 */ 2993 for (j = 0; j < I40E_RING_WAIT_NTRIES; j++) { 2994 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2995 2996 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) 2997 break; 2998 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2999 } 3000 3001 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { 3002 i40e_error(i40e, "failed to enable tx queue %d, timed " 3003 "out", i); 3004 return (B_FALSE); 3005 } 3006 } 3007 3008 return (B_TRUE); 3009 } 3010 3011 void 3012 i40e_stop(i40e_t *i40e, boolean_t free_allocations) 3013 { 3014 uint_t i; 3015 i40e_hw_t *hw = &i40e->i40e_hw_space; 3016 3017 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 3018 3019 /* 3020 * Shutdown and drain the tx and rx pipeline. We do this using the 3021 * following steps. 3022 * 3023 * 1) Shutdown interrupts to all the queues (trying to keep the admin 3024 * queue alive). 3025 * 3026 * 2) Remove all of the interrupt tx and rx causes by setting the 3027 * interrupt linked lists to zero. 3028 * 3029 * 2) Shutdown the tx and rx rings. Because i40e_shutdown_rings() should 3030 * wait for all the queues to be disabled, once we reach that point 3031 * it should be safe to free associated data. 3032 * 3033 * 4) Wait 50ms after all that is done. This ensures that the rings are 3034 * ready for programming again and we don't have to think about this 3035 * in other parts of the driver. 3036 * 3037 * 5) Disable remaining chip interrupts, (admin queue, etc.) 3038 * 3039 * 6) Verify that FM is happy with all the register accesses we 3040 * performed. 3041 */ 3042 i40e_intr_io_disable_all(i40e); 3043 i40e_intr_io_clear_cause(i40e); 3044 3045 if (i40e_shutdown_rings(i40e) == B_FALSE) { 3046 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3047 } 3048 3049 delay(50 * drv_usectohz(1000)); 3050 3051 /* 3052 * We don't delete the default VSI because it replaces the VEB 3053 * after VEB deletion (see the "Delete Element" section). 3054 * Furthermore, since the default VSI is provided by the 3055 * firmware, we never attempt to delete it. 3056 */ 3057 for (i = 1; i < i40e->i40e_num_rx_groups; i++) { 3058 i40e_delete_vsi(i40e, i); 3059 } 3060 3061 if (i40e->i40e_veb_seid != 0) { 3062 int rc = i40e_aq_delete_element(hw, i40e->i40e_veb_seid, NULL); 3063 3064 if (rc != I40E_SUCCESS) { 3065 i40e_error(i40e, "Failed to delete VEB %d: %d", rc, 3066 hw->aq.asq_last_status); 3067 } 3068 3069 i40e->i40e_veb_seid = 0; 3070 } 3071 3072 i40e_intr_chip_fini(i40e); 3073 3074 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3075 mutex_enter(&i40e->i40e_trqpairs[i].itrq_rx_lock); 3076 mutex_enter(&i40e->i40e_trqpairs[i].itrq_tx_lock); 3077 } 3078 3079 /* 3080 * We should consider refactoring this to be part of the ring start / 3081 * stop routines at some point. 3082 */ 3083 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3084 i40e_stats_trqpair_fini(&i40e->i40e_trqpairs[i]); 3085 } 3086 3087 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 3088 DDI_FM_OK) { 3089 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3090 } 3091 3092 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3093 i40e_tx_cleanup_ring(&i40e->i40e_trqpairs[i]); 3094 } 3095 3096 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3097 mutex_exit(&i40e->i40e_trqpairs[i].itrq_rx_lock); 3098 mutex_exit(&i40e->i40e_trqpairs[i].itrq_tx_lock); 3099 } 3100 3101 for (i = 0; i < i40e->i40e_num_rx_groups; i++) { 3102 i40e_stat_vsi_fini(i40e, i); 3103 } 3104 3105 i40e->i40e_link_speed = 0; 3106 i40e->i40e_link_duplex = 0; 3107 i40e_link_state_set(i40e, LINK_STATE_UNKNOWN); 3108 3109 if (free_allocations) { 3110 i40e_free_ring_mem(i40e, B_FALSE); 3111 } 3112 } 3113 3114 boolean_t 3115 i40e_start(i40e_t *i40e, boolean_t alloc) 3116 { 3117 i40e_hw_t *hw = &i40e->i40e_hw_space; 3118 boolean_t rc = B_TRUE; 3119 int i, err; 3120 3121 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 3122 3123 if (alloc) { 3124 if (i40e_alloc_ring_mem(i40e) == B_FALSE) { 3125 i40e_error(i40e, 3126 "Failed to allocate ring memory"); 3127 return (B_FALSE); 3128 } 3129 } 3130 3131 /* 3132 * This should get refactored to be part of ring start and stop at 3133 * some point, along with most of the logic here. 3134 */ 3135 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3136 if (i40e_stats_trqpair_init(&i40e->i40e_trqpairs[i]) == 3137 B_FALSE) { 3138 int j; 3139 3140 for (j = 0; j < i; j++) { 3141 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[j]; 3142 i40e_stats_trqpair_fini(itrq); 3143 } 3144 return (B_FALSE); 3145 } 3146 } 3147 3148 if (!i40e_chip_start(i40e)) { 3149 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 3150 rc = B_FALSE; 3151 goto done; 3152 } 3153 3154 if (i40e_setup_rx_rings(i40e) == B_FALSE) { 3155 rc = B_FALSE; 3156 goto done; 3157 } 3158 3159 if (i40e_setup_tx_rings(i40e) == B_FALSE) { 3160 rc = B_FALSE; 3161 goto done; 3162 } 3163 3164 /* 3165 * Enable broadcast traffic; however, do not enable multicast traffic. 3166 * That's handle exclusively through MAC's mc_multicst routines. 3167 */ 3168 err = i40e_aq_set_vsi_broadcast(hw, I40E_DEF_VSI_SEID(i40e), B_TRUE, 3169 NULL); 3170 if (err != I40E_SUCCESS) { 3171 i40e_error(i40e, "failed to set default VSI: %d", err); 3172 rc = B_FALSE; 3173 goto done; 3174 } 3175 3176 err = i40e_aq_set_mac_config(hw, i40e->i40e_frame_max, B_TRUE, 0, NULL); 3177 if (err != I40E_SUCCESS) { 3178 i40e_error(i40e, "failed to set MAC config: %d", err); 3179 rc = B_FALSE; 3180 goto done; 3181 } 3182 3183 /* 3184 * Finally, make sure that we're happy from an FM perspective. 3185 */ 3186 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 3187 DDI_FM_OK) { 3188 rc = B_FALSE; 3189 goto done; 3190 } 3191 3192 /* Clear state bits prior to final interrupt enabling. */ 3193 atomic_and_32(&i40e->i40e_state, 3194 ~(I40E_ERROR | I40E_STALL | I40E_OVERTEMP)); 3195 3196 i40e_intr_io_enable_all(i40e); 3197 3198 done: 3199 if (rc == B_FALSE) { 3200 i40e_stop(i40e, B_FALSE); 3201 if (alloc == B_TRUE) { 3202 i40e_free_ring_mem(i40e, B_TRUE); 3203 } 3204 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3205 } 3206 3207 return (rc); 3208 } 3209 3210 /* 3211 * We may have loaned up descriptors to the stack. As such, if we still have 3212 * them outstanding, then we will not continue with detach. 3213 */ 3214 static boolean_t 3215 i40e_drain_rx(i40e_t *i40e) 3216 { 3217 mutex_enter(&i40e->i40e_rx_pending_lock); 3218 while (i40e->i40e_rx_pending > 0) { 3219 if (cv_reltimedwait(&i40e->i40e_rx_pending_cv, 3220 &i40e->i40e_rx_pending_lock, 3221 drv_usectohz(I40E_DRAIN_RX_WAIT), TR_CLOCK_TICK) == -1) { 3222 mutex_exit(&i40e->i40e_rx_pending_lock); 3223 return (B_FALSE); 3224 } 3225 } 3226 mutex_exit(&i40e->i40e_rx_pending_lock); 3227 3228 return (B_TRUE); 3229 } 3230 3231 /* 3232 * DDI UFM Callbacks 3233 */ 3234 static int 3235 i40e_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 3236 ddi_ufm_image_t *img) 3237 { 3238 if (imgno != 0) 3239 return (EINVAL); 3240 3241 ddi_ufm_image_set_desc(img, "Firmware"); 3242 ddi_ufm_image_set_nslots(img, 1); 3243 3244 return (0); 3245 } 3246 3247 static int 3248 i40e_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 3249 uint_t slotno, ddi_ufm_slot_t *slot) 3250 { 3251 i40e_t *i40e = (i40e_t *)arg; 3252 char *fw_ver = NULL, *fw_bld = NULL, *api_ver = NULL; 3253 nvlist_t *misc = NULL; 3254 uint_t flags = DDI_PROP_DONTPASS; 3255 int err; 3256 3257 if (imgno != 0 || slotno != 0 || 3258 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3259 "firmware-version", &fw_ver) != DDI_PROP_SUCCESS || 3260 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3261 "firmware-build", &fw_bld) != DDI_PROP_SUCCESS || 3262 ddi_prop_lookup_string(DDI_DEV_T_ANY, i40e->i40e_dip, flags, 3263 "api-version", &api_ver) != DDI_PROP_SUCCESS) { 3264 err = EINVAL; 3265 goto err; 3266 } 3267 3268 ddi_ufm_slot_set_attrs(slot, DDI_UFM_ATTR_ACTIVE); 3269 ddi_ufm_slot_set_version(slot, fw_ver); 3270 3271 (void) nvlist_alloc(&misc, NV_UNIQUE_NAME, KM_SLEEP); 3272 if ((err = nvlist_add_string(misc, "firmware-build", fw_bld)) != 0 || 3273 (err = nvlist_add_string(misc, "api-version", api_ver)) != 0) { 3274 goto err; 3275 } 3276 ddi_ufm_slot_set_misc(slot, misc); 3277 3278 ddi_prop_free(fw_ver); 3279 ddi_prop_free(fw_bld); 3280 ddi_prop_free(api_ver); 3281 3282 return (0); 3283 err: 3284 nvlist_free(misc); 3285 if (fw_ver != NULL) 3286 ddi_prop_free(fw_ver); 3287 if (fw_bld != NULL) 3288 ddi_prop_free(fw_bld); 3289 if (api_ver != NULL) 3290 ddi_prop_free(api_ver); 3291 3292 return (err); 3293 } 3294 3295 static int 3296 i40e_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps) 3297 { 3298 *caps = DDI_UFM_CAP_REPORT; 3299 3300 return (0); 3301 } 3302 3303 static ddi_ufm_ops_t i40e_ufm_ops = { 3304 NULL, 3305 i40e_ufm_fill_image, 3306 i40e_ufm_fill_slot, 3307 i40e_ufm_getcaps 3308 }; 3309 3310 static int 3311 i40e_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 3312 { 3313 i40e_t *i40e; 3314 struct i40e_osdep *osdep; 3315 i40e_hw_t *hw; 3316 int instance; 3317 3318 if (cmd != DDI_ATTACH) 3319 return (DDI_FAILURE); 3320 3321 instance = ddi_get_instance(devinfo); 3322 i40e = kmem_zalloc(sizeof (i40e_t), KM_SLEEP); 3323 3324 i40e->i40e_aqbuf = kmem_zalloc(I40E_ADMINQ_BUFSZ, KM_SLEEP); 3325 i40e->i40e_instance = instance; 3326 i40e->i40e_dip = devinfo; 3327 3328 hw = &i40e->i40e_hw_space; 3329 osdep = &i40e->i40e_osdep_space; 3330 hw->back = osdep; 3331 osdep->ios_i40e = i40e; 3332 3333 ddi_set_driver_private(devinfo, i40e); 3334 3335 i40e_fm_init(i40e); 3336 i40e->i40e_attach_progress |= I40E_ATTACH_FM_INIT; 3337 3338 if (pci_config_setup(devinfo, &osdep->ios_cfg_handle) != DDI_SUCCESS) { 3339 i40e_error(i40e, "Failed to map PCI configurations."); 3340 goto attach_fail; 3341 } 3342 i40e->i40e_attach_progress |= I40E_ATTACH_PCI_CONFIG; 3343 3344 i40e_identify_hardware(i40e); 3345 3346 if (!i40e_regs_map(i40e)) { 3347 i40e_error(i40e, "Failed to map device registers."); 3348 goto attach_fail; 3349 } 3350 i40e->i40e_attach_progress |= I40E_ATTACH_REGS_MAP; 3351 3352 i40e_init_properties(i40e); 3353 i40e->i40e_attach_progress |= I40E_ATTACH_PROPS; 3354 3355 if (!i40e_common_code_init(i40e, hw)) 3356 goto attach_fail; 3357 i40e->i40e_attach_progress |= I40E_ATTACH_COMMON_CODE; 3358 3359 /* 3360 * When we participate in IRM, we should make sure that we register 3361 * ourselves with it before callbacks. 3362 */ 3363 if (!i40e_alloc_intrs(i40e, devinfo)) { 3364 i40e_error(i40e, "Failed to allocate interrupts."); 3365 goto attach_fail; 3366 } 3367 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_INTR; 3368 3369 if (!i40e_alloc_trqpairs(i40e)) { 3370 i40e_error(i40e, 3371 "Failed to allocate receive & transmit rings."); 3372 goto attach_fail; 3373 } 3374 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_RINGSLOCKS; 3375 3376 if (!i40e_map_intrs_to_vectors(i40e)) { 3377 i40e_error(i40e, "Failed to map interrupts to vectors."); 3378 goto attach_fail; 3379 } 3380 3381 if (!i40e_add_intr_handlers(i40e)) { 3382 i40e_error(i40e, "Failed to add the interrupt handlers."); 3383 goto attach_fail; 3384 } 3385 i40e->i40e_attach_progress |= I40E_ATTACH_ADD_INTR; 3386 3387 if (!i40e_final_init(i40e)) { 3388 i40e_error(i40e, "Final initialization failed."); 3389 goto attach_fail; 3390 } 3391 i40e->i40e_attach_progress |= I40E_ATTACH_INIT; 3392 3393 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 3394 DDI_FM_OK) { 3395 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 3396 goto attach_fail; 3397 } 3398 3399 if (!i40e_stats_init(i40e)) { 3400 i40e_error(i40e, "Stats initialization failed."); 3401 goto attach_fail; 3402 } 3403 i40e->i40e_attach_progress |= I40E_ATTACH_STATS; 3404 3405 if (!i40e_register_mac(i40e)) { 3406 i40e_error(i40e, "Failed to register to MAC/GLDv3"); 3407 goto attach_fail; 3408 } 3409 i40e->i40e_attach_progress |= I40E_ATTACH_MAC; 3410 3411 i40e->i40e_periodic_id = ddi_periodic_add(i40e_timer, i40e, 3412 I40E_CYCLIC_PERIOD, DDI_IPL_0); 3413 if (i40e->i40e_periodic_id == 0) { 3414 i40e_error(i40e, "Failed to add the link-check timer"); 3415 goto attach_fail; 3416 } 3417 i40e->i40e_attach_progress |= I40E_ATTACH_LINK_TIMER; 3418 3419 if (!i40e_enable_interrupts(i40e)) { 3420 i40e_error(i40e, "Failed to enable DDI interrupts"); 3421 goto attach_fail; 3422 } 3423 i40e->i40e_attach_progress |= I40E_ATTACH_ENABLE_INTR; 3424 3425 if (i40e->i40e_hw_space.bus.func == 0) { 3426 if (ddi_ufm_init(i40e->i40e_dip, DDI_UFM_CURRENT_VERSION, 3427 &i40e_ufm_ops, &i40e->i40e_ufmh, i40e) != 0) { 3428 i40e_error(i40e, "failed to initialize UFM subsystem"); 3429 goto attach_fail; 3430 } 3431 ddi_ufm_update(i40e->i40e_ufmh); 3432 i40e->i40e_attach_progress |= I40E_ATTACH_UFM_INIT; 3433 } 3434 3435 atomic_or_32(&i40e->i40e_state, I40E_INITIALIZED); 3436 3437 mutex_enter(&i40e_glock); 3438 list_insert_tail(&i40e_glist, i40e); 3439 mutex_exit(&i40e_glock); 3440 3441 return (DDI_SUCCESS); 3442 3443 attach_fail: 3444 i40e_unconfigure(devinfo, i40e); 3445 return (DDI_FAILURE); 3446 } 3447 3448 static int 3449 i40e_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 3450 { 3451 i40e_t *i40e; 3452 3453 if (cmd != DDI_DETACH) 3454 return (DDI_FAILURE); 3455 3456 i40e = (i40e_t *)ddi_get_driver_private(devinfo); 3457 if (i40e == NULL) { 3458 i40e_log(NULL, "i40e_detach() called with no i40e pointer!"); 3459 return (DDI_FAILURE); 3460 } 3461 3462 if (i40e_drain_rx(i40e) == B_FALSE) { 3463 i40e_log(i40e, "timed out draining DMA resources, %d buffers " 3464 "remain", i40e->i40e_rx_pending); 3465 return (DDI_FAILURE); 3466 } 3467 3468 mutex_enter(&i40e_glock); 3469 list_remove(&i40e_glist, i40e); 3470 mutex_exit(&i40e_glock); 3471 3472 i40e_unconfigure(devinfo, i40e); 3473 3474 return (DDI_SUCCESS); 3475 } 3476 3477 static struct cb_ops i40e_cb_ops = { 3478 nulldev, /* cb_open */ 3479 nulldev, /* cb_close */ 3480 nodev, /* cb_strategy */ 3481 nodev, /* cb_print */ 3482 nodev, /* cb_dump */ 3483 nodev, /* cb_read */ 3484 nodev, /* cb_write */ 3485 nodev, /* cb_ioctl */ 3486 nodev, /* cb_devmap */ 3487 nodev, /* cb_mmap */ 3488 nodev, /* cb_segmap */ 3489 nochpoll, /* cb_chpoll */ 3490 ddi_prop_op, /* cb_prop_op */ 3491 NULL, /* cb_stream */ 3492 D_MP | D_HOTPLUG, /* cb_flag */ 3493 CB_REV, /* cb_rev */ 3494 nodev, /* cb_aread */ 3495 nodev /* cb_awrite */ 3496 }; 3497 3498 static struct dev_ops i40e_dev_ops = { 3499 DEVO_REV, /* devo_rev */ 3500 0, /* devo_refcnt */ 3501 NULL, /* devo_getinfo */ 3502 nulldev, /* devo_identify */ 3503 nulldev, /* devo_probe */ 3504 i40e_attach, /* devo_attach */ 3505 i40e_detach, /* devo_detach */ 3506 nodev, /* devo_reset */ 3507 &i40e_cb_ops, /* devo_cb_ops */ 3508 NULL, /* devo_bus_ops */ 3509 ddi_power, /* devo_power */ 3510 ddi_quiesce_not_supported /* devo_quiesce */ 3511 }; 3512 3513 static struct modldrv i40e_modldrv = { 3514 &mod_driverops, 3515 i40e_ident, 3516 &i40e_dev_ops 3517 }; 3518 3519 static struct modlinkage i40e_modlinkage = { 3520 MODREV_1, 3521 &i40e_modldrv, 3522 NULL 3523 }; 3524 3525 /* 3526 * Module Initialization Functions. 3527 */ 3528 int 3529 _init(void) 3530 { 3531 int status; 3532 3533 list_create(&i40e_glist, sizeof (i40e_t), offsetof(i40e_t, i40e_glink)); 3534 list_create(&i40e_dlist, sizeof (i40e_device_t), 3535 offsetof(i40e_device_t, id_link)); 3536 mutex_init(&i40e_glock, NULL, MUTEX_DRIVER, NULL); 3537 mac_init_ops(&i40e_dev_ops, I40E_MODULE_NAME); 3538 3539 status = mod_install(&i40e_modlinkage); 3540 if (status != DDI_SUCCESS) { 3541 mac_fini_ops(&i40e_dev_ops); 3542 mutex_destroy(&i40e_glock); 3543 list_destroy(&i40e_dlist); 3544 list_destroy(&i40e_glist); 3545 } 3546 3547 return (status); 3548 } 3549 3550 int 3551 _info(struct modinfo *modinfop) 3552 { 3553 return (mod_info(&i40e_modlinkage, modinfop)); 3554 } 3555 3556 int 3557 _fini(void) 3558 { 3559 int status; 3560 3561 status = mod_remove(&i40e_modlinkage); 3562 if (status == DDI_SUCCESS) { 3563 mac_fini_ops(&i40e_dev_ops); 3564 mutex_destroy(&i40e_glock); 3565 list_destroy(&i40e_dlist); 3566 list_destroy(&i40e_glist); 3567 } 3568 3569 return (status); 3570 } 3571