1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved. 14 * Copyright 2016 Joyent, Inc. 15 */ 16 17 /* 18 * i40e - Intel 10/40 Gb Ethernet driver 19 * 20 * The i40e driver is the main software device driver for the Intel 40 Gb family 21 * of devices. Note that these devices come in many flavors with both 40 GbE 22 * ports and 10 GbE ports. This device is the successor to the 82599 family of 23 * devices (ixgbe). 24 * 25 * Unlike previous generations of Intel 1 GbE and 10 GbE devices, the 40 GbE 26 * devices defined in the XL710 controller (previously known as Fortville) are a 27 * rather different beast and have a small switch embedded inside of them. In 28 * addition, the way that most of the programming is done has been overhauled. 29 * As opposed to just using PCIe memory mapped registers, it also has an 30 * administrative queue which is used to communicate with firmware running on 31 * the chip. 32 * 33 * Each physical function in the hardware shows up as a device that this driver 34 * will bind to. The hardware splits many resources evenly across all of the 35 * physical functions present on the device, while other resources are instead 36 * shared across the entire card and its up to the device driver to 37 * intelligently partition them. 38 * 39 * ------------ 40 * Organization 41 * ------------ 42 * 43 * This driver is made up of several files which have their own theory 44 * statements spread across them. We'll touch on the high level purpose of each 45 * file here, and then we'll get into more discussion on how the device is 46 * generally modelled with respect to the interfaces in illumos. 47 * 48 * i40e_gld.c: This file contains all of the bindings to MAC and the networking 49 * stack. 50 * 51 * i40e_intr.c: This file contains all of the interrupt service routines and 52 * contains logic to enable and disable interrupts on the hardware. 53 * It also contains the logic to map hardware resources such as the 54 * rings to and from interrupts and controls their ability to fire. 55 * 56 * There is a big theory statement on interrupts present there. 57 * 58 * i40e_main.c: The file that you're currently in. It interfaces with the 59 * traditional OS DDI interfaces and is in charge of configuring 60 * the device. 61 * 62 * i40e_osdep.[ch]: These files contain interfaces and definitions needed to 63 * work with Intel's common code for the device. 64 * 65 * i40e_stats.c: This file contains the general work and logic around our 66 * kstats. A theory statement on their organization and use of the 67 * hardware exists there. 68 * 69 * i40e_sw.h: This header file contains all of the primary structure definitions 70 * and constants that are used across the entire driver. 71 * 72 * i40e_transceiver.c: This file contains all of the logic for sending and 73 * receiving data. It contains all of the ring and DMA 74 * allocation logic, as well as, the actual interfaces to 75 * send and receive data. 76 * 77 * A big theory statement on ring management, descriptors, 78 * and how it ties into the OS is present there. 79 * 80 * -------------- 81 * General Design 82 * -------------- 83 * 84 * Before we go too far into the general way we've laid out data structures and 85 * the like, it's worth taking some time to explain how the hardware is 86 * organized. This organization informs a lot of how we do things at this time 87 * in the driver. 88 * 89 * Each physical device consists of a number of one or more ports, which are 90 * considered physical functions in the PCI sense and thus each get enumerated 91 * by the system, resulting in an instance being created and attached to. While 92 * there are many resources that are unique to each physical function eg. 93 * instance of the device, there are many that are shared across all of them. 94 * Several resources have an amount reserved for each Virtual Station Interface 95 * (VSI) and then a static pool of resources, available for all functions on the 96 * card. 97 * 98 * The most important resource in hardware are its transmit and receive queue 99 * pairs (i40e_trqpair_t). These should be thought of as rings in GLDv3 100 * parlance. There are a set number of these on each device; however, they are 101 * statically partitioned among all of the different physical functions. 102 * 103 * 'Fortville' (the code name for this device family) is basically a switch. To 104 * map MAC addresses and other things to queues, we end up having to create 105 * Virtual Station Interfaces (VSIs) and establish forwarding rules that direct 106 * traffic to a queue. A VSI owns a collection of queues and has a series of 107 * forwarding rules that point to it. One way to think of this is to treat it 108 * like MAC does a VNIC. When MAC refers to a group, a collection of rings and 109 * classification resources, that is a VSI in i40e. 110 * 111 * The sets of VSIs is shared across the entire device, though there may be some 112 * amount that are reserved to each PF. Because the GLDv3 does not let us change 113 * the number of groups dynamically, we instead statically divide this amount 114 * evenly between all the functions that exist. In addition, we have the same 115 * problem with the mac address forwarding rules. There are a static number that 116 * exist shared across all the functions. 117 * 118 * To handle both of these resources, what we end up doing is going through and 119 * determining which functions belong to the same device. Nominally one might do 120 * this by having a nexus driver; however, a prime requirement for a nexus 121 * driver is identifying the various children and activating them. While it is 122 * possible to get this information from NVRAM, we would end up duplicating a 123 * lot of the PCI enumeration logic. Really, at the end of the day, the device 124 * doesn't give us the traditional identification properties we want from a 125 * nexus driver. 126 * 127 * Instead, we rely on some properties that are guaranteed to be unique. While 128 * it might be tempting to leverage the PBA or serial number of the device from 129 * NVRAM, there is nothing that says that two devices can't be mis-programmed to 130 * have the same values in NVRAM. Instead, we uniquely identify a group of 131 * functions based on their parent in the /devices tree, their PCI bus and PCI 132 * function identifiers. Using either on their own may not be sufficient. 133 * 134 * For each unique PCI device that we encounter, we'll create a i40e_device_t. 135 * From there, because we don't have a good way to tell the GLDv3 about sharing 136 * resources between everything, we'll end up just dividing the resources 137 * evenly between all of the functions. Longer term, if we don't have to declare 138 * to the GLDv3 that these resources are shared, then we'll maintain a pool and 139 * hae each PF allocate from the pool in the device, thus if only two of four 140 * ports are being used, for example, then all of the resources can still be 141 * used. 142 * 143 * ------------------------------------------- 144 * Transmit and Receive Queue Pair Allocations 145 * ------------------------------------------- 146 * 147 * NVRAM ends up assigning each PF its own share of the transmit and receive LAN 148 * queue pairs, we have no way of modifying it, only observing it. From there, 149 * it's up to us to map these queues to VSIs and VFs. Since we don't support any 150 * VFs at this time, we only focus on assignments to VSIs. 151 * 152 * At the moment, we used a static mapping of transmit/receive queue pairs to a 153 * given VSI (eg. rings to a group). Though in the fullness of time, we want to 154 * make this something which is fully dynamic and take advantage of documented, 155 * but not yet available functionality for adding filters based on VXLAN and 156 * other encapsulation technologies. 157 * 158 * ------------------------------------- 159 * Broadcast, Multicast, and Promiscuous 160 * ------------------------------------- 161 * 162 * As part of the GLDv3, we need to make sure that we can handle receiving 163 * broadcast and multicast traffic. As well as enabling promiscuous mode when 164 * requested. GLDv3 requires that all broadcast and multicast traffic be 165 * retrieved by the default group, eg. the first one. This is the same thing as 166 * the default VSI. 167 * 168 * To receieve broadcast traffic, we enable it through the admin queue, rather 169 * than use one of our filters for it. For multicast traffic, we reserve a 170 * certain number of the hash filters and assign them to a given PF. When we 171 * exceed those, we then switch to using promicuous mode for multicast traffic. 172 * 173 * More specifically, once we exceed the number of filters (indicated because 174 * the i40e_t`i40e_resources.ifr_nmcastfilt == 175 * i40e_t`i40e_resources.ifr_nmcastfilt_used), we then instead need to toggle 176 * promiscuous mode. If promiscuous mode is toggled then we keep track of the 177 * number of MACs added to it by incrementing i40e_t`i40e_mcast_promisc_count. 178 * That will stay enabled until that count reaches zero indicating that we have 179 * only added multicast addresses that we have a corresponding entry for. 180 * 181 * Because MAC itself wants to toggle promiscuous mode, which includes both 182 * unicast and multicast traffic, we go through and keep track of that 183 * ourselves. That is maintained through the use of the i40e_t`i40e_promisc_on 184 * member. 185 * 186 * -------------- 187 * VSI Management 188 * -------------- 189 * 190 * At this time, we currently only support a single MAC group, and thus a single 191 * VSI. This VSI is considered the default VSI and should be the only one that 192 * exists after a reset. Currently it is stored as the member 193 * i40e_t`i40e_vsi_id. While this works for the moment and for an initial 194 * driver, it's not sufficient for the longer-term path of the driver. Instead, 195 * we'll want to actually have a unique i40e_vsi_t structure which is used 196 * everywhere. Note that this means that every place that uses the 197 * i40e_t`i40e_vsi_id will need to be refactored. 198 * 199 * ---------------- 200 * Structure Layout 201 * ---------------- 202 * 203 * The following images relates the core data structures together. The primary 204 * structure in the system is the i40e_t. It itself contains multiple rings, 205 * i40e_trqpair_t's which contain the various transmit and receive data. The 206 * receive data is stored outside of the i40e_trqpair_t and instead in the 207 * i40e_rx_data_t. The i40e_t has a corresponding i40e_device_t which keeps 208 * track of per-physical device state. Finally, for every active descriptor, 209 * there is a corresponding control block, which is where the 210 * i40e_rx_control_block_t and the i40e_tx_control_block_t come from. 211 * 212 * +-----------------------+ +-----------------------+ 213 * | Global i40e_t list | | Global Device list | 214 * | | +--| | 215 * | i40e_glist | | | i40e_dlist | 216 * +-----------------------+ | +-----------------------+ 217 * | v 218 * | +------------------------+ +-----------------------+ 219 * | | Device-wide Structure |----->| Device-wide Structure |--> ... 220 * | | i40e_device_t | | i40e_device_t | 221 * | | | +-----------------------+ 222 * | | dev_info_t * ------+--> Parent in devices tree. 223 * | | uint_t ------+--> PCI bus number 224 * | | uint_t ------+--> PCI device number 225 * | | uint_t ------+--> Number of functions 226 * | | i40e_switch_rsrcs_t ---+--> Captured total switch resources 227 * | | list_t ------+-------------+ 228 * | +------------------------+ | 229 * | ^ | 230 * | +--------+ | 231 * | | v 232 * | +---------------------------+ | +-------------------+ 233 * +->| GLDv3 Device, per PF |-----|-->| GLDv3 Device (PF) |--> ... 234 * | i40e_t | | | i40e_t | 235 * | **Primary Structure** | | +-------------------+ 236 * | | | 237 * | i40e_device_t * --+-----+ 238 * | i40e_state_t --+---> Device State 239 * | i40e_hw_t --+---> Intel common code structure 240 * | mac_handle_t --+---> GLDv3 handle to MAC 241 * | ddi_periodic_t --+---> Link activity timer 242 * | int (vsi_id) --+---> VSI ID, main identifier 243 * | i40e_func_rsrc_t --+---> Available hardware resources 244 * | i40e_switch_rsrc_t * --+---> Switch resource snapshot 245 * | i40e_sdu --+---> Current MTU 246 * | i40e_frame_max --+---> Current HW frame size 247 * | i40e_uaddr_t * --+---> Array of assigned unicast MACs 248 * | i40e_maddr_t * --+---> Array of assigned multicast MACs 249 * | i40e_mcast_promisccount --+---> Active multicast state 250 * | i40e_promisc_on --+---> Current promiscuous mode state 251 * | int --+---> Number of transmit/receive pairs 252 * | kstat_t * --+---> PF kstats 253 * | kstat_t * --+---> VSI kstats 254 * | i40e_pf_stats_t --+---> PF kstat backing data 255 * | i40e_vsi_stats_t --+---> VSI kstat backing data 256 * | i40e_trqpair_t * --+---------+ 257 * +---------------------------+ | 258 * | 259 * v 260 * +-------------------------------+ +-----------------------------+ 261 * | Transmit/Receive Queue Pair |-------| Transmit/Receive Queue Pair |->... 262 * | i40e_trqpair_t | | i40e_trqpair_t | 263 * + Ring Data Structure | +-----------------------------+ 264 * | | 265 * | mac_ring_handle_t +--> MAC RX ring handle 266 * | mac_ring_handle_t +--> MAC TX ring handle 267 * | i40e_rxq_stat_t --+--> RX Queue stats 268 * | i40e_txq_stat_t --+--> TX Queue stats 269 * | uint32_t (tx ring size) +--> TX Ring Size 270 * | uint32_t (tx free list size) +--> TX Free List Size 271 * | i40e_dma_buffer_t --------+--> TX Descriptor ring DMA 272 * | i40e_tx_desc_t * --------+--> TX descriptor ring 273 * | volatile unt32_t * +--> TX Write back head 274 * | uint32_t -------+--> TX ring head 275 * | uint32_t -------+--> TX ring tail 276 * | uint32_t -------+--> Num TX desc free 277 * | i40e_tx_control_block_t * --+--> TX control block array ---+ 278 * | i40e_tx_control_block_t ** --+--> TCB work list ----+ 279 * | i40e_tx_control_block_t ** --+--> TCB free list ---+ 280 * | uint32_t -------+--> Free TCB count | 281 * | i40e_rx_data_t * -------+--+ v 282 * +-------------------------------+ | +---------------------------+ 283 * | | Per-TX Frame Metadata | 284 * | | i40e_tx_control_block_t | 285 * +--------------------+ | | 286 * | mblk to transmit <--+--- mblk_t * | 287 * | type of transmit <--+--- i40e_tx_type_t | 288 * | TX DMA handle <--+--- ddi_dma_handle_t | 289 * v TX DMA buffer <--+--- i40e_dma_buffer_t | 290 * +------------------------------+ +---------------------------+ 291 * | Core Receive Data | 292 * | i40e_rx_data_t | 293 * | | 294 * | i40e_dma_buffer_t --+--> RX descriptor DMA Data 295 * | i40e_rx_desc_t --+--> RX descriptor ring 296 * | uint32_t --+--> Next free desc. 297 * | i40e_rx_control_block_t * --+--> RX Control Block Array ---+ 298 * | i40e_rx_control_block_t ** --+--> RCB work list ---+ 299 * | i40e_rx_control_block_t ** --+--> RCB free list ---+ 300 * +------------------------------+ | 301 * ^ | 302 * | +---------------------------+ | 303 * | | Per-RX Frame Metadata |<---------------+ 304 * | | i40e_rx_control_block_t | 305 * | | | 306 * | | mblk_t * ----+--> Received mblk_t data 307 * | | uint32_t ----+--> Reference count 308 * | | i40e_dma_buffer_t ----+--> Receive data DMA info 309 * | | frtn_t ----+--> mblk free function info 310 * +-----+-- i40e_rx_data_t * | 311 * +---------------------------+ 312 * 313 * ------------- 314 * Lock Ordering 315 * ------------- 316 * 317 * In order to ensure that we don't deadlock, the following represents the 318 * lock order being used. When grabbing locks, follow the following order. Lower 319 * numbers are more important. Thus, the i40e_glock which is number 0, must be 320 * taken before any other locks in the driver. On the other hand, the 321 * i40e_t`i40e_stat_lock, has the highest number because it's the least 322 * important lock. Note, that just because one lock is higher than another does 323 * not mean that all intermediary locks are required. 324 * 325 * 0) i40e_glock 326 * 1) i40e_t`i40e_general_lock 327 * 328 * 2) i40e_trqpair_t`itrq_rx_lock 329 * 3) i40e_trqpair_t`itrq_tx_lock 330 * 4) i40e_t`i40e_rx_pending_lock 331 * 5) i40e_trqpair_t`itrq_tcb_lock 332 * 333 * 6) i40e_t`i40e_stat_lock 334 * 335 * Rules and expectations: 336 * 337 * 1) A thread holding locks belong to one PF should not hold locks belonging to 338 * a second. If for some reason this becomes necessary, locks should be grabbed 339 * based on the list order in the i40e_device_t, which implies that the 340 * i40e_glock is held. 341 * 342 * 2) When grabbing locks between multiple transmit and receive queues, the 343 * locks for the lowest number transmit/receive queue should be grabbed first. 344 * 345 * 3) When grabbing both the transmit and receive lock for a given queue, always 346 * grab i40e_trqpair_t`itrq_rx_lock before the i40e_trqpair_t`itrq_tx_lock. 347 * 348 * 4) The following pairs of locks are not expected to be held at the same time: 349 * 350 * o i40e_t`i40e_rx_pending_lock and i40e_trqpair_t`itrq_tcb_lock 351 * 352 * ----------- 353 * Future Work 354 * ----------- 355 * 356 * At the moment the i40e_t driver is rather bare bones, allowing us to start 357 * getting data flowing and folks using it while we develop additional features. 358 * While bugs have been filed to cover this future work, the following gives an 359 * overview of expected work: 360 * 361 * o TSO support 362 * o RSS / multiple ring support 363 * o Multiple group support 364 * o DMA binding and breaking up the locking in ring recycling. 365 * o Enhanced detection of device errors 366 * o Participation in IRM 367 * o FMA device reset 368 * o Stall detection, temperature error detection, etc. 369 * o More dynamic resource pools 370 */ 371 372 #include "i40e_sw.h" 373 374 static char i40e_ident[] = "Intel 10/40Gb Ethernet v1.0.0"; 375 376 /* 377 * The i40e_glock primarily protects the lists below and the i40e_device_t 378 * structures. 379 */ 380 static kmutex_t i40e_glock; 381 static list_t i40e_glist; 382 static list_t i40e_dlist; 383 384 /* 385 * Access attributes for register mapping. 386 */ 387 static ddi_device_acc_attr_t i40e_regs_acc_attr = { 388 DDI_DEVICE_ATTR_V1, 389 DDI_STRUCTURE_LE_ACC, 390 DDI_STRICTORDER_ACC, 391 DDI_FLAGERR_ACC 392 }; 393 394 /* 395 * Logging function for this driver. 396 */ 397 static void 398 i40e_dev_err(i40e_t *i40e, int level, boolean_t console, const char *fmt, 399 va_list ap) 400 { 401 char buf[1024]; 402 403 (void) vsnprintf(buf, sizeof (buf), fmt, ap); 404 405 if (i40e == NULL) { 406 cmn_err(level, (console) ? "%s: %s" : "!%s: %s", 407 I40E_MODULE_NAME, buf); 408 } else { 409 dev_err(i40e->i40e_dip, level, (console) ? "%s" : "!%s", 410 buf); 411 } 412 } 413 414 /* 415 * Because there's the stupid trailing-comma problem with the C preprocessor 416 * and variable arguments, I need to instantiate these. Pardon the redundant 417 * code. 418 */ 419 /*PRINTFLIKE2*/ 420 void 421 i40e_error(i40e_t *i40e, const char *fmt, ...) 422 { 423 va_list ap; 424 425 va_start(ap, fmt); 426 i40e_dev_err(i40e, CE_WARN, B_FALSE, fmt, ap); 427 va_end(ap); 428 } 429 430 /*PRINTFLIKE2*/ 431 void 432 i40e_log(i40e_t *i40e, const char *fmt, ...) 433 { 434 va_list ap; 435 436 va_start(ap, fmt); 437 i40e_dev_err(i40e, CE_NOTE, B_FALSE, fmt, ap); 438 va_end(ap); 439 } 440 441 /*PRINTFLIKE2*/ 442 void 443 i40e_notice(i40e_t *i40e, const char *fmt, ...) 444 { 445 va_list ap; 446 447 va_start(ap, fmt); 448 i40e_dev_err(i40e, CE_NOTE, B_TRUE, fmt, ap); 449 va_end(ap); 450 } 451 452 static void 453 i40e_device_rele(i40e_t *i40e) 454 { 455 i40e_device_t *idp = i40e->i40e_device; 456 457 if (idp == NULL) 458 return; 459 460 mutex_enter(&i40e_glock); 461 VERIFY(idp->id_nreg > 0); 462 list_remove(&idp->id_i40e_list, i40e); 463 idp->id_nreg--; 464 if (idp->id_nreg == 0) { 465 list_remove(&i40e_dlist, idp); 466 list_destroy(&idp->id_i40e_list); 467 kmem_free(idp->id_rsrcs, sizeof (i40e_switch_rsrc_t) * 468 idp->id_rsrcs_alloc); 469 kmem_free(idp, sizeof (i40e_device_t)); 470 } 471 i40e->i40e_device = NULL; 472 mutex_exit(&i40e_glock); 473 } 474 475 static i40e_device_t * 476 i40e_device_find(i40e_t *i40e, dev_info_t *parent, uint_t bus, uint_t device) 477 { 478 i40e_device_t *idp; 479 mutex_enter(&i40e_glock); 480 for (idp = list_head(&i40e_dlist); idp != NULL; 481 idp = list_next(&i40e_dlist, idp)) { 482 if (idp->id_parent == parent && idp->id_pci_bus == bus && 483 idp->id_pci_device == device) { 484 break; 485 } 486 } 487 488 if (idp != NULL) { 489 VERIFY(idp->id_nreg < idp->id_nfuncs); 490 idp->id_nreg++; 491 } else { 492 i40e_hw_t *hw = &i40e->i40e_hw_space; 493 ASSERT(hw->num_ports > 0); 494 ASSERT(hw->num_partitions > 0); 495 496 /* 497 * The Intel common code doesn't exactly keep the number of PCI 498 * functions. But it calculates it during discovery of 499 * partitions and ports. So what we do is undo the calculation 500 * that it does originally, as functions are evenly spread 501 * across ports in the rare case of partitions. 502 */ 503 idp = kmem_alloc(sizeof (i40e_device_t), KM_SLEEP); 504 idp->id_parent = parent; 505 idp->id_pci_bus = bus; 506 idp->id_pci_device = device; 507 idp->id_nfuncs = hw->num_ports * hw->num_partitions; 508 idp->id_nreg = 1; 509 idp->id_rsrcs_alloc = i40e->i40e_switch_rsrc_alloc; 510 idp->id_rsrcs_act = i40e->i40e_switch_rsrc_actual; 511 idp->id_rsrcs = kmem_alloc(sizeof (i40e_switch_rsrc_t) * 512 idp->id_rsrcs_alloc, KM_SLEEP); 513 bcopy(i40e->i40e_switch_rsrcs, idp->id_rsrcs, 514 sizeof (i40e_switch_rsrc_t) * idp->id_rsrcs_alloc); 515 list_create(&idp->id_i40e_list, sizeof (i40e_t), 516 offsetof(i40e_t, i40e_dlink)); 517 518 list_insert_tail(&i40e_dlist, idp); 519 } 520 521 list_insert_tail(&idp->id_i40e_list, i40e); 522 mutex_exit(&i40e_glock); 523 524 return (idp); 525 } 526 527 static void 528 i40e_link_state_set(i40e_t *i40e, link_state_t state) 529 { 530 if (i40e->i40e_link_state == state) 531 return; 532 533 i40e->i40e_link_state = state; 534 mac_link_update(i40e->i40e_mac_hdl, i40e->i40e_link_state); 535 } 536 537 /* 538 * This is a basic link check routine. Mostly we're using this just to see 539 * if we can get any accurate information about the state of the link being 540 * up or down, as well as updating the link state, speed, etc. information. 541 */ 542 void 543 i40e_link_check(i40e_t *i40e) 544 { 545 i40e_hw_t *hw = &i40e->i40e_hw_space; 546 boolean_t ls; 547 int ret; 548 549 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 550 551 hw->phy.get_link_info = B_TRUE; 552 if ((ret = i40e_get_link_status(hw, &ls)) != I40E_SUCCESS) { 553 i40e->i40e_s_link_status_errs++; 554 i40e->i40e_s_link_status_lasterr = ret; 555 return; 556 } 557 558 /* 559 * Firmware abstracts all of the mac and phy information for us, so we 560 * can use i40e_get_link_status to determine the current state. 561 */ 562 if (ls == B_TRUE) { 563 enum i40e_aq_link_speed speed; 564 565 speed = i40e_get_link_speed(hw); 566 567 /* 568 * Translate from an i40e value to a value in Mbits/s. 569 */ 570 switch (speed) { 571 case I40E_LINK_SPEED_100MB: 572 i40e->i40e_link_speed = 100; 573 break; 574 case I40E_LINK_SPEED_1GB: 575 i40e->i40e_link_speed = 1000; 576 break; 577 case I40E_LINK_SPEED_10GB: 578 i40e->i40e_link_speed = 10000; 579 break; 580 case I40E_LINK_SPEED_20GB: 581 i40e->i40e_link_speed = 20000; 582 break; 583 case I40E_LINK_SPEED_40GB: 584 i40e->i40e_link_speed = 40000; 585 break; 586 default: 587 i40e->i40e_link_speed = 0; 588 break; 589 } 590 591 /* 592 * At this time, hardware does not support half-duplex 593 * operation, hence why we don't ask the hardware about our 594 * current speed. 595 */ 596 i40e->i40e_link_duplex = LINK_DUPLEX_FULL; 597 i40e_link_state_set(i40e, LINK_STATE_UP); 598 } else { 599 i40e->i40e_link_speed = 0; 600 i40e->i40e_link_duplex = 0; 601 i40e_link_state_set(i40e, LINK_STATE_DOWN); 602 } 603 } 604 605 static void 606 i40e_rem_intrs(i40e_t *i40e) 607 { 608 int i, rc; 609 610 for (i = 0; i < i40e->i40e_intr_count; i++) { 611 rc = ddi_intr_free(i40e->i40e_intr_handles[i]); 612 if (rc != DDI_SUCCESS) { 613 i40e_log(i40e, "failed to free interrupt %d: %d", 614 i, rc); 615 } 616 } 617 618 kmem_free(i40e->i40e_intr_handles, i40e->i40e_intr_size); 619 i40e->i40e_intr_handles = NULL; 620 } 621 622 static void 623 i40e_rem_intr_handlers(i40e_t *i40e) 624 { 625 int i, rc; 626 627 for (i = 0; i < i40e->i40e_intr_count; i++) { 628 rc = ddi_intr_remove_handler(i40e->i40e_intr_handles[i]); 629 if (rc != DDI_SUCCESS) { 630 i40e_log(i40e, "failed to remove interrupt %d: %d", 631 i, rc); 632 } 633 } 634 } 635 636 /* 637 * illumos Fault Management Architecture (FMA) support. 638 */ 639 640 int 641 i40e_check_acc_handle(ddi_acc_handle_t handle) 642 { 643 ddi_fm_error_t de; 644 645 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 646 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 647 return (de.fme_status); 648 } 649 650 int 651 i40e_check_dma_handle(ddi_dma_handle_t handle) 652 { 653 ddi_fm_error_t de; 654 655 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 656 return (de.fme_status); 657 } 658 659 /* 660 * Fault service error handling callback function. 661 */ 662 /* ARGSUSED */ 663 static int 664 i40e_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 665 { 666 pci_ereport_post(dip, err, NULL); 667 return (err->fme_status); 668 } 669 670 static void 671 i40e_fm_init(i40e_t *i40e) 672 { 673 ddi_iblock_cookie_t iblk; 674 675 i40e->i40e_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY, 676 i40e->i40e_dip, DDI_PROP_DONTPASS, "fm_capable", 677 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 678 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 679 680 if (i40e->i40e_fm_capabilities < 0) { 681 i40e->i40e_fm_capabilities = 0; 682 } else if (i40e->i40e_fm_capabilities > 0xf) { 683 i40e->i40e_fm_capabilities = DDI_FM_EREPORT_CAPABLE | 684 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE | 685 DDI_FM_ERRCB_CAPABLE; 686 } 687 688 /* 689 * Only register with IO Fault Services if we have some capability 690 */ 691 if (i40e->i40e_fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 692 i40e_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 693 } else { 694 i40e_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 695 } 696 697 if (i40e->i40e_fm_capabilities) { 698 ddi_fm_init(i40e->i40e_dip, &i40e->i40e_fm_capabilities, &iblk); 699 700 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 701 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 702 pci_ereport_setup(i40e->i40e_dip); 703 } 704 705 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) { 706 ddi_fm_handler_register(i40e->i40e_dip, 707 i40e_fm_error_cb, (void*)i40e); 708 } 709 } 710 711 if (i40e->i40e_fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 712 i40e_init_dma_attrs(i40e, B_TRUE); 713 } else { 714 i40e_init_dma_attrs(i40e, B_FALSE); 715 } 716 } 717 718 static void 719 i40e_fm_fini(i40e_t *i40e) 720 { 721 if (i40e->i40e_fm_capabilities) { 722 723 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) || 724 DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 725 pci_ereport_teardown(i40e->i40e_dip); 726 727 if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) 728 ddi_fm_handler_unregister(i40e->i40e_dip); 729 730 ddi_fm_fini(i40e->i40e_dip); 731 } 732 } 733 734 void 735 i40e_fm_ereport(i40e_t *i40e, char *detail) 736 { 737 uint64_t ena; 738 char buf[FM_MAX_CLASS]; 739 740 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 741 ena = fm_ena_generate(0, FM_ENA_FMT1); 742 if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities)) { 743 ddi_fm_ereport_post(i40e->i40e_dip, buf, ena, DDI_NOSLEEP, 744 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 745 } 746 } 747 748 /* 749 * Here we're trying to get the ID of the default VSI. In general, when we come 750 * through and look at this shortly after attach, we expect there to only be a 751 * single element present, which is the default VSI. Importantly, each PF seems 752 * to not see any other devices, in part because of the simple switch mode that 753 * we're using. If for some reason, we see more artifact, we'll need to revisit 754 * what we're doing here. 755 */ 756 static int 757 i40e_get_vsi_id(i40e_t *i40e) 758 { 759 i40e_hw_t *hw = &i40e->i40e_hw_space; 760 struct i40e_aqc_get_switch_config_resp *sw_config; 761 uint8_t aq_buf[I40E_AQ_LARGE_BUF]; 762 uint16_t next = 0; 763 int rc; 764 765 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 766 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 767 rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next, 768 NULL); 769 if (rc != I40E_SUCCESS) { 770 i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d", 771 rc, hw->aq.asq_last_status); 772 return (-1); 773 } 774 775 if (LE_16(sw_config->header.num_reported) != 1) { 776 i40e_error(i40e, "encountered multiple (%d) switching units " 777 "during attach, not proceeding", 778 LE_16(sw_config->header.num_reported)); 779 return (-1); 780 } 781 782 return (sw_config->element[0].seid); 783 } 784 785 /* 786 * We need to fill the i40e_hw_t structure with the capabilities of this PF. We 787 * must also provide the memory for it; however, we don't need to keep it around 788 * to the call to the common code. It takes it and parses it into an internal 789 * structure. 790 */ 791 static boolean_t 792 i40e_get_hw_capabilities(i40e_t *i40e, i40e_hw_t *hw) 793 { 794 struct i40e_aqc_list_capabilities_element_resp *buf; 795 int rc; 796 size_t len; 797 uint16_t needed; 798 int nelems = I40E_HW_CAP_DEFAULT; 799 800 len = nelems * sizeof (*buf); 801 802 for (;;) { 803 ASSERT(len > 0); 804 buf = kmem_alloc(len, KM_SLEEP); 805 rc = i40e_aq_discover_capabilities(hw, buf, len, 806 &needed, i40e_aqc_opc_list_func_capabilities, NULL); 807 kmem_free(buf, len); 808 809 if (hw->aq.asq_last_status == I40E_AQ_RC_ENOMEM && 810 nelems == I40E_HW_CAP_DEFAULT) { 811 if (nelems == needed) { 812 i40e_error(i40e, "Capability discovery failed " 813 "due to byzantine common code"); 814 return (B_FALSE); 815 } 816 len = needed; 817 continue; 818 } else if (rc != I40E_SUCCESS || 819 hw->aq.asq_last_status != I40E_AQ_RC_OK) { 820 i40e_error(i40e, "Capability discovery failed: %d", rc); 821 return (B_FALSE); 822 } 823 824 break; 825 } 826 827 return (B_TRUE); 828 } 829 830 /* 831 * Obtain the switch's capabilities as seen by this PF and keep it around for 832 * our later use. 833 */ 834 static boolean_t 835 i40e_get_switch_resources(i40e_t *i40e) 836 { 837 i40e_hw_t *hw = &i40e->i40e_hw_space; 838 uint8_t cnt = 2; 839 uint8_t act; 840 size_t size; 841 i40e_switch_rsrc_t *buf; 842 843 for (;;) { 844 enum i40e_status_code ret; 845 size = cnt * sizeof (i40e_switch_rsrc_t); 846 ASSERT(size > 0); 847 if (size > UINT16_MAX) 848 return (B_FALSE); 849 buf = kmem_alloc(size, KM_SLEEP); 850 851 ret = i40e_aq_get_switch_resource_alloc(hw, &act, buf, 852 cnt, NULL); 853 if (ret == I40E_ERR_ADMIN_QUEUE_ERROR && 854 hw->aq.asq_last_status == I40E_AQ_RC_EINVAL) { 855 kmem_free(buf, size); 856 cnt += I40E_SWITCH_CAP_DEFAULT; 857 continue; 858 } else if (ret != I40E_SUCCESS) { 859 kmem_free(buf, size); 860 i40e_error(i40e, 861 "failed to retrieve switch statistics: %d", ret); 862 return (B_FALSE); 863 } 864 865 break; 866 } 867 868 i40e->i40e_switch_rsrc_alloc = cnt; 869 i40e->i40e_switch_rsrc_actual = act; 870 i40e->i40e_switch_rsrcs = buf; 871 872 return (B_TRUE); 873 } 874 875 static void 876 i40e_cleanup_resources(i40e_t *i40e) 877 { 878 if (i40e->i40e_uaddrs != NULL) { 879 kmem_free(i40e->i40e_uaddrs, sizeof (i40e_uaddr_t) * 880 i40e->i40e_resources.ifr_nmacfilt); 881 i40e->i40e_uaddrs = NULL; 882 } 883 884 if (i40e->i40e_maddrs != NULL) { 885 kmem_free(i40e->i40e_maddrs, sizeof (i40e_maddr_t) * 886 i40e->i40e_resources.ifr_nmcastfilt); 887 i40e->i40e_maddrs = NULL; 888 } 889 890 if (i40e->i40e_switch_rsrcs != NULL) { 891 size_t sz = sizeof (i40e_switch_rsrc_t) * 892 i40e->i40e_switch_rsrc_alloc; 893 ASSERT(sz > 0); 894 kmem_free(i40e->i40e_switch_rsrcs, sz); 895 i40e->i40e_switch_rsrcs = NULL; 896 } 897 898 if (i40e->i40e_device != NULL) 899 i40e_device_rele(i40e); 900 } 901 902 static boolean_t 903 i40e_get_available_resources(i40e_t *i40e) 904 { 905 dev_info_t *parent; 906 uint16_t bus, device, func; 907 uint_t nregs; 908 int *regs, i; 909 i40e_device_t *idp; 910 i40e_hw_t *hw = &i40e->i40e_hw_space; 911 912 parent = ddi_get_parent(i40e->i40e_dip); 913 914 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, i40e->i40e_dip, 0, "reg", 915 ®s, &nregs) != DDI_PROP_SUCCESS) { 916 return (B_FALSE); 917 } 918 919 if (nregs < 1) { 920 ddi_prop_free(regs); 921 return (B_FALSE); 922 } 923 924 bus = PCI_REG_BUS_G(regs[0]); 925 device = PCI_REG_DEV_G(regs[0]); 926 func = PCI_REG_FUNC_G(regs[0]); 927 ddi_prop_free(regs); 928 929 i40e->i40e_hw_space.bus.func = func; 930 i40e->i40e_hw_space.bus.device = device; 931 932 if (i40e_get_switch_resources(i40e) == B_FALSE) { 933 return (B_FALSE); 934 } 935 936 /* 937 * To calculate the total amount of a resource we have available, we 938 * need to add how many our i40e_t thinks it has guaranteed, if any, and 939 * then we need to go through and divide the number of available on the 940 * device, which was snapshotted before anyone should have allocated 941 * anything, and use that to derive how many are available from the 942 * pool. Longer term, we may want to turn this into something that's 943 * more of a pool-like resource that everything can share (though that 944 * may require some more assistance from MAC). 945 * 946 * Though for transmit and receive queue pairs, we just have to ask 947 * firmware instead. 948 */ 949 idp = i40e_device_find(i40e, parent, bus, device); 950 i40e->i40e_device = idp; 951 i40e->i40e_resources.ifr_nvsis = 0; 952 i40e->i40e_resources.ifr_nvsis_used = 0; 953 i40e->i40e_resources.ifr_nmacfilt = 0; 954 i40e->i40e_resources.ifr_nmacfilt_used = 0; 955 i40e->i40e_resources.ifr_nmcastfilt = 0; 956 i40e->i40e_resources.ifr_nmcastfilt_used = 0; 957 958 for (i = 0; i < i40e->i40e_switch_rsrc_actual; i++) { 959 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 960 961 switch (srp->resource_type) { 962 case I40E_AQ_RESOURCE_TYPE_VSI: 963 i40e->i40e_resources.ifr_nvsis += 964 LE_16(srp->guaranteed); 965 i40e->i40e_resources.ifr_nvsis_used = LE_16(srp->used); 966 break; 967 case I40E_AQ_RESOURCE_TYPE_MACADDR: 968 i40e->i40e_resources.ifr_nmacfilt += 969 LE_16(srp->guaranteed); 970 i40e->i40e_resources.ifr_nmacfilt_used = 971 LE_16(srp->used); 972 break; 973 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 974 i40e->i40e_resources.ifr_nmcastfilt += 975 LE_16(srp->guaranteed); 976 i40e->i40e_resources.ifr_nmcastfilt_used = 977 LE_16(srp->used); 978 break; 979 default: 980 break; 981 } 982 } 983 984 for (i = 0; i < idp->id_rsrcs_act; i++) { 985 i40e_switch_rsrc_t *srp = &i40e->i40e_switch_rsrcs[i]; 986 switch (srp->resource_type) { 987 case I40E_AQ_RESOURCE_TYPE_VSI: 988 i40e->i40e_resources.ifr_nvsis += 989 LE_16(srp->total_unalloced) / idp->id_nfuncs; 990 break; 991 case I40E_AQ_RESOURCE_TYPE_MACADDR: 992 i40e->i40e_resources.ifr_nmacfilt += 993 LE_16(srp->total_unalloced) / idp->id_nfuncs; 994 break; 995 case I40E_AQ_RESOURCE_TYPE_MULTICAST_HASH: 996 i40e->i40e_resources.ifr_nmcastfilt += 997 LE_16(srp->total_unalloced) / idp->id_nfuncs; 998 default: 999 break; 1000 } 1001 } 1002 1003 i40e->i40e_resources.ifr_nrx_queue = hw->func_caps.num_rx_qp; 1004 i40e->i40e_resources.ifr_ntx_queue = hw->func_caps.num_tx_qp; 1005 1006 i40e->i40e_uaddrs = kmem_zalloc(sizeof (i40e_uaddr_t) * 1007 i40e->i40e_resources.ifr_nmacfilt, KM_SLEEP); 1008 i40e->i40e_maddrs = kmem_zalloc(sizeof (i40e_maddr_t) * 1009 i40e->i40e_resources.ifr_nmcastfilt, KM_SLEEP); 1010 1011 /* 1012 * Initialize these as multicast addresses to indicate it's invalid for 1013 * sanity purposes. Think of it like 0xdeadbeef. 1014 */ 1015 for (i = 0; i < i40e->i40e_resources.ifr_nmacfilt; i++) 1016 i40e->i40e_uaddrs[i].iua_mac[0] = 0x01; 1017 1018 return (B_TRUE); 1019 } 1020 1021 static boolean_t 1022 i40e_enable_interrupts(i40e_t *i40e) 1023 { 1024 int i, rc; 1025 1026 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1027 rc = ddi_intr_block_enable(i40e->i40e_intr_handles, 1028 i40e->i40e_intr_count); 1029 if (rc != DDI_SUCCESS) { 1030 i40e_error(i40e, "Interrupt block-enable failed: %d", 1031 rc); 1032 return (B_FALSE); 1033 } 1034 } else { 1035 for (i = 0; i < i40e->i40e_intr_count; i++) { 1036 rc = ddi_intr_enable(i40e->i40e_intr_handles[i]); 1037 if (rc != DDI_SUCCESS) { 1038 i40e_error(i40e, 1039 "Failed to enable interrupt %d: %d", i, rc); 1040 while (--i >= 0) { 1041 (void) ddi_intr_disable( 1042 i40e->i40e_intr_handles[i]); 1043 } 1044 return (B_FALSE); 1045 } 1046 } 1047 } 1048 1049 return (B_TRUE); 1050 } 1051 1052 static boolean_t 1053 i40e_disable_interrupts(i40e_t *i40e) 1054 { 1055 int i, rc; 1056 1057 if (i40e->i40e_intr_cap & DDI_INTR_FLAG_BLOCK) { 1058 rc = ddi_intr_block_disable(i40e->i40e_intr_handles, 1059 i40e->i40e_intr_count); 1060 if (rc != DDI_SUCCESS) { 1061 i40e_error(i40e, 1062 "Interrupt block-disabled failed: %d", rc); 1063 return (B_FALSE); 1064 } 1065 } else { 1066 for (i = 0; i < i40e->i40e_intr_count; i++) { 1067 rc = ddi_intr_disable(i40e->i40e_intr_handles[i]); 1068 if (rc != DDI_SUCCESS) { 1069 i40e_error(i40e, 1070 "Failed to disable interrupt %d: %d", 1071 i, rc); 1072 return (B_FALSE); 1073 } 1074 } 1075 } 1076 1077 return (B_TRUE); 1078 } 1079 1080 /* 1081 * Free receive & transmit rings. 1082 */ 1083 static void 1084 i40e_free_trqpairs(i40e_t *i40e) 1085 { 1086 int i; 1087 i40e_trqpair_t *itrq; 1088 1089 if (i40e->i40e_trqpairs != NULL) { 1090 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 1091 itrq = &i40e->i40e_trqpairs[i]; 1092 mutex_destroy(&itrq->itrq_rx_lock); 1093 mutex_destroy(&itrq->itrq_tx_lock); 1094 mutex_destroy(&itrq->itrq_tcb_lock); 1095 1096 /* 1097 * Should have already been cleaned up by start/stop, 1098 * etc. 1099 */ 1100 ASSERT(itrq->itrq_txkstat == NULL); 1101 ASSERT(itrq->itrq_rxkstat == NULL); 1102 } 1103 1104 kmem_free(i40e->i40e_trqpairs, 1105 sizeof (i40e_trqpair_t) * i40e->i40e_num_trqpairs); 1106 i40e->i40e_trqpairs = NULL; 1107 } 1108 1109 cv_destroy(&i40e->i40e_rx_pending_cv); 1110 mutex_destroy(&i40e->i40e_rx_pending_lock); 1111 mutex_destroy(&i40e->i40e_general_lock); 1112 } 1113 1114 /* 1115 * Allocate transmit and receive rings, as well as other data structures that we 1116 * need. 1117 */ 1118 static boolean_t 1119 i40e_alloc_trqpairs(i40e_t *i40e) 1120 { 1121 int i; 1122 void *mutexpri = DDI_INTR_PRI(i40e->i40e_intr_pri); 1123 1124 /* 1125 * Now that we have the priority for the interrupts, initialize 1126 * all relevant locks. 1127 */ 1128 mutex_init(&i40e->i40e_general_lock, NULL, MUTEX_DRIVER, mutexpri); 1129 mutex_init(&i40e->i40e_rx_pending_lock, NULL, MUTEX_DRIVER, mutexpri); 1130 cv_init(&i40e->i40e_rx_pending_cv, NULL, CV_DRIVER, NULL); 1131 1132 i40e->i40e_trqpairs = kmem_zalloc(sizeof (i40e_trqpair_t) * 1133 i40e->i40e_num_trqpairs, KM_SLEEP); 1134 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 1135 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 1136 1137 itrq->itrq_i40e = i40e; 1138 mutex_init(&itrq->itrq_rx_lock, NULL, MUTEX_DRIVER, mutexpri); 1139 mutex_init(&itrq->itrq_tx_lock, NULL, MUTEX_DRIVER, mutexpri); 1140 mutex_init(&itrq->itrq_tcb_lock, NULL, MUTEX_DRIVER, mutexpri); 1141 itrq->itrq_index = i; 1142 } 1143 1144 return (B_TRUE); 1145 } 1146 1147 1148 1149 /* 1150 * Unless a .conf file already overrode i40e_t structure values, they will 1151 * be 0, and need to be set in conjunction with the now-available HW report. 1152 * 1153 * However, at the moment, we cap all of these resources as we only support a 1154 * single receive ring and a single group. 1155 */ 1156 /* ARGSUSED */ 1157 static void 1158 i40e_hw_to_instance(i40e_t *i40e, i40e_hw_t *hw) 1159 { 1160 if (i40e->i40e_num_trqpairs == 0) { 1161 i40e->i40e_num_trqpairs = I40E_TRQPAIR_MAX; 1162 } 1163 1164 if (i40e->i40e_num_rx_groups == 0) { 1165 i40e->i40e_num_rx_groups = I40E_GROUP_MAX; 1166 } 1167 } 1168 1169 /* 1170 * Free any resources required by, or setup by, the Intel common code. 1171 */ 1172 static void 1173 i40e_common_code_fini(i40e_t *i40e) 1174 { 1175 i40e_hw_t *hw = &i40e->i40e_hw_space; 1176 int rc; 1177 1178 rc = i40e_shutdown_lan_hmc(hw); 1179 if (rc != I40E_SUCCESS) 1180 i40e_error(i40e, "failed to shutdown LAN hmc: %d", rc); 1181 1182 rc = i40e_shutdown_adminq(hw); 1183 if (rc != I40E_SUCCESS) 1184 i40e_error(i40e, "failed to shutdown admin queue: %d", rc); 1185 } 1186 1187 /* 1188 * Initialize and call Intel common-code routines, includes some setup 1189 * the common code expects from the driver. Also prints on failure, so 1190 * the caller doesn't have to. 1191 */ 1192 static boolean_t 1193 i40e_common_code_init(i40e_t *i40e, i40e_hw_t *hw) 1194 { 1195 int rc; 1196 1197 i40e_clear_hw(hw); 1198 rc = i40e_pf_reset(hw); 1199 if (rc != 0) { 1200 i40e_error(i40e, "failed to reset hardware: %d", rc); 1201 i40e_fm_ereport(i40e, DDI_FM_DEVICE_NO_RESPONSE); 1202 return (B_FALSE); 1203 } 1204 1205 rc = i40e_init_shared_code(hw); 1206 if (rc != 0) { 1207 i40e_error(i40e, "failed to initialize i40e core: %d", rc); 1208 return (B_FALSE); 1209 } 1210 1211 hw->aq.num_arq_entries = I40E_DEF_ADMINQ_SIZE; 1212 hw->aq.num_asq_entries = I40E_DEF_ADMINQ_SIZE; 1213 hw->aq.arq_buf_size = I40E_ADMINQ_BUFSZ; 1214 hw->aq.asq_buf_size = I40E_ADMINQ_BUFSZ; 1215 1216 rc = i40e_init_adminq(hw); 1217 if (rc != 0) { 1218 i40e_error(i40e, "failed to initialize firmware admin queue: " 1219 "%d, potential firmware version mismatch", rc); 1220 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 1221 return (B_FALSE); 1222 } 1223 1224 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 1225 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) { 1226 i40e_notice(i40e, "The driver for the device detected a newer " 1227 "version of the NVM image (%d.%d) than expected (%d.%d).\n" 1228 "Please install the most recent version of the network " 1229 "driver.\n", hw->aq.api_maj_ver, hw->aq.api_min_ver, 1230 I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR); 1231 } else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 1232 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) { 1233 i40e_notice(i40e, "The driver for the device detected an older" 1234 " version of the NVM image (%d.%d) than expected (%d.%d)." 1235 "\nPlease update the NVM image.\n", 1236 hw->aq.api_maj_ver, hw->aq.api_min_ver, 1237 I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR - 1); 1238 } 1239 1240 i40e_clear_pxe_mode(hw); 1241 1242 /* 1243 * We need to call this so that the common code can discover 1244 * capabilities of the hardware, which it uses throughout the rest. 1245 */ 1246 if (!i40e_get_hw_capabilities(i40e, hw)) { 1247 i40e_error(i40e, "failed to obtain hardware capabilities"); 1248 return (B_FALSE); 1249 } 1250 1251 if (i40e_get_available_resources(i40e) == B_FALSE) { 1252 i40e_error(i40e, "failed to obtain hardware resources"); 1253 return (B_FALSE); 1254 } 1255 1256 i40e_hw_to_instance(i40e, hw); 1257 1258 rc = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 1259 hw->func_caps.num_rx_qp, 0, 0); 1260 if (rc != 0) { 1261 i40e_error(i40e, "failed to initialize hardware memory cache: " 1262 "%d", rc); 1263 return (B_FALSE); 1264 } 1265 1266 rc = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 1267 if (rc != 0) { 1268 i40e_error(i40e, "failed to configure hardware memory cache: " 1269 "%d", rc); 1270 return (B_FALSE); 1271 } 1272 1273 (void) i40e_aq_stop_lldp(hw, TRUE, NULL); 1274 1275 rc = i40e_get_mac_addr(hw, hw->mac.addr); 1276 if (rc != I40E_SUCCESS) { 1277 i40e_error(i40e, "failed to retrieve hardware mac address: %d", 1278 rc); 1279 return (B_FALSE); 1280 } 1281 1282 rc = i40e_validate_mac_addr(hw->mac.addr); 1283 if (rc != 0) { 1284 i40e_error(i40e, "failed to validate internal mac address: " 1285 "%d", rc); 1286 return (B_FALSE); 1287 } 1288 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 1289 if ((rc = i40e_get_port_mac_addr(hw, hw->mac.port_addr)) != 1290 I40E_SUCCESS) { 1291 i40e_error(i40e, "failed to retrieve port mac address: %d", 1292 rc); 1293 return (B_FALSE); 1294 } 1295 1296 /* 1297 * We need to obtain the Virtual Station ID (VSI) before we can 1298 * perform other operations on the device. 1299 */ 1300 i40e->i40e_vsi_id = i40e_get_vsi_id(i40e); 1301 if (i40e->i40e_vsi_id == -1) { 1302 i40e_error(i40e, "failed to obtain VSI ID"); 1303 return (B_FALSE); 1304 } 1305 1306 return (B_TRUE); 1307 } 1308 1309 static void 1310 i40e_unconfigure(dev_info_t *devinfo, i40e_t *i40e) 1311 { 1312 int rc; 1313 1314 if (i40e->i40e_attach_progress & I40E_ATTACH_ENABLE_INTR) 1315 (void) i40e_disable_interrupts(i40e); 1316 1317 if ((i40e->i40e_attach_progress & I40E_ATTACH_LINK_TIMER) && 1318 i40e->i40e_periodic_id != 0) { 1319 ddi_periodic_delete(i40e->i40e_periodic_id); 1320 i40e->i40e_periodic_id = 0; 1321 } 1322 1323 if (i40e->i40e_attach_progress & I40E_ATTACH_MAC) { 1324 rc = mac_unregister(i40e->i40e_mac_hdl); 1325 if (rc != 0) { 1326 i40e_error(i40e, "failed to unregister from mac: %d", 1327 rc); 1328 } 1329 } 1330 1331 if (i40e->i40e_attach_progress & I40E_ATTACH_STATS) { 1332 i40e_stats_fini(i40e); 1333 } 1334 1335 if (i40e->i40e_attach_progress & I40E_ATTACH_ADD_INTR) 1336 i40e_rem_intr_handlers(i40e); 1337 1338 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_RINGSLOCKS) 1339 i40e_free_trqpairs(i40e); 1340 1341 if (i40e->i40e_attach_progress & I40E_ATTACH_ALLOC_INTR) 1342 i40e_rem_intrs(i40e); 1343 1344 if (i40e->i40e_attach_progress & I40E_ATTACH_COMMON_CODE) 1345 i40e_common_code_fini(i40e); 1346 1347 i40e_cleanup_resources(i40e); 1348 1349 if (i40e->i40e_attach_progress & I40E_ATTACH_PROPS) 1350 (void) ddi_prop_remove_all(devinfo); 1351 1352 if (i40e->i40e_attach_progress & I40E_ATTACH_REGS_MAP && 1353 i40e->i40e_osdep_space.ios_reg_handle != NULL) { 1354 ddi_regs_map_free(&i40e->i40e_osdep_space.ios_reg_handle); 1355 i40e->i40e_osdep_space.ios_reg_handle = NULL; 1356 } 1357 1358 if ((i40e->i40e_attach_progress & I40E_ATTACH_PCI_CONFIG) && 1359 i40e->i40e_osdep_space.ios_cfg_handle != NULL) { 1360 pci_config_teardown(&i40e->i40e_osdep_space.ios_cfg_handle); 1361 i40e->i40e_osdep_space.ios_cfg_handle = NULL; 1362 } 1363 1364 if (i40e->i40e_attach_progress & I40E_ATTACH_FM_INIT) 1365 i40e_fm_fini(i40e); 1366 1367 kmem_free(i40e->i40e_aqbuf, I40E_ADMINQ_BUFSZ); 1368 kmem_free(i40e, sizeof (i40e_t)); 1369 1370 ddi_set_driver_private(devinfo, NULL); 1371 } 1372 1373 static boolean_t 1374 i40e_final_init(i40e_t *i40e) 1375 { 1376 i40e_hw_t *hw = &i40e->i40e_hw_space; 1377 struct i40e_osdep *osdep = OS_DEP(hw); 1378 uint8_t pbanum[I40E_PBANUM_STRLEN]; 1379 enum i40e_status_code irc; 1380 char buf[I40E_DDI_PROP_LEN]; 1381 1382 pbanum[0] = '\0'; 1383 irc = i40e_read_pba_string(hw, pbanum, sizeof (pbanum)); 1384 if (irc != I40E_SUCCESS) { 1385 i40e_log(i40e, "failed to read PBA string: %d", irc); 1386 } else { 1387 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1388 "printed-board-assembly", (char *)pbanum); 1389 } 1390 1391 #ifdef DEBUG 1392 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.fw_maj_ver, 1393 hw->aq.fw_min_ver) < sizeof (buf)); 1394 ASSERT(snprintf(NULL, 0, "%x", hw->aq.fw_build) < sizeof (buf)); 1395 ASSERT(snprintf(NULL, 0, "%d.%d", hw->aq.api_maj_ver, 1396 hw->aq.api_min_ver) < sizeof (buf)); 1397 #endif 1398 1399 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.fw_maj_ver, 1400 hw->aq.fw_min_ver); 1401 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1402 "firmware-version", buf); 1403 (void) snprintf(buf, sizeof (buf), "%x", hw->aq.fw_build); 1404 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1405 "firmware-build", buf); 1406 (void) snprintf(buf, sizeof (buf), "%d.%d", hw->aq.api_maj_ver, 1407 hw->aq.api_min_ver); 1408 (void) ddi_prop_update_string(DDI_DEV_T_NONE, i40e->i40e_dip, 1409 "api-version", buf); 1410 1411 if (!i40e_set_hw_bus_info(hw)) 1412 return (B_FALSE); 1413 1414 if (i40e_check_acc_handle(osdep->ios_reg_handle) != DDI_FM_OK) { 1415 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 1416 return (B_FALSE); 1417 } 1418 1419 return (B_TRUE); 1420 } 1421 1422 static boolean_t 1423 i40e_identify_hardware(i40e_t *i40e) 1424 { 1425 i40e_hw_t *hw = &i40e->i40e_hw_space; 1426 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1427 1428 hw->vendor_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_VENID); 1429 hw->device_id = pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_DEVID); 1430 hw->revision_id = pci_config_get8(osdep->ios_cfg_handle, 1431 PCI_CONF_REVID); 1432 hw->subsystem_device_id = 1433 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBSYSID); 1434 hw->subsystem_vendor_id = 1435 pci_config_get16(osdep->ios_cfg_handle, PCI_CONF_SUBVENID); 1436 1437 /* 1438 * Note that we set the hardware's bus information later on, in 1439 * i40e_get_available_resources(). The common code doesn't seem to 1440 * require that it be set in any ways, it seems to be mostly for 1441 * book-keeping. 1442 */ 1443 1444 /* Call common code to set the MAC type for this adapter. */ 1445 if (i40e_set_mac_type(hw) != I40E_SUCCESS) 1446 return (B_FALSE); 1447 1448 return (B_TRUE); 1449 } 1450 1451 static boolean_t 1452 i40e_regs_map(i40e_t *i40e) 1453 { 1454 dev_info_t *devinfo = i40e->i40e_dip; 1455 i40e_hw_t *hw = &i40e->i40e_hw_space; 1456 struct i40e_osdep *osdep = &i40e->i40e_osdep_space; 1457 off_t memsize; 1458 int ret; 1459 1460 if (ddi_dev_regsize(devinfo, I40E_ADAPTER_REGSET, &memsize) != 1461 DDI_SUCCESS) { 1462 i40e_error(i40e, "Used invalid register set to map PCIe regs"); 1463 return (B_FALSE); 1464 } 1465 1466 if ((ret = ddi_regs_map_setup(devinfo, I40E_ADAPTER_REGSET, 1467 (caddr_t *)&hw->hw_addr, 0, memsize, &i40e_regs_acc_attr, 1468 &osdep->ios_reg_handle)) != DDI_SUCCESS) { 1469 i40e_error(i40e, "failed to map device registers: %d", ret); 1470 return (B_FALSE); 1471 } 1472 1473 osdep->ios_reg_size = memsize; 1474 return (B_TRUE); 1475 } 1476 1477 /* 1478 * Update parameters required when a new MTU has been configured. Calculate the 1479 * maximum frame size, as well as, size our DMA buffers which we size in 1480 * increments of 1K. 1481 */ 1482 void 1483 i40e_update_mtu(i40e_t *i40e) 1484 { 1485 uint32_t rx, tx; 1486 1487 i40e->i40e_frame_max = i40e->i40e_sdu + 1488 sizeof (struct ether_vlan_header) + ETHERFCSL; 1489 1490 rx = i40e->i40e_frame_max + I40E_BUF_IPHDR_ALIGNMENT; 1491 i40e->i40e_rx_buf_size = ((rx >> 10) + 1492 ((rx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1493 1494 tx = i40e->i40e_frame_max; 1495 i40e->i40e_tx_buf_size = ((tx >> 10) + 1496 ((tx & (((uint32_t)1 << 10) -1)) > 0 ? 1 : 0)) << 10; 1497 } 1498 1499 static int 1500 i40e_get_prop(i40e_t *i40e, char *prop, int min, int max, int def) 1501 { 1502 int val; 1503 1504 val = ddi_prop_get_int(DDI_DEV_T_ANY, i40e->i40e_dip, DDI_PROP_DONTPASS, 1505 prop, def); 1506 if (val > max) 1507 val = max; 1508 if (val < min) 1509 val = min; 1510 return (val); 1511 } 1512 1513 static void 1514 i40e_init_properties(i40e_t *i40e) 1515 { 1516 i40e->i40e_sdu = i40e_get_prop(i40e, "default_mtu", 1517 I40E_MIN_MTU, I40E_MAX_MTU, I40E_DEF_MTU); 1518 1519 i40e->i40e_intr_force = i40e_get_prop(i40e, "intr_force", 1520 I40E_INTR_NONE, I40E_INTR_LEGACY, I40E_INTR_NONE); 1521 1522 i40e->i40e_mr_enable = i40e_get_prop(i40e, "mr_enable", 1523 B_FALSE, B_TRUE, B_TRUE); 1524 1525 i40e->i40e_tx_ring_size = i40e_get_prop(i40e, "tx_ring_size", 1526 I40E_MIN_TX_RING_SIZE, I40E_MAX_TX_RING_SIZE, 1527 I40E_DEF_TX_RING_SIZE); 1528 if ((i40e->i40e_tx_ring_size % I40E_DESC_ALIGN) != 0) { 1529 i40e->i40e_tx_ring_size = P2ROUNDUP(i40e->i40e_tx_ring_size, 1530 I40E_DESC_ALIGN); 1531 } 1532 1533 i40e->i40e_tx_block_thresh = i40e_get_prop(i40e, "tx_resched_threshold", 1534 I40E_MIN_TX_BLOCK_THRESH, 1535 i40e->i40e_tx_ring_size - I40E_TX_MAX_COOKIE, 1536 I40E_DEF_TX_BLOCK_THRESH); 1537 1538 i40e->i40e_rx_ring_size = i40e_get_prop(i40e, "rx_ring_size", 1539 I40E_MIN_RX_RING_SIZE, I40E_MAX_RX_RING_SIZE, 1540 I40E_DEF_RX_RING_SIZE); 1541 if ((i40e->i40e_rx_ring_size % I40E_DESC_ALIGN) != 0) { 1542 i40e->i40e_rx_ring_size = P2ROUNDUP(i40e->i40e_rx_ring_size, 1543 I40E_DESC_ALIGN); 1544 } 1545 1546 i40e->i40e_rx_limit_per_intr = i40e_get_prop(i40e, "rx_limit_per_intr", 1547 I40E_MIN_RX_LIMIT_PER_INTR, I40E_MAX_RX_LIMIT_PER_INTR, 1548 I40E_DEF_RX_LIMIT_PER_INTR); 1549 1550 i40e->i40e_tx_hcksum_enable = i40e_get_prop(i40e, "tx_hcksum_enable", 1551 B_FALSE, B_TRUE, B_TRUE); 1552 1553 i40e->i40e_rx_hcksum_enable = i40e_get_prop(i40e, "rx_hcksum_enable", 1554 B_FALSE, B_TRUE, B_TRUE); 1555 1556 i40e->i40e_rx_dma_min = i40e_get_prop(i40e, "rx_dma_threshold", 1557 I40E_MIN_RX_DMA_THRESH, I40E_MAX_RX_DMA_THRESH, 1558 I40E_DEF_RX_DMA_THRESH); 1559 1560 i40e->i40e_tx_dma_min = i40e_get_prop(i40e, "tx_dma_threshold", 1561 I40E_MIN_TX_DMA_THRESH, I40E_MAX_TX_DMA_THRESH, 1562 I40E_DEF_TX_DMA_THRESH); 1563 1564 i40e->i40e_tx_itr = i40e_get_prop(i40e, "tx_intr_throttle", 1565 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_TX_ITR); 1566 1567 i40e->i40e_rx_itr = i40e_get_prop(i40e, "rx_intr_throttle", 1568 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_RX_ITR); 1569 1570 i40e->i40e_other_itr = i40e_get_prop(i40e, "other_intr_throttle", 1571 I40E_MIN_ITR, I40E_MAX_ITR, I40E_DEF_OTHER_ITR); 1572 1573 if (!i40e->i40e_mr_enable) { 1574 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1575 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1576 } 1577 1578 i40e_update_mtu(i40e); 1579 } 1580 1581 /* 1582 * There are a few constraints on interrupts that we're currently imposing, some 1583 * of which are restrictions from hardware. For a fuller treatment, see 1584 * i40e_intr.c. 1585 * 1586 * Currently, to use MSI-X we require two interrupts be available though in 1587 * theory we should participate in IRM and happily use more interrupts. 1588 * 1589 * Hardware only supports a single MSI being programmed and therefore if we 1590 * don't have MSI-X interrupts available at this time, then we ratchet down the 1591 * number of rings and groups available. Obviously, we only bother with a single 1592 * fixed interrupt. 1593 */ 1594 static boolean_t 1595 i40e_alloc_intr_handles(i40e_t *i40e, dev_info_t *devinfo, int intr_type) 1596 { 1597 int request, count, actual, rc, min; 1598 1599 switch (intr_type) { 1600 case DDI_INTR_TYPE_FIXED: 1601 case DDI_INTR_TYPE_MSI: 1602 request = 1; 1603 min = 1; 1604 break; 1605 case DDI_INTR_TYPE_MSIX: 1606 /* 1607 * At the moment, we always request two MSI-X while we still 1608 * only support a single interrupt. The upper bound on what's 1609 * supported by a given device is defined by MSI_X_PF_N in 1610 * GLPCI_CNF2. When we evolve, we should read it to determine 1611 * what the real max is. 1612 */ 1613 ASSERT(i40e->i40e_num_trqpairs == 1); 1614 request = 2; 1615 min = 2; 1616 break; 1617 default: 1618 panic("bad interrupt type passed to i40e_alloc_intr_handles: " 1619 "%d", intr_type); 1620 return (B_FALSE); 1621 } 1622 1623 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 1624 if (rc != DDI_SUCCESS || count < min) { 1625 i40e_log(i40e, "Get interrupt number failed, " 1626 "returned %d, count %d", rc, count); 1627 return (B_FALSE); 1628 } 1629 1630 rc = ddi_intr_get_navail(devinfo, intr_type, &count); 1631 if (rc != DDI_SUCCESS || count < min) { 1632 i40e_log(i40e, "Get AVAILABLE interrupt number failed, " 1633 "returned %d, count %d", rc, count); 1634 return (B_FALSE); 1635 } 1636 1637 actual = 0; 1638 i40e->i40e_intr_count = 0; 1639 i40e->i40e_intr_count_max = 0; 1640 i40e->i40e_intr_count_min = 0; 1641 1642 i40e->i40e_intr_size = request * sizeof (ddi_intr_handle_t); 1643 ASSERT(i40e->i40e_intr_size != 0); 1644 i40e->i40e_intr_handles = kmem_alloc(i40e->i40e_intr_size, KM_SLEEP); 1645 1646 rc = ddi_intr_alloc(devinfo, i40e->i40e_intr_handles, intr_type, 0, 1647 min(request, count), &actual, DDI_INTR_ALLOC_NORMAL); 1648 if (rc != DDI_SUCCESS) { 1649 i40e_log(i40e, "Interrupt allocation failed with %d.", rc); 1650 goto alloc_handle_fail; 1651 } 1652 1653 i40e->i40e_intr_count = actual; 1654 i40e->i40e_intr_count_max = request; 1655 i40e->i40e_intr_count_min = min; 1656 1657 if (actual < min) { 1658 i40e_log(i40e, "actual (%d) is less than minimum (%d).", 1659 actual, min); 1660 goto alloc_handle_fail; 1661 } 1662 1663 /* 1664 * Record the priority and capabilities for our first vector. Once 1665 * we have it, that's our priority until detach time. Even if we 1666 * eventually participate in IRM, our priority shouldn't change. 1667 */ 1668 rc = ddi_intr_get_pri(i40e->i40e_intr_handles[0], &i40e->i40e_intr_pri); 1669 if (rc != DDI_SUCCESS) { 1670 i40e_log(i40e, 1671 "Getting interrupt priority failed with %d.", rc); 1672 goto alloc_handle_fail; 1673 } 1674 1675 rc = ddi_intr_get_cap(i40e->i40e_intr_handles[0], &i40e->i40e_intr_cap); 1676 if (rc != DDI_SUCCESS) { 1677 i40e_log(i40e, 1678 "Getting interrupt capabilities failed with %d.", rc); 1679 goto alloc_handle_fail; 1680 } 1681 1682 i40e->i40e_intr_type = intr_type; 1683 return (B_TRUE); 1684 1685 alloc_handle_fail: 1686 1687 i40e_rem_intrs(i40e); 1688 return (B_FALSE); 1689 } 1690 1691 static boolean_t 1692 i40e_alloc_intrs(i40e_t *i40e, dev_info_t *devinfo) 1693 { 1694 int intr_types, rc; 1695 1696 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 1697 if (rc != DDI_SUCCESS) { 1698 i40e_error(i40e, "failed to get supported interrupt types: %d", 1699 rc); 1700 return (B_FALSE); 1701 } 1702 1703 i40e->i40e_intr_type = 0; 1704 1705 if ((intr_types & DDI_INTR_TYPE_MSIX) && 1706 i40e->i40e_intr_force <= I40E_INTR_MSIX) { 1707 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSIX)) 1708 return (B_TRUE); 1709 } 1710 1711 /* 1712 * We only use multiple transmit/receive pairs when MSI-X interrupts are 1713 * available due to the fact that the device basically only supports a 1714 * single MSI interrupt. 1715 */ 1716 i40e->i40e_num_trqpairs = I40E_TRQPAIR_NOMSIX; 1717 i40e->i40e_num_rx_groups = I40E_GROUP_NOMSIX; 1718 1719 if ((intr_types & DDI_INTR_TYPE_MSI) && 1720 (i40e->i40e_intr_force <= I40E_INTR_MSI)) { 1721 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_MSI)) 1722 return (B_TRUE); 1723 } 1724 1725 if (intr_types & DDI_INTR_TYPE_FIXED) { 1726 if (i40e_alloc_intr_handles(i40e, devinfo, DDI_INTR_TYPE_FIXED)) 1727 return (B_TRUE); 1728 } 1729 1730 return (B_FALSE); 1731 } 1732 1733 /* 1734 * Map different interrupts to MSI-X vectors. 1735 */ 1736 static boolean_t 1737 i40e_map_intrs_to_vectors(i40e_t *i40e) 1738 { 1739 if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 1740 return (B_TRUE); 1741 } 1742 1743 /* 1744 * At the moment, we only have one queue and one interrupt thus both are 1745 * on that one interrupt. However, longer term we need to go back to 1746 * using the ixgbe style map of queues to vectors or walk the linked 1747 * list from the device to know what to go handle. Therefore for the 1748 * moment, since we need to map our single set of rings to the one 1749 * I/O interrupt that exists for MSI-X. 1750 */ 1751 ASSERT(i40e->i40e_intr_count == 2); 1752 ASSERT(i40e->i40e_num_trqpairs == 1); 1753 1754 i40e->i40e_trqpairs[0].itrq_rx_intrvec = 1; 1755 i40e->i40e_trqpairs[0].itrq_tx_intrvec = 1; 1756 1757 return (B_TRUE); 1758 } 1759 1760 static boolean_t 1761 i40e_add_intr_handlers(i40e_t *i40e) 1762 { 1763 int rc, vector; 1764 1765 switch (i40e->i40e_intr_type) { 1766 case DDI_INTR_TYPE_MSIX: 1767 for (vector = 0; vector < i40e->i40e_intr_count; vector++) { 1768 rc = ddi_intr_add_handler( 1769 i40e->i40e_intr_handles[vector], 1770 (ddi_intr_handler_t *)i40e_intr_msix, i40e, 1771 (void *)(uintptr_t)vector); 1772 if (rc != DDI_SUCCESS) { 1773 i40e_log(i40e, "Add interrupt handler (MSI-X) " 1774 "failed: return %d, vector %d", rc, vector); 1775 for (vector--; vector >= 0; vector--) { 1776 (void) ddi_intr_remove_handler( 1777 i40e->i40e_intr_handles[vector]); 1778 } 1779 return (B_FALSE); 1780 } 1781 } 1782 break; 1783 case DDI_INTR_TYPE_MSI: 1784 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1785 (ddi_intr_handler_t *)i40e_intr_msi, i40e, NULL); 1786 if (rc != DDI_SUCCESS) { 1787 i40e_log(i40e, "Add interrupt handler (MSI) failed: " 1788 "return %d", rc); 1789 return (B_FALSE); 1790 } 1791 break; 1792 case DDI_INTR_TYPE_FIXED: 1793 rc = ddi_intr_add_handler(i40e->i40e_intr_handles[0], 1794 (ddi_intr_handler_t *)i40e_intr_legacy, i40e, NULL); 1795 if (rc != DDI_SUCCESS) { 1796 i40e_log(i40e, "Add interrupt handler (legacy) failed:" 1797 " return %d", rc); 1798 return (B_FALSE); 1799 } 1800 break; 1801 default: 1802 /* Cast to pacify lint */ 1803 panic("i40e_intr_type %p contains an unknown type: %d", 1804 (void *)i40e, i40e->i40e_intr_type); 1805 } 1806 1807 return (B_TRUE); 1808 } 1809 1810 /* 1811 * Perform periodic checks. Longer term, we should be thinking about additional 1812 * things here: 1813 * 1814 * o Stall Detection 1815 * o Temperature sensor detection 1816 * o Device resetting 1817 * o Statistics updating to avoid wraparound 1818 */ 1819 static void 1820 i40e_timer(void *arg) 1821 { 1822 i40e_t *i40e = arg; 1823 1824 mutex_enter(&i40e->i40e_general_lock); 1825 i40e_link_check(i40e); 1826 mutex_exit(&i40e->i40e_general_lock); 1827 } 1828 1829 /* 1830 * Get the hardware state, and scribble away anything that needs scribbling. 1831 */ 1832 static void 1833 i40e_get_hw_state(i40e_t *i40e, i40e_hw_t *hw) 1834 { 1835 int rc; 1836 1837 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 1838 1839 (void) i40e_aq_get_link_info(hw, TRUE, NULL, NULL); 1840 i40e_link_check(i40e); 1841 1842 /* 1843 * Try and determine our PHY. Note that we may have to retry to and 1844 * delay to detect fiber correctly. 1845 */ 1846 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, &i40e->i40e_phy, 1847 NULL); 1848 if (rc == I40E_ERR_UNKNOWN_PHY) { 1849 i40e_msec_delay(200); 1850 rc = i40e_aq_get_phy_capabilities(hw, B_FALSE, B_TRUE, 1851 &i40e->i40e_phy, NULL); 1852 } 1853 1854 if (rc != I40E_SUCCESS) { 1855 if (rc == I40E_ERR_UNKNOWN_PHY) { 1856 i40e_error(i40e, "encountered unknown PHY type, " 1857 "not attaching."); 1858 } else { 1859 i40e_error(i40e, "error getting physical capabilities: " 1860 "%d, %d", rc, hw->aq.asq_last_status); 1861 } 1862 } 1863 1864 rc = i40e_update_link_info(hw); 1865 if (rc != I40E_SUCCESS) { 1866 i40e_error(i40e, "failed to update link information: %d", rc); 1867 } 1868 1869 /* 1870 * In general, we don't want to mask off (as in stop from being a cause) 1871 * any of the interrupts that the phy might be able to generate. 1872 */ 1873 rc = i40e_aq_set_phy_int_mask(hw, 0, NULL); 1874 if (rc != I40E_SUCCESS) { 1875 i40e_error(i40e, "failed to update phy link mask: %d", rc); 1876 } 1877 } 1878 1879 /* 1880 * Go through and re-initialize any existing filters that we may have set up for 1881 * this device. Note that we would only expect them to exist if hardware had 1882 * already been initialized and we had just reset it. While we're not 1883 * implementing this yet, we're keeping this around for when we add reset 1884 * capabilities, so this isn't forgotten. 1885 */ 1886 /* ARGSUSED */ 1887 static void 1888 i40e_init_macaddrs(i40e_t *i40e, i40e_hw_t *hw) 1889 { 1890 } 1891 1892 /* 1893 * Configure the hardware for the Virtual Station Interface (VSI). Currently 1894 * we only support one, but in the future we could instantiate more than one 1895 * per attach-point. 1896 */ 1897 static boolean_t 1898 i40e_config_vsi(i40e_t *i40e, i40e_hw_t *hw) 1899 { 1900 struct i40e_vsi_context context; 1901 int err; 1902 1903 bzero(&context, sizeof (struct i40e_vsi_context)); 1904 context.seid = i40e->i40e_vsi_id; 1905 context.pf_num = hw->pf_id; 1906 err = i40e_aq_get_vsi_params(hw, &context, NULL); 1907 if (err != I40E_SUCCESS) { 1908 i40e_error(i40e, "get VSI params failed with %d", err); 1909 return (B_FALSE); 1910 } 1911 1912 /* 1913 * Set the queue and traffic class bits. Keep it simple for now. 1914 */ 1915 context.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1916 context.info.mapping_flags = I40E_AQ_VSI_QUE_MAP_CONTIG; 1917 context.info.queue_mapping[0] = I40E_ASSIGN_ALL_QUEUES; 1918 context.info.tc_mapping[0] = I40E_TRAFFIC_CLASS_NO_QUEUES; 1919 1920 context.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; 1921 context.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 1922 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 1923 1924 context.flags = LE16_TO_CPU(I40E_AQ_VSI_TYPE_PF); 1925 1926 i40e->i40e_vsi_stat_id = LE16_TO_CPU(context.info.stat_counter_idx); 1927 if (i40e_stat_vsi_init(i40e) == B_FALSE) 1928 return (B_FALSE); 1929 1930 err = i40e_aq_update_vsi_params(hw, &context, NULL); 1931 if (err != I40E_SUCCESS) { 1932 i40e_error(i40e, "Update VSI params failed with %d", err); 1933 return (B_FALSE); 1934 } 1935 1936 1937 return (B_TRUE); 1938 } 1939 1940 /* 1941 * Wrapper to kick the chipset on. 1942 */ 1943 static boolean_t 1944 i40e_chip_start(i40e_t *i40e) 1945 { 1946 i40e_hw_t *hw = &i40e->i40e_hw_space; 1947 struct i40e_filter_control_settings filter; 1948 int rc; 1949 1950 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || 1951 (hw->aq.fw_maj_ver < 4)) { 1952 i40e_msec_delay(75); 1953 if (i40e_aq_set_link_restart_an(hw, TRUE, NULL) != 1954 I40E_SUCCESS) { 1955 i40e_error(i40e, "failed to restart link: admin queue " 1956 "error: %d", hw->aq.asq_last_status); 1957 return (B_FALSE); 1958 } 1959 } 1960 1961 /* Determine hardware state */ 1962 i40e_get_hw_state(i40e, hw); 1963 1964 /* Initialize mac addresses. */ 1965 i40e_init_macaddrs(i40e, hw); 1966 1967 /* 1968 * Set up the filter control. 1969 */ 1970 bzero(&filter, sizeof (filter)); 1971 filter.enable_ethtype = TRUE; 1972 filter.enable_macvlan = TRUE; 1973 1974 rc = i40e_set_filter_control(hw, &filter); 1975 if (rc != I40E_SUCCESS) { 1976 i40e_error(i40e, "i40e_set_filter_control() returned %d", rc); 1977 return (B_FALSE); 1978 } 1979 1980 i40e_intr_chip_init(i40e); 1981 1982 if (!i40e_config_vsi(i40e, hw)) 1983 return (B_FALSE); 1984 1985 i40e_flush(hw); 1986 1987 return (B_TRUE); 1988 } 1989 1990 /* 1991 * Take care of tearing down the rx ring. See 8.3.3.1.2 for more information. 1992 */ 1993 static void 1994 i40e_shutdown_rx_rings(i40e_t *i40e) 1995 { 1996 int i; 1997 uint32_t reg; 1998 1999 i40e_hw_t *hw = &i40e->i40e_hw_space; 2000 2001 /* 2002 * Step 1. The interrupt linked list (see i40e_intr.c for more 2003 * information) should have already been cleared before calling this 2004 * function. 2005 */ 2006 #ifdef DEBUG 2007 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 2008 for (i = 1; i < i40e->i40e_intr_count; i++) { 2009 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i - 1)); 2010 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2011 } 2012 } else { 2013 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLST0); 2014 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2015 } 2016 2017 #endif /* DEBUG */ 2018 2019 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2020 /* 2021 * Step 1. Request the queue by clearing QENA_REQ. It may not be 2022 * set due to unwinding from failures and a partially enabled 2023 * ring set. 2024 */ 2025 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2026 if (!(reg & I40E_QRX_ENA_QENA_REQ_MASK)) 2027 continue; 2028 VERIFY((reg & I40E_QRX_ENA_QENA_REQ_MASK) == 2029 I40E_QRX_ENA_QENA_REQ_MASK); 2030 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 2031 I40E_WRITE_REG(hw, I40E_QRX_ENA(i), reg); 2032 } 2033 2034 /* 2035 * Step 2. Wait for the disable to take, by having QENA_STAT in the FPM 2036 * be cleared. Note that we could still receive data in the queue during 2037 * this time. We don't actually wait for this now and instead defer this 2038 * to i40e_shutdown_rings_wait(), after we've interleaved disabling the 2039 * TX queues as well. 2040 */ 2041 } 2042 2043 static void 2044 i40e_shutdown_tx_rings(i40e_t *i40e) 2045 { 2046 int i; 2047 uint32_t reg; 2048 2049 i40e_hw_t *hw = &i40e->i40e_hw_space; 2050 2051 /* 2052 * Step 1. The interrupt linked list should already have been cleared. 2053 */ 2054 #ifdef DEBUG 2055 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 2056 for (i = 1; i < i40e->i40e_intr_count; i++) { 2057 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i - 1)); 2058 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2059 } 2060 } else { 2061 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLST0); 2062 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 2063 2064 } 2065 #endif /* DEBUG */ 2066 2067 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2068 /* 2069 * Step 2. Set the SET_QDIS flag for every queue. 2070 */ 2071 i40e_pre_tx_queue_cfg(hw, i, B_FALSE); 2072 } 2073 2074 /* 2075 * Step 3. Wait at least 400 usec (can be done once for all queues). 2076 */ 2077 drv_usecwait(500); 2078 2079 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2080 /* 2081 * Step 4. Clear the QENA_REQ flag which tells hardware to 2082 * quiesce. If QENA_REQ is not already set then that means that 2083 * we likely already tried to disable this queue. 2084 */ 2085 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2086 if (!(reg & I40E_QTX_ENA_QENA_REQ_MASK)) 2087 continue; 2088 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 2089 I40E_WRITE_REG(hw, I40E_QTX_ENA(i), reg); 2090 } 2091 2092 /* 2093 * Step 5. Wait for all drains to finish. This will be done by the 2094 * hardware removing the QENA_STAT flag from the queue. Rather than 2095 * waiting here, we interleave it with all the others in 2096 * i40e_shutdown_rings_wait(). 2097 */ 2098 } 2099 2100 /* 2101 * Wait for all the rings to be shut down. e.g. Steps 2 and 5 from the above 2102 * functions. 2103 */ 2104 static boolean_t 2105 i40e_shutdown_rings_wait(i40e_t *i40e) 2106 { 2107 int i, try; 2108 i40e_hw_t *hw = &i40e->i40e_hw_space; 2109 2110 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2111 uint32_t reg; 2112 2113 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2114 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2115 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) 2116 break; 2117 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2118 } 2119 2120 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) != 0) { 2121 i40e_error(i40e, "timed out disabling rx queue %d", 2122 i); 2123 return (B_FALSE); 2124 } 2125 2126 for (try = 0; try < I40E_RING_WAIT_NTRIES; try++) { 2127 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2128 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) 2129 break; 2130 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2131 } 2132 2133 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) != 0) { 2134 i40e_error(i40e, "timed out disabling tx queue %d", 2135 i); 2136 return (B_FALSE); 2137 } 2138 } 2139 2140 return (B_TRUE); 2141 } 2142 2143 static boolean_t 2144 i40e_shutdown_rings(i40e_t *i40e) 2145 { 2146 i40e_shutdown_rx_rings(i40e); 2147 i40e_shutdown_tx_rings(i40e); 2148 return (i40e_shutdown_rings_wait(i40e)); 2149 } 2150 2151 static void 2152 i40e_setup_rx_descs(i40e_trqpair_t *itrq) 2153 { 2154 int i; 2155 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2156 2157 for (i = 0; i < rxd->rxd_ring_size; i++) { 2158 i40e_rx_control_block_t *rcb; 2159 i40e_rx_desc_t *rdesc; 2160 2161 rcb = rxd->rxd_work_list[i]; 2162 rdesc = &rxd->rxd_desc_ring[i]; 2163 2164 rdesc->read.pkt_addr = 2165 CPU_TO_LE64((uintptr_t)rcb->rcb_dma.dmab_dma_address); 2166 rdesc->read.hdr_addr = 0; 2167 } 2168 } 2169 2170 static boolean_t 2171 i40e_setup_rx_hmc(i40e_trqpair_t *itrq) 2172 { 2173 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2174 i40e_t *i40e = itrq->itrq_i40e; 2175 i40e_hw_t *hw = &i40e->i40e_hw_space; 2176 2177 struct i40e_hmc_obj_rxq rctx; 2178 int err; 2179 2180 bzero(&rctx, sizeof (struct i40e_hmc_obj_rxq)); 2181 rctx.base = rxd->rxd_desc_area.dmab_dma_address / 2182 I40E_HMC_RX_CTX_UNIT; 2183 rctx.qlen = rxd->rxd_ring_size; 2184 VERIFY(i40e->i40e_rx_buf_size >= I40E_HMC_RX_DBUFF_MIN); 2185 VERIFY(i40e->i40e_rx_buf_size <= I40E_HMC_RX_DBUFF_MAX); 2186 rctx.dbuff = i40e->i40e_rx_buf_size >> I40E_RXQ_CTX_DBUFF_SHIFT; 2187 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 2188 rctx.dtype = I40E_HMC_RX_DTYPE_NOSPLIT; 2189 rctx.dsize = I40E_HMC_RX_DSIZE_32BYTE; 2190 rctx.crcstrip = I40E_HMC_RX_CRCSTRIP_ENABLE; 2191 rctx.fc_ena = I40E_HMC_RX_FC_DISABLE; 2192 rctx.l2tsel = I40E_HMC_RX_L2TAGORDER; 2193 rctx.hsplit_0 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2194 rctx.hsplit_1 = I40E_HMC_RX_HDRSPLIT_DISABLE; 2195 rctx.showiv = I40E_HMC_RX_INVLAN_DONTSTRIP; 2196 rctx.rxmax = i40e->i40e_frame_max; 2197 rctx.tphrdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2198 rctx.tphwdesc_ena = I40E_HMC_RX_TPH_DISABLE; 2199 rctx.tphdata_ena = I40E_HMC_RX_TPH_DISABLE; 2200 rctx.tphhead_ena = I40E_HMC_RX_TPH_DISABLE; 2201 rctx.lrxqthresh = I40E_HMC_RX_LOWRXQ_NOINTR; 2202 2203 /* 2204 * This must be set to 0x1, see Table 8-12 in section 8.3.3.2.2. 2205 */ 2206 rctx.prefena = I40E_HMC_RX_PREFENA; 2207 2208 err = i40e_clear_lan_rx_queue_context(hw, itrq->itrq_index); 2209 if (err != I40E_SUCCESS) { 2210 i40e_error(i40e, "failed to clear rx queue %d context: %d", 2211 itrq->itrq_index, err); 2212 return (B_FALSE); 2213 } 2214 2215 err = i40e_set_lan_rx_queue_context(hw, itrq->itrq_index, &rctx); 2216 if (err != I40E_SUCCESS) { 2217 i40e_error(i40e, "failed to set rx queue %d context: %d", 2218 itrq->itrq_index, err); 2219 return (B_FALSE); 2220 } 2221 2222 return (B_TRUE); 2223 } 2224 2225 /* 2226 * Take care of setting up the descriptor rings and actually programming the 2227 * device. See 8.3.3.1.1 for the full list of steps we need to do to enable the 2228 * rx rings. 2229 */ 2230 static boolean_t 2231 i40e_setup_rx_rings(i40e_t *i40e) 2232 { 2233 int i; 2234 i40e_hw_t *hw = &i40e->i40e_hw_space; 2235 2236 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2237 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 2238 i40e_rx_data_t *rxd = itrq->itrq_rxdata; 2239 uint32_t reg; 2240 2241 /* 2242 * Step 1. Program all receive ring descriptors. 2243 */ 2244 i40e_setup_rx_descs(itrq); 2245 2246 /* 2247 * Step 2. Program the queue's FPM/HMC context. 2248 */ 2249 if (i40e_setup_rx_hmc(itrq) == B_FALSE) 2250 return (B_FALSE); 2251 2252 /* 2253 * Step 3. Clear the queue's tail pointer and set it to the end 2254 * of the space. 2255 */ 2256 I40E_WRITE_REG(hw, I40E_QRX_TAIL(i), 0); 2257 I40E_WRITE_REG(hw, I40E_QRX_TAIL(i), rxd->rxd_ring_size - 1); 2258 2259 /* 2260 * Step 4. Enable the queue via the QENA_REQ. 2261 */ 2262 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2263 VERIFY0(reg & (I40E_QRX_ENA_QENA_REQ_MASK | 2264 I40E_QRX_ENA_QENA_STAT_MASK)); 2265 reg |= I40E_QRX_ENA_QENA_REQ_MASK; 2266 I40E_WRITE_REG(hw, I40E_QRX_ENA(i), reg); 2267 } 2268 2269 /* 2270 * Note, we wait for every queue to be enabled before we start checking. 2271 * This will hopefully cause most queues to be enabled at this point. 2272 */ 2273 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2274 uint32_t j, reg; 2275 2276 /* 2277 * Step 5. Verify that QENA_STAT has been set. It's promised 2278 * that this should occur within about 10 us, but like other 2279 * systems, we give the card a bit more time. 2280 */ 2281 for (j = 0; j < I40E_RING_WAIT_NTRIES; j++) { 2282 reg = I40E_READ_REG(hw, I40E_QRX_ENA(i)); 2283 2284 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) 2285 break; 2286 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2287 } 2288 2289 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { 2290 i40e_error(i40e, "failed to enable rx queue %d, timed " 2291 "out.", i); 2292 return (B_FALSE); 2293 } 2294 } 2295 2296 return (B_TRUE); 2297 } 2298 2299 static boolean_t 2300 i40e_setup_tx_hmc(i40e_trqpair_t *itrq) 2301 { 2302 i40e_t *i40e = itrq->itrq_i40e; 2303 i40e_hw_t *hw = &i40e->i40e_hw_space; 2304 2305 struct i40e_hmc_obj_txq tctx; 2306 struct i40e_vsi_context context; 2307 int err; 2308 2309 bzero(&tctx, sizeof (struct i40e_hmc_obj_txq)); 2310 tctx.new_context = I40E_HMC_TX_NEW_CONTEXT; 2311 tctx.base = itrq->itrq_desc_area.dmab_dma_address / 2312 I40E_HMC_TX_CTX_UNIT; 2313 tctx.fc_ena = I40E_HMC_TX_FC_DISABLE; 2314 tctx.timesync_ena = I40E_HMC_TX_TS_DISABLE; 2315 tctx.fd_ena = I40E_HMC_TX_FD_DISABLE; 2316 tctx.alt_vlan_ena = I40E_HMC_TX_ALT_VLAN_DISABLE; 2317 tctx.head_wb_ena = I40E_HMC_TX_WB_ENABLE; 2318 tctx.qlen = itrq->itrq_tx_ring_size; 2319 tctx.tphrdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2320 tctx.tphrpacket_ena = I40E_HMC_TX_TPH_DISABLE; 2321 tctx.tphwdesc_ena = I40E_HMC_TX_TPH_DISABLE; 2322 tctx.head_wb_addr = itrq->itrq_desc_area.dmab_dma_address + 2323 sizeof (i40e_tx_desc_t) * itrq->itrq_tx_ring_size; 2324 2325 /* 2326 * This field isn't actually documented, like crc, but it suggests that 2327 * it should be zeroed. We leave both of these here because of that for 2328 * now. We should check with Intel on why these are here even. 2329 */ 2330 tctx.crc = 0; 2331 tctx.rdylist_act = 0; 2332 2333 /* 2334 * We're supposed to assign the rdylist field with the value of the 2335 * traffic class index for the first device. We query the VSI parameters 2336 * again to get what the handle is. Note that every queue is always 2337 * assigned to traffic class zero, because we don't actually use them. 2338 */ 2339 bzero(&context, sizeof (struct i40e_vsi_context)); 2340 context.seid = i40e->i40e_vsi_id; 2341 context.pf_num = hw->pf_id; 2342 err = i40e_aq_get_vsi_params(hw, &context, NULL); 2343 if (err != I40E_SUCCESS) { 2344 i40e_error(i40e, "get VSI params failed with %d", err); 2345 return (B_FALSE); 2346 } 2347 tctx.rdylist = LE_16(context.info.qs_handle[0]); 2348 2349 err = i40e_clear_lan_tx_queue_context(hw, itrq->itrq_index); 2350 if (err != I40E_SUCCESS) { 2351 i40e_error(i40e, "failed to clear tx queue %d context: %d", 2352 itrq->itrq_index, err); 2353 return (B_FALSE); 2354 } 2355 2356 err = i40e_set_lan_tx_queue_context(hw, itrq->itrq_index, &tctx); 2357 if (err != I40E_SUCCESS) { 2358 i40e_error(i40e, "failed to set tx queue %d context: %d", 2359 itrq->itrq_index, err); 2360 return (B_FALSE); 2361 } 2362 2363 return (B_TRUE); 2364 } 2365 2366 /* 2367 * Take care of setting up the descriptor rings and actually programming the 2368 * device. See 8.4.3.1.1 for what we need to do here. 2369 */ 2370 static boolean_t 2371 i40e_setup_tx_rings(i40e_t *i40e) 2372 { 2373 int i; 2374 i40e_hw_t *hw = &i40e->i40e_hw_space; 2375 2376 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2377 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 2378 uint32_t reg; 2379 2380 /* 2381 * Step 1. Clear the queue disable flag and verify that the 2382 * index is set correctly. 2383 */ 2384 i40e_pre_tx_queue_cfg(hw, i, B_TRUE); 2385 2386 /* 2387 * Step 2. Prepare the queue's FPM/HMC context. 2388 */ 2389 if (i40e_setup_tx_hmc(itrq) == B_FALSE) 2390 return (B_FALSE); 2391 2392 /* 2393 * Step 3. Verify that it's clear that this PF owns this queue. 2394 */ 2395 reg = I40E_QTX_CTL_PF_QUEUE; 2396 reg |= (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2397 I40E_QTX_CTL_PF_INDX_MASK; 2398 I40E_WRITE_REG(hw, I40E_QTX_CTL(itrq->itrq_index), reg); 2399 i40e_flush(hw); 2400 2401 /* 2402 * Step 4. Set the QENA_REQ flag. 2403 */ 2404 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2405 VERIFY0(reg & (I40E_QTX_ENA_QENA_REQ_MASK | 2406 I40E_QTX_ENA_QENA_STAT_MASK)); 2407 reg |= I40E_QTX_ENA_QENA_REQ_MASK; 2408 I40E_WRITE_REG(hw, I40E_QTX_ENA(i), reg); 2409 } 2410 2411 /* 2412 * Note, we wait for every queue to be enabled before we start checking. 2413 * This will hopefully cause most queues to be enabled at this point. 2414 */ 2415 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2416 uint32_t j, reg; 2417 2418 /* 2419 * Step 5. Verify that QENA_STAT has been set. It's promised 2420 * that this should occur within about 10 us, but like BSD, 2421 * we'll try for up to 100 ms for this queue. 2422 */ 2423 for (j = 0; j < I40E_RING_WAIT_NTRIES; j++) { 2424 reg = I40E_READ_REG(hw, I40E_QTX_ENA(i)); 2425 2426 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) 2427 break; 2428 i40e_msec_delay(I40E_RING_WAIT_PAUSE); 2429 } 2430 2431 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { 2432 i40e_error(i40e, "failed to enable tx queue %d, timed " 2433 "out", i); 2434 return (B_FALSE); 2435 } 2436 } 2437 2438 return (B_TRUE); 2439 } 2440 2441 void 2442 i40e_stop(i40e_t *i40e, boolean_t free_allocations) 2443 { 2444 int i; 2445 2446 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 2447 2448 /* 2449 * Shutdown and drain the tx and rx pipeline. We do this using the 2450 * following steps. 2451 * 2452 * 1) Shutdown interrupts to all the queues (trying to keep the admin 2453 * queue alive). 2454 * 2455 * 2) Remove all of the interrupt tx and rx causes by setting the 2456 * interrupt linked lists to zero. 2457 * 2458 * 2) Shutdown the tx and rx rings. Because i40e_shutdown_rings() should 2459 * wait for all the queues to be disabled, once we reach that point 2460 * it should be safe to free associated data. 2461 * 2462 * 4) Wait 50ms after all that is done. This ensures that the rings are 2463 * ready for programming again and we don't have to think about this 2464 * in other parts of the driver. 2465 * 2466 * 5) Disable remaining chip interrupts, (admin queue, etc.) 2467 * 2468 * 6) Verify that FM is happy with all the register accesses we 2469 * performed. 2470 */ 2471 i40e_intr_io_disable_all(i40e); 2472 i40e_intr_io_clear_cause(i40e); 2473 2474 if (i40e_shutdown_rings(i40e) == B_FALSE) { 2475 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 2476 } 2477 2478 delay(50 * drv_usectohz(1000)); 2479 2480 i40e_intr_chip_fini(i40e); 2481 2482 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2483 mutex_enter(&i40e->i40e_trqpairs[i].itrq_rx_lock); 2484 mutex_enter(&i40e->i40e_trqpairs[i].itrq_tx_lock); 2485 } 2486 2487 /* 2488 * We should consider refactoring this to be part of the ring start / 2489 * stop routines at some point. 2490 */ 2491 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2492 i40e_stats_trqpair_fini(&i40e->i40e_trqpairs[i]); 2493 } 2494 2495 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 2496 DDI_FM_OK) { 2497 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 2498 } 2499 2500 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2501 i40e_tx_cleanup_ring(&i40e->i40e_trqpairs[i]); 2502 } 2503 2504 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2505 mutex_exit(&i40e->i40e_trqpairs[i].itrq_rx_lock); 2506 mutex_exit(&i40e->i40e_trqpairs[i].itrq_tx_lock); 2507 } 2508 2509 i40e_stat_vsi_fini(i40e); 2510 2511 i40e->i40e_link_speed = 0; 2512 i40e->i40e_link_duplex = 0; 2513 i40e_link_state_set(i40e, LINK_STATE_UNKNOWN); 2514 2515 if (free_allocations) { 2516 i40e_free_ring_mem(i40e, B_FALSE); 2517 } 2518 } 2519 2520 boolean_t 2521 i40e_start(i40e_t *i40e, boolean_t alloc) 2522 { 2523 i40e_hw_t *hw = &i40e->i40e_hw_space; 2524 boolean_t rc = B_TRUE; 2525 int i, err; 2526 2527 ASSERT(MUTEX_HELD(&i40e->i40e_general_lock)); 2528 2529 if (alloc) { 2530 if (i40e_alloc_ring_mem(i40e) == B_FALSE) { 2531 i40e_error(i40e, 2532 "Failed to allocate ring memory"); 2533 return (B_FALSE); 2534 } 2535 } 2536 2537 /* 2538 * This should get refactored to be part of ring start and stop at 2539 * some point, along with most of the logic here. 2540 */ 2541 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 2542 if (i40e_stats_trqpair_init(&i40e->i40e_trqpairs[i]) == 2543 B_FALSE) { 2544 int j; 2545 2546 for (j = 0; j < i; j++) { 2547 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[j]; 2548 i40e_stats_trqpair_fini(itrq); 2549 } 2550 return (B_FALSE); 2551 } 2552 } 2553 2554 if (!i40e_chip_start(i40e)) { 2555 i40e_fm_ereport(i40e, DDI_FM_DEVICE_INVAL_STATE); 2556 rc = B_FALSE; 2557 goto done; 2558 } 2559 2560 if (i40e_setup_rx_rings(i40e) == B_FALSE) { 2561 rc = B_FALSE; 2562 goto done; 2563 } 2564 2565 if (i40e_setup_tx_rings(i40e) == B_FALSE) { 2566 rc = B_FALSE; 2567 goto done; 2568 } 2569 2570 /* 2571 * Enable broadcast traffic; however, do not enable multicast traffic. 2572 * That's handle exclusively through MAC's mc_multicst routines. 2573 */ 2574 err = i40e_aq_set_vsi_broadcast(hw, i40e->i40e_vsi_id, B_TRUE, NULL); 2575 if (err != I40E_SUCCESS) { 2576 i40e_error(i40e, "failed to set default VSI: %d", err); 2577 rc = B_FALSE; 2578 goto done; 2579 } 2580 2581 err = i40e_aq_set_mac_config(hw, i40e->i40e_frame_max, B_TRUE, 0, NULL); 2582 if (err != I40E_SUCCESS) { 2583 i40e_error(i40e, "failed to set MAC config: %d", err); 2584 rc = B_FALSE; 2585 goto done; 2586 } 2587 2588 /* 2589 * Finally, make sure that we're happy from an FM perspective. 2590 */ 2591 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 2592 DDI_FM_OK) { 2593 rc = B_FALSE; 2594 goto done; 2595 } 2596 2597 /* Clear state bits prior to final interrupt enabling. */ 2598 atomic_and_32(&i40e->i40e_state, 2599 ~(I40E_ERROR | I40E_STALL | I40E_OVERTEMP)); 2600 2601 i40e_intr_io_enable_all(i40e); 2602 2603 done: 2604 if (rc == B_FALSE) { 2605 i40e_stop(i40e, B_FALSE); 2606 if (alloc == B_TRUE) { 2607 i40e_free_ring_mem(i40e, B_TRUE); 2608 } 2609 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 2610 } 2611 2612 return (rc); 2613 } 2614 2615 /* 2616 * We may have loaned up descriptors to the stack. As such, if we still have 2617 * them outstanding, then we will not continue with detach. 2618 */ 2619 static boolean_t 2620 i40e_drain_rx(i40e_t *i40e) 2621 { 2622 mutex_enter(&i40e->i40e_rx_pending_lock); 2623 while (i40e->i40e_rx_pending > 0) { 2624 if (cv_reltimedwait(&i40e->i40e_rx_pending_cv, 2625 &i40e->i40e_rx_pending_lock, 2626 drv_usectohz(I40E_DRAIN_RX_WAIT), TR_CLOCK_TICK) == -1) { 2627 mutex_exit(&i40e->i40e_rx_pending_lock); 2628 return (B_FALSE); 2629 } 2630 } 2631 mutex_exit(&i40e->i40e_rx_pending_lock); 2632 2633 return (B_TRUE); 2634 } 2635 2636 static int 2637 i40e_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 2638 { 2639 i40e_t *i40e; 2640 struct i40e_osdep *osdep; 2641 i40e_hw_t *hw; 2642 int instance; 2643 2644 if (cmd != DDI_ATTACH) 2645 return (DDI_FAILURE); 2646 2647 instance = ddi_get_instance(devinfo); 2648 i40e = kmem_zalloc(sizeof (i40e_t), KM_SLEEP); 2649 2650 i40e->i40e_aqbuf = kmem_zalloc(I40E_ADMINQ_BUFSZ, KM_SLEEP); 2651 i40e->i40e_instance = instance; 2652 i40e->i40e_dip = devinfo; 2653 2654 hw = &i40e->i40e_hw_space; 2655 osdep = &i40e->i40e_osdep_space; 2656 hw->back = osdep; 2657 osdep->ios_i40e = i40e; 2658 2659 ddi_set_driver_private(devinfo, i40e); 2660 2661 i40e_fm_init(i40e); 2662 i40e->i40e_attach_progress |= I40E_ATTACH_FM_INIT; 2663 2664 if (pci_config_setup(devinfo, &osdep->ios_cfg_handle) != DDI_SUCCESS) { 2665 i40e_error(i40e, "Failed to map PCI configurations."); 2666 goto attach_fail; 2667 } 2668 i40e->i40e_attach_progress |= I40E_ATTACH_PCI_CONFIG; 2669 2670 if (!i40e_identify_hardware(i40e)) { 2671 i40e_error(i40e, "Failed to identify hardware"); 2672 goto attach_fail; 2673 } 2674 2675 if (!i40e_regs_map(i40e)) { 2676 i40e_error(i40e, "Failed to map device registers."); 2677 goto attach_fail; 2678 } 2679 i40e->i40e_attach_progress |= I40E_ATTACH_REGS_MAP; 2680 2681 i40e_init_properties(i40e); 2682 i40e->i40e_attach_progress |= I40E_ATTACH_PROPS; 2683 2684 if (!i40e_common_code_init(i40e, hw)) 2685 goto attach_fail; 2686 i40e->i40e_attach_progress |= I40E_ATTACH_COMMON_CODE; 2687 2688 /* 2689 * When we participate in IRM, we should make sure that we register 2690 * ourselves with it before callbacks. 2691 */ 2692 if (!i40e_alloc_intrs(i40e, devinfo)) { 2693 i40e_error(i40e, "Failed to allocate interrupts."); 2694 goto attach_fail; 2695 } 2696 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_INTR; 2697 2698 if (!i40e_alloc_trqpairs(i40e)) { 2699 i40e_error(i40e, 2700 "Failed to allocate receive & transmit rings."); 2701 goto attach_fail; 2702 } 2703 i40e->i40e_attach_progress |= I40E_ATTACH_ALLOC_RINGSLOCKS; 2704 2705 if (!i40e_map_intrs_to_vectors(i40e)) { 2706 i40e_error(i40e, "Failed to map interrupts to vectors."); 2707 goto attach_fail; 2708 } 2709 2710 if (!i40e_add_intr_handlers(i40e)) { 2711 i40e_error(i40e, "Failed to add the interrupt handlers."); 2712 goto attach_fail; 2713 } 2714 i40e->i40e_attach_progress |= I40E_ATTACH_ADD_INTR; 2715 2716 if (!i40e_final_init(i40e)) { 2717 i40e_error(i40e, "Final initialization failed."); 2718 goto attach_fail; 2719 } 2720 i40e->i40e_attach_progress |= I40E_ATTACH_INIT; 2721 2722 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_cfg_handle) != 2723 DDI_FM_OK) { 2724 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_LOST); 2725 goto attach_fail; 2726 } 2727 2728 if (!i40e_stats_init(i40e)) { 2729 i40e_error(i40e, "Stats initialization failed."); 2730 goto attach_fail; 2731 } 2732 i40e->i40e_attach_progress |= I40E_ATTACH_STATS; 2733 2734 if (!i40e_register_mac(i40e)) { 2735 i40e_error(i40e, "Failed to register to MAC/GLDv3"); 2736 goto attach_fail; 2737 } 2738 i40e->i40e_attach_progress |= I40E_ATTACH_MAC; 2739 2740 i40e->i40e_periodic_id = ddi_periodic_add(i40e_timer, i40e, 2741 I40E_CYCLIC_PERIOD, DDI_IPL_0); 2742 if (i40e->i40e_periodic_id == 0) { 2743 i40e_error(i40e, "Failed to add the link-check timer"); 2744 goto attach_fail; 2745 } 2746 i40e->i40e_attach_progress |= I40E_ATTACH_LINK_TIMER; 2747 2748 if (!i40e_enable_interrupts(i40e)) { 2749 i40e_error(i40e, "Failed to enable DDI interrupts"); 2750 goto attach_fail; 2751 } 2752 i40e->i40e_attach_progress |= I40E_ATTACH_ENABLE_INTR; 2753 2754 atomic_or_32(&i40e->i40e_state, I40E_INITIALIZED); 2755 2756 mutex_enter(&i40e_glock); 2757 list_insert_tail(&i40e_glist, i40e); 2758 mutex_exit(&i40e_glock); 2759 2760 return (DDI_SUCCESS); 2761 2762 attach_fail: 2763 i40e_unconfigure(devinfo, i40e); 2764 return (DDI_FAILURE); 2765 } 2766 2767 static int 2768 i40e_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 2769 { 2770 i40e_t *i40e; 2771 2772 if (cmd != DDI_DETACH) 2773 return (DDI_FAILURE); 2774 2775 i40e = (i40e_t *)ddi_get_driver_private(devinfo); 2776 if (i40e == NULL) { 2777 i40e_log(NULL, "i40e_detach() called with no i40e pointer!"); 2778 return (DDI_FAILURE); 2779 } 2780 2781 if (i40e_drain_rx(i40e) == B_FALSE) { 2782 i40e_log(i40e, "timed out draining DMA resources, %d buffers " 2783 "remain", i40e->i40e_rx_pending); 2784 return (DDI_FAILURE); 2785 } 2786 2787 mutex_enter(&i40e_glock); 2788 list_remove(&i40e_glist, i40e); 2789 mutex_exit(&i40e_glock); 2790 2791 i40e_unconfigure(devinfo, i40e); 2792 2793 return (DDI_SUCCESS); 2794 } 2795 2796 static struct cb_ops i40e_cb_ops = { 2797 nulldev, /* cb_open */ 2798 nulldev, /* cb_close */ 2799 nodev, /* cb_strategy */ 2800 nodev, /* cb_print */ 2801 nodev, /* cb_dump */ 2802 nodev, /* cb_read */ 2803 nodev, /* cb_write */ 2804 nodev, /* cb_ioctl */ 2805 nodev, /* cb_devmap */ 2806 nodev, /* cb_mmap */ 2807 nodev, /* cb_segmap */ 2808 nochpoll, /* cb_chpoll */ 2809 ddi_prop_op, /* cb_prop_op */ 2810 NULL, /* cb_stream */ 2811 D_MP | D_HOTPLUG, /* cb_flag */ 2812 CB_REV, /* cb_rev */ 2813 nodev, /* cb_aread */ 2814 nodev /* cb_awrite */ 2815 }; 2816 2817 static struct dev_ops i40e_dev_ops = { 2818 DEVO_REV, /* devo_rev */ 2819 0, /* devo_refcnt */ 2820 NULL, /* devo_getinfo */ 2821 nulldev, /* devo_identify */ 2822 nulldev, /* devo_probe */ 2823 i40e_attach, /* devo_attach */ 2824 i40e_detach, /* devo_detach */ 2825 nodev, /* devo_reset */ 2826 &i40e_cb_ops, /* devo_cb_ops */ 2827 NULL, /* devo_bus_ops */ 2828 ddi_power, /* devo_power */ 2829 ddi_quiesce_not_supported /* devo_quiesce */ 2830 }; 2831 2832 static struct modldrv i40e_modldrv = { 2833 &mod_driverops, 2834 i40e_ident, 2835 &i40e_dev_ops 2836 }; 2837 2838 static struct modlinkage i40e_modlinkage = { 2839 MODREV_1, 2840 &i40e_modldrv, 2841 NULL 2842 }; 2843 2844 /* 2845 * Module Initialization Functions. 2846 */ 2847 int 2848 _init(void) 2849 { 2850 int status; 2851 2852 list_create(&i40e_glist, sizeof (i40e_t), offsetof(i40e_t, i40e_glink)); 2853 list_create(&i40e_dlist, sizeof (i40e_device_t), 2854 offsetof(i40e_device_t, id_link)); 2855 mutex_init(&i40e_glock, NULL, MUTEX_DRIVER, NULL); 2856 mac_init_ops(&i40e_dev_ops, I40E_MODULE_NAME); 2857 2858 status = mod_install(&i40e_modlinkage); 2859 if (status != DDI_SUCCESS) { 2860 mac_fini_ops(&i40e_dev_ops); 2861 mutex_destroy(&i40e_glock); 2862 list_destroy(&i40e_dlist); 2863 list_destroy(&i40e_glist); 2864 } 2865 2866 return (status); 2867 } 2868 2869 int 2870 _info(struct modinfo *modinfop) 2871 { 2872 return (mod_info(&i40e_modlinkage, modinfop)); 2873 } 2874 2875 int 2876 _fini(void) 2877 { 2878 int status; 2879 2880 status = mod_remove(&i40e_modlinkage); 2881 if (status == DDI_SUCCESS) { 2882 mac_fini_ops(&i40e_dev_ops); 2883 mutex_destroy(&i40e_glock); 2884 list_destroy(&i40e_dlist); 2885 list_destroy(&i40e_glist); 2886 } 2887 2888 return (status); 2889 } 2890