xref: /illumos-gate/usr/src/uts/common/io/i40e/i40e_main.c (revision df36e06d)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
 * Copyright 2020 RackTop Systems, Inc.
 * Copyright 2020 Ryan Zezeski
 * Copyright 2021 Oxide Computer Company
 */

/*
 * i40e - Intel 10/40 Gb Ethernet driver
 *
 * The i40e driver is the main software device driver for the Intel 40 Gb family
 * of devices. Note that these devices come in many flavors with both 40 GbE
 * ports and 10 GbE ports. This device is the successor to the 82599 family of
 * devices (ixgbe).
 *
 * Unlike previous generations of Intel 1 GbE and 10 GbE devices, the 40 GbE
 * devices defined in the XL710 controller (previously known as Fortville) are a
 * rather different beast and have a small switch embedded inside of them. In
 * addition, the way that most of the programming is done has been overhauled.
 * As opposed to just using PCIe memory mapped registers, it also has an
 * administrative queue which is used to communicate with firmware running on
 * the chip.
 *
 * Each physical function in the hardware shows up as a device that this driver
 * will bind to. The hardware splits many resources evenly across all of the
 * physical functions present on the device, while other resources are instead
 * shared across the entire card and its up to the device driver to
 * intelligently partition them.
 *
 * ------------
 * Organization
 * ------------
 *
 * This driver is made up of several files which have their own theory
 * statements spread across them. We'll touch on the high level purpose of each
 * file here, and then we'll get into more discussion on how the device is
 * generally modelled with respect to the interfaces in illumos.
 *
 * i40e_gld.c: This file contains all of the bindings to MAC and the networking
 *             stack.
 *
 * i40e_intr.c: This file contains all of the interrupt service routines and
 *              contains logic to enable and disable interrupts on the hardware.
 *              It also contains the logic to map hardware resources such as the
 *              rings to and from interrupts and controls their ability to fire.
 *
 *              There is a big theory statement on interrupts present there.
 *
 * i40e_main.c: The file that you're currently in. It interfaces with the
 *              traditional OS DDI interfaces and is in charge of configuring
 *              the device.
 *
 * i40e_osdep.[ch]: These files contain interfaces and definitions needed to
 *                  work with Intel's common code for the device.
 *
 * i40e_stats.c: This file contains the general work and logic around our
 *               kstats. A theory statement on their organization and use of the
 *               hardware exists there.
 *
 * i40e_sw.h: This header file contains all of the primary structure definitions
 *            and constants that are used across the entire driver.
 *
 * i40e_transceiver.c: This file contains all of the logic for sending and
 *                     receiving data. It contains all of the ring and DMA
 *                     allocation logic, as well as, the actual interfaces to
 *                     send and receive data.
 *
 *                     A big theory statement on ring management, descriptors,
 *                     and how it ties into the OS is present there.
 *
 * --------------
 * General Design
 * --------------
 *
 * Before we go too far into the general way we've laid out data structures and
 * the like, it's worth taking some time to explain how the hardware is
 * organized. This organization informs a lot of how we do things at this time
 * in the driver.
 *
 * Each physical device consists of a number of one or more ports, which are
 * considered physical functions in the PCI sense and thus each get enumerated
 * by the system, resulting in an instance being created and attached to. While
 * there are many resources that are unique to each physical function eg.
 * instance of the device, there are many that are shared across all of them.
 * Several resources have an amount reserved for each Virtual Station Interface
 * (VSI) and then a static pool of resources, available for all functions on the
 * card.
 *
 * The most important resource in hardware are its transmit and receive queue
 * pairs (i40e_trqpair_t). These should be thought of as rings in GLDv3
 * parlance. There are a set number of these on each device; however, they are
 * statically partitioned among all of the different physical functions.
 *
 * 'Fortville' (the code name for this device family) is basically a switch. To
 * map MAC addresses and other things to queues, we end up having to create
 * Virtual Station Interfaces (VSIs) and establish forwarding rules that direct
 * traffic to a queue. A VSI owns a collection of queues and has a series of
 * forwarding rules that point to it. One way to think of this is to treat it
 * like MAC does a VNIC. When MAC refers to a group, a collection of rings and
 * classification resources, that is a VSI in i40e.
 *
 * The sets of VSIs is shared across the entire device, though there may be some
 * amount that are reserved to each PF. Because the GLDv3 does not let us change
 * the number of groups dynamically, we instead statically divide this amount
 * evenly between all the functions that exist. In addition, we have the same
 * problem with the mac address forwarding rules. There are a static number that
 * exist shared across all the functions.
 *
 * To handle both of these resources, what we end up doing is going through and
 * determining which functions belong to the same device. Nominally one might do
 * this by having a nexus driver; however, a prime requirement for a nexus
 * driver is identifying the various children and activating them. While it is
 * possible to get this information from NVRAM, we would end up duplicating a
 * lot of the PCI enumeration logic. Really, at the end of the day, the device
 * doesn't give us the traditional identification properties we want from a
 * nexus driver.
 *
 * Instead, we rely on some properties that are guaranteed to be unique. While
 * it might be tempting to leverage the PBA or serial number of the device from
 * NVRAM, there is nothing that says that two devices can't be mis-programmed to
 * have the same values in NVRAM. Instead, we uniquely identify a group of
 * functions based on their parent in the /devices tree, their PCI bus and PCI
 * function identifiers. Using either on their own may not be sufficient.
 *
 * For each unique PCI device that we encounter, we'll create a i40e_device_t.
 * From there, because we don't have a good way to tell the GLDv3 about sharing
 * resources between everything, we'll end up just dividing the resources
 * evenly between all of the functions. Longer term, if we don't have to declare
 * to the GLDv3 that these resources are shared, then we'll maintain a pool and
 * have each PF allocate from the pool in the device, thus if only two of four
 * ports are being used, for example, then all of the resources can still be
 * used.
 *
 * -------------------------------------------
 * Transmit and Receive Queue Pair Allocations
 * -------------------------------------------
 *
 * NVRAM ends up assigning each PF its own share of the transmit and receive LAN
 * queue pairs, we have no way of modifying it, only observing it. From there,
 * it's up to us to map these queues to VSIs and VFs. Since we don't support any
 * VFs at this time, we only focus on assignments to VSIs.
 *
 * At the moment, we used a static mapping of transmit/receive queue pairs to a
 * given VSI (eg. rings to a group). Though in the fullness of time, we want to
 * make this something which is fully dynamic and take advantage of documented,
 * but not yet available functionality for adding filters based on VXLAN and
 * other encapsulation technologies.
 *
 * -------------------------------------
 * Broadcast, Multicast, and Promiscuous
 * -------------------------------------
 *
 * As part of the GLDv3, we need to make sure that we can handle receiving
 * broadcast and multicast traffic. As well as enabling promiscuous mode when
 * requested. GLDv3 requires that all broadcast and multicast traffic be
 * retrieved by the default group, eg. the first one. This is the same thing as
 * the default VSI.
 *
 * To receieve broadcast traffic, we enable it through the admin queue, rather
 * than use one of our filters for it. For multicast traffic, we reserve a
 * certain number of the hash filters and assign them to a given PF. When we
 * exceed those, we then switch to using promiscuous mode for multicast traffic.
 *
 * More specifically, once we exceed the number of filters (indicated because
 * the i40e_t`i40e_resources.ifr_nmcastfilt ==
 * i40e_t`i40e_resources.ifr_nmcastfilt_used), we then instead need to toggle
 * promiscuous mode. If promiscuous mode is toggled then we keep track of the
 * number of MACs added to it by incrementing i40e_t`i40e_mcast_promisc_count.
 * That will stay enabled until that count reaches zero indicating that we have
 * only added multicast addresses that we have a corresponding entry for.
 *
 * Because MAC itself wants to toggle promiscuous mode, which includes both
 * unicast and multicast traffic, we go through and keep track of that
 * ourselves. That is maintained through the use of the i40e_t`i40e_promisc_on
 * member.
 *
 * --------------
 * VSI Management
 * --------------
 *
 * The PFs share 384 VSIs. The firmware creates one VSI per PF by default.
 * During chip start we retrieve the SEID of this VSI and assign it as the
 * default VSI for our VEB (one VEB per PF). We then add additional VSIs to
 * the VEB up to the determined number of rx groups: i40e_t`i40e_num_rx_groups.
 * We currently cap this number to I40E_GROUP_MAX to a) make sure all PFs can
 * allocate the same number of VSIs, and b) to keep the interrupt multiplexing
 * under control. In the future, when we improve the interrupt allocation, we
 * may want to revisit this cap to make better use of the available VSIs. The
 * VSI allocation and configuration can be found in i40e_chip_start().
 *
 * ----------------
 * Structure Layout
 * ----------------
 *
 * The following images relates the core data structures together. The primary
 * structure in the system is the i40e_t. It itself contains multiple rings,
 * i40e_trqpair_t's which contain the various transmit and receive data. The
 * receive data is stored outside of the i40e_trqpair_t and instead in the
 * i40e_rx_data_t. The i40e_t has a corresponding i40e_device_t which keeps
 * track of per-physical device state. Finally, for every active descriptor,
 * there is a corresponding control block, which is where the
 * i40e_rx_control_block_t and the i40e_tx_control_block_t come from.
 *
 *   +-----------------------+       +-----------------------+
 *   | Global i40e_t list    |       | Global Device list    |
 *   |                       |    +--|                       |
 *   | i40e_glist            |    |  | i40e_dlist            |
 *   +-----------------------+    |  +-----------------------+
 *       |                        v
 *       |      +------------------------+      +-----------------------+
 *       |      | Device-wide Structure  |----->| Device-wide Structure |--> ...
 *       |      | i40e_device_t          |      | i40e_device_t         |
 *       |      |                        |      +-----------------------+
 *       |      | dev_info_t *     ------+--> Parent in devices tree.
 *       |      | uint_t           ------+--> PCI bus number
 *       |      | uint_t           ------+--> PCI device number
 *       |      | uint_t           ------+--> Number of functions
 *       |      | i40e_switch_rsrcs_t ---+--> Captured total switch resources
 *       |      | list_t           ------+-------------+
 *       |      +------------------------+             |
 *       |                           ^                 |
 *       |                           +--------+        |
 *       |                                    |        v
 *       |  +---------------------------+     |   +-------------------+
 *       +->| GLDv3 Device, per PF      |-----|-->| GLDv3 Device (PF) |--> ...
 *          | i40e_t                    |     |   | i40e_t            |
 *          | **Primary Structure**     |     |   +-------------------+
 *          |                           |     |
 *          | i40e_device_t *         --+-----+
 *          | i40e_state_t            --+---> Device State
 *          | i40e_hw_t               --+---> Intel common code structure
 *          | mac_handle_t            --+---> GLDv3 handle to MAC
 *          | ddi_periodic_t          --+---> Link activity timer
 *          | i40e_vsi_t *            --+---> Array of VSIs
 *          | i40e_func_rsrc_t        --+---> Available hardware resources
 *          | i40e_switch_rsrc_t *    --+---> Switch resource snapshot
 *          | i40e_sdu                --+---> Current MTU
 *          | i40e_frame_max          --+---> Current HW frame size
 *          | i40e_uaddr_t *          --+---> Array of assigned unicast MACs
 *          | i40e_maddr_t *          --+---> Array of assigned multicast MACs
 *          | i40e_mcast_promisccount --+---> Active multicast state
 *          | i40e_promisc_on         --+---> Current promiscuous mode state
 *          | uint_t                  --+---> Number of transmit/receive pairs
 *          | i40e_rx_group_t *       --+---> Array of Rx groups
 *          | kstat_t *               --+---> PF kstats
 *          | i40e_pf_stats_t         --+---> PF kstat backing data
 *          | i40e_trqpair_t *        --+---------+
 *          +---------------------------+         |
 *                                                |
 *                                                v
 *  +-------------------------------+       +-----------------------------+
 *  | Transmit/Receive Queue Pair   |-------| Transmit/Receive Queue Pair |->...
 *  | i40e_trqpair_t                |       | i40e_trqpair_t              |
 *  + Ring Data Structure           |       +-----------------------------+
 *  |                               |
 *  | mac_ring_handle_t             +--> MAC RX ring handle
 *  | mac_ring_handle_t             +--> MAC TX ring handle
 *  | i40e_rxq_stat_t             --+--> RX Queue stats
 *  | i40e_txq_stat_t             --+--> TX Queue stats
 *  | uint32_t (tx ring size)       +--> TX Ring Size
 *  | uint32_t (tx free list size)  +--> TX Free List Size
 *  | i40e_dma_buffer_t     --------+--> TX Descriptor ring DMA
 *  | i40e_tx_desc_t *      --------+--> TX descriptor ring
 *  | volatile unt32_t *            +--> TX Write back head
 *  | uint32_t               -------+--> TX ring head
 *  | uint32_t               -------+--> TX ring tail
 *  | uint32_t               -------+--> Num TX desc free
 *  | i40e_tx_control_block_t *   --+--> TX control block array  ---+
 *  | i40e_tx_control_block_t **  --+--> TCB work list          ----+
 *  | i40e_tx_control_block_t **  --+--> TCB free list           ---+
 *  | uint32_t               -------+--> Free TCB count             |
 *  | i40e_rx_data_t *       -------+--+                            v
 *  +-------------------------------+  |          +---------------------------+
 *                                     |          | Per-TX Frame Metadata     |
 *                                     |          | i40e_tx_control_block_t   |
 *                +--------------------+          |                           |
 *                |           mblk to transmit <--+---      mblk_t *          |
 *                |           type of transmit <--+---      i40e_tx_type_t    |
 *                |              TX DMA handle <--+---      ddi_dma_handle_t  |
 *                v              TX DMA buffer <--+---      i40e_dma_buffer_t |
 *    +------------------------------+            +---------------------------+
 *    | Core Receive Data            |
 *    | i40e_rx_data_t               |
 *    |                              |
 *    | i40e_dma_buffer_t          --+--> RX descriptor DMA Data
 *    | i40e_rx_desc_t             --+--> RX descriptor ring
 *    | uint32_t                   --+--> Next free desc.
 *    | i40e_rx_control_block_t *  --+--> RX Control Block Array  ---+
 *    | i40e_rx_control_block_t ** --+--> RCB work list           ---+
 *    | i40e_rx_control_block_t ** --+--> RCB free list           ---+
 *    +------------------------------+                               |
 *                ^                                                  |
 *                |     +---------------------------+                |
 *                |     | Per-RX Frame Metadata     |<---------------+
 *                |     | i40e_rx_control_block_t   |
 *                |     |                           |
 *                |     | mblk_t *              ----+--> Received mblk_t data
 *                |     | uint32_t              ----+--> Reference count
 *                |     | i40e_dma_buffer_t     ----+--> Receive data DMA info
 *                |     | frtn_t                ----+--> mblk free function info
 *                +-----+-- i40e_rx_data_t *        |
 *                      +---------------------------+
 *
 * -------------
 * Lock Ordering
 * -------------
 *
 * In order to ensure that we don't deadlock, the following represents the
 * lock order being used. When grabbing locks, follow the following order. Lower
 * numbers are more important. Thus, the i40e_glock which is number 0, must be
 * taken before any other locks in the driver. On the other hand, the
 * i40e_t`i40e_stat_lock, has the highest number because it's the least
 * important lock. Note, that just because one lock is higher than another does
 * not mean that all intermediary locks are required.
 *
 * 0) i40e_glock
 * 1) i40e_t`i40e_general_lock
 *
 * 2) i40e_trqpair_t`itrq_rx_lock
 * 3) i40e_trqpair_t`itrq_tx_lock
 * 4) i40e_trqpair_t`itrq_intr_lock
 * 5) i40e_t`i40e_rx_pending_lock
 * 6) i40e_trqpair_t`itrq_tcb_lock
 *
 * 7) i40e_t`i40e_stat_lock
 *
 * Rules and expectations:
 *
 * 1) A thread holding locks belong to one PF should not hold locks belonging to
 * a second. If for some reason this becomes necessary, locks should be grabbed
 * based on the list order in the i40e_device_t, which implies that the
 * i40e_glock is held.
 *
 * 2) When grabbing locks between multiple transmit and receive queues, the
 * locks for the lowest number transmit/receive queue should be grabbed first.
 *
 * 3) When grabbing both the transmit and receive lock for a given queue, always
 * grab i40e_trqpair_t`itrq_rx_lock before the i40e_trqpair_t`itrq_tx_lock.
 *
 * 4) The following pairs of locks are not expected to be held at the same time:
 *
 * o i40e_t`i40e_rx_pending_lock and i40e_trqpair_t`itrq_tcb_lock
 * o i40e_trqpair_t`itrq_intr_lock is not expected to be held with any
 *   other lock except i40e_t`i40e_general_lock in mc_start(9E) and
 *   mc_stop(9e).
 *
 * -----------
 * Future Work
 * -----------
 *
 * At the moment the i40e_t driver is rather bare bones, allowing us to start
 * getting data flowing and folks using it while we develop additional features.
 * While bugs have been filed to cover this future work, the following gives an
 * overview of expected work:
 *
 *  o DMA binding and breaking up the locking in ring recycling.
 *  o Enhanced detection of device errors
 *  o Participation in IRM
 *  o FMA device reset
 *  o Stall detection, temperature error detection, etc.
 *  o More dynamic resource pools
 */

#include "i40e_sw.h"

static char i40e_ident[] = "Intel 10/40Gb Ethernet v1.0.3";

/*
 * The i40e_glock primarily protects the lists below and the i40e_device_t
 * structures.
 */
static kmutex_t i40e_glock;
static list_t i40e_glist;
static list_t i40e_dlist;

/*
 * Access attributes for register mapping.
 */
static ddi_device_acc_attr_t i40e_regs_acc_attr = {
	DDI_DEVICE_ATTR_V1,
	DDI_STRUCTURE_LE_ACC,
	DDI_STRICTORDER_ACC,
	DDI_FLAGERR_ACC
};

/*
 * Logging function for this driver.
 */
static void
i40e_dev_err(i40e_t *i40e, int level, boolean_t console, const char *fmt,
    va_list ap)
{
	char buf[1024];

	(void) vsnprintf(buf, sizeof (buf), fmt, ap);

	if (i40e == NULL) {
		cmn_err(level, (console) ? "%s: %s" : "!%s: %s",
		    I40E_MODULE_NAME, buf);
	} else {
		dev_err(i40e->i40e_dip, level, (console) ? "%s" : "!%s",
		    buf);
	}
}

/*
 * Because there's the stupid trailing-comma problem with the C preprocessor
 * and variable arguments, I need to instantiate these.	 Pardon the redundant
 * code.
 */
/*PRINTFLIKE2*/
void
i40e_error(i40e_t *i40e, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	i40e_dev_err(i40e, CE_WARN, B_FALSE, fmt, ap);
	va_end(ap);
}

/*PRINTFLIKE2*/
void
i40e_log(i40e_t *i40e, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	i40e_dev_err(i40e, CE_NOTE, B_FALSE, fmt, ap);
	va_end(ap);
}

/*PRINTFLIKE2*/
void
i40e_notice(i40e_t *i40e, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	i40e_dev_err(i40e, CE_NOTE, B_TRUE, fmt, ap);
	va_end(ap);
}

/*
 * Various parts of the driver need to know if the controller is from the X722
 * family, which has a few additional capabilities and different programming
 * means. We don't consider virtual functions as part of this as they are quite
 * different and will require substantially more work.
 */
static boolean_t
i40e_is_x722(i40e_t *i40e)
{
	return (i40e->i40e_hw_space.mac.type == I40E_MAC_X722);
}

static void
i40e_device_rele(i40e_t *i40e)
{
	i40e_device_t *idp = i40e->i40e_device;

	if (idp == NULL)
		return;

	mutex_enter(&i40e_glock);
	VERIFY(idp->id_nreg > 0);
	list_remove(&idp->id_i40e_list, i40e);
	idp->id_nreg--;
	if (idp->id_nreg == 0) {
		list_remove(&i40e_dlist, idp);
		list_destroy(&idp->id_i40e_list);
		kmem_free(idp->id_rsrcs, sizeof (i40e_switch_rsrc_t) *
		    idp->id_rsrcs_alloc);
		kmem_free(idp, sizeof (i40e_device_t));
	}
	i40e->i40e_device = NULL;
	mutex_exit(&i40e_glock);
}

static i40e_device_t *
i40e_device_find(i40e_t *i40e, dev_info_t *parent, uint_t bus, uint_t device)
{
	i40e_device_t *idp;
	mutex_enter(&i40e_glock);
	for (idp = list_head(&i40e_dlist); idp != NULL;
	    idp = list_next(&i40e_dlist, idp)) {
		if (idp->id_parent == parent && idp->id_pci_bus == bus &&
		    idp->id_pci_device == device) {
			break;
		}
	}

	if (idp != NULL) {
		VERIFY(idp->id_nreg < idp->id_nfuncs);
		idp->id_nreg++;
	} else {
		i40e_hw_t *hw = &i40e->i40e_hw_space;
		ASSERT(hw->num_ports > 0);
		ASSERT(hw->num_partitions > 0);

		/*
		 * The Intel common code doesn't exactly keep the number of PCI
		 * functions. But it calculates it during discovery of
		 * partitions and ports. So what we do is undo the calculation
		 * that it does originally, as functions are evenly spread
		 * across ports in the rare case of partitions.
		 */
		idp = kmem_alloc(sizeof (i40e_device_t), KM_SLEEP);
		idp->id_parent = parent;
		idp->id_pci_bus = bus;
		idp->id_pci_device = device;
		idp->id_nfuncs = hw->num_ports * hw->num_partitions;
		idp->id_nreg = 1;
		idp->id_rsrcs_alloc = i40e->i40e_switch_rsrc_alloc;
		idp->id_rsrcs_act = i40e->i40e_switch_rsrc_actual;
		idp->id_rsrcs = kmem_alloc(sizeof (i40e_switch_rsrc_t) *
		    idp->id_rsrcs_alloc, KM_SLEEP);
		bcopy(i40e->i40e_switch_rsrcs, idp->id_rsrcs,
		    sizeof (i40e_switch_rsrc_t) * idp->id_rsrcs_alloc);
		list_create(&idp->id_i40e_list, sizeof (i40e_t),
		    offsetof(i40e_t, i40e_dlink));

		list_insert_tail(&i40e_dlist, idp);
	}

	list_insert_tail(&idp->id_i40e_list, i40e);
	mutex_exit(&i40e_glock);

	return (idp);
}

static void
i40e_link_state_set(i40e_t *i40e, link_state_t state)
{
	if (i40e->i40e_link_state == state)
		return;

	i40e->i40e_link_state = state;
	mac_link_update(i40e->i40e_mac_hdl, i40e->i40e_link_state);
}

/*
 * This is a basic link check routine. Mostly we're using this just to see
 * if we can get any accurate information about the state of the link being
 * up or down, as well as updating the link state, speed, etc. information.
 */
void
i40e_link_check(i40e_t *i40e)
{
	i40e_hw_t *hw = &i40e->i40e_hw_space;
	boolean_t ls;
	int ret;

	ASSERT(MUTEX_HELD(&i40e->i40e_general_lock));

	hw->phy.get_link_info = B_TRUE;
	if ((ret = i40e_get_link_status(hw, &ls)) != I40E_SUCCESS) {
		i40e->i40e_s_link_status_errs++;
		i40e->i40e_s_link_status_lasterr = ret;
		return;
	}

	/*
	 * Firmware abstracts all of the mac and phy information for us, so we
	 * can use i40e_get_link_status to determine the current state.
	 */
	if (ls == B_TRUE) {
		enum i40e_aq_link_speed speed;

		speed = i40e_get_link_speed(hw);

		/*
		 * Translate from an i40e value to a value in Mbits/s.
		 */
		switch (speed) {
		case I40E_LINK_SPEED_100MB:
			i40e->i40e_link_speed = 100;
			break;
		case I40E_LINK_SPEED_1GB:
			i40e->i40e_link_speed = 1000;
			break;
		case I40E_LINK_SPEED_2_5GB:
			i40e->i40e_link_speed = 2500;
			break;
		case I40E_LINK_SPEED_5GB:
			i40e->i40e_link_speed = 5000;
			break;
		case I40E_LINK_SPEED_10GB:
			i40e->i40e_link_speed = 10000;
			break;
		case I40E_LINK_SPEED_20GB:
			i40e->i40e_link_speed = 20000;
			break;
		case I40E_LINK_SPEED_40GB:
			i40e->i40e_link_speed = 40000;
			break;
		case I40E_LINK_SPEED_25GB:
			i40e->i40e_link_speed = 25000;
			break;
		default:
			i40e->i40e_link_speed = 0;
			break;
		}

		/*
		 * At this time, hardware does not support half-duplex
		 * operation, hence why we don't ask the hardware about our
		 * current speed.
		 */
		i40e->i40e_link_duplex = LINK_DUPLEX_FULL;
		i40e_link_state_set(i40e, LINK_STATE_UP);
	} else {
		i40e->i40e_link_speed = 0;
		i40e->i40e_link_duplex = 0;
		i40e_link_state_set(i40e, LINK_STATE_DOWN);
	}
}

static void
i40e_rem_intrs(i40e_t *i40e)
{
	int i, rc;

	for (i = 0; i < i40e->i40e_intr_count; i++) {
		rc = ddi_intr_free(i40e->i40e_intr_handles[i]);
		if (rc != DDI_SUCCESS) {
			i40e_log(i40e, "failed to free interrupt %d: %d",
			    i, rc);
		}
	}

	kmem_free(i40e->i40e_intr_handles, i40e->i40e_intr_size);
	i40e->i40e_intr_handles = NULL;
}

static void
i40e_rem_intr_handlers(i40e_t *i40e)
{
	int i, rc;

	for (i = 0; i < i40e->i40e_intr_count; i++) {
		rc = ddi_intr_remove_handler(i40e->i40e_intr_handles[i]);
		if (rc != DDI_SUCCESS) {
			i40e_log(i40e, "failed to remove interrupt %d: %d",
			    i, rc);
		}
	}
}

/*
 * illumos Fault Management Architecture (FMA) support.
 */

int
i40e_check_acc_handle(ddi_acc_handle_t handle)
{
	ddi_fm_error_t de;

	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
	ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
	return (de.fme_status);
}

int
i40e_check_dma_handle(ddi_dma_handle_t handle)
{
	ddi_fm_error_t de;

	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
	return (de.fme_status);
}

/*
 * Fault service error handling callback function.
 */
/* ARGSUSED */
static int
i40e_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
{
	pci_ereport_post(dip, err, NULL);
	return (err->fme_status);
}

static void
i40e_fm_init(i40e_t *i40e)
{
	ddi_iblock_cookie_t iblk;

	i40e->i40e_fm_capabilities = ddi_prop_get_int(DDI_DEV_T_ANY,
	    i40e->i40e_dip, DDI_PROP_DONTPASS, "fm_capable",
	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);

	if (i40e->i40e_fm_capabilities < 0) {
		i40e->i40e_fm_capabilities = 0;
	} else if (i40e->i40e_fm_capabilities > 0xf) {
		i40e->i40e_fm_capabilities = DDI_FM_EREPORT_CAPABLE |
		    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE |
		    DDI_FM_ERRCB_CAPABLE;
	}

	/*
	 * Only register with IO Fault Services if we have some capability
	 */
	if (i40e->i40e_fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
		i40e_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
	} else {
		i40e_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
	}

	if (i40e->i40e_fm_capabilities) {
		ddi_fm_init(i40e->i40e_dip, &i40e->i40e_fm_capabilities, &iblk);

		if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) ||
		    DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) {
			pci_ereport_setup(i40e->i40e_dip);
		}

		if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities)) {
			ddi_fm_handler_register(i40e->i40e_dip,
			    i40e_fm_error_cb, (void*)i40e);
		}
	}

	if (i40e->i40e_fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
		i40e_init_dma_attrs(i40e, B_TRUE);
	} else {
		i40e_init_dma_attrs(i40e, B_FALSE);
	}
}

static void
i40e_fm_fini(i40e_t *i40e)
{
	if (i40e->i40e_fm_capabilities) {

		if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities) ||
		    DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities))
			pci_ereport_teardown(i40e->i40e_dip);

		if (DDI_FM_ERRCB_CAP(i40e->i40e_fm_capabilities))
			ddi_fm_handler_unregister(i40e->i40e_dip);

		ddi_fm_fini(i40e->i40e_dip);
	}
}

void
i40e_fm_ereport(i40e_t *i40e, char *detail)
{
	uint64_t ena;
	char buf[FM_MAX_CLASS];

	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
	ena = fm_ena_generate(0, FM_ENA_FMT1);
	if (DDI_FM_EREPORT_CAP(i40e->i40e_fm_capabilities)) {
		ddi_fm_ereport_post(i40e->i40e_dip, buf, ena, DDI_NOSLEEP,
		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
	}
}

/*
 * Here we're trying to set the SEID of the default VSI. In general,
 * when we come through and look at this shortly after attach, we
 * expect there to only be a single element present, which is the
 * default VSI. Importantly, each PF seems to not see any other
 * devices, in part because of the simple switch mode that we're
 * using. If for some reason, we see more artifacts, we'll need to
 * revisit what we're doing here.
 */
static boolean_t
i40e_set_def_vsi_seid(i40e_t *i40e)
{
	i40e_hw_t *hw = &i40e->i40e_hw_space;
	struct i40e_aqc_get_switch_config_resp *sw_config;
	uint8_t aq_buf[I40E_AQ_LARGE_BUF];
	uint16_t next = 0;
	int rc;

	/* LINTED: E_BAD_PTR_CAST_ALIGN */
	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
	rc = i40e_aq_get_switch_config(hw, sw_config, sizeof (aq_buf), &next,
	    NULL);
	if (rc != I40E_SUCCESS) {
		i40e_error(i40e, "i40e_aq_get_switch_config() failed %d: %d",
		    rc, hw->aq.asq_last_status);
		return (B_FALSE);
	}

	if (LE_16(sw_config->header.num_reported) != 1) {
		i40e_error(i40e, "encountered multiple (%d) switching units "
		    "during attach, not proceeding",
		    LE_16(sw_config->header.num_reported));
		return (B_FALSE);
	}

	I40E_DEF_VSI_SEID(i40e) = sw_config->element[0].seid;
	return (B_TRUE);
}

/*
 * Get the SEID of the uplink MAC.
 */
static int
i40e_get_mac_seid(i40e_t *i40e)
{
	i40e_hw_t *hw = &i40e->i40e_hw_space;
	struct i40e_aqc_get_switch_config_resp *sw_config;
	uint8_t aq_buf[I40E_AQ_LARGE_BUF];