xref: /illumos-gate/usr/src/uts/common/io/i40e/i40e_intr.c (revision aa2a44af)
19d26e4fcSRobert Mustacchi /*
29d26e4fcSRobert Mustacchi  * This file and its contents are supplied under the terms of the
39d26e4fcSRobert Mustacchi  * Common Development and Distribution License ("CDDL"), version 1.0.
49d26e4fcSRobert Mustacchi  * You may only use this file in accordance with the terms of version
59d26e4fcSRobert Mustacchi  * 1.0 of the CDDL.
69d26e4fcSRobert Mustacchi  *
79d26e4fcSRobert Mustacchi  * A full copy of the text of the CDDL should have accompanied this
89d26e4fcSRobert Mustacchi  * source.  A copy of the CDDL is also available via the Internet at
99d26e4fcSRobert Mustacchi  * http://www.illumos.org/license/CDDL.
109d26e4fcSRobert Mustacchi  */
119d26e4fcSRobert Mustacchi 
129d26e4fcSRobert Mustacchi /*
13093e8453SRobert Mustacchi  * Copyright 2019 Joyent, Inc.
14396505afSPaul Winder  * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
15*aa2a44afSPaul Winder  * Copyright 2020 RackTop Systems, Inc.
169d26e4fcSRobert Mustacchi  */
179d26e4fcSRobert Mustacchi 
189d26e4fcSRobert Mustacchi /*
199d26e4fcSRobert Mustacchi  * -------------------------
209d26e4fcSRobert Mustacchi  * Interrupt Handling Theory
219d26e4fcSRobert Mustacchi  * -------------------------
229d26e4fcSRobert Mustacchi  *
239d26e4fcSRobert Mustacchi  * There are a couple different sets of interrupts that we need to worry about:
249d26e4fcSRobert Mustacchi  *
259d26e4fcSRobert Mustacchi  *   - Interrupts from receive queues
269d26e4fcSRobert Mustacchi  *   - Interrupts from transmit queues
279d26e4fcSRobert Mustacchi  *   - 'Other Interrupts', such as the administrative queue
289d26e4fcSRobert Mustacchi  *
299d26e4fcSRobert Mustacchi  * 'Other Interrupts' are asynchronous events such as a link status change event
309d26e4fcSRobert Mustacchi  * being posted to the administrative queue, unrecoverable ECC errors, and more.
319d26e4fcSRobert Mustacchi  * If we have something being posted to the administrative queue, then we go
329d26e4fcSRobert Mustacchi  * through and process it, because it's generally enabled as a separate logical
339d26e4fcSRobert Mustacchi  * interrupt. Note, we may need to do more here eventually. To re-enable the
349d26e4fcSRobert Mustacchi  * interrupts from the 'Other Interrupts' section, we need to clear the PBA and
359d26e4fcSRobert Mustacchi  * write ENA to PFINT_ICR0.
369d26e4fcSRobert Mustacchi  *
379d26e4fcSRobert Mustacchi  * Interrupts from the transmit and receive queues indicates that our requests
389d26e4fcSRobert Mustacchi  * have been processed. In the rx case, it means that we have data that we
399d26e4fcSRobert Mustacchi  * should take a look at and send up the stack. In the tx case, it means that
409d26e4fcSRobert Mustacchi  * data which we got from MAC has now been sent out on the wire and we can free
419d26e4fcSRobert Mustacchi  * the associated data. Most of the logic for acting upon the presence of this
429d26e4fcSRobert Mustacchi  * data can be found in i40e_transciever.c which handles all of the DMA, rx, and
439d26e4fcSRobert Mustacchi  * tx operations. This file is dedicated to handling and dealing with interrupt
449d26e4fcSRobert Mustacchi  * processing.
459d26e4fcSRobert Mustacchi  *
469d26e4fcSRobert Mustacchi  * All devices supported by this driver support three kinds of interrupts:
479d26e4fcSRobert Mustacchi  *
489d26e4fcSRobert Mustacchi  *   o Extended Message Signaled Interrupts (MSI-X)
499d26e4fcSRobert Mustacchi  *   o Message Signaled Interrupts (MSI)
509d26e4fcSRobert Mustacchi  *   o Legacy PCI interrupts (INTx)
519d26e4fcSRobert Mustacchi  *
529d26e4fcSRobert Mustacchi  * Generally speaking the hardware logically handles MSI and INTx the same and
539d26e4fcSRobert Mustacchi  * restricts us to only using a single interrupt, which isn't the interesting
549d26e4fcSRobert Mustacchi  * case. With MSI-X available, each physical function of the device provides the
559d26e4fcSRobert Mustacchi  * opportunity for multiple interrupts which is what we'll focus on.
569d26e4fcSRobert Mustacchi  *
579d26e4fcSRobert Mustacchi  * --------------------
589d26e4fcSRobert Mustacchi  * Interrupt Management
599d26e4fcSRobert Mustacchi  * --------------------
609d26e4fcSRobert Mustacchi  *
619d26e4fcSRobert Mustacchi  * By default, the admin queue, which consists of the asynchronous other
629d26e4fcSRobert Mustacchi  * interrupts is always bound to MSI-X vector zero. Next, we spread out all of
639d26e4fcSRobert Mustacchi  * the other interrupts that we have available to us over the remaining
649d26e4fcSRobert Mustacchi  * interrupt vectors.
659d26e4fcSRobert Mustacchi  *
669d26e4fcSRobert Mustacchi  * This means that there may be multiple queues, both tx and rx, which are
679d26e4fcSRobert Mustacchi  * mapped to the same interrupt. When the interrupt fires, we'll have to check
689d26e4fcSRobert Mustacchi  * all of them for servicing, before we go through and indicate that the
699d26e4fcSRobert Mustacchi  * interrupt is claimed.
709d26e4fcSRobert Mustacchi  *
719d26e4fcSRobert Mustacchi  * The hardware provides the means of mapping various queues to MSI-X interrupts
729d26e4fcSRobert Mustacchi  * by programming the I40E_QINT_RQCTL() and I4OE_QINT_TQCTL() registers. These
739d26e4fcSRobert Mustacchi  * registers can also be used to enable and disable whether or not the queue is
749d26e4fcSRobert Mustacchi  * a source of interrupts. As part of this, the hardware requires that we
759d26e4fcSRobert Mustacchi  * maintain a linked list of queues for each interrupt vector. While it may seem
769d26e4fcSRobert Mustacchi  * like this is only there for the purproses of ITRs, that's not the case. The
779d26e4fcSRobert Mustacchi  * first queue must be programmed in I40E_QINT_LNKLSTN(%vector) register. Each
789d26e4fcSRobert Mustacchi  * queue defines the next one in either the I40E_QINT_RQCTL or I40E_QINT_TQCTL
799d26e4fcSRobert Mustacchi  * register.
809d26e4fcSRobert Mustacchi  *
819d26e4fcSRobert Mustacchi  * Finally, the individual interrupt vector itself has the ability to be enabled
829d26e4fcSRobert Mustacchi  * and disabled. The overall interrupt is controlled through the
839d26e4fcSRobert Mustacchi  * I40E_PFINT_DYN_CTLN() register. This is used to turn on and off the interrupt
849d26e4fcSRobert Mustacchi  * as a whole.
859d26e4fcSRobert Mustacchi  *
869d26e4fcSRobert Mustacchi  * Note that this means that both the individual queue and the interrupt as a
879d26e4fcSRobert Mustacchi  * whole can be toggled and re-enabled.
889d26e4fcSRobert Mustacchi  *
899d26e4fcSRobert Mustacchi  * -------------------
909d26e4fcSRobert Mustacchi  * Non-MSIX Management
919d26e4fcSRobert Mustacchi  * -------------------
929d26e4fcSRobert Mustacchi  *
939d26e4fcSRobert Mustacchi  * We may have a case where the Operating System is unable to actually allocate
949d26e4fcSRobert Mustacchi  * any MSI-X to the system. In such a world, there is only one transmit/receive
959d26e4fcSRobert Mustacchi  * queue pair and it is bound to the same interrupt with index zero. The
969d26e4fcSRobert Mustacchi  * hardware doesn't allow us access to additional interrupt vectors in these
979d26e4fcSRobert Mustacchi  * modes. Note that technically we could support more transmit/receive queues if
989d26e4fcSRobert Mustacchi  * we wanted.
999d26e4fcSRobert Mustacchi  *
1009d26e4fcSRobert Mustacchi  * In this world, because the interrupts for the admin queue and traffic are
1019d26e4fcSRobert Mustacchi  * mixed together, we have to consult ICR0 to determine what has occurred. The
1029d26e4fcSRobert Mustacchi  * QINT_TQCTL and QINT_RQCTL registers have a field, 'MSI-X 0 index' which
1039d26e4fcSRobert Mustacchi  * allows us to set a specific bit in ICR0. There are up to seven such bits;
1049d26e4fcSRobert Mustacchi  * however, we only use the bit 0 and 1 for the rx and tx queue respectively.
1059d26e4fcSRobert Mustacchi  * These are contained by the I40E_INTR_NOTX_{R|T}X_QUEUE and
1069d26e4fcSRobert Mustacchi  * I40E_INTR_NOTX_{R|T}X_MASK registers respectively.
1079d26e4fcSRobert Mustacchi  *
1089d26e4fcSRobert Mustacchi  * Unfortunately, these corresponding queue bits have no corresponding entry in
1099d26e4fcSRobert Mustacchi  * the ICR0_ENA register. So instead, when enabling interrupts on the queues, we
1109d26e4fcSRobert Mustacchi  * end up enabling it on the queue registers rather than on the MSI-X registers.
1119d26e4fcSRobert Mustacchi  * In the MSI-X world, because they can be enabled and disabled, this is
1129d26e4fcSRobert Mustacchi  * different and the queues can always be enabled and disabled, but the
1139d26e4fcSRobert Mustacchi  * interrupts themselves are toggled (ignoring the question of interrupt
1149d26e4fcSRobert Mustacchi  * blanking for polling on rings).
1159d26e4fcSRobert Mustacchi  *
1169d26e4fcSRobert Mustacchi  * Finally, we still have to set up the interrupt linked list, but the list is
1179d26e4fcSRobert Mustacchi  * instead rooted at the register I40E_PFINT_LNKLST0, rather than being tied to
1189d26e4fcSRobert Mustacchi  * one of the other MSI-X registers.
1199d26e4fcSRobert Mustacchi  *
1209d26e4fcSRobert Mustacchi  * --------------------
1219d26e4fcSRobert Mustacchi  * Interrupt Moderation
1229d26e4fcSRobert Mustacchi  * --------------------
1239d26e4fcSRobert Mustacchi  *
1249d26e4fcSRobert Mustacchi  * The XL710 hardware has three different interrupt moderation registers per
1259d26e4fcSRobert Mustacchi  * interrupt. Unsurprisingly, we use these for:
1269d26e4fcSRobert Mustacchi  *
1279d26e4fcSRobert Mustacchi  *   o RX interrupts
1289d26e4fcSRobert Mustacchi  *   o TX interrupts
1299d26e4fcSRobert Mustacchi  *   o 'Other interrupts' (link status change, admin queue, etc.)
1309d26e4fcSRobert Mustacchi  *
1319d26e4fcSRobert Mustacchi  * By default, we throttle 'other interrupts' the most, then TX interrupts, and
1329d26e4fcSRobert Mustacchi  * then RX interrupts. The default values for these were based on trying to
1339d26e4fcSRobert Mustacchi  * reason about both the importance and frequency of events. Generally speaking
1349d26e4fcSRobert Mustacchi  * 'other interrupts' are not very frequent and they're not important for the
1359d26e4fcSRobert Mustacchi  * I/O data path in and of itself (though they may indicate issues with the I/O
1369d26e4fcSRobert Mustacchi  * data path).
1379d26e4fcSRobert Mustacchi  *
1389d26e4fcSRobert Mustacchi  * On the flip side, when we're not polling, RX interrupts are very important.
1399d26e4fcSRobert Mustacchi  * The longer we wait for them, the more latency that we inject into the system.
1409d26e4fcSRobert Mustacchi  * However, if we allow interrupts to occur too frequently, we risk a few
1419d26e4fcSRobert Mustacchi  * problems:
1429d26e4fcSRobert Mustacchi  *
1439d26e4fcSRobert Mustacchi  *  1) Abusing system resources. Without proper interrupt blanking and polling,
1449d26e4fcSRobert Mustacchi  *     we can see upwards of 200k-300k interrupts per second on the system.
1459d26e4fcSRobert Mustacchi  *
1469d26e4fcSRobert Mustacchi  *  2) Not enough data coalescing to enable polling. In other words, the more
1479d26e4fcSRobert Mustacchi  *     data that we allow to build up, the more likely we'll be able to enable
1489d26e4fcSRobert Mustacchi  *     polling mode and allowing us to better handle bulk data.
1499d26e4fcSRobert Mustacchi  *
1509d26e4fcSRobert Mustacchi  * In-between the 'other interrupts' and the TX interrupts we have the
1519d26e4fcSRobert Mustacchi  * reclamation of TX buffers. This operation is not quite as important as we
1529d26e4fcSRobert Mustacchi  * generally size the ring large enough that we should be able to reclaim a
1539d26e4fcSRobert Mustacchi  * substantial amount of the descriptors that we have used per interrupt. So
1549d26e4fcSRobert Mustacchi  * while it's important that this interrupt occur, we don't necessarily need it
1559d26e4fcSRobert Mustacchi  * firing as frequently as RX; it doesn't, on its own, induce additional latency
1569d26e4fcSRobert Mustacchi  * into the system.
1579d26e4fcSRobert Mustacchi  *
1589d26e4fcSRobert Mustacchi  * Based on all this we currently assign static ITR values for the system. While
1599d26e4fcSRobert Mustacchi  * we could move to a dynamic system (the hardware supports that), we'd want to
1609d26e4fcSRobert Mustacchi  * make sure that we're seeing problems from this that we believe would be
1619d26e4fcSRobert Mustacchi  * generally helped by the added complexity.
1629d26e4fcSRobert Mustacchi  *
1639d26e4fcSRobert Mustacchi  * Based on this, the default values that we have allow for the following
1649d26e4fcSRobert Mustacchi  * interrupt thresholds:
1659d26e4fcSRobert Mustacchi  *
1669d26e4fcSRobert Mustacchi  *    o 20k interrupts/s for RX
1679d26e4fcSRobert Mustacchi  *    o 5k interrupts/s for TX
1689d26e4fcSRobert Mustacchi  *    o 2k interupts/s for 'Other Interrupts'
1699d26e4fcSRobert Mustacchi  */
1709d26e4fcSRobert Mustacchi 
1719d26e4fcSRobert Mustacchi #include "i40e_sw.h"
1729d26e4fcSRobert Mustacchi 
1739d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_QUEUE	0
1749d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_INTR	0
1759d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_RX_QUEUE	0
1769d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_RX_MASK	(1 << I40E_PFINT_ICR0_QUEUE_0_SHIFT)
1779d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_TX_QUEUE	1
1789d26e4fcSRobert Mustacchi #define	I40E_INTR_NOTX_TX_MASK	(1 << I40E_PFINT_ICR0_QUEUE_1_SHIFT)
1799d26e4fcSRobert Mustacchi 
1809d26e4fcSRobert Mustacchi void
i40e_intr_set_itr(i40e_t * i40e,i40e_itr_index_t itr,uint_t val)1819d26e4fcSRobert Mustacchi i40e_intr_set_itr(i40e_t *i40e, i40e_itr_index_t itr, uint_t val)
1829d26e4fcSRobert Mustacchi {
1839d26e4fcSRobert Mustacchi 	int i;
1849d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
1859d26e4fcSRobert Mustacchi 
1869d26e4fcSRobert Mustacchi 	VERIFY3U(val, <=, I40E_MAX_ITR);
1879d26e4fcSRobert Mustacchi 	VERIFY3U(itr, <, I40E_ITR_INDEX_NONE);
1889d26e4fcSRobert Mustacchi 
1899d26e4fcSRobert Mustacchi 	/*
1909d26e4fcSRobert Mustacchi 	 * No matter the interrupt mode, the ITR for other interrupts is always
1919d26e4fcSRobert Mustacchi 	 * on interrupt zero and the same is true if we're not using MSI-X.
1929d26e4fcSRobert Mustacchi 	 */
1939d26e4fcSRobert Mustacchi 	if (itr == I40E_ITR_INDEX_OTHER ||
1949d26e4fcSRobert Mustacchi 	    i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) {
1959d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_PFINT_ITR0(itr), val);
1969d26e4fcSRobert Mustacchi 		return;
1979d26e4fcSRobert Mustacchi 	}
1989d26e4fcSRobert Mustacchi 
199396505afSPaul Winder 	for (i = 0; i < i40e->i40e_num_trqpairs; i++) {
200396505afSPaul Winder 		I40E_WRITE_REG(hw, I40E_PFINT_ITRN(itr, i), val);
2019d26e4fcSRobert Mustacchi 	}
2029d26e4fcSRobert Mustacchi }
2039d26e4fcSRobert Mustacchi 
2049d26e4fcSRobert Mustacchi /*
2059d26e4fcSRobert Mustacchi  * Re-enable the adminq. Note that the adminq doesn't have a traditional queue
2069d26e4fcSRobert Mustacchi  * associated with it from an interrupt perspective and just lives on ICR0.
2079d26e4fcSRobert Mustacchi  * However when MSI-X interrupts are not being used, then this also enables and
2089d26e4fcSRobert Mustacchi  * disables those interrupts.
2099d26e4fcSRobert Mustacchi  */
2109d26e4fcSRobert Mustacchi static void
i40e_intr_adminq_enable(i40e_t * i40e)2119d26e4fcSRobert Mustacchi i40e_intr_adminq_enable(i40e_t *i40e)
2129d26e4fcSRobert Mustacchi {
2139d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
2149d26e4fcSRobert Mustacchi 	uint32_t reg;
2159d26e4fcSRobert Mustacchi 
2169d26e4fcSRobert Mustacchi 	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
2179d26e4fcSRobert Mustacchi 	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2189d26e4fcSRobert Mustacchi 	    (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2199d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg);
2209d26e4fcSRobert Mustacchi 	i40e_flush(hw);
2219d26e4fcSRobert Mustacchi }
2229d26e4fcSRobert Mustacchi 
2239d26e4fcSRobert Mustacchi static void
i40e_intr_adminq_disable(i40e_t * i40e)2249d26e4fcSRobert Mustacchi i40e_intr_adminq_disable(i40e_t *i40e)
2259d26e4fcSRobert Mustacchi {
2269d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
2279d26e4fcSRobert Mustacchi 	uint32_t reg;
2289d26e4fcSRobert Mustacchi 
2299d26e4fcSRobert Mustacchi 	reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
2309d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg);
2319d26e4fcSRobert Mustacchi }
2329d26e4fcSRobert Mustacchi 
23309aee612SRyan Zezeski /*
23409aee612SRyan Zezeski  * The next two functions enable/disable the reception of interrupts
23509aee612SRyan Zezeski  * on the given vector. Only vectors 1..N are programmed by these
23609aee612SRyan Zezeski  * functions; vector 0 is special and handled by a different register.
23709aee612SRyan Zezeski  * We must subtract one from the vector because i40e implicitly adds
23809aee612SRyan Zezeski  * one to the vector value. See section for more details.
23909aee612SRyan Zezeski  */
2409d26e4fcSRobert Mustacchi static void
i40e_intr_io_enable(i40e_t * i40e,int vector)2419d26e4fcSRobert Mustacchi i40e_intr_io_enable(i40e_t *i40e, int vector)
2429d26e4fcSRobert Mustacchi {
2439d26e4fcSRobert Mustacchi 	uint32_t reg;
2449d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
2459d26e4fcSRobert Mustacchi 
24609aee612SRyan Zezeski 	ASSERT3S(vector, >, 0);
2479d26e4fcSRobert Mustacchi 	reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2489d26e4fcSRobert Mustacchi 	    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2499d26e4fcSRobert Mustacchi 	    (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2509d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg);
2519d26e4fcSRobert Mustacchi }
2529d26e4fcSRobert Mustacchi 
2539d26e4fcSRobert Mustacchi static void
i40e_intr_io_disable(i40e_t * i40e,int vector)2549d26e4fcSRobert Mustacchi i40e_intr_io_disable(i40e_t *i40e, int vector)
2559d26e4fcSRobert Mustacchi {
2569d26e4fcSRobert Mustacchi 	uint32_t reg;
2579d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
2589d26e4fcSRobert Mustacchi 
25909aee612SRyan Zezeski 	ASSERT3S(vector, >, 0);
2609d26e4fcSRobert Mustacchi 	reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
2619d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg);
2629d26e4fcSRobert Mustacchi }
2639d26e4fcSRobert Mustacchi 
2649d26e4fcSRobert Mustacchi /*
2659d26e4fcSRobert Mustacchi  * When MSI-X interrupts are being used, then we can enable the actual
2669d26e4fcSRobert Mustacchi  * interrupts themselves. However, when they are not, we instead have to turn
2679d26e4fcSRobert Mustacchi  * towards the queue's CAUSE_ENA bit and enable that.
2689d26e4fcSRobert Mustacchi  */
2699d26e4fcSRobert Mustacchi void
i40e_intr_io_enable_all(i40e_t * i40e)2709d26e4fcSRobert Mustacchi i40e_intr_io_enable_all(i40e_t *i40e)
2719d26e4fcSRobert Mustacchi {
2729d26e4fcSRobert Mustacchi 	if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
2739d26e4fcSRobert Mustacchi 		int i;
2749d26e4fcSRobert Mustacchi 
2759d26e4fcSRobert Mustacchi 		for (i = 1; i < i40e->i40e_intr_count; i++) {
2769d26e4fcSRobert Mustacchi 			i40e_intr_io_enable(i40e, i);
2779d26e4fcSRobert Mustacchi 		}
2789d26e4fcSRobert Mustacchi 	} else {
2799d26e4fcSRobert Mustacchi 		uint32_t reg;
2809d26e4fcSRobert Mustacchi 		i40e_hw_t *hw = &i40e->i40e_hw_space;
2819d26e4fcSRobert Mustacchi 
2829d26e4fcSRobert Mustacchi 		reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE));
2839d26e4fcSRobert Mustacchi 		reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
2849d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
2859d26e4fcSRobert Mustacchi 
2869d26e4fcSRobert Mustacchi 		reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE));
2879d26e4fcSRobert Mustacchi 		reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
2889d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
2899d26e4fcSRobert Mustacchi 	}
2909d26e4fcSRobert Mustacchi }
2919d26e4fcSRobert Mustacchi 
2929d26e4fcSRobert Mustacchi /*
2939d26e4fcSRobert Mustacchi  * When MSI-X interrupts are being used, then we can disable the actual
2949d26e4fcSRobert Mustacchi  * interrupts themselves. However, when they are not, we instead have to turn
2959d26e4fcSRobert Mustacchi  * towards the queue's CAUSE_ENA bit and disable that.
2969d26e4fcSRobert Mustacchi  */
2979d26e4fcSRobert Mustacchi void
i40e_intr_io_disable_all(i40e_t * i40e)2989d26e4fcSRobert Mustacchi i40e_intr_io_disable_all(i40e_t *i40e)
2999d26e4fcSRobert Mustacchi {
3009d26e4fcSRobert Mustacchi 	if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
3019d26e4fcSRobert Mustacchi 		int i;
3029d26e4fcSRobert Mustacchi 
3039d26e4fcSRobert Mustacchi 		for (i = 1; i < i40e->i40e_intr_count; i++) {
3049d26e4fcSRobert Mustacchi 			i40e_intr_io_disable(i40e, i);
3059d26e4fcSRobert Mustacchi 		}
3069d26e4fcSRobert Mustacchi 	} else {
3079d26e4fcSRobert Mustacchi 		uint32_t reg;
3089d26e4fcSRobert Mustacchi 		i40e_hw_t *hw = &i40e->i40e_hw_space;
3099d26e4fcSRobert Mustacchi 
3109d26e4fcSRobert Mustacchi 		reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE));
3119d26e4fcSRobert Mustacchi 		reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3129d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
3139d26e4fcSRobert Mustacchi 
3149d26e4fcSRobert Mustacchi 		reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE));
3159d26e4fcSRobert Mustacchi 		reg &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3169d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
3179d26e4fcSRobert Mustacchi 	}
3189d26e4fcSRobert Mustacchi }
3199d26e4fcSRobert Mustacchi 
3209d26e4fcSRobert Mustacchi /*
3219d26e4fcSRobert Mustacchi  * As part of disabling the tx and rx queue's we're technically supposed to
3229d26e4fcSRobert Mustacchi  * remove the linked list entries. The simplest way is to clear the LNKLSTN
3239d26e4fcSRobert Mustacchi  * register by setting it to I40E_QUEUE_TYPE_EOL (0x7FF).
3249d26e4fcSRobert Mustacchi  *
3259d26e4fcSRobert Mustacchi  * Note all of the FM register access checks are performed by the caller.
3269d26e4fcSRobert Mustacchi  */
3279d26e4fcSRobert Mustacchi void
i40e_intr_io_clear_cause(i40e_t * i40e)3289d26e4fcSRobert Mustacchi i40e_intr_io_clear_cause(i40e_t *i40e)
3299d26e4fcSRobert Mustacchi {
330093e8453SRobert Mustacchi 	uint32_t i;
3319d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
3329d26e4fcSRobert Mustacchi 
3339d26e4fcSRobert Mustacchi 	if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) {
3349d26e4fcSRobert Mustacchi 		uint32_t reg;
3359d26e4fcSRobert Mustacchi 		reg = I40E_QUEUE_TYPE_EOL;
3369d26e4fcSRobert Mustacchi 		I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg);
3379d26e4fcSRobert Mustacchi 		return;
3389d26e4fcSRobert Mustacchi 	}
3399d26e4fcSRobert Mustacchi 
340093e8453SRobert Mustacchi 	for (i = 0; i < i40e->i40e_intr_count - 1; i++) {
3419d26e4fcSRobert Mustacchi 		uint32_t reg;
342093e8453SRobert Mustacchi 
3439d26e4fcSRobert Mustacchi 		reg = I40E_QUEUE_TYPE_EOL;
344396505afSPaul Winder 		I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(i), reg);
3459d26e4fcSRobert Mustacchi 	}
3469d26e4fcSRobert Mustacchi 
3479d26e4fcSRobert Mustacchi 	i40e_flush(hw);
3489d26e4fcSRobert Mustacchi }
3499d26e4fcSRobert Mustacchi 
3509d26e4fcSRobert Mustacchi /*
3519d26e4fcSRobert Mustacchi  * Finalize interrupt handling. Mostly this disables the admin queue.
3529d26e4fcSRobert Mustacchi  */
3539d26e4fcSRobert Mustacchi void
i40e_intr_chip_fini(i40e_t * i40e)3549d26e4fcSRobert Mustacchi i40e_intr_chip_fini(i40e_t *i40e)
3559d26e4fcSRobert Mustacchi {
3569d26e4fcSRobert Mustacchi #ifdef DEBUG
3579d26e4fcSRobert Mustacchi 	int i;
3589d26e4fcSRobert Mustacchi 	uint32_t reg;
3599d26e4fcSRobert Mustacchi 
3609d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
3619d26e4fcSRobert Mustacchi 
3629d26e4fcSRobert Mustacchi 	/*
3639d26e4fcSRobert Mustacchi 	 * Take a look and verify that all other interrupts have been disabled
3649d26e4fcSRobert Mustacchi 	 * and the interrupt linked lists have been zeroed.
3659d26e4fcSRobert Mustacchi 	 */
3669d26e4fcSRobert Mustacchi 	if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
367093e8453SRobert Mustacchi 		for (i = 0; i < i40e->i40e_intr_count - 1; i++) {
368396505afSPaul Winder 			reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i));
3699d26e4fcSRobert Mustacchi 			VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK);
3709d26e4fcSRobert Mustacchi 
371396505afSPaul Winder 			reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i));
3729d26e4fcSRobert Mustacchi 			VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL);
3739d26e4fcSRobert Mustacchi 		}
3749d26e4fcSRobert Mustacchi 	}
3759d26e4fcSRobert Mustacchi #endif
3769d26e4fcSRobert Mustacchi 
3779d26e4fcSRobert Mustacchi 	i40e_intr_adminq_disable(i40e);
3789d26e4fcSRobert Mustacchi }
3799d26e4fcSRobert Mustacchi 
3809d26e4fcSRobert Mustacchi /*
38109aee612SRyan Zezeski  * Set the head of the interrupt linked list. The PFINT_LNKLSTN[N]
38209aee612SRyan Zezeski  * register actually refers to the 'N + 1' interrupt vector. E.g.,
38309aee612SRyan Zezeski  * PFINT_LNKLSTN[0] refers to interrupt vector 1.
38409aee612SRyan Zezeski  */
38509aee612SRyan Zezeski static void
i40e_set_lnklstn(i40e_t * i40e,uint_t vector,uint_t queue)38609aee612SRyan Zezeski i40e_set_lnklstn(i40e_t *i40e, uint_t vector, uint_t queue)
38709aee612SRyan Zezeski {
38809aee612SRyan Zezeski 	uint32_t	reg;
38909aee612SRyan Zezeski 	i40e_hw_t	*hw = &i40e->i40e_hw_space;
39009aee612SRyan Zezeski 
39109aee612SRyan Zezeski 	reg = (queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
39209aee612SRyan Zezeski 	    (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
39309aee612SRyan Zezeski 
39409aee612SRyan Zezeski 	I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(vector), reg);
39509aee612SRyan Zezeski 	DEBUGOUT2("PFINT_LNKLSTN[%u] = 0x%x", vector, reg);
39609aee612SRyan Zezeski }
39709aee612SRyan Zezeski 
39809aee612SRyan Zezeski /*
39909aee612SRyan Zezeski  * Set the QINT_RQCTL[queue] register. The next queue is always the Tx
40009aee612SRyan Zezeski  * queue associated with this Rx queue. Unlike PFINT_LNKLSTN, the
40109aee612SRyan Zezeski  * vector should be the actual vector this queue is on -- i.e., it
40209aee612SRyan Zezeski  * should be equal to itrq_rx_intrvec.
40309aee612SRyan Zezeski  */
40409aee612SRyan Zezeski static void
i40e_set_rqctl(i40e_t * i40e,uint_t vector,uint_t queue)40509aee612SRyan Zezeski i40e_set_rqctl(i40e_t *i40e, uint_t vector, uint_t queue)
40609aee612SRyan Zezeski {
40709aee612SRyan Zezeski 	uint32_t	reg;
40809aee612SRyan Zezeski 	i40e_hw_t	*hw = &i40e->i40e_hw_space;
40909aee612SRyan Zezeski 
41009aee612SRyan Zezeski 	ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_rx_intrvec);
41109aee612SRyan Zezeski 
41209aee612SRyan Zezeski 	reg = (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
41309aee612SRyan Zezeski 	    (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
41409aee612SRyan Zezeski 	    (queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
41509aee612SRyan Zezeski 	    (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
41609aee612SRyan Zezeski 	    I40E_QINT_RQCTL_CAUSE_ENA_MASK;
41709aee612SRyan Zezeski 
41809aee612SRyan Zezeski 	I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
41909aee612SRyan Zezeski 	DEBUGOUT2("QINT_RQCTL[%u] = 0x%x", queue, reg);
42009aee612SRyan Zezeski }
42109aee612SRyan Zezeski 
42209aee612SRyan Zezeski /*
42309aee612SRyan Zezeski  * Like i40e_set_rqctl(), but for QINT_TQCTL[queue]. The next queue is
42409aee612SRyan Zezeski  * either the Rx queue of another TRQP, or EOL.
42509aee612SRyan Zezeski  */
42609aee612SRyan Zezeski static void
i40e_set_tqctl(i40e_t * i40e,uint_t vector,uint_t queue,uint_t next_queue)42709aee612SRyan Zezeski i40e_set_tqctl(i40e_t *i40e, uint_t vector, uint_t queue, uint_t next_queue)
42809aee612SRyan Zezeski {
42909aee612SRyan Zezeski 	uint32_t	reg;
43009aee612SRyan Zezeski 	i40e_hw_t	*hw = &i40e->i40e_hw_space;
43109aee612SRyan Zezeski 
43209aee612SRyan Zezeski 	ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_tx_intrvec);
43309aee612SRyan Zezeski 
43409aee612SRyan Zezeski 	reg = (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
43509aee612SRyan Zezeski 	    (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
43609aee612SRyan Zezeski 	    (next_queue << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
43709aee612SRyan Zezeski 	    (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) |
43809aee612SRyan Zezeski 	    I40E_QINT_TQCTL_CAUSE_ENA_MASK;
43909aee612SRyan Zezeski 
44009aee612SRyan Zezeski 	I40E_WRITE_REG(hw, I40E_QINT_TQCTL(queue), reg);
44109aee612SRyan Zezeski 	DEBUGOUT2("QINT_TQCTL[%u] = 0x%x", queue, reg);
44209aee612SRyan Zezeski }
44309aee612SRyan Zezeski 
44409aee612SRyan Zezeski /*
44509aee612SRyan Zezeski  * Program the interrupt linked list. Each vector has a linked list of
44609aee612SRyan Zezeski  * queues which act as event sources for that vector. When one of
44709aee612SRyan Zezeski  * those sources has an event the associated interrupt vector is
44809aee612SRyan Zezeski  * fired. This mapping must match the mapping found in
44909aee612SRyan Zezeski  * i40e_map_intrs_to_vectors().
45009aee612SRyan Zezeski  *
45109aee612SRyan Zezeski  * See section 7.5.3 for more information about the configuration of
45209aee612SRyan Zezeski  * the interrupt linked list.
4539d26e4fcSRobert Mustacchi  */
4549d26e4fcSRobert Mustacchi static void
i40e_intr_init_queue_msix(i40e_t * i40e)4559d26e4fcSRobert Mustacchi i40e_intr_init_queue_msix(i40e_t *i40e)
4569d26e4fcSRobert Mustacchi {
45709aee612SRyan Zezeski 	uint_t intr_count;
4589d26e4fcSRobert Mustacchi 
4599d26e4fcSRobert Mustacchi 	/*
46009aee612SRyan Zezeski 	 * The 0th vector is for 'Other Interrupts' only (subject to
46109aee612SRyan Zezeski 	 * change in the future).
4629d26e4fcSRobert Mustacchi 	 */
46309aee612SRyan Zezeski 	intr_count = i40e->i40e_intr_count - 1;
464396505afSPaul Winder 
46509aee612SRyan Zezeski 	for (uint_t vec = 0; vec < intr_count; vec++) {
46609aee612SRyan Zezeski 		boolean_t head = B_TRUE;
467396505afSPaul Winder 
46809aee612SRyan Zezeski 		for (uint_t qidx = vec; qidx < i40e->i40e_num_trqpairs;
469093e8453SRobert Mustacchi 		    qidx += intr_count) {
47009aee612SRyan Zezeski 			uint_t next_qidx = qidx + intr_count;
471396505afSPaul Winder 
47209aee612SRyan Zezeski 			next_qidx = (next_qidx > i40e->i40e_num_trqpairs) ?
47309aee612SRyan Zezeski 			    I40E_QUEUE_TYPE_EOL : next_qidx;
474396505afSPaul Winder 
47509aee612SRyan Zezeski 			if (head) {
47609aee612SRyan Zezeski 				i40e_set_lnklstn(i40e, vec, qidx);
47709aee612SRyan Zezeski 				head = B_FALSE;
47809aee612SRyan Zezeski 			}
479396505afSPaul Winder 
48009aee612SRyan Zezeski 			i40e_set_rqctl(i40e, vec + 1, qidx);
48109aee612SRyan Zezeski 			i40e_set_tqctl(i40e, vec + 1, qidx, next_qidx);
48209aee612SRyan Zezeski 		}
483396505afSPaul Winder 	}
4849d26e4fcSRobert Mustacchi }
4859d26e4fcSRobert Mustacchi 
4869d26e4fcSRobert Mustacchi /*
4879d26e4fcSRobert Mustacchi  * Set up a single queue to share the admin queue interrupt in the non-MSI-X
4889d26e4fcSRobert Mustacchi  * world. Note we do not enable the queue as an interrupt cause at this time. We
4899d26e4fcSRobert Mustacchi  * don't have any other vector of control here, unlike with the MSI-X interrupt
4909d26e4fcSRobert Mustacchi  * case.
4919d26e4fcSRobert Mustacchi  */
4929d26e4fcSRobert Mustacchi static void
i40e_intr_init_queue_shared(i40e_t * i40e)4939d26e4fcSRobert Mustacchi i40e_intr_init_queue_shared(i40e_t *i40e)
4949d26e4fcSRobert Mustacchi {
4959d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
4969d26e4fcSRobert Mustacchi 	uint32_t reg;
4979d26e4fcSRobert Mustacchi 
4989d26e4fcSRobert Mustacchi 	VERIFY(i40e->i40e_intr_type == DDI_INTR_TYPE_FIXED ||
4999d26e4fcSRobert Mustacchi 	    i40e->i40e_intr_type == DDI_INTR_TYPE_MSI);
5009d26e4fcSRobert Mustacchi 
5019d26e4fcSRobert Mustacchi 	reg = (I40E_INTR_NOTX_QUEUE << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5029d26e4fcSRobert Mustacchi 	    (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT);
5039d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg);
5049d26e4fcSRobert Mustacchi 
5059d26e4fcSRobert Mustacchi 	reg = (I40E_INTR_NOTX_INTR << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5069d26e4fcSRobert Mustacchi 	    (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
5079d26e4fcSRobert Mustacchi 	    (I40E_INTR_NOTX_RX_QUEUE << I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) |
5089d26e4fcSRobert Mustacchi 	    (I40E_INTR_NOTX_QUEUE << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5099d26e4fcSRobert Mustacchi 	    (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
5109d26e4fcSRobert Mustacchi 
5119d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg);
5129d26e4fcSRobert Mustacchi 
5139d26e4fcSRobert Mustacchi 	reg = (I40E_INTR_NOTX_INTR << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
5149d26e4fcSRobert Mustacchi 	    (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
5159d26e4fcSRobert Mustacchi 	    (I40E_INTR_NOTX_TX_QUEUE << I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) |
5169d26e4fcSRobert Mustacchi 	    (I40E_QUEUE_TYPE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
5179d26e4fcSRobert Mustacchi 	    (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
5189d26e4fcSRobert Mustacchi 
5199d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg);
5209d26e4fcSRobert Mustacchi }
5219d26e4fcSRobert Mustacchi 
5229d26e4fcSRobert Mustacchi /*
5239d26e4fcSRobert Mustacchi  * Enable the specified queue as a valid source of interrupts. Note, this should
5249d26e4fcSRobert Mustacchi  * only be used as part of the GLDv3's interrupt blanking routines. The debug
5259d26e4fcSRobert Mustacchi  * build assertions are specific to that.
5269d26e4fcSRobert Mustacchi  */
5279d26e4fcSRobert Mustacchi void
i40e_intr_rx_queue_enable(i40e_trqpair_t * itrq)528338749cbSRobert Mustacchi i40e_intr_rx_queue_enable(i40e_trqpair_t *itrq)
5299d26e4fcSRobert Mustacchi {
5309d26e4fcSRobert Mustacchi 	uint32_t reg;
531338749cbSRobert Mustacchi 	uint_t queue = itrq->itrq_index;
532338749cbSRobert Mustacchi 	i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space;
5339d26e4fcSRobert Mustacchi 
534338749cbSRobert Mustacchi 	ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock));
535338749cbSRobert Mustacchi 	ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs);
5369d26e4fcSRobert Mustacchi 
5379d26e4fcSRobert Mustacchi 	reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue));
5389d26e4fcSRobert Mustacchi 	ASSERT0(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK);
5399d26e4fcSRobert Mustacchi 	reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
5409d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
5419d26e4fcSRobert Mustacchi }
5429d26e4fcSRobert Mustacchi 
5439d26e4fcSRobert Mustacchi /*
5449d26e4fcSRobert Mustacchi  * Disable the specified queue as a valid source of interrupts. Note, this
5459d26e4fcSRobert Mustacchi  * should only be used as part of the GLDv3's interrupt blanking routines. The
5469d26e4fcSRobert Mustacchi  * debug build assertions are specific to that.
5479d26e4fcSRobert Mustacchi  */
5489d26e4fcSRobert Mustacchi void
i40e_intr_rx_queue_disable(i40e_trqpair_t * itrq)549338749cbSRobert Mustacchi i40e_intr_rx_queue_disable(i40e_trqpair_t *itrq)
5509d26e4fcSRobert Mustacchi {
5519d26e4fcSRobert Mustacchi 	uint32_t reg;
552338749cbSRobert Mustacchi 	uint_t queue = itrq->itrq_index;
553338749cbSRobert Mustacchi 	i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space;
5549d26e4fcSRobert Mustacchi 
555338749cbSRobert Mustacchi 	ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock));
556338749cbSRobert Mustacchi 	ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs);
5579d26e4fcSRobert Mustacchi 
5589d26e4fcSRobert Mustacchi 	reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue));
5599d26e4fcSRobert Mustacchi 	ASSERT3U(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK, ==,
5609d26e4fcSRobert Mustacchi 	    I40E_QINT_RQCTL_CAUSE_ENA_MASK);
5619d26e4fcSRobert Mustacchi 	reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
5629d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg);
5639d26e4fcSRobert Mustacchi }
5649d26e4fcSRobert Mustacchi 
5659d26e4fcSRobert Mustacchi /*
5669d26e4fcSRobert Mustacchi  * Start up the various chip's interrupt handling. We not only configure the
5679d26e4fcSRobert Mustacchi  * adminq here, but we also go through and configure all of the actual queues,
5689d26e4fcSRobert Mustacchi  * the interrupt linked lists, and others.
5699d26e4fcSRobert Mustacchi  */
5709d26e4fcSRobert Mustacchi void
i40e_intr_chip_init(i40e_t * i40e)5719d26e4fcSRobert Mustacchi i40e_intr_chip_init(i40e_t *i40e)
5729d26e4fcSRobert Mustacchi {
5739d26e4fcSRobert Mustacchi 	i40e_hw_t *hw = &i40e->i40e_hw_space;
5749d26e4fcSRobert Mustacchi 	uint32_t reg;
5759d26e4fcSRobert Mustacchi 
5769d26e4fcSRobert Mustacchi 	/*
5779d26e4fcSRobert Mustacchi 	 * Ensure that all non adminq interrupts are disabled at the chip level.
5789d26e4fcSRobert Mustacchi 	 */
5799d26e4fcSRobert Mustacchi 	i40e_intr_io_disable_all(i40e);
5809d26e4fcSRobert Mustacchi 
5819d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, 0);
5829d26e4fcSRobert Mustacchi 	(void) I40E_READ_REG(hw, I40E_PFINT_ICR0);
5839d26e4fcSRobert Mustacchi 
5849d26e4fcSRobert Mustacchi 	/*
5859d26e4fcSRobert Mustacchi 	 * Always enable all of the other-class interrupts to be on their own
5869d26e4fcSRobert Mustacchi 	 * ITR. This only needs to be set on interrupt zero, which has its own
5879d26e4fcSRobert Mustacchi 	 * special setting.
5889d26e4fcSRobert Mustacchi 	 */
5899d26e4fcSRobert Mustacchi 	reg = I40E_ITR_INDEX_OTHER << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT;
5909d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0, reg);
5919d26e4fcSRobert Mustacchi 
5929d26e4fcSRobert Mustacchi 	/*
5939d26e4fcSRobert Mustacchi 	 * Enable interrupt types we expect to receive. At the moment, this
5949d26e4fcSRobert Mustacchi 	 * is limited to the adminq; however, we'll want to review
5959d26e4fcSRobert Mustacchi 	 * for more types here as we add support for detecting them, handling
5969d26e4fcSRobert Mustacchi 	 * them, and resetting the device as appropriate.
5979d26e4fcSRobert Mustacchi 	 */
5989d26e4fcSRobert Mustacchi 	reg = I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
5999d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg);
6009d26e4fcSRobert Mustacchi 
6019d26e4fcSRobert Mustacchi 	/*
6029d26e4fcSRobert Mustacchi 	 * Always set the interrupt linked list to empty. We'll come back and
6039d26e4fcSRobert Mustacchi 	 * change this if MSI-X are actually on the scene.
6049d26e4fcSRobert Mustacchi 	 */
6059d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_TYPE_EOL);
6069d26e4fcSRobert Mustacchi 
6079d26e4fcSRobert Mustacchi 	i40e_intr_adminq_enable(i40e);
6089d26e4fcSRobert Mustacchi 
6099d26e4fcSRobert Mustacchi 	/*
6109d26e4fcSRobert Mustacchi 	 * Set up all of the queues and map them to interrupts based on the bit
6119d26e4fcSRobert Mustacchi 	 * assignments.
6129d26e4fcSRobert Mustacchi 	 */
6139d26e4fcSRobert Mustacchi 	if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) {
6149d26e4fcSRobert Mustacchi 		i40e_intr_init_queue_msix(i40e);
6159d26e4fcSRobert Mustacchi 	} else {
6169d26e4fcSRobert Mustacchi 		i40e_intr_init_queue_shared(i40e);
6179d26e4fcSRobert Mustacchi 	}
6189d26e4fcSRobert Mustacchi 
6199d26e4fcSRobert Mustacchi 	/*
6209d26e4fcSRobert Mustacchi 	 * Finally set all of the default ITRs for the interrupts. Note that the
6219d26e4fcSRobert Mustacchi 	 * queues will have been set up above.
6229d26e4fcSRobert Mustacchi 	 */
6239d26e4fcSRobert Mustacchi 	i40e_intr_set_itr(i40e, I40E_ITR_INDEX_RX, i40e->i40e_rx_itr);
6249d26e4fcSRobert Mustacchi 	i40e_intr_set_itr(i40e, I40E_ITR_INDEX_TX, i40e->i40e_tx_itr);
6259d26e4fcSRobert Mustacchi 	i40e_intr_set_itr(i40e, I40E_ITR_INDEX_OTHER, i40e->i40e_other_itr);
6269d26e4fcSRobert Mustacchi }
6279d26e4fcSRobert Mustacchi 
6289d26e4fcSRobert Mustacchi static void
i40e_intr_adminq_work(i40e_t * i40e)6299d26e4fcSRobert Mustacchi i40e_intr_adminq_work(i40e_t *i40e)
6309d26e4fcSRobert Mustacchi {
6319d26e4fcSRobert Mustacchi 	struct i40e_hw *hw = &i40e->i40e_hw_space;
6329d26e4fcSRobert Mustacchi 	struct i40e_arq_event_info evt;
6339d26e4fcSRobert Mustacchi 	uint16_t remain = 1;
6349d26e4fcSRobert Mustacchi 
6359d26e4fcSRobert Mustacchi 	bzero(&evt, sizeof (struct i40e_arq_event_info));
6369d26e4fcSRobert Mustacchi 	evt.buf_len = I40E_ADMINQ_BUFSZ;
6379d26e4fcSRobert Mustacchi 	evt.msg_buf = i40e->i40e_aqbuf;
6389d26e4fcSRobert Mustacchi 
6399d26e4fcSRobert Mustacchi 	while (remain != 0) {
6409d26e4fcSRobert Mustacchi 		enum i40e_status_code ret;
6419d26e4fcSRobert Mustacchi 		uint16_t opcode;
6429d26e4fcSRobert Mustacchi 
6439d26e4fcSRobert Mustacchi 		/*
6449d26e4fcSRobert Mustacchi 		 * At the moment, the only error code that seems to be returned
6459d26e4fcSRobert Mustacchi 		 * is one saying that there's no work. In such a case we leave
6469d26e4fcSRobert Mustacchi 		 * this be.
6479d26e4fcSRobert Mustacchi 		 */
6489d26e4fcSRobert Mustacchi 		ret = i40e_clean_arq_element(hw, &evt, &remain);
6499d26e4fcSRobert Mustacchi 		if (ret != I40E_SUCCESS)
6509d26e4fcSRobert Mustacchi 			break;
6519d26e4fcSRobert Mustacchi 
6529d26e4fcSRobert Mustacchi 		opcode = LE_16(evt.desc.opcode);
6539d26e4fcSRobert Mustacchi 		switch (opcode) {
6549d26e4fcSRobert Mustacchi 		case i40e_aqc_opc_get_link_status:
6559d26e4fcSRobert Mustacchi 			mutex_enter(&i40e->i40e_general_lock);
6569d26e4fcSRobert Mustacchi 			i40e_link_check(i40e);
6579d26e4fcSRobert Mustacchi 			mutex_exit(&i40e->i40e_general_lock);
6589d26e4fcSRobert Mustacchi 			break;
6599d26e4fcSRobert Mustacchi 		default:
6609d26e4fcSRobert Mustacchi 			/*
6619d26e4fcSRobert Mustacchi 			 * Longer term we'll want to enable other causes here
6629d26e4fcSRobert Mustacchi 			 * and get these cleaned up and doing something.
6639d26e4fcSRobert Mustacchi 			 */
6649d26e4fcSRobert Mustacchi 			break;
6659d26e4fcSRobert Mustacchi 		}
6669d26e4fcSRobert Mustacchi 	}
6679d26e4fcSRobert Mustacchi }
6689d26e4fcSRobert Mustacchi 
6699d26e4fcSRobert Mustacchi static void
i40e_intr_rx_work(i40e_t * i40e,i40e_trqpair_t * itrq)67009aee612SRyan Zezeski i40e_intr_rx_work(i40e_t *i40e, i40e_trqpair_t *itrq)
6719d26e4fcSRobert Mustacchi {
672396505afSPaul Winder 	mblk_t *mp = NULL;
6739d26e4fcSRobert Mustacchi 
6749d26e4fcSRobert Mustacchi 	mutex_enter(&itrq->itrq_rx_lock);
675396505afSPaul Winder 	if (!itrq->itrq_intr_poll)
676396505afSPaul Winder 		mp = i40e_ring_rx(itrq, I40E_POLL_NULL);
6779d26e4fcSRobert Mustacchi 	mutex_exit(&itrq->itrq_rx_lock);
6789d26e4fcSRobert Mustacchi 
67909aee612SRyan Zezeski 	if (mp == NULL)
68009aee612SRyan Zezeski 		return;
68109aee612SRyan Zezeski 
68209aee612SRyan Zezeski 	mac_rx_ring(i40e->i40e_mac_hdl, itrq->itrq_macrxring, mp,
68309aee612SRyan Zezeski 	    itrq->itrq_rxgen);
6849d26e4fcSRobert Mustacchi }
6859d26e4fcSRobert Mustacchi 
68609aee612SRyan Zezeski /* ARGSUSED */
6879d26e4fcSRobert Mustacchi static void
i40e_intr_tx_work(i40e_t * i40e,i40e_trqpair_t * itrq)68809aee612SRyan Zezeski i40e_intr_tx_work(i40e_t *i40e, i40e_trqpair_t *itrq)
6899d26e4fcSRobert Mustacchi {
6909d26e4fcSRobert Mustacchi 	i40e_tx_recycle_ring(itrq);
6919d26e4fcSRobert Mustacchi }
6929d26e4fcSRobert Mustacchi 
6939d26e4fcSRobert Mustacchi /*
6949d26e4fcSRobert Mustacchi  * At the moment, the only 'other' interrupt on ICR0 that we handle is the
6959d26e4fcSRobert Mustacchi  * adminq. We should go through and support the other notifications at some
6969d26e4fcSRobert Mustacchi  * point.
6979d26e4fcSRobert Mustacchi  */
6989d26e4fcSRobert Mustacchi static void
i40e_intr_other_work(i40e_t * i40e)6999d26e4fcSRobert Mustacchi i40e_intr_other_work(i40e_t *i40e)
7009d26e4fcSRobert Mustacchi {
7019d26e4fcSRobert Mustacchi 	struct i40e_hw *hw = &i40e->i40e_hw_space;
7029d26e4fcSRobert Mustacchi 	uint32_t reg;
7039d26e4fcSRobert Mustacchi 
7049d26e4fcSRobert Mustacchi 	reg = I40E_READ_REG(hw, I40E_PFINT_ICR0);
7059d26e4fcSRobert Mustacchi 	if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) !=
7069d26e4fcSRobert Mustacchi 	    DDI_FM_OK) {
7079d26e4fcSRobert Mustacchi 		ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED);
7089d26e4fcSRobert Mustacchi 		atomic_or_32(&i40e->i40e_state, I40E_ERROR);
7099d26e4fcSRobert Mustacchi 		return;
7109d26e4fcSRobert Mustacchi 	}
7119d26e4fcSRobert Mustacchi 
7129d26e4fcSRobert Mustacchi 	if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
7139d26e4fcSRobert Mustacchi 		i40e_intr_adminq_work(i40e);
7149d26e4fcSRobert Mustacchi 
7159d26e4fcSRobert Mustacchi 	/*
7169d26e4fcSRobert Mustacchi 	 * Make sure that the adminq interrupt is not masked and then explicitly
7179d26e4fcSRobert Mustacchi 	 * enable the adminq and thus the other interrupt.
7189d26e4fcSRobert Mustacchi 	 */
7199d26e4fcSRobert Mustacchi 	reg = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA);
7209d26e4fcSRobert Mustacchi 	reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
7219d26e4fcSRobert Mustacchi 	I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg);
7229d26e4fcSRobert Mustacchi 
7239d26e4fcSRobert Mustacchi 	i40e_intr_adminq_enable(i40e);
7249d26e4fcSRobert Mustacchi }
7259d26e4fcSRobert Mustacchi 
726*aa2a44afSPaul Winder /*
727*aa2a44afSPaul Winder  * The prolog/epilog pair of functions ensure the integrity of the trqpair
728*aa2a44afSPaul Winder  * across ring stop/start operations.
729*aa2a44afSPaul Winder  *
730*aa2a44afSPaul Winder  * A ring stop operation will wait whilst an interrupt is processing a
731*aa2a44afSPaul Winder  * trqpair, and when a ring is stopped the interrupt handler will skip
732*aa2a44afSPaul Winder  * the trqpair.
733*aa2a44afSPaul Winder  */
734*aa2a44afSPaul Winder static boolean_t
i40e_intr_trqpair_prolog(i40e_trqpair_t * itrq)735*aa2a44afSPaul Winder i40e_intr_trqpair_prolog(i40e_trqpair_t *itrq)
736*aa2a44afSPaul Winder {
737*aa2a44afSPaul Winder 	boolean_t enabled;
738*aa2a44afSPaul Winder 
739*aa2a44afSPaul Winder 	mutex_enter(&itrq->itrq_intr_lock);
740*aa2a44afSPaul Winder 	enabled = !itrq->itrq_intr_quiesce;
741*aa2a44afSPaul Winder 	if (enabled)
742*aa2a44afSPaul Winder 		itrq->itrq_intr_busy = B_TRUE;
743*aa2a44afSPaul Winder 	mutex_exit(&itrq->itrq_intr_lock);
744*aa2a44afSPaul Winder 
745*aa2a44afSPaul Winder 	return (enabled);
746*aa2a44afSPaul Winder }
747*aa2a44afSPaul Winder 
748*aa2a44afSPaul Winder static void
i40e_intr_trqpair_epilog(i40e_trqpair_t * itrq)749*aa2a44afSPaul Winder i40e_intr_trqpair_epilog(i40e_trqpair_t *itrq)
750*aa2a44afSPaul Winder {
751*aa2a44afSPaul Winder 	mutex_enter(&itrq->itrq_intr_lock);
752*aa2a44afSPaul Winder 	itrq->itrq_intr_busy = B_FALSE;
753*aa2a44afSPaul Winder 	if (itrq->itrq_intr_quiesce)
754*aa2a44afSPaul Winder 		cv_signal(&itrq->itrq_intr_cv);
755*aa2a44afSPaul Winder 	mutex_exit(&itrq->itrq_intr_lock);
756*aa2a44afSPaul Winder }
757*aa2a44afSPaul Winder 
758*aa2a44afSPaul Winder /*
759*aa2a44afSPaul Winder  * Tell any active interrupt vectors the ring is quiescing, then
760*aa2a44afSPaul Winder  * wait until any active interrupt thread has finished with this