19d26e4fcSRobert Mustacchi /* 29d26e4fcSRobert Mustacchi * This file and its contents are supplied under the terms of the 39d26e4fcSRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0. 49d26e4fcSRobert Mustacchi * You may only use this file in accordance with the terms of version 59d26e4fcSRobert Mustacchi * 1.0 of the CDDL. 69d26e4fcSRobert Mustacchi * 79d26e4fcSRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this 89d26e4fcSRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at 99d26e4fcSRobert Mustacchi * http://www.illumos.org/license/CDDL. 109d26e4fcSRobert Mustacchi */ 119d26e4fcSRobert Mustacchi 129d26e4fcSRobert Mustacchi /* 13*09aee612SRyan Zezeski * Copyright 2018 Joyent, Inc. 14396505afSPaul Winder * Copyright 2017 Tegile Systems, Inc. All rights reserved. 159d26e4fcSRobert Mustacchi */ 169d26e4fcSRobert Mustacchi 179d26e4fcSRobert Mustacchi /* 189d26e4fcSRobert Mustacchi * ------------------------- 199d26e4fcSRobert Mustacchi * Interrupt Handling Theory 209d26e4fcSRobert Mustacchi * ------------------------- 219d26e4fcSRobert Mustacchi * 229d26e4fcSRobert Mustacchi * There are a couple different sets of interrupts that we need to worry about: 239d26e4fcSRobert Mustacchi * 249d26e4fcSRobert Mustacchi * - Interrupts from receive queues 259d26e4fcSRobert Mustacchi * - Interrupts from transmit queues 269d26e4fcSRobert Mustacchi * - 'Other Interrupts', such as the administrative queue 279d26e4fcSRobert Mustacchi * 289d26e4fcSRobert Mustacchi * 'Other Interrupts' are asynchronous events such as a link status change event 299d26e4fcSRobert Mustacchi * being posted to the administrative queue, unrecoverable ECC errors, and more. 309d26e4fcSRobert Mustacchi * If we have something being posted to the administrative queue, then we go 319d26e4fcSRobert Mustacchi * through and process it, because it's generally enabled as a separate logical 329d26e4fcSRobert Mustacchi * interrupt. Note, we may need to do more here eventually. To re-enable the 339d26e4fcSRobert Mustacchi * interrupts from the 'Other Interrupts' section, we need to clear the PBA and 349d26e4fcSRobert Mustacchi * write ENA to PFINT_ICR0. 359d26e4fcSRobert Mustacchi * 369d26e4fcSRobert Mustacchi * Interrupts from the transmit and receive queues indicates that our requests 379d26e4fcSRobert Mustacchi * have been processed. In the rx case, it means that we have data that we 389d26e4fcSRobert Mustacchi * should take a look at and send up the stack. In the tx case, it means that 399d26e4fcSRobert Mustacchi * data which we got from MAC has now been sent out on the wire and we can free 409d26e4fcSRobert Mustacchi * the associated data. Most of the logic for acting upon the presence of this 419d26e4fcSRobert Mustacchi * data can be found in i40e_transciever.c which handles all of the DMA, rx, and 429d26e4fcSRobert Mustacchi * tx operations. This file is dedicated to handling and dealing with interrupt 439d26e4fcSRobert Mustacchi * processing. 449d26e4fcSRobert Mustacchi * 459d26e4fcSRobert Mustacchi * All devices supported by this driver support three kinds of interrupts: 469d26e4fcSRobert Mustacchi * 479d26e4fcSRobert Mustacchi * o Extended Message Signaled Interrupts (MSI-X) 489d26e4fcSRobert Mustacchi * o Message Signaled Interrupts (MSI) 499d26e4fcSRobert Mustacchi * o Legacy PCI interrupts (INTx) 509d26e4fcSRobert Mustacchi * 519d26e4fcSRobert Mustacchi * Generally speaking the hardware logically handles MSI and INTx the same and 529d26e4fcSRobert Mustacchi * restricts us to only using a single interrupt, which isn't the interesting 539d26e4fcSRobert Mustacchi * case. With MSI-X available, each physical function of the device provides the 549d26e4fcSRobert Mustacchi * opportunity for multiple interrupts which is what we'll focus on. 559d26e4fcSRobert Mustacchi * 569d26e4fcSRobert Mustacchi * -------------------- 579d26e4fcSRobert Mustacchi * Interrupt Management 589d26e4fcSRobert Mustacchi * -------------------- 599d26e4fcSRobert Mustacchi * 609d26e4fcSRobert Mustacchi * By default, the admin queue, which consists of the asynchronous other 619d26e4fcSRobert Mustacchi * interrupts is always bound to MSI-X vector zero. Next, we spread out all of 629d26e4fcSRobert Mustacchi * the other interrupts that we have available to us over the remaining 639d26e4fcSRobert Mustacchi * interrupt vectors. 649d26e4fcSRobert Mustacchi * 659d26e4fcSRobert Mustacchi * This means that there may be multiple queues, both tx and rx, which are 669d26e4fcSRobert Mustacchi * mapped to the same interrupt. When the interrupt fires, we'll have to check 679d26e4fcSRobert Mustacchi * all of them for servicing, before we go through and indicate that the 689d26e4fcSRobert Mustacchi * interrupt is claimed. 699d26e4fcSRobert Mustacchi * 709d26e4fcSRobert Mustacchi * The hardware provides the means of mapping various queues to MSI-X interrupts 719d26e4fcSRobert Mustacchi * by programming the I40E_QINT_RQCTL() and I4OE_QINT_TQCTL() registers. These 729d26e4fcSRobert Mustacchi * registers can also be used to enable and disable whether or not the queue is 739d26e4fcSRobert Mustacchi * a source of interrupts. As part of this, the hardware requires that we 749d26e4fcSRobert Mustacchi * maintain a linked list of queues for each interrupt vector. While it may seem 759d26e4fcSRobert Mustacchi * like this is only there for the purproses of ITRs, that's not the case. The 769d26e4fcSRobert Mustacchi * first queue must be programmed in I40E_QINT_LNKLSTN(%vector) register. Each 779d26e4fcSRobert Mustacchi * queue defines the next one in either the I40E_QINT_RQCTL or I40E_QINT_TQCTL 789d26e4fcSRobert Mustacchi * register. 799d26e4fcSRobert Mustacchi * 809d26e4fcSRobert Mustacchi * Finally, the individual interrupt vector itself has the ability to be enabled 819d26e4fcSRobert Mustacchi * and disabled. The overall interrupt is controlled through the 829d26e4fcSRobert Mustacchi * I40E_PFINT_DYN_CTLN() register. This is used to turn on and off the interrupt 839d26e4fcSRobert Mustacchi * as a whole. 849d26e4fcSRobert Mustacchi * 859d26e4fcSRobert Mustacchi * Note that this means that both the individual queue and the interrupt as a 869d26e4fcSRobert Mustacchi * whole can be toggled and re-enabled. 879d26e4fcSRobert Mustacchi * 889d26e4fcSRobert Mustacchi * ------------------- 899d26e4fcSRobert Mustacchi * Non-MSIX Management 909d26e4fcSRobert Mustacchi * ------------------- 919d26e4fcSRobert Mustacchi * 929d26e4fcSRobert Mustacchi * We may have a case where the Operating System is unable to actually allocate 939d26e4fcSRobert Mustacchi * any MSI-X to the system. In such a world, there is only one transmit/receive 949d26e4fcSRobert Mustacchi * queue pair and it is bound to the same interrupt with index zero. The 959d26e4fcSRobert Mustacchi * hardware doesn't allow us access to additional interrupt vectors in these 969d26e4fcSRobert Mustacchi * modes. Note that technically we could support more transmit/receive queues if 979d26e4fcSRobert Mustacchi * we wanted. 989d26e4fcSRobert Mustacchi * 999d26e4fcSRobert Mustacchi * In this world, because the interrupts for the admin queue and traffic are 1009d26e4fcSRobert Mustacchi * mixed together, we have to consult ICR0 to determine what has occurred. The 1019d26e4fcSRobert Mustacchi * QINT_TQCTL and QINT_RQCTL registers have a field, 'MSI-X 0 index' which 1029d26e4fcSRobert Mustacchi * allows us to set a specific bit in ICR0. There are up to seven such bits; 1039d26e4fcSRobert Mustacchi * however, we only use the bit 0 and 1 for the rx and tx queue respectively. 1049d26e4fcSRobert Mustacchi * These are contained by the I40E_INTR_NOTX_{R|T}X_QUEUE and 1059d26e4fcSRobert Mustacchi * I40E_INTR_NOTX_{R|T}X_MASK registers respectively. 1069d26e4fcSRobert Mustacchi * 1079d26e4fcSRobert Mustacchi * Unfortunately, these corresponding queue bits have no corresponding entry in 1089d26e4fcSRobert Mustacchi * the ICR0_ENA register. So instead, when enabling interrupts on the queues, we 1099d26e4fcSRobert Mustacchi * end up enabling it on the queue registers rather than on the MSI-X registers. 1109d26e4fcSRobert Mustacchi * In the MSI-X world, because they can be enabled and disabled, this is 1119d26e4fcSRobert Mustacchi * different and the queues can always be enabled and disabled, but the 1129d26e4fcSRobert Mustacchi * interrupts themselves are toggled (ignoring the question of interrupt 1139d26e4fcSRobert Mustacchi * blanking for polling on rings). 1149d26e4fcSRobert Mustacchi * 1159d26e4fcSRobert Mustacchi * Finally, we still have to set up the interrupt linked list, but the list is 1169d26e4fcSRobert Mustacchi * instead rooted at the register I40E_PFINT_LNKLST0, rather than being tied to 1179d26e4fcSRobert Mustacchi * one of the other MSI-X registers. 1189d26e4fcSRobert Mustacchi * 1199d26e4fcSRobert Mustacchi * -------------------- 1209d26e4fcSRobert Mustacchi * Interrupt Moderation 1219d26e4fcSRobert Mustacchi * -------------------- 1229d26e4fcSRobert Mustacchi * 1239d26e4fcSRobert Mustacchi * The XL710 hardware has three different interrupt moderation registers per 1249d26e4fcSRobert Mustacchi * interrupt. Unsurprisingly, we use these for: 1259d26e4fcSRobert Mustacchi * 1269d26e4fcSRobert Mustacchi * o RX interrupts 1279d26e4fcSRobert Mustacchi * o TX interrupts 1289d26e4fcSRobert Mustacchi * o 'Other interrupts' (link status change, admin queue, etc.) 1299d26e4fcSRobert Mustacchi * 1309d26e4fcSRobert Mustacchi * By default, we throttle 'other interrupts' the most, then TX interrupts, and 1319d26e4fcSRobert Mustacchi * then RX interrupts. The default values for these were based on trying to 1329d26e4fcSRobert Mustacchi * reason about both the importance and frequency of events. Generally speaking 1339d26e4fcSRobert Mustacchi * 'other interrupts' are not very frequent and they're not important for the 1349d26e4fcSRobert Mustacchi * I/O data path in and of itself (though they may indicate issues with the I/O 1359d26e4fcSRobert Mustacchi * data path). 1369d26e4fcSRobert Mustacchi * 1379d26e4fcSRobert Mustacchi * On the flip side, when we're not polling, RX interrupts are very important. 1389d26e4fcSRobert Mustacchi * The longer we wait for them, the more latency that we inject into the system. 1399d26e4fcSRobert Mustacchi * However, if we allow interrupts to occur too frequently, we risk a few 1409d26e4fcSRobert Mustacchi * problems: 1419d26e4fcSRobert Mustacchi * 1429d26e4fcSRobert Mustacchi * 1) Abusing system resources. Without proper interrupt blanking and polling, 1439d26e4fcSRobert Mustacchi * we can see upwards of 200k-300k interrupts per second on the system. 1449d26e4fcSRobert Mustacchi * 1459d26e4fcSRobert Mustacchi * 2) Not enough data coalescing to enable polling. In other words, the more 1469d26e4fcSRobert Mustacchi * data that we allow to build up, the more likely we'll be able to enable 1479d26e4fcSRobert Mustacchi * polling mode and allowing us to better handle bulk data. 1489d26e4fcSRobert Mustacchi * 1499d26e4fcSRobert Mustacchi * In-between the 'other interrupts' and the TX interrupts we have the 1509d26e4fcSRobert Mustacchi * reclamation of TX buffers. This operation is not quite as important as we 1519d26e4fcSRobert Mustacchi * generally size the ring large enough that we should be able to reclaim a 1529d26e4fcSRobert Mustacchi * substantial amount of the descriptors that we have used per interrupt. So 1539d26e4fcSRobert Mustacchi * while it's important that this interrupt occur, we don't necessarily need it 1549d26e4fcSRobert Mustacchi * firing as frequently as RX; it doesn't, on its own, induce additional latency 1559d26e4fcSRobert Mustacchi * into the system. 1569d26e4fcSRobert Mustacchi * 1579d26e4fcSRobert Mustacchi * Based on all this we currently assign static ITR values for the system. While 1589d26e4fcSRobert Mustacchi * we could move to a dynamic system (the hardware supports that), we'd want to 1599d26e4fcSRobert Mustacchi * make sure that we're seeing problems from this that we believe would be 1609d26e4fcSRobert Mustacchi * generally helped by the added complexity. 1619d26e4fcSRobert Mustacchi * 1629d26e4fcSRobert Mustacchi * Based on this, the default values that we have allow for the following 1639d26e4fcSRobert Mustacchi * interrupt thresholds: 1649d26e4fcSRobert Mustacchi * 1659d26e4fcSRobert Mustacchi * o 20k interrupts/s for RX 1669d26e4fcSRobert Mustacchi * o 5k interrupts/s for TX 1679d26e4fcSRobert Mustacchi * o 2k interupts/s for 'Other Interrupts' 1689d26e4fcSRobert Mustacchi */ 1699d26e4fcSRobert Mustacchi 1709d26e4fcSRobert Mustacchi #include "i40e_sw.h" 1719d26e4fcSRobert Mustacchi 1729d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_QUEUE 0 1739d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_INTR 0 1749d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_RX_QUEUE 0 1759d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_RX_MASK (1 << I40E_PFINT_ICR0_QUEUE_0_SHIFT) 1769d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_TX_QUEUE 1 1779d26e4fcSRobert Mustacchi #define I40E_INTR_NOTX_TX_MASK (1 << I40E_PFINT_ICR0_QUEUE_1_SHIFT) 1789d26e4fcSRobert Mustacchi 1799d26e4fcSRobert Mustacchi void 1809d26e4fcSRobert Mustacchi i40e_intr_set_itr(i40e_t *i40e, i40e_itr_index_t itr, uint_t val) 1819d26e4fcSRobert Mustacchi { 1829d26e4fcSRobert Mustacchi int i; 1839d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 1849d26e4fcSRobert Mustacchi 1859d26e4fcSRobert Mustacchi VERIFY3U(val, <=, I40E_MAX_ITR); 1869d26e4fcSRobert Mustacchi VERIFY3U(itr, <, I40E_ITR_INDEX_NONE); 1879d26e4fcSRobert Mustacchi 1889d26e4fcSRobert Mustacchi /* 1899d26e4fcSRobert Mustacchi * No matter the interrupt mode, the ITR for other interrupts is always 1909d26e4fcSRobert Mustacchi * on interrupt zero and the same is true if we're not using MSI-X. 1919d26e4fcSRobert Mustacchi */ 1929d26e4fcSRobert Mustacchi if (itr == I40E_ITR_INDEX_OTHER || 1939d26e4fcSRobert Mustacchi i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 1949d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_ITR0(itr), val); 1959d26e4fcSRobert Mustacchi return; 1969d26e4fcSRobert Mustacchi } 1979d26e4fcSRobert Mustacchi 198396505afSPaul Winder for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 199396505afSPaul Winder I40E_WRITE_REG(hw, I40E_PFINT_ITRN(itr, i), val); 2009d26e4fcSRobert Mustacchi } 2019d26e4fcSRobert Mustacchi } 2029d26e4fcSRobert Mustacchi 2039d26e4fcSRobert Mustacchi /* 2049d26e4fcSRobert Mustacchi * Re-enable the adminq. Note that the adminq doesn't have a traditional queue 2059d26e4fcSRobert Mustacchi * associated with it from an interrupt perspective and just lives on ICR0. 2069d26e4fcSRobert Mustacchi * However when MSI-X interrupts are not being used, then this also enables and 2079d26e4fcSRobert Mustacchi * disables those interrupts. 2089d26e4fcSRobert Mustacchi */ 2099d26e4fcSRobert Mustacchi static void 2109d26e4fcSRobert Mustacchi i40e_intr_adminq_enable(i40e_t *i40e) 2119d26e4fcSRobert Mustacchi { 2129d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 2139d26e4fcSRobert Mustacchi uint32_t reg; 2149d26e4fcSRobert Mustacchi 2159d26e4fcSRobert Mustacchi reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 2169d26e4fcSRobert Mustacchi I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 2179d26e4fcSRobert Mustacchi (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 2189d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg); 2199d26e4fcSRobert Mustacchi i40e_flush(hw); 2209d26e4fcSRobert Mustacchi } 2219d26e4fcSRobert Mustacchi 2229d26e4fcSRobert Mustacchi static void 2239d26e4fcSRobert Mustacchi i40e_intr_adminq_disable(i40e_t *i40e) 2249d26e4fcSRobert Mustacchi { 2259d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 2269d26e4fcSRobert Mustacchi uint32_t reg; 2279d26e4fcSRobert Mustacchi 2289d26e4fcSRobert Mustacchi reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 2299d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg); 2309d26e4fcSRobert Mustacchi } 2319d26e4fcSRobert Mustacchi 232*09aee612SRyan Zezeski /* 233*09aee612SRyan Zezeski * The next two functions enable/disable the reception of interrupts 234*09aee612SRyan Zezeski * on the given vector. Only vectors 1..N are programmed by these 235*09aee612SRyan Zezeski * functions; vector 0 is special and handled by a different register. 236*09aee612SRyan Zezeski * We must subtract one from the vector because i40e implicitly adds 237*09aee612SRyan Zezeski * one to the vector value. See section 10.2.2.10.13 for more details. 238*09aee612SRyan Zezeski */ 2399d26e4fcSRobert Mustacchi static void 2409d26e4fcSRobert Mustacchi i40e_intr_io_enable(i40e_t *i40e, int vector) 2419d26e4fcSRobert Mustacchi { 2429d26e4fcSRobert Mustacchi uint32_t reg; 2439d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 2449d26e4fcSRobert Mustacchi 245*09aee612SRyan Zezeski ASSERT3S(vector, >, 0); 2469d26e4fcSRobert Mustacchi reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2479d26e4fcSRobert Mustacchi I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 2489d26e4fcSRobert Mustacchi (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 2499d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg); 2509d26e4fcSRobert Mustacchi } 2519d26e4fcSRobert Mustacchi 2529d26e4fcSRobert Mustacchi static void 2539d26e4fcSRobert Mustacchi i40e_intr_io_disable(i40e_t *i40e, int vector) 2549d26e4fcSRobert Mustacchi { 2559d26e4fcSRobert Mustacchi uint32_t reg; 2569d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 2579d26e4fcSRobert Mustacchi 258*09aee612SRyan Zezeski ASSERT3S(vector, >, 0); 2599d26e4fcSRobert Mustacchi reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 2609d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg); 2619d26e4fcSRobert Mustacchi } 2629d26e4fcSRobert Mustacchi 2639d26e4fcSRobert Mustacchi /* 2649d26e4fcSRobert Mustacchi * When MSI-X interrupts are being used, then we can enable the actual 2659d26e4fcSRobert Mustacchi * interrupts themselves. However, when they are not, we instead have to turn 2669d26e4fcSRobert Mustacchi * towards the queue's CAUSE_ENA bit and enable that. 2679d26e4fcSRobert Mustacchi */ 2689d26e4fcSRobert Mustacchi void 2699d26e4fcSRobert Mustacchi i40e_intr_io_enable_all(i40e_t *i40e) 2709d26e4fcSRobert Mustacchi { 2719d26e4fcSRobert Mustacchi if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 2729d26e4fcSRobert Mustacchi int i; 2739d26e4fcSRobert Mustacchi 2749d26e4fcSRobert Mustacchi for (i = 1; i < i40e->i40e_intr_count; i++) { 2759d26e4fcSRobert Mustacchi i40e_intr_io_enable(i40e, i); 2769d26e4fcSRobert Mustacchi } 2779d26e4fcSRobert Mustacchi } else { 2789d26e4fcSRobert Mustacchi uint32_t reg; 2799d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 2809d26e4fcSRobert Mustacchi 2819d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE)); 2829d26e4fcSRobert Mustacchi reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 2839d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 2849d26e4fcSRobert Mustacchi 2859d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE)); 2869d26e4fcSRobert Mustacchi reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 2879d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 2889d26e4fcSRobert Mustacchi } 2899d26e4fcSRobert Mustacchi } 2909d26e4fcSRobert Mustacchi 2919d26e4fcSRobert Mustacchi /* 2929d26e4fcSRobert Mustacchi * When MSI-X interrupts are being used, then we can disable the actual 2939d26e4fcSRobert Mustacchi * interrupts themselves. However, when they are not, we instead have to turn 2949d26e4fcSRobert Mustacchi * towards the queue's CAUSE_ENA bit and disable that. 2959d26e4fcSRobert Mustacchi */ 2969d26e4fcSRobert Mustacchi void 2979d26e4fcSRobert Mustacchi i40e_intr_io_disable_all(i40e_t *i40e) 2989d26e4fcSRobert Mustacchi { 2999d26e4fcSRobert Mustacchi if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 3009d26e4fcSRobert Mustacchi int i; 3019d26e4fcSRobert Mustacchi 3029d26e4fcSRobert Mustacchi for (i = 1; i < i40e->i40e_intr_count; i++) { 3039d26e4fcSRobert Mustacchi i40e_intr_io_disable(i40e, i); 3049d26e4fcSRobert Mustacchi } 3059d26e4fcSRobert Mustacchi } else { 3069d26e4fcSRobert Mustacchi uint32_t reg; 3079d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 3089d26e4fcSRobert Mustacchi 3099d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE)); 3109d26e4fcSRobert Mustacchi reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3119d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 3129d26e4fcSRobert Mustacchi 3139d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE)); 3149d26e4fcSRobert Mustacchi reg &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3159d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 3169d26e4fcSRobert Mustacchi } 3179d26e4fcSRobert Mustacchi } 3189d26e4fcSRobert Mustacchi 3199d26e4fcSRobert Mustacchi /* 3209d26e4fcSRobert Mustacchi * As part of disabling the tx and rx queue's we're technically supposed to 3219d26e4fcSRobert Mustacchi * remove the linked list entries. The simplest way is to clear the LNKLSTN 3229d26e4fcSRobert Mustacchi * register by setting it to I40E_QUEUE_TYPE_EOL (0x7FF). 3239d26e4fcSRobert Mustacchi * 3249d26e4fcSRobert Mustacchi * Note all of the FM register access checks are performed by the caller. 3259d26e4fcSRobert Mustacchi */ 3269d26e4fcSRobert Mustacchi void 3279d26e4fcSRobert Mustacchi i40e_intr_io_clear_cause(i40e_t *i40e) 3289d26e4fcSRobert Mustacchi { 3299d26e4fcSRobert Mustacchi int i; 3309d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 3319d26e4fcSRobert Mustacchi 3329d26e4fcSRobert Mustacchi if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 3339d26e4fcSRobert Mustacchi uint32_t reg; 3349d26e4fcSRobert Mustacchi reg = I40E_QUEUE_TYPE_EOL; 3359d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg); 3369d26e4fcSRobert Mustacchi return; 3379d26e4fcSRobert Mustacchi } 3389d26e4fcSRobert Mustacchi 339396505afSPaul Winder for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 3409d26e4fcSRobert Mustacchi uint32_t reg; 3419d26e4fcSRobert Mustacchi #ifdef DEBUG 3429d26e4fcSRobert Mustacchi /* 3439d26e4fcSRobert Mustacchi * Verify that the interrupt in question is disabled. This is a 3449d26e4fcSRobert Mustacchi * prerequisite of modifying the data in question. 3459d26e4fcSRobert Mustacchi */ 346396505afSPaul Winder reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i)); 3479d26e4fcSRobert Mustacchi VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK); 3489d26e4fcSRobert Mustacchi #endif 3499d26e4fcSRobert Mustacchi reg = I40E_QUEUE_TYPE_EOL; 350396505afSPaul Winder I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(i), reg); 3519d26e4fcSRobert Mustacchi } 3529d26e4fcSRobert Mustacchi 3539d26e4fcSRobert Mustacchi i40e_flush(hw); 3549d26e4fcSRobert Mustacchi } 3559d26e4fcSRobert Mustacchi 3569d26e4fcSRobert Mustacchi /* 3579d26e4fcSRobert Mustacchi * Finalize interrupt handling. Mostly this disables the admin queue. 3589d26e4fcSRobert Mustacchi */ 3599d26e4fcSRobert Mustacchi void 3609d26e4fcSRobert Mustacchi i40e_intr_chip_fini(i40e_t *i40e) 3619d26e4fcSRobert Mustacchi { 3629d26e4fcSRobert Mustacchi #ifdef DEBUG 3639d26e4fcSRobert Mustacchi int i; 3649d26e4fcSRobert Mustacchi uint32_t reg; 3659d26e4fcSRobert Mustacchi 3669d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 3679d26e4fcSRobert Mustacchi 3689d26e4fcSRobert Mustacchi /* 3699d26e4fcSRobert Mustacchi * Take a look and verify that all other interrupts have been disabled 3709d26e4fcSRobert Mustacchi * and the interrupt linked lists have been zeroed. 3719d26e4fcSRobert Mustacchi */ 3729d26e4fcSRobert Mustacchi if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 373396505afSPaul Winder for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 374396505afSPaul Winder reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i)); 3759d26e4fcSRobert Mustacchi VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK); 3769d26e4fcSRobert Mustacchi 377396505afSPaul Winder reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i)); 3789d26e4fcSRobert Mustacchi VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 3799d26e4fcSRobert Mustacchi } 3809d26e4fcSRobert Mustacchi } 3819d26e4fcSRobert Mustacchi #endif 3829d26e4fcSRobert Mustacchi 3839d26e4fcSRobert Mustacchi i40e_intr_adminq_disable(i40e); 3849d26e4fcSRobert Mustacchi } 3859d26e4fcSRobert Mustacchi 3869d26e4fcSRobert Mustacchi /* 387*09aee612SRyan Zezeski * Set the head of the interrupt linked list. The PFINT_LNKLSTN[N] 388*09aee612SRyan Zezeski * register actually refers to the 'N + 1' interrupt vector. E.g., 389*09aee612SRyan Zezeski * PFINT_LNKLSTN[0] refers to interrupt vector 1. 390*09aee612SRyan Zezeski */ 391*09aee612SRyan Zezeski static void 392*09aee612SRyan Zezeski i40e_set_lnklstn(i40e_t *i40e, uint_t vector, uint_t queue) 393*09aee612SRyan Zezeski { 394*09aee612SRyan Zezeski uint32_t reg; 395*09aee612SRyan Zezeski i40e_hw_t *hw = &i40e->i40e_hw_space; 396*09aee612SRyan Zezeski 397*09aee612SRyan Zezeski reg = (queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 398*09aee612SRyan Zezeski (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT); 399*09aee612SRyan Zezeski 400*09aee612SRyan Zezeski I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(vector), reg); 401*09aee612SRyan Zezeski DEBUGOUT2("PFINT_LNKLSTN[%u] = 0x%x", vector, reg); 402*09aee612SRyan Zezeski } 403*09aee612SRyan Zezeski 404*09aee612SRyan Zezeski /* 405*09aee612SRyan Zezeski * Set the QINT_RQCTL[queue] register. The next queue is always the Tx 406*09aee612SRyan Zezeski * queue associated with this Rx queue. Unlike PFINT_LNKLSTN, the 407*09aee612SRyan Zezeski * vector should be the actual vector this queue is on -- i.e., it 408*09aee612SRyan Zezeski * should be equal to itrq_rx_intrvec. 409*09aee612SRyan Zezeski */ 410*09aee612SRyan Zezeski static void 411*09aee612SRyan Zezeski i40e_set_rqctl(i40e_t *i40e, uint_t vector, uint_t queue) 412*09aee612SRyan Zezeski { 413*09aee612SRyan Zezeski uint32_t reg; 414*09aee612SRyan Zezeski i40e_hw_t *hw = &i40e->i40e_hw_space; 415*09aee612SRyan Zezeski 416*09aee612SRyan Zezeski ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_rx_intrvec); 417*09aee612SRyan Zezeski 418*09aee612SRyan Zezeski reg = (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 419*09aee612SRyan Zezeski (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 420*09aee612SRyan Zezeski (queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 421*09aee612SRyan Zezeski (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 422*09aee612SRyan Zezeski I40E_QINT_RQCTL_CAUSE_ENA_MASK; 423*09aee612SRyan Zezeski 424*09aee612SRyan Zezeski I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 425*09aee612SRyan Zezeski DEBUGOUT2("QINT_RQCTL[%u] = 0x%x", queue, reg); 426*09aee612SRyan Zezeski } 427*09aee612SRyan Zezeski 428*09aee612SRyan Zezeski /* 429*09aee612SRyan Zezeski * Like i40e_set_rqctl(), but for QINT_TQCTL[queue]. The next queue is 430*09aee612SRyan Zezeski * either the Rx queue of another TRQP, or EOL. 431*09aee612SRyan Zezeski */ 432*09aee612SRyan Zezeski static void 433*09aee612SRyan Zezeski i40e_set_tqctl(i40e_t *i40e, uint_t vector, uint_t queue, uint_t next_queue) 434*09aee612SRyan Zezeski { 435*09aee612SRyan Zezeski uint32_t reg; 436*09aee612SRyan Zezeski i40e_hw_t *hw = &i40e->i40e_hw_space; 437*09aee612SRyan Zezeski 438*09aee612SRyan Zezeski ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_tx_intrvec); 439*09aee612SRyan Zezeski 440*09aee612SRyan Zezeski reg = (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 441*09aee612SRyan Zezeski (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 442*09aee612SRyan Zezeski (next_queue << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 443*09aee612SRyan Zezeski (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 444*09aee612SRyan Zezeski I40E_QINT_TQCTL_CAUSE_ENA_MASK; 445*09aee612SRyan Zezeski 446*09aee612SRyan Zezeski I40E_WRITE_REG(hw, I40E_QINT_TQCTL(queue), reg); 447*09aee612SRyan Zezeski DEBUGOUT2("QINT_TQCTL[%u] = 0x%x", queue, reg); 448*09aee612SRyan Zezeski } 449*09aee612SRyan Zezeski 450*09aee612SRyan Zezeski /* 451*09aee612SRyan Zezeski * Program the interrupt linked list. Each vector has a linked list of 452*09aee612SRyan Zezeski * queues which act as event sources for that vector. When one of 453*09aee612SRyan Zezeski * those sources has an event the associated interrupt vector is 454*09aee612SRyan Zezeski * fired. This mapping must match the mapping found in 455*09aee612SRyan Zezeski * i40e_map_intrs_to_vectors(). 456*09aee612SRyan Zezeski * 457*09aee612SRyan Zezeski * See section 7.5.3 for more information about the configuration of 458*09aee612SRyan Zezeski * the interrupt linked list. 4599d26e4fcSRobert Mustacchi */ 4609d26e4fcSRobert Mustacchi static void 4619d26e4fcSRobert Mustacchi i40e_intr_init_queue_msix(i40e_t *i40e) 4629d26e4fcSRobert Mustacchi { 463*09aee612SRyan Zezeski uint_t intr_count; 4649d26e4fcSRobert Mustacchi 4659d26e4fcSRobert Mustacchi /* 466*09aee612SRyan Zezeski * The 0th vector is for 'Other Interrupts' only (subject to 467*09aee612SRyan Zezeski * change in the future). 4689d26e4fcSRobert Mustacchi */ 469*09aee612SRyan Zezeski intr_count = i40e->i40e_intr_count - 1; 470396505afSPaul Winder 471*09aee612SRyan Zezeski for (uint_t vec = 0; vec < intr_count; vec++) { 472*09aee612SRyan Zezeski boolean_t head = B_TRUE; 473396505afSPaul Winder 474*09aee612SRyan Zezeski for (uint_t qidx = vec; qidx < i40e->i40e_num_trqpairs; 475*09aee612SRyan Zezeski qidx += intr_count) { 476*09aee612SRyan Zezeski uint_t next_qidx = qidx + intr_count; 477396505afSPaul Winder 478*09aee612SRyan Zezeski next_qidx = (next_qidx > i40e->i40e_num_trqpairs) ? 479*09aee612SRyan Zezeski I40E_QUEUE_TYPE_EOL : next_qidx; 480396505afSPaul Winder 481*09aee612SRyan Zezeski if (head) { 482*09aee612SRyan Zezeski i40e_set_lnklstn(i40e, vec, qidx); 483*09aee612SRyan Zezeski head = B_FALSE; 484*09aee612SRyan Zezeski } 485396505afSPaul Winder 486*09aee612SRyan Zezeski i40e_set_rqctl(i40e, vec + 1, qidx); 487*09aee612SRyan Zezeski i40e_set_tqctl(i40e, vec + 1, qidx, next_qidx); 488*09aee612SRyan Zezeski } 489396505afSPaul Winder } 4909d26e4fcSRobert Mustacchi } 4919d26e4fcSRobert Mustacchi 4929d26e4fcSRobert Mustacchi /* 4939d26e4fcSRobert Mustacchi * Set up a single queue to share the admin queue interrupt in the non-MSI-X 4949d26e4fcSRobert Mustacchi * world. Note we do not enable the queue as an interrupt cause at this time. We 4959d26e4fcSRobert Mustacchi * don't have any other vector of control here, unlike with the MSI-X interrupt 4969d26e4fcSRobert Mustacchi * case. 4979d26e4fcSRobert Mustacchi */ 4989d26e4fcSRobert Mustacchi static void 4999d26e4fcSRobert Mustacchi i40e_intr_init_queue_shared(i40e_t *i40e) 5009d26e4fcSRobert Mustacchi { 5019d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 5029d26e4fcSRobert Mustacchi uint32_t reg; 5039d26e4fcSRobert Mustacchi 5049d26e4fcSRobert Mustacchi VERIFY(i40e->i40e_intr_type == DDI_INTR_TYPE_FIXED || 5059d26e4fcSRobert Mustacchi i40e->i40e_intr_type == DDI_INTR_TYPE_MSI); 5069d26e4fcSRobert Mustacchi 5079d26e4fcSRobert Mustacchi reg = (I40E_INTR_NOTX_QUEUE << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5089d26e4fcSRobert Mustacchi (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT); 5099d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg); 5109d26e4fcSRobert Mustacchi 5119d26e4fcSRobert Mustacchi reg = (I40E_INTR_NOTX_INTR << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5129d26e4fcSRobert Mustacchi (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 5139d26e4fcSRobert Mustacchi (I40E_INTR_NOTX_RX_QUEUE << I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 5149d26e4fcSRobert Mustacchi (I40E_INTR_NOTX_QUEUE << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5159d26e4fcSRobert Mustacchi (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 5169d26e4fcSRobert Mustacchi 5179d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 5189d26e4fcSRobert Mustacchi 5199d26e4fcSRobert Mustacchi reg = (I40E_INTR_NOTX_INTR << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 5209d26e4fcSRobert Mustacchi (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 5219d26e4fcSRobert Mustacchi (I40E_INTR_NOTX_TX_QUEUE << I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 5229d26e4fcSRobert Mustacchi (I40E_QUEUE_TYPE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 5239d26e4fcSRobert Mustacchi (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 5249d26e4fcSRobert Mustacchi 5259d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 5269d26e4fcSRobert Mustacchi } 5279d26e4fcSRobert Mustacchi 5289d26e4fcSRobert Mustacchi /* 5299d26e4fcSRobert Mustacchi * Enable the specified queue as a valid source of interrupts. Note, this should 5309d26e4fcSRobert Mustacchi * only be used as part of the GLDv3's interrupt blanking routines. The debug 5319d26e4fcSRobert Mustacchi * build assertions are specific to that. 5329d26e4fcSRobert Mustacchi */ 5339d26e4fcSRobert Mustacchi void 534338749cbSRobert Mustacchi i40e_intr_rx_queue_enable(i40e_trqpair_t *itrq) 5359d26e4fcSRobert Mustacchi { 5369d26e4fcSRobert Mustacchi uint32_t reg; 537338749cbSRobert Mustacchi uint_t queue = itrq->itrq_index; 538338749cbSRobert Mustacchi i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space; 5399d26e4fcSRobert Mustacchi 540338749cbSRobert Mustacchi ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); 541338749cbSRobert Mustacchi ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs); 5429d26e4fcSRobert Mustacchi 5439d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue)); 5449d26e4fcSRobert Mustacchi ASSERT0(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5459d26e4fcSRobert Mustacchi reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 5469d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 5479d26e4fcSRobert Mustacchi } 5489d26e4fcSRobert Mustacchi 5499d26e4fcSRobert Mustacchi /* 5509d26e4fcSRobert Mustacchi * Disable the specified queue as a valid source of interrupts. Note, this 5519d26e4fcSRobert Mustacchi * should only be used as part of the GLDv3's interrupt blanking routines. The 5529d26e4fcSRobert Mustacchi * debug build assertions are specific to that. 5539d26e4fcSRobert Mustacchi */ 5549d26e4fcSRobert Mustacchi void 555338749cbSRobert Mustacchi i40e_intr_rx_queue_disable(i40e_trqpair_t *itrq) 5569d26e4fcSRobert Mustacchi { 5579d26e4fcSRobert Mustacchi uint32_t reg; 558338749cbSRobert Mustacchi uint_t queue = itrq->itrq_index; 559338749cbSRobert Mustacchi i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space; 5609d26e4fcSRobert Mustacchi 561338749cbSRobert Mustacchi ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); 562338749cbSRobert Mustacchi ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs); 5639d26e4fcSRobert Mustacchi 5649d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue)); 5659d26e4fcSRobert Mustacchi ASSERT3U(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK, ==, 5669d26e4fcSRobert Mustacchi I40E_QINT_RQCTL_CAUSE_ENA_MASK); 5679d26e4fcSRobert Mustacchi reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 5689d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 5699d26e4fcSRobert Mustacchi } 5709d26e4fcSRobert Mustacchi 5719d26e4fcSRobert Mustacchi /* 5729d26e4fcSRobert Mustacchi * Start up the various chip's interrupt handling. We not only configure the 5739d26e4fcSRobert Mustacchi * adminq here, but we also go through and configure all of the actual queues, 5749d26e4fcSRobert Mustacchi * the interrupt linked lists, and others. 5759d26e4fcSRobert Mustacchi */ 5769d26e4fcSRobert Mustacchi void 5779d26e4fcSRobert Mustacchi i40e_intr_chip_init(i40e_t *i40e) 5789d26e4fcSRobert Mustacchi { 5799d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 5809d26e4fcSRobert Mustacchi uint32_t reg; 5819d26e4fcSRobert Mustacchi 5829d26e4fcSRobert Mustacchi /* 5839d26e4fcSRobert Mustacchi * Ensure that all non adminq interrupts are disabled at the chip level. 5849d26e4fcSRobert Mustacchi */ 5859d26e4fcSRobert Mustacchi i40e_intr_io_disable_all(i40e); 5869d26e4fcSRobert Mustacchi 5879d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, 0); 5889d26e4fcSRobert Mustacchi (void) I40E_READ_REG(hw, I40E_PFINT_ICR0); 5899d26e4fcSRobert Mustacchi 5909d26e4fcSRobert Mustacchi /* 5919d26e4fcSRobert Mustacchi * Always enable all of the other-class interrupts to be on their own 5929d26e4fcSRobert Mustacchi * ITR. This only needs to be set on interrupt zero, which has its own 5939d26e4fcSRobert Mustacchi * special setting. 5949d26e4fcSRobert Mustacchi */ 5959d26e4fcSRobert Mustacchi reg = I40E_ITR_INDEX_OTHER << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT; 5969d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0, reg); 5979d26e4fcSRobert Mustacchi 5989d26e4fcSRobert Mustacchi /* 5999d26e4fcSRobert Mustacchi * Enable interrupt types we expect to receive. At the moment, this 6009d26e4fcSRobert Mustacchi * is limited to the adminq; however, we'll want to review 11.2.2.9.22 6019d26e4fcSRobert Mustacchi * for more types here as we add support for detecting them, handling 6029d26e4fcSRobert Mustacchi * them, and resetting the device as appropriate. 6039d26e4fcSRobert Mustacchi */ 6049d26e4fcSRobert Mustacchi reg = I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6059d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg); 6069d26e4fcSRobert Mustacchi 6079d26e4fcSRobert Mustacchi /* 6089d26e4fcSRobert Mustacchi * Always set the interrupt linked list to empty. We'll come back and 6099d26e4fcSRobert Mustacchi * change this if MSI-X are actually on the scene. 6109d26e4fcSRobert Mustacchi */ 6119d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_TYPE_EOL); 6129d26e4fcSRobert Mustacchi 6139d26e4fcSRobert Mustacchi i40e_intr_adminq_enable(i40e); 6149d26e4fcSRobert Mustacchi 6159d26e4fcSRobert Mustacchi /* 6169d26e4fcSRobert Mustacchi * Set up all of the queues and map them to interrupts based on the bit 6179d26e4fcSRobert Mustacchi * assignments. 6189d26e4fcSRobert Mustacchi */ 6199d26e4fcSRobert Mustacchi if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 6209d26e4fcSRobert Mustacchi i40e_intr_init_queue_msix(i40e); 6219d26e4fcSRobert Mustacchi } else { 6229d26e4fcSRobert Mustacchi i40e_intr_init_queue_shared(i40e); 6239d26e4fcSRobert Mustacchi } 6249d26e4fcSRobert Mustacchi 6259d26e4fcSRobert Mustacchi /* 6269d26e4fcSRobert Mustacchi * Finally set all of the default ITRs for the interrupts. Note that the 6279d26e4fcSRobert Mustacchi * queues will have been set up above. 6289d26e4fcSRobert Mustacchi */ 6299d26e4fcSRobert Mustacchi i40e_intr_set_itr(i40e, I40E_ITR_INDEX_RX, i40e->i40e_rx_itr); 6309d26e4fcSRobert Mustacchi i40e_intr_set_itr(i40e, I40E_ITR_INDEX_TX, i40e->i40e_tx_itr); 6319d26e4fcSRobert Mustacchi i40e_intr_set_itr(i40e, I40E_ITR_INDEX_OTHER, i40e->i40e_other_itr); 6329d26e4fcSRobert Mustacchi } 6339d26e4fcSRobert Mustacchi 6349d26e4fcSRobert Mustacchi static void 6359d26e4fcSRobert Mustacchi i40e_intr_adminq_work(i40e_t *i40e) 6369d26e4fcSRobert Mustacchi { 6379d26e4fcSRobert Mustacchi struct i40e_hw *hw = &i40e->i40e_hw_space; 6389d26e4fcSRobert Mustacchi struct i40e_arq_event_info evt; 6399d26e4fcSRobert Mustacchi uint16_t remain = 1; 6409d26e4fcSRobert Mustacchi 6419d26e4fcSRobert Mustacchi bzero(&evt, sizeof (struct i40e_arq_event_info)); 6429d26e4fcSRobert Mustacchi evt.buf_len = I40E_ADMINQ_BUFSZ; 6439d26e4fcSRobert Mustacchi evt.msg_buf = i40e->i40e_aqbuf; 6449d26e4fcSRobert Mustacchi 6459d26e4fcSRobert Mustacchi while (remain != 0) { 6469d26e4fcSRobert Mustacchi enum i40e_status_code ret; 6479d26e4fcSRobert Mustacchi uint16_t opcode; 6489d26e4fcSRobert Mustacchi 6499d26e4fcSRobert Mustacchi /* 6509d26e4fcSRobert Mustacchi * At the moment, the only error code that seems to be returned 6519d26e4fcSRobert Mustacchi * is one saying that there's no work. In such a case we leave 6529d26e4fcSRobert Mustacchi * this be. 6539d26e4fcSRobert Mustacchi */ 6549d26e4fcSRobert Mustacchi ret = i40e_clean_arq_element(hw, &evt, &remain); 6559d26e4fcSRobert Mustacchi if (ret != I40E_SUCCESS) 6569d26e4fcSRobert Mustacchi break; 6579d26e4fcSRobert Mustacchi 6589d26e4fcSRobert Mustacchi opcode = LE_16(evt.desc.opcode); 6599d26e4fcSRobert Mustacchi switch (opcode) { 6609d26e4fcSRobert Mustacchi case i40e_aqc_opc_get_link_status: 6619d26e4fcSRobert Mustacchi mutex_enter(&i40e->i40e_general_lock); 6629d26e4fcSRobert Mustacchi i40e_link_check(i40e); 6639d26e4fcSRobert Mustacchi mutex_exit(&i40e->i40e_general_lock); 6649d26e4fcSRobert Mustacchi break; 6659d26e4fcSRobert Mustacchi default: 6669d26e4fcSRobert Mustacchi /* 6679d26e4fcSRobert Mustacchi * Longer term we'll want to enable other causes here 6689d26e4fcSRobert Mustacchi * and get these cleaned up and doing something. 6699d26e4fcSRobert Mustacchi */ 6709d26e4fcSRobert Mustacchi break; 6719d26e4fcSRobert Mustacchi } 6729d26e4fcSRobert Mustacchi } 6739d26e4fcSRobert Mustacchi } 6749d26e4fcSRobert Mustacchi 6759d26e4fcSRobert Mustacchi static void 676*09aee612SRyan Zezeski i40e_intr_rx_work(i40e_t *i40e, i40e_trqpair_t *itrq) 6779d26e4fcSRobert Mustacchi { 678396505afSPaul Winder mblk_t *mp = NULL; 6799d26e4fcSRobert Mustacchi 6809d26e4fcSRobert Mustacchi mutex_enter(&itrq->itrq_rx_lock); 681396505afSPaul Winder if (!itrq->itrq_intr_poll) 682396505afSPaul Winder mp = i40e_ring_rx(itrq, I40E_POLL_NULL); 6839d26e4fcSRobert Mustacchi mutex_exit(&itrq->itrq_rx_lock); 6849d26e4fcSRobert Mustacchi 685*09aee612SRyan Zezeski if (mp == NULL) 686*09aee612SRyan Zezeski return; 687*09aee612SRyan Zezeski 688*09aee612SRyan Zezeski mac_rx_ring(i40e->i40e_mac_hdl, itrq->itrq_macrxring, mp, 689*09aee612SRyan Zezeski itrq->itrq_rxgen); 6909d26e4fcSRobert Mustacchi } 6919d26e4fcSRobert Mustacchi 692*09aee612SRyan Zezeski /* ARGSUSED */ 6939d26e4fcSRobert Mustacchi static void 694*09aee612SRyan Zezeski i40e_intr_tx_work(i40e_t *i40e, i40e_trqpair_t *itrq) 6959d26e4fcSRobert Mustacchi { 6969d26e4fcSRobert Mustacchi i40e_tx_recycle_ring(itrq); 6979d26e4fcSRobert Mustacchi } 6989d26e4fcSRobert Mustacchi 6999d26e4fcSRobert Mustacchi /* 7009d26e4fcSRobert Mustacchi * At the moment, the only 'other' interrupt on ICR0 that we handle is the 7019d26e4fcSRobert Mustacchi * adminq. We should go through and support the other notifications at some 7029d26e4fcSRobert Mustacchi * point. 7039d26e4fcSRobert Mustacchi */ 7049d26e4fcSRobert Mustacchi static void 7059d26e4fcSRobert Mustacchi i40e_intr_other_work(i40e_t *i40e) 7069d26e4fcSRobert Mustacchi { 7079d26e4fcSRobert Mustacchi struct i40e_hw *hw = &i40e->i40e_hw_space; 7089d26e4fcSRobert Mustacchi uint32_t reg; 7099d26e4fcSRobert Mustacchi 7109d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_PFINT_ICR0); 7119d26e4fcSRobert Mustacchi if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 7129d26e4fcSRobert Mustacchi DDI_FM_OK) { 7139d26e4fcSRobert Mustacchi ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED); 7149d26e4fcSRobert Mustacchi atomic_or_32(&i40e->i40e_state, I40E_ERROR); 7159d26e4fcSRobert Mustacchi return; 7169d26e4fcSRobert Mustacchi } 7179d26e4fcSRobert Mustacchi 7189d26e4fcSRobert Mustacchi if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 7199d26e4fcSRobert Mustacchi i40e_intr_adminq_work(i40e); 7209d26e4fcSRobert Mustacchi 7219d26e4fcSRobert Mustacchi /* 7229d26e4fcSRobert Mustacchi * Make sure that the adminq interrupt is not masked and then explicitly 7239d26e4fcSRobert Mustacchi * enable the adminq and thus the other interrupt. 7249d26e4fcSRobert Mustacchi */ 7259d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA); 7269d26e4fcSRobert Mustacchi reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 7279d26e4fcSRobert Mustacchi I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg); 7289d26e4fcSRobert Mustacchi 7299d26e4fcSRobert Mustacchi i40e_intr_adminq_enable(i40e); 7309d26e4fcSRobert Mustacchi } 7319d26e4fcSRobert Mustacchi 732*09aee612SRyan Zezeski /* 733*09aee612SRyan Zezeski * Handle an MSI-X interrupt. See section 7.5.1.3 for an overview of 734*09aee612SRyan Zezeski * the MSI-X interrupt sequence. 735*09aee612SRyan Zezeski */ 7369d26e4fcSRobert Mustacchi uint_t 7379d26e4fcSRobert Mustacchi i40e_intr_msix(void *arg1, void *arg2) 7389d26e4fcSRobert Mustacchi { 7399d26e4fcSRobert Mustacchi i40e_t *i40e = (i40e_t *)arg1; 740*09aee612SRyan Zezeski uint_t vector_idx = (uint_t)(uintptr_t)arg2; 741*09aee612SRyan Zezeski 742*09aee612SRyan Zezeski ASSERT3U(vector_idx, <, i40e->i40e_intr_count); 7439d26e4fcSRobert Mustacchi 7449d26e4fcSRobert Mustacchi /* 7459d26e4fcSRobert Mustacchi * When using MSI-X interrupts, vector 0 is always reserved for the 7469d26e4fcSRobert Mustacchi * adminq at this time. Though longer term, we'll want to also bridge 7479d26e4fcSRobert Mustacchi * some I/O to them. 7489d26e4fcSRobert Mustacchi */ 7499d26e4fcSRobert Mustacchi if (vector_idx == 0) { 7509d26e4fcSRobert Mustacchi i40e_intr_other_work(i40e); 7519d26e4fcSRobert Mustacchi return (DDI_INTR_CLAIMED); 7529d26e4fcSRobert Mustacchi } 7539d26e4fcSRobert Mustacchi 754*09aee612SRyan Zezeski ASSERT3U(vector_idx, >, 0); 7559d26e4fcSRobert Mustacchi 756*09aee612SRyan Zezeski /* 757*09aee612SRyan Zezeski * We determine the queue indexes via simple arithmetic (as 758*09aee612SRyan Zezeski * opposed to keeping explicit state like a bitmap). While 759*09aee612SRyan Zezeski * conveinent, it does mean that i40e_map_intrs_to_vectors(), 760*09aee612SRyan Zezeski * i40e_intr_init_queue_msix(), and this function must be 761*09aee612SRyan Zezeski * modified as a unit. 762*09aee612SRyan Zezeski * 763*09aee612SRyan Zezeski * We subtract 1 from the vector to offset the addition we 764*09aee612SRyan Zezeski * performed during i40e_map_intrs_to_vectors(). 765*09aee612SRyan Zezeski */ 766*09aee612SRyan Zezeski for (uint_t i = vector_idx - 1; i < i40e->i40e_num_trqpairs; 767*09aee612SRyan Zezeski i += (i40e->i40e_intr_count - 1)) { 768*09aee612SRyan Zezeski i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 769*09aee612SRyan Zezeski 770*09aee612SRyan Zezeski ASSERT3U(i, <, i40e->i40e_num_trqpairs); 771*09aee612SRyan Zezeski ASSERT3P(itrq, !=, NULL); 772*09aee612SRyan Zezeski i40e_intr_rx_work(i40e, itrq); 773*09aee612SRyan Zezeski i40e_intr_tx_work(i40e, itrq); 774*09aee612SRyan Zezeski } 775*09aee612SRyan Zezeski 776*09aee612SRyan Zezeski i40e_intr_io_enable(i40e, vector_idx); 7779d26e4fcSRobert Mustacchi return (DDI_INTR_CLAIMED); 7789d26e4fcSRobert Mustacchi } 7799d26e4fcSRobert Mustacchi 7809d26e4fcSRobert Mustacchi static uint_t 7819d26e4fcSRobert Mustacchi i40e_intr_notx(i40e_t *i40e, boolean_t shared) 7829d26e4fcSRobert Mustacchi { 7839d26e4fcSRobert Mustacchi i40e_hw_t *hw = &i40e->i40e_hw_space; 7849d26e4fcSRobert Mustacchi uint32_t reg; 785*09aee612SRyan Zezeski i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[0]; 7869d26e4fcSRobert Mustacchi int ret = DDI_INTR_CLAIMED; 7879d26e4fcSRobert Mustacchi 7889d26e4fcSRobert Mustacchi if (shared == B_TRUE) { 7899d26e4fcSRobert Mustacchi mutex_enter(&i40e->i40e_general_lock); 7909d26e4fcSRobert Mustacchi if (i40e->i40e_state & I40E_SUSPENDED) { 7919d26e4fcSRobert Mustacchi mutex_exit(&i40e->i40e_general_lock); 7929d26e4fcSRobert Mustacchi return (DDI_INTR_UNCLAIMED); 7939d26e4fcSRobert Mustacchi } 7949d26e4fcSRobert Mustacchi mutex_exit(&i40e->i40e_general_lock); 7959d26e4fcSRobert Mustacchi } 7969d26e4fcSRobert Mustacchi 7979d26e4fcSRobert Mustacchi reg = I40E_READ_REG(hw, I40E_PFINT_ICR0); 7989d26e4fcSRobert Mustacchi if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 7999d26e4fcSRobert Mustacchi DDI_FM_OK) { 8009d26e4fcSRobert Mustacchi ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED); 8019d26e4fcSRobert Mustacchi atomic_or_32(&i40e->i40e_state, I40E_ERROR); 8029d26e4fcSRobert Mustacchi return (DDI_INTR_CLAIMED); 8039d26e4fcSRobert Mustacchi } 8049d26e4fcSRobert Mustacchi 8059d26e4fcSRobert Mustacchi if (reg == 0) { 8069d26e4fcSRobert Mustacchi if (shared == B_TRUE) 8079d26e4fcSRobert Mustacchi ret = DDI_INTR_UNCLAIMED; 8089d26e4fcSRobert Mustacchi goto done; 8099d26e4fcSRobert Mustacchi } 8109d26e4fcSRobert Mustacchi 8119d26e4fcSRobert Mustacchi if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 8129d26e4fcSRobert Mustacchi i40e_intr_adminq_work(i40e); 8139d26e4fcSRobert Mustacchi 8149d26e4fcSRobert Mustacchi if (reg & I40E_INTR_NOTX_RX_MASK) 815*09aee612SRyan Zezeski i40e_intr_rx_work(i40e, itrq); 8169d26e4fcSRobert Mustacchi 8179d26e4fcSRobert Mustacchi if (reg & I40E_INTR_NOTX_TX_MASK) 818*09aee612SRyan Zezeski i40e_intr_tx_work(i40e, itrq); 8199d26e4fcSRobert Mustacchi 8209d26e4fcSRobert Mustacchi done: 8219d26e4fcSRobert Mustacchi i40e_intr_adminq_enable(i40e); 8229d26e4fcSRobert Mustacchi return (ret); 8239d26e4fcSRobert Mustacchi 8249d26e4fcSRobert Mustacchi } 8259d26e4fcSRobert Mustacchi 8269d26e4fcSRobert Mustacchi /* ARGSUSED */ 8279d26e4fcSRobert Mustacchi uint_t 8289d26e4fcSRobert Mustacchi i40e_intr_msi(void *arg1, void *arg2) 8299d26e4fcSRobert Mustacchi { 8309d26e4fcSRobert Mustacchi i40e_t *i40e = (i40e_t *)arg1; 8319d26e4fcSRobert Mustacchi 8329d26e4fcSRobert Mustacchi return (i40e_intr_notx(i40e, B_FALSE)); 8339d26e4fcSRobert Mustacchi } 8349d26e4fcSRobert Mustacchi 8359d26e4fcSRobert Mustacchi /* ARGSUSED */ 8369d26e4fcSRobert Mustacchi uint_t 8379d26e4fcSRobert Mustacchi i40e_intr_legacy(void *arg1, void *arg2) 8389d26e4fcSRobert Mustacchi { 8399d26e4fcSRobert Mustacchi i40e_t *i40e = (i40e_t *)arg1; 8409d26e4fcSRobert Mustacchi 8419d26e4fcSRobert Mustacchi return (i40e_intr_notx(i40e, B_TRUE)); 8429d26e4fcSRobert Mustacchi } 843