/* * Copyright (c) 2008-2016 Solarflare Communications Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and documentation are * those of the authors and should not be interpreted as representing official * policies, either expressed or implied, of the FreeBSD Project. */ #include #include #include #include #include #include #include #include #include #include "sfxge.h" #include "efx.h" /* Timeout to wait for DRIVER_EV_START event at EVQ startup */ #define SFXGE_EV_QSTART_TIMEOUT_USEC (2000000) /* Event queue DMA attributes */ static ddi_device_acc_attr_t sfxge_evq_devacc = { DDI_DEVICE_ATTR_V0, /* devacc_attr_version */ DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */ DDI_STRICTORDER_ACC /* devacc_attr_dataorder */ }; static ddi_dma_attr_t sfxge_evq_dma_attr = { DMA_ATTR_V0, /* dma_attr_version */ 0, /* dma_attr_addr_lo */ 0xffffffffffffffffull, /* dma_attr_addr_hi */ 0xffffffffffffffffull, /* dma_attr_count_max */ EFX_BUF_SIZE, /* dma_attr_align */ 0xffffffff, /* dma_attr_burstsizes */ 1, /* dma_attr_minxfer */ 0xffffffffffffffffull, /* dma_attr_maxxfer */ 0xffffffffffffffffull, /* dma_attr_seg */ 1, /* dma_attr_sgllen */ 1, /* dma_attr_granular */ 0 /* dma_attr_flags */ }; static int _sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size) { efsys_mem_t *esmp = &(sep->se_mem); sfxge_dma_buffer_attr_t dma_attr; int rc; /* Compile-time structure layout checks */ EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <= sizeof (sep->__se_u1.__se_pad)); EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <= sizeof (sep->__se_u2.__se_pad)); EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <= sizeof (sep->__se_u3.__se_pad)); bzero(sep, sizeof (sfxge_evq_t)); sep->se_sp = sp; dma_attr.sdba_dip = sp->s_dip; dma_attr.sdba_dattrp = &sfxge_evq_dma_attr; dma_attr.sdba_callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT; dma_attr.sdba_length = EFX_EVQ_SIZE(evq_size); dma_attr.sdba_memflags = DDI_DMA_CONSISTENT; dma_attr.sdba_devaccp = &sfxge_evq_devacc; dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT; dma_attr.sdba_maxcookies = 1; dma_attr.sdba_zeroinit = B_FALSE; if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0) goto fail1; /* Allocate some buffer table entries */ if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size), &(sep->se_id))) != 0) goto fail2; sep->se_stpp = &(sep->se_stp); return (0); fail2: DTRACE_PROBE(fail2); /* Tear down DMA setup */ esmp->esm_addr = 0; sfxge_dma_buffer_destroy(esmp); fail1: DTRACE_PROBE1(fail1, int, rc); sep->se_sp = NULL; SFXGE_OBJ_CHECK(sep, sfxge_evq_t); return (-1); } static int sfxge_ev_q0ctor(void *buf, void *arg, int kmflags) { sfxge_evq_t *sep = buf; sfxge_t *sp = arg; return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size)); } static int sfxge_ev_qXctor(void *buf, void *arg, int kmflags) { sfxge_evq_t *sep = buf; sfxge_t *sp = arg; return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size)); } static void _sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size) { efsys_mem_t *esmp = &(sep->se_mem); ASSERT3P(sep->se_sp, ==, sp); ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); sep->se_stpp = NULL; /* Free the buffer table entries */ sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); sep->se_id = 0; /* Tear down DMA setup */ sfxge_dma_buffer_destroy(esmp); sep->se_sp = NULL; SFXGE_OBJ_CHECK(sep, sfxge_evq_t); } static void sfxge_ev_q0dtor(void *buf, void *arg) { sfxge_evq_t *sep = buf; sfxge_t *sp = arg; _sfxge_ev_qdtor(sp, sep, sp->s_evq0_size); } static void sfxge_ev_qXdtor(void *buf, void *arg) { sfxge_evq_t *sep = buf; sfxge_t *sp = arg; _sfxge_ev_qdtor(sp, sep, sp->s_evqX_size); } static boolean_t sfxge_ev_initialized(void *arg) { sfxge_evq_t *sep = arg; ASSERT(mutex_owned(&(sep->se_lock))); /* Init done events may be duplicated on 7xxx (see SFCbug31631) */ if (sep->se_state == SFXGE_EVQ_STARTED) goto done; ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING); sep->se_state = SFXGE_EVQ_STARTED; cv_broadcast(&(sep->se_init_kv)); done: return (B_FALSE); } static void sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop) { sfxge_t *sp = sep->se_sp; unsigned int index = sep->se_index; sfxge_rxq_t *srp = sp->s_srp[index]; sfxge_txq_t *stp; if ((stp = sep->se_stp) != NULL) { sep->se_stp = NULL; sep->se_stpp = &(sep->se_stp); do { sfxge_txq_t *next; next = stp->st_next; stp->st_next = NULL; ASSERT3U(stp->st_evq, ==, index); if (stp->st_pending != stp->st_completed) sfxge_tx_qcomplete(stp); stp = next; } while (stp != NULL); } if (srp != NULL) { if (srp->sr_pending != srp->sr_completed) sfxge_rx_qcomplete(srp, eop); } } static boolean_t sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, uint16_t flags) { sfxge_evq_t *sep = arg; sfxge_t *sp = sep->se_sp; sfxge_rxq_t *srp; sfxge_rx_packet_t *srpp; unsigned int prefetch; unsigned int stop; unsigned int delta; ASSERT(mutex_owned(&(sep->se_lock))); if (sep->se_exception) goto done; srp = sp->s_srp[label]; if (srp == NULL) goto done; ASSERT3U(sep->se_index, ==, srp->sr_index); ASSERT3U(id, <, sp->s_rxq_size); /* * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset * So the return missing srp->sr_pending increase is safe */ if (srp->sr_state != SFXGE_RXQ_STARTED) goto done; stop = (id + 1) & (sp->s_rxq_size - 1); id = srp->sr_pending & (sp->s_rxq_size - 1); delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop); srp->sr_pending += delta; if (delta != 1) { if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) || (delta == 0) || (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) { /* * FIXME: This does not take into account scatter * aborts. See Bug40811 */ sep->se_exception = B_TRUE; DTRACE_PROBE(restart_ev_rx_id); /* sfxge_evq_t->se_lock held */ (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR, "Out of order RX event", delta); goto done; } } prefetch = (id + 4) & (sp->s_rxq_size - 1); if ((srpp = srp->sr_srpp[prefetch]) != NULL) prefetch_read_many(srpp); srpp = srp->sr_srpp[id]; ASSERT(srpp != NULL); prefetch_read_many(srpp->srp_mp); for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) { srpp = srp->sr_srpp[id]; ASSERT(srpp != NULL); ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD); srpp->srp_flags = flags; ASSERT3U(size, <, (1 << 16)); srpp->srp_size = (uint16_t)size; } sep->se_rx++; DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index, unsigned int, srp->sr_added - srp->sr_pending); if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH) sfxge_ev_qcomplete(sep, B_FALSE); done: /* returning B_TRUE makes efx_ev_qpoll() stop processing events */ return (sep->se_rx >= sep->se_ev_batch); } static boolean_t sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) { sfxge_evq_t *sep = arg; sfxge_t *sp = sep->se_sp; _NOTE(ARGUNUSED(code)) _NOTE(ARGUNUSED(data)) ASSERT(mutex_owned(&(sep->se_lock))); sep->se_exception = B_TRUE; if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { DTRACE_PROBE(restart_ev_exception); /* sfxge_evq_t->se_lock held */ (void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR, "Unknown EV", code); } return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) { sfxge_evq_t *sep_targetq, *sep = arg; sfxge_t *sp = sep->se_sp; sfxge_rxq_t *srp; unsigned int index; unsigned int label; uint16_t magic; ASSERT(mutex_owned(&(sep->se_lock))); /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */ srp = sp->s_srp[rxq_index]; if (srp == NULL) goto done; /* Process right now if it is the correct event queue */ index = srp->sr_index; if (index == sep->se_index) { sfxge_rx_qflush_done(srp); goto done; } /* Resend a software event on the correct queue */ sep_targetq = sp->s_sep[index]; if (sep_targetq->se_state != SFXGE_EVQ_STARTED) goto done; /* TBD: state test not under the lock */ label = rxq_index; ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; efx_ev_qpost(sep_targetq->se_eep, magic); done: return (B_FALSE); } static boolean_t sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) { sfxge_evq_t *sep_targetq, *sep = arg; sfxge_t *sp = sep->se_sp; sfxge_rxq_t *srp; unsigned int index; unsigned int label; uint16_t magic; ASSERT(mutex_owned(&(sep->se_lock))); /* Ensure RXQ exists, as events may arrive after RXQ was destroyed */ srp = sp->s_srp[rxq_index]; if (srp == NULL) goto done; /* Process right now if it is the correct event queue */ index = srp->sr_index; if (index == sep->se_index) { sfxge_rx_qflush_failed(srp); goto done; } /* Resend a software event on the correct queue */ sep_targetq = sp->s_sep[index]; label = rxq_index; ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; if (sep_targetq->se_state != SFXGE_EVQ_STARTED) goto done; /* TBD: state test not under the lock */ efx_ev_qpost(sep_targetq->se_eep, magic); done: return (B_FALSE); } static boolean_t sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) { sfxge_evq_t *sep = arg; sfxge_txq_t *stp; unsigned int stop; unsigned int delta; ASSERT(mutex_owned(&(sep->se_lock))); stp = sep->se_label_stp[label]; if (stp == NULL) goto done; if (stp->st_state != SFXGE_TXQ_STARTED) goto done; ASSERT3U(sep->se_index, ==, stp->st_evq); stop = (id + 1) & (SFXGE_TX_NDESCS - 1); id = stp->st_pending & (SFXGE_TX_NDESCS - 1); delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop); stp->st_pending += delta; sep->se_tx++; if (stp->st_next == NULL && sep->se_stpp != &(stp->st_next)) { *(sep->se_stpp) = stp; sep->se_stpp = &(stp->st_next); } DTRACE_PROBE2(qlevel, unsigned int, stp->st_index, unsigned int, stp->st_added - stp->st_pending); if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH) sfxge_tx_qcomplete(stp); done: /* returning B_TRUE makes efx_ev_qpoll() stop processing events */ return (sep->se_tx >= sep->se_ev_batch); } static boolean_t sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) { sfxge_evq_t *sep = arg; sfxge_t *sp = sep->se_sp; sfxge_txq_t *stp; unsigned int evq; unsigned int label; uint16_t magic; ASSERT(mutex_owned(&(sep->se_lock))); /* Ensure TXQ exists, as events may arrive after TXQ was destroyed */ stp = sp->s_stp[txq_index]; if (stp == NULL) goto done; /* Process right now if it is the correct event queue */ evq = stp->st_evq; if (evq == sep->se_index) { sfxge_tx_qflush_done(stp); goto done; } /* Resend a software event on the correct queue */ sep = sp->s_sep[evq]; label = stp->st_label; ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label); magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); efx_ev_qpost(sep->se_eep, magic); done: return (B_FALSE); } static boolean_t sfxge_ev_software(void *arg, uint16_t magic) { sfxge_evq_t *sep = arg; sfxge_t *sp = sep->se_sp; dev_info_t *dip = sp->s_dip; unsigned int label; ASSERT(mutex_owned(&(sep->se_lock))); EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH == FSF_AZ_RX_EV_Q_LABEL_WIDTH); EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH == FSF_AZ_TX_EV_Q_LABEL_WIDTH); label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; switch (magic) { case SFXGE_MAGIC_RX_QFLUSH_DONE: { sfxge_rxq_t *srp = sp->s_srp[label]; if (srp != NULL) { ASSERT3U(sep->se_index, ==, srp->sr_index); sfxge_rx_qflush_done(srp); } break; } case SFXGE_MAGIC_RX_QFLUSH_FAILED: { sfxge_rxq_t *srp = sp->s_srp[label]; if (srp != NULL) { ASSERT3U(sep->se_index, ==, srp->sr_index); sfxge_rx_qflush_failed(srp); } break; } case SFXGE_MAGIC_RX_QFPP_TRIM: { sfxge_rxq_t *srp = sp->s_srp[label]; if (srp != NULL) { ASSERT3U(sep->se_index, ==, srp->sr_index); sfxge_rx_qfpp_trim(srp); } break; } case SFXGE_MAGIC_TX_QFLUSH_DONE: { sfxge_txq_t *stp = sep->se_label_stp[label]; if (stp != NULL) { ASSERT3U(sep->se_index, ==, stp->st_evq); sfxge_tx_qflush_done(stp); } break; } default: dev_err(dip, CE_NOTE, SFXGE_CMN_ERR "unknown software event 0x%x", magic); break; } return (B_FALSE); } static boolean_t sfxge_ev_sram(void *arg, uint32_t code) { _NOTE(ARGUNUSED(arg)) switch (code) { case EFX_SRAM_UPDATE: DTRACE_PROBE(sram_update); break; case EFX_SRAM_CLEAR: DTRACE_PROBE(sram_clear); break; case EFX_SRAM_ILLEGAL_CLEAR: DTRACE_PROBE(sram_illegal_clear); break; default: ASSERT(B_FALSE); break; } return (B_FALSE); } static boolean_t sfxge_ev_timer(void *arg, uint32_t index) { _NOTE(ARGUNUSED(arg, index)) return (B_FALSE); } static boolean_t sfxge_ev_wake_up(void *arg, uint32_t index) { _NOTE(ARGUNUSED(arg, index)) return (B_FALSE); } static boolean_t sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) { sfxge_evq_t *sep = arg; sfxge_t *sp = sep->se_sp; sfxge_mac_link_update(sp, link_mode); return (B_FALSE); } static int sfxge_ev_kstat_update(kstat_t *ksp, int rw) { sfxge_evq_t *sep = ksp->ks_private; kstat_named_t *knp; int rc; if (rw != KSTAT_READ) { rc = EACCES; goto fail1; } ASSERT(mutex_owned(&(sep->se_lock))); if (sep->se_state != SFXGE_EVQ_STARTED) goto done; efx_ev_qstats_update(sep->se_eep, sep->se_stat); knp = ksp->ks_data; knp += EV_NQSTATS; knp->value.ui64 = sep->se_cpu_id; done: return (0); fail1: DTRACE_PROBE1(fail1, int, rc); return (rc); } static int sfxge_ev_kstat_init(sfxge_evq_t *sep) { sfxge_t *sp = sep->se_sp; unsigned int index = sep->se_index; dev_info_t *dip = sp->s_dip; kstat_t *ksp; kstat_named_t *knp; char name[MAXNAMELEN]; unsigned int id; int rc; /* Determine the name */ (void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d", ddi_driver_name(dip), index); /* Create the set */ if ((ksp = kstat_create((char *)ddi_driver_name(dip), ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED, EV_NQSTATS + 1, 0)) == NULL) { rc = ENOMEM; goto fail1; } sep->se_ksp = ksp; ksp->ks_update = sfxge_ev_kstat_update; ksp->ks_private = sep; ksp->ks_lock = &(sep->se_lock); /* Initialise the named stats */ sep->se_stat = knp = ksp->ks_data; for (id = 0; id < EV_NQSTATS; id++) { kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id), KSTAT_DATA_UINT64); knp++; } kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64); kstat_install(ksp); return (0); fail1: DTRACE_PROBE1(fail1, int, rc); return (rc); } static void sfxge_ev_kstat_fini(sfxge_evq_t *sep) { /* Destroy the set */ kstat_delete(sep->se_ksp); sep->se_ksp = NULL; sep->se_stat = NULL; } inline unsigned pow2_ge(unsigned int n) { unsigned int order = 0; ASSERT3U(n, >, 0); while ((1ul << order) < n) ++order; return (1ul << (order)); } static int sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch) { sfxge_evq_t *sep; int rc; ASSERT3U(index, <, SFXGE_RX_SCALE_MAX); sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP); if (sep == NULL) { rc = ENOMEM; goto fail1; } ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED); sep->se_index = index; mutex_init(&(sep->se_lock), NULL, MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri)); cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL); /* Initialize the statistics */ if ((rc = sfxge_ev_kstat_init(sep)) != 0) goto fail2; sep->se_state = SFXGE_EVQ_INITIALIZED; sep->se_ev_batch = (uint16_t)ev_batch; sp->s_sep[index] = sep; return (0); fail2: DTRACE_PROBE(fail2); sep->se_index = 0; cv_destroy(&(sep->se_init_kv)); mutex_destroy(&(sep->se_lock)); kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); fail1: DTRACE_PROBE1(fail1, int, rc); return (rc); } static int sfxge_ev_qstart(sfxge_t *sp, unsigned int index) { sfxge_evq_t *sep = sp->s_sep[index]; sfxge_intr_t *sip = &(sp->s_intr); efx_nic_t *enp = sp->s_enp; efx_ev_callbacks_t *eecp; efsys_mem_t *esmp; clock_t timeout; int rc; uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; mutex_enter(&(sep->se_lock)); esmp = &(sep->se_mem); ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); /* Set the memory to all ones */ (void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size)); /* Program the buffer table */ if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp, EFX_EVQ_NBUFS(evq_size))) != 0) goto fail1; /* Set up the event callbacks */ eecp = &(sep->se_eec); eecp->eec_initialized = sfxge_ev_initialized; eecp->eec_rx = sfxge_ev_rx; eecp->eec_tx = sfxge_ev_tx; eecp->eec_exception = sfxge_ev_exception; eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done; eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed; eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done; eecp->eec_software = sfxge_ev_software; eecp->eec_sram = sfxge_ev_sram; eecp->eec_wake_up = sfxge_ev_wake_up; eecp->eec_timer = sfxge_ev_timer; eecp->eec_link_change = sfxge_ev_link_change; /* Create the event queue */ if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id, &(sep->se_eep))) != 0) goto fail2; /* Set the default moderation */ if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0) goto fail3; /* Check that interrupts are enabled at the NIC */ if (sip->si_state != SFXGE_INTR_STARTED) { rc = EINVAL; goto fail4; } sep->se_state = SFXGE_EVQ_STARTING; /* Prime the event queue for interrupts */ if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) goto fail5; /* Wait for the initialization event */ timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC); while (sep->se_state != SFXGE_EVQ_STARTED) { if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock), timeout) < 0) { /* Timeout waiting for initialization */ dev_info_t *dip = sp->s_dip; DTRACE_PROBE(timeout); dev_err(dip, CE_NOTE, SFXGE_CMN_ERR "evq[%d] qstart timeout", index); rc = ETIMEDOUT; goto fail6; } } mutex_exit(&(sep->se_lock)); return (0); fail6: DTRACE_PROBE(fail6); fail5: DTRACE_PROBE(fail5); sep->se_state = SFXGE_EVQ_INITIALIZED; fail4: DTRACE_PROBE(fail4); fail3: DTRACE_PROBE(fail3); /* Destroy the event queue */ efx_ev_qdestroy(sep->se_eep); sep->se_eep = NULL; fail2: DTRACE_PROBE(fail2); /* Zero out the event handlers */ bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); /* Clear entries from the buffer table */ sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); fail1: DTRACE_PROBE1(fail1, int, rc); mutex_exit(&(sep->se_lock)); return (rc); } int sfxge_ev_qpoll(sfxge_t *sp, unsigned int index) { sfxge_evq_t *sep = sp->s_sep[index]; processorid_t cpu_id; int rc; uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; mutex_enter(&(sep->se_lock)); if (sep->se_state != SFXGE_EVQ_STARTING && sep->se_state != SFXGE_EVQ_STARTED) { rc = EINVAL; goto fail1; } /* Make sure the CPU information is up to date */ cpu_id = CPU->cpu_id; if (cpu_id != sep->se_cpu_id) { sep->se_cpu_id = cpu_id; /* sfxge_evq_t->se_lock held */ (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp, DDI_NOSLEEP); } /* Synchronize the DMA memory for reading */ (void) ddi_dma_sync(sep->se_mem.esm_dma_handle, 0, EFX_EVQ_SIZE(evq_size), DDI_DMA_SYNC_FORKERNEL); ASSERT3U(sep->se_rx, ==, 0); ASSERT3U(sep->se_tx, ==, 0); ASSERT3P(sep->se_stp, ==, NULL); ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); /* Poll the queue */ efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec), sep); sep->se_rx = 0; sep->se_tx = 0; /* Perform any pending completion processing */ sfxge_ev_qcomplete(sep, B_TRUE); /* Re-prime the event queue for interrupts */ if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) goto fail2; mutex_exit(&(sep->se_lock)); return (0); fail2: DTRACE_PROBE(fail2); fail1: DTRACE_PROBE1(fail1, int, rc); mutex_exit(&(sep->se_lock)); return (rc); } int sfxge_ev_qprime(sfxge_t *sp, unsigned int index) { sfxge_evq_t *sep = sp->s_sep[index]; int rc; mutex_enter(&(sep->se_lock)); if (sep->se_state != SFXGE_EVQ_STARTING && sep->se_state != SFXGE_EVQ_STARTED) { rc = EINVAL; goto fail1; } if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) goto fail2; mutex_exit(&(sep->se_lock)); return (0); fail2: DTRACE_PROBE(fail2); fail1: DTRACE_PROBE1(fail1, int, rc); mutex_exit(&(sep->se_lock)); return (rc); } int sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us) { sfxge_evq_t *sep = sp->s_sep[index]; efx_evq_t *eep = sep->se_eep; ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); return (efx_ev_qmoderate(eep, us)); } static void sfxge_ev_qstop(sfxge_t *sp, unsigned int index) { sfxge_evq_t *sep = sp->s_sep[index]; uint16_t evq_size; mutex_enter(&(sep->se_lock)); ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); sep->se_state = SFXGE_EVQ_INITIALIZED; evq_size = index ? sp->s_evqX_size : sp->s_evq0_size; /* Clear the CPU information */ sep->se_cpu_id = 0; /* Clear the event count */ sep->se_count = 0; /* Reset the exception flag */ sep->se_exception = B_FALSE; /* Destroy the event queue */ efx_ev_qdestroy(sep->se_eep); sep->se_eep = NULL; mutex_exit(&(sep->se_lock)); /* Zero out the event handlers */ bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); /* Clear entries from the buffer table */ sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); } static void sfxge_ev_qfini(sfxge_t *sp, unsigned int index) { sfxge_evq_t *sep = sp->s_sep[index]; ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); sp->s_sep[index] = NULL; sep->se_state = SFXGE_EVQ_UNINITIALIZED; /* Tear down the statistics */ sfxge_ev_kstat_fini(sep); cv_destroy(&(sep->se_init_kv)); mutex_destroy(&(sep->se_lock)); sep->se_index = 0; kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); } int sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp) { sfxge_evq_t *sep = sp->s_sep[evq]; sfxge_txq_t **stpp; unsigned int label; int rc; mutex_enter(&(sep->se_lock)); if (stp == NULL || labelp == NULL) { rc = EINVAL; goto fail1; } stpp = NULL; for (label = 0; label < SFXGE_TX_NLABELS; label++) { if (sep->se_label_stp[label] == stp) { rc = EEXIST; goto fail2; } if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) { stpp = &sep->se_label_stp[label]; } } if (stpp == NULL) { rc = ENOSPC; goto fail3; } *stpp = stp; label = stpp - sep->se_label_stp; ASSERT3U(label, <, SFXGE_TX_NLABELS); *labelp = label; mutex_exit(&(sep->se_lock)); return (0); fail3: DTRACE_PROBE(fail3); fail2: DTRACE_PROBE(fail2); fail1: DTRACE_PROBE1(fail1, int, rc); mutex_exit(&(sep->se_lock)); return (rc); } int sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp, unsigned int label) { sfxge_evq_t *sep = sp->s_sep[evq]; int rc; mutex_enter(&(sep->se_lock)); if (stp == NULL || label > SFXGE_TX_NLABELS) { rc = EINVAL; goto fail1; } if (sep->se_label_stp[label] != stp) { rc = EINVAL; goto fail2; } sep->se_label_stp[label] = NULL; mutex_exit(&(sep->se_lock)); return (0); fail2: DTRACE_PROBE(fail2); fail1: DTRACE_PROBE1(fail1, int, rc); mutex_exit(&(sep->se_lock)); return (rc); } static kmem_cache_t * sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname, int (*ctor)(void *, void *, int), void (*dtor)(void *, void *)) { char name[MAXNAMELEN]; kmem_cache_t *eqc; (void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache", ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname); eqc = kmem_cache_create(name, sizeof (sfxge_evq_t), SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0); ASSERT(eqc != NULL); return (eqc); } int sfxge_ev_init(sfxge_t *sp) { sfxge_intr_t *sip = &(sp->s_intr); unsigned int evq0_size; unsigned int evqX_size; unsigned int ev_batch; int index; int rc; ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); /* * Must account for RXQ, TXQ(s); MCDI not event completed at present * Note that common code does not completely fill descriptor queues */ evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS; evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */ evq0_size += SFXGE_TX_NDESCS; /* no checksums */ ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS); ASSERT3U(evq0_size, >, evqX_size); if (evq0_size > EFX_EVQ_MAXNEVS) { rc = EINVAL; goto fail1; } sp->s_evq0_size = pow2_ge(evq0_size); sp->s_evqX_size = pow2_ge(evqX_size); /* Read driver parameters */ sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION); ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip, DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH); /* * It is slightly peverse to have a cache for one item. But it allows * for simple alignment control without increasing the allocation size */ sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor, sfxge_ev_q0dtor); sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor, sfxge_ev_qXdtor); /* Initialize the event queue(s) */ for (index = 0; index < sip->si_nalloc; index++) { if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0) goto fail2; } return (0); fail2: DTRACE_PROBE(fail2); while (--index >= 0) sfxge_ev_qfini(sp, index); sp->s_ev_moderation = 0; fail1: DTRACE_PROBE1(fail1, int, rc); kmem_cache_destroy(sp->s_eqXc); kmem_cache_destroy(sp->s_eq0c); sp->s_eqXc = NULL; sp->s_eq0c = NULL; return (rc); } int sfxge_ev_start(sfxge_t *sp) { sfxge_intr_t *sip = &(sp->s_intr); efx_nic_t *enp = sp->s_enp; int index; int rc; ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED); /* Initialize the event module */ if ((rc = efx_ev_init(enp)) != 0) goto fail1; /* Start the event queues */ for (index = 0; index < sip->si_nalloc; index++) { if ((rc = sfxge_ev_qstart(sp, index)) != 0) goto fail2; } return (0); fail2: DTRACE_PROBE(fail2); /* Stop the event queue(s) */ while (--index >= 0) sfxge_ev_qstop(sp, index); /* Tear down the event module */ efx_ev_fini(enp); fail1: DTRACE_PROBE1(fail1, int, rc); return (rc); } void sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp) { *usp = sp->s_ev_moderation; } int sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us) { sfxge_intr_t *sip = &(sp->s_intr); int index; int rc; if (sip->si_state != SFXGE_INTR_STARTED) return (ENODEV); for (index = 0; index < sip->si_nalloc; index++) { if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0) goto fail1; } sp->s_ev_moderation = us; return (0); fail1: DTRACE_PROBE1(fail1, int, rc); /* The only error path is if the value to set to is invalid. */ ASSERT3U(index, ==, 0); return (rc); } void sfxge_ev_stop(sfxge_t *sp) { sfxge_intr_t *sip = &(sp->s_intr); efx_nic_t *enp = sp->s_enp; int index; ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED); /* Stop the event queue(s) */ index = sip->si_nalloc; while (--index >= 0) sfxge_ev_qstop(sp, index); /* Tear down the event module */ efx_ev_fini(enp); } void sfxge_ev_fini(sfxge_t *sp) { sfxge_intr_t *sip = &(sp->s_intr); int index; ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED); sp->s_ev_moderation = 0; /* Tear down the event queue(s) */ index = sip->si_nalloc; while (--index >= 0) sfxge_ev_qfini(sp, index); kmem_cache_destroy(sp->s_eqXc); kmem_cache_destroy(sp->s_eq0c); sp->s_eqXc = NULL; sp->s_eq0c = NULL; }