xref: /illumos-gate/usr/src/uts/common/io/sfxge/sfxge_ev.c (revision 49ef7e06)

/*
 * Copyright (c) 2008-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/cpu.h>
#include <sys/pghw.h>

#include "sfxge.h"

#include "efx.h"


/* Timeout to wait for DRIVER_EV_START event at EVQ startup */
#define	SFXGE_EV_QSTART_TIMEOUT_USEC	(2000000)


/* Event queue DMA attributes */
static ddi_device_acc_attr_t sfxge_evq_devacc = {

	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
};

static ddi_dma_attr_t sfxge_evq_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version	*/
	0,			/* dma_attr_addr_lo	*/
	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
	0xffffffffffffffffull,	/* dma_attr_count_max	*/
	EFX_BUF_SIZE,		/* dma_attr_align	*/
	0xffffffff,		/* dma_attr_burstsizes	*/
	1,			/* dma_attr_minxfer	*/
	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
	0xffffffffffffffffull,	/* dma_attr_seg		*/
	1,			/* dma_attr_sgllen	*/
	1,			/* dma_attr_granular	*/
	0			/* dma_attr_flags	*/
};

static int
_sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size)
{
	efsys_mem_t *esmp = &(sep->se_mem);
	sfxge_dma_buffer_attr_t dma_attr;
	int rc;

	/* Compile-time structure layout checks */
	EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <=
	    sizeof (sep->__se_u1.__se_pad));
	EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <=
	    sizeof (sep->__se_u2.__se_pad));
	EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <=
	    sizeof (sep->__se_u3.__se_pad));

	bzero(sep, sizeof (sfxge_evq_t));

	sep->se_sp = sp;

	dma_attr.sdba_dip	 = sp->s_dip;
	dma_attr.sdba_dattrp	 = &sfxge_evq_dma_attr;
	dma_attr.sdba_callback	 = (kmflags == KM_SLEEP) ?
	    DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
	dma_attr.sdba_length	 = EFX_EVQ_SIZE(evq_size);
	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
	dma_attr.sdba_devaccp	 = &sfxge_evq_devacc;
	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
	dma_attr.sdba_maxcookies = 1;
	dma_attr.sdba_zeroinit	 = B_FALSE;

	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
		goto fail1;

	/* Allocate some buffer table entries */
	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size),
	    &(sep->se_id))) != 0)
		goto fail2;

	sep->se_stpp = &(sep->se_stp);

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	/* Tear down DMA setup */
	esmp->esm_addr = 0;
	sfxge_dma_buffer_destroy(esmp);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	sep->se_sp = NULL;

	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);

	return (-1);
}

static int
sfxge_ev_q0ctor(void *buf, void *arg, int kmflags)
{
	sfxge_evq_t *sep = buf;
	sfxge_t *sp = arg;
	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size));
}

static int
sfxge_ev_qXctor(void *buf, void *arg, int kmflags)
{
	sfxge_evq_t *sep = buf;
	sfxge_t *sp = arg;
	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size));
}
static void
_sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size)
{
	efsys_mem_t *esmp = &(sep->se_mem);
	ASSERT3P(sep->se_sp, ==, sp);
	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
	sep->se_stpp = NULL;

	/* Free the buffer table entries */
	sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
	sep->se_id = 0;

	/* Tear down DMA setup */
	sfxge_dma_buffer_destroy(esmp);

	sep->se_sp = NULL;

	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
}

static void
sfxge_ev_q0dtor(void *buf, void *arg)
{
	sfxge_evq_t *sep = buf;
	sfxge_t *sp = arg;
	_sfxge_ev_qdtor(sp, sep, sp->s_evq0_size);
}

static void
sfxge_ev_qXdtor(void *buf, void *arg)
{
	sfxge_evq_t *sep = buf;
	sfxge_t *sp = arg;
	_sfxge_ev_qdtor(sp, sep, sp->s_evqX_size);
}

static boolean_t
sfxge_ev_initialized(void *arg)
{
	sfxge_evq_t *sep = arg;

	ASSERT(mutex_owned(&(sep->se_lock)));

	/* Init done events may be duplicated on 7xxx (see SFCbug31631) */
	if (sep->se_state == SFXGE_EVQ_STARTED)
		goto done;

	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING);
	sep->se_state = SFXGE_EVQ_STARTED;

	cv_broadcast(&(sep->se_init_kv));

done:
	return (B_FALSE);
}

static void
sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop)
{
	sfxge_t *sp = sep->se_sp;
	unsigned int index = sep->se_index;
	sfxge_rxq_t *srp = sp->s_srp[index];
	sfxge_txq_t *stp;

	if ((stp = sep->se_stp) != NULL) {
		sep->se_stp = NULL;
		sep->se_stpp = &(sep->se_stp);

		do {
			sfxge_txq_t *next;

			next = stp->st_next;
			stp->st_next = NULL;

			ASSERT3U(stp->st_evq, ==, index);

			if (stp->st_pending != stp->st_completed)
				sfxge_tx_qcomplete(stp);

			stp = next;
		} while (stp != NULL);
	}

	if (srp != NULL) {
		if (srp->sr_pending != srp->sr_completed)
			sfxge_rx_qcomplete(srp, eop);
	}
}

static boolean_t
sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
    uint16_t flags)
{
	sfxge_evq_t *sep = arg;
	sfxge_t *sp = sep->se_sp;
	sfxge_rxq_t *srp;
	sfxge_rx_packet_t *srpp;
	unsigned int prefetch;
	unsigned int stop;
	unsigned int delta;

	ASSERT(mutex_owned(&(sep->se_lock)));

	if (sep->se_exception)
		goto done;

	srp = sp->s_srp[label];
	if (srp == NULL)
		goto done;

	ASSERT3U(sep->se_index, ==, srp->sr_index);
	ASSERT3U(id, <, sp->s_rxq_size);

	/*
	 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset
	 * So the return missing srp->sr_pending increase is safe
	 */
	if (srp->sr_state != SFXGE_RXQ_STARTED)
		goto done;

	stop = (id + 1) & (sp->s_rxq_size - 1);
	id = srp->sr_pending & (sp->s_rxq_size - 1);

	delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop);
	srp->sr_pending += delta;

	if (delta != 1) {
		if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) ||
		    (delta == 0) ||
		    (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) {
			/*
			 * FIXME: This does not take into account scatter
			 * aborts.  See Bug40811
			 */
			sep->se_exception = B_TRUE;

			DTRACE_PROBE(restart_ev_rx_id);
			/* sfxge_evq_t->se_lock held */
			(void) sfxge_restart_dispatch(sp, DDI_SLEEP,
			    SFXGE_HW_ERR, "Out of order RX event", delta);

			goto done;
		}
	}

	prefetch = (id + 4) & (sp->s_rxq_size - 1);
	if ((srpp = srp->sr_srpp[prefetch]) != NULL)
		prefetch_read_many(srpp);

	srpp = srp->sr_srpp[id];
	ASSERT(srpp != NULL);
	prefetch_read_many(srpp->srp_mp);

	for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) {
		srpp = srp->sr_srpp[id];
		ASSERT(srpp != NULL);

		ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD);
		srpp->srp_flags = flags;

		ASSERT3U(size, <, (1 << 16));
		srpp->srp_size = (uint16_t)size;
	}

	sep->se_rx++;

	DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index,
	    unsigned int, srp->sr_added - srp->sr_pending);

	if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH)
		sfxge_ev_qcomplete(sep, B_FALSE);

done:
	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
	return (sep->se_rx >= sep->se_ev_batch);
}

static boolean_t
sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
{
	sfxge_evq_t *sep = arg;
	sfxge_t *sp = sep->se_sp;

	_NOTE(ARGUNUSED(code))
	_NOTE(ARGUNUSED(data))

	ASSERT(mutex_owned(&(sep->se_lock)));
	sep->se_exception = B_TRUE;

	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {

		DTRACE_PROBE(restart_ev_exception);

		/* sfxge_evq_t->se_lock held */
		(void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
		    "Unknown EV", code);
	}

	return (B_FALSE);
}

static boolean_t
sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
{
	sfxge_evq_t *sep_targetq, *sep = arg;
	sfxge_t *sp = sep->se_sp;
	sfxge_rxq_t *srp;
	unsigned int index;
	unsigned int label;
	uint16_t magic;

	ASSERT(mutex_owned(&(sep->se_lock)));

	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
	srp = sp->s_srp[rxq_index];
	if (srp == NULL)
		goto done;

	/* Process right now if it is the correct event queue */
	index = srp->sr_index;
	if (index == sep->se_index) {
		sfxge_rx_qflush_done(srp);
		goto done;
	}

	/* Resend a software event on the correct queue */
	sep_targetq = sp->s_sep[index];

	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
		goto done; /* TBD: state test not under the lock */

	label = rxq_index;
	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;

	efx_ev_qpost(sep_targetq->se_eep, magic);

done:
	return (B_FALSE);
}

static boolean_t
sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
{
	sfxge_evq_t *sep_targetq, *sep = arg;
	sfxge_t *sp = sep->se_sp;
	sfxge_rxq_t *srp;
	unsigned int index;
	unsigned int label;
	uint16_t magic;

	ASSERT(mutex_owned(&(sep->se_lock)));

	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
	srp = sp->s_srp[rxq_index];
	if (srp == NULL)
		goto done;

	/* Process right now if it is the correct event queue */
	index = srp->sr_index;
	if (index == sep->se_index) {
		sfxge_rx_qflush_failed(srp);
		goto done;
	}

	/* Resend a software event on the correct queue */
	sep_targetq = sp->s_sep[index];

	label = rxq_index;
	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;

	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
		goto done; /* TBD: state test not under the lock */

	efx_ev_qpost(sep_targetq->se_eep, magic);

done:
	return (B_FALSE);
}

static boolean_t
sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
{
	sfxge_evq_t *sep = arg;
	sfxge_txq_t *stp;
	unsigned int stop;
	unsigned int delta;

	ASSERT(mutex_owned(&(sep->se_lock)));

	stp = sep->se_label_stp[label];
	if (stp == NULL)
		goto done;

	if (stp->st_state != SFXGE_TXQ_STARTED)
		goto done;

	ASSERT3U(sep->se_index, ==, stp->st_evq);

	stop = (id + 1) & (SFXGE_TX_NDESCS - 1);
	id = stp->st_pending & (SFXGE_TX_NDESCS - 1);

	delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop);
	stp->st_pending += delta;

	sep->se_tx++;

	if (stp->st_next == NULL &&
	    sep->se_stpp != &(stp->st_next)) {
		*(sep->se_stpp) = stp;
		sep->se_stpp = &(stp->st_next);
	}

	DTRACE_PROBE2(qlevel, unsigned int, stp->st_index,
	    unsigned int, stp->st_added - stp->st_pending);

	if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH)
		sfxge_tx_qcomplete(stp);

done:
	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
	return (sep->se_tx >= sep->se_ev_batch);
}

static boolean_t
sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
{
	sfxge_evq_t *sep = arg;
	sfxge_t *sp = sep->se_sp;
	sfxge_txq_t *stp;
	unsigned int evq;
	unsigned int label;
	uint16_t magic;

	ASSERT(mutex_owned(&(sep->se_lock)));

	/* Ensure TXQ exists, as events may arrive after TXQ was destroyed */
	stp = sp->s_stp[txq_index];
	if (stp == NULL)
		goto done;

	/* Process right now if it is the correct event queue */
	evq = stp->st_evq;
	if (evq == sep->se_index) {
		sfxge_tx_qflush_done(stp);
		goto done;
	}

	/* Resend a software event on the correct queue */
	sep = sp->s_sep[evq];

	label = stp->st_label;

	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;

	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
	efx_ev_qpost(sep->se_eep, magic);

done:
	return (B_FALSE);
}

static boolean_t
sfxge_ev_software(void *arg, uint16_t magic)
{
	sfxge_evq_t *sep = arg;
	sfxge_t *sp = sep->se_sp;
	dev_info_t *dip = sp->s_dip;
	unsigned int label;

	ASSERT(mutex_owned(&(sep->se_lock)));

	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
	    FSF_AZ_RX_EV_Q_LABEL_WIDTH);
	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
	    FSF_AZ_TX_EV_Q_LABEL_WIDTH);

	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;

	switch (magic) {
	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
		sfxge_rxq_t *srp = sp->s_srp[label];

		if (srp != NULL) {
			ASSERT3U(sep->se_index, ==, srp->sr_index);

			sfxge_rx_qflush_done(srp);
		}
		break;
	}
	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
		sfxge_rxq_t *srp = sp->s_srp[label];

		if (srp != NULL) {
			ASSERT3U(sep->se_index, ==, srp->sr_index);

			sfxge_rx_qflush_failed(srp);
		}
		break;
	}
	case SFXGE_MAGIC_RX_QFPP_TRIM: {
		sfxge_rxq_t *srp = sp->s_srp[label];

		if (srp != NULL) {
			ASSERT3U(sep->se_index, ==, srp->sr_index);

			sfxge_rx_qfpp_trim(srp);
		}
		break;
	}
	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
		sfxge_txq_t *stp = sep->se_label_stp[label];

		if (stp != NULL) {
			ASSERT3U(sep->se_index, ==, stp->st_evq);

			sfxge_tx_qflush_done(stp);
		}
		break;
	}
	default:
		dev_err(dip, CE_NOTE,
		    SFXGE_CMN_ERR "unknown software event 0x%x", magic);
		break;
	}

	return (B_FALSE);
}

static boolean_t
sfxge_ev_sram(void *arg, uint32_t code)
{
	_NOTE(ARGUNUSED(arg))

	switch (code) {
	case EFX_SRAM_UPDATE:
		DTRACE_PROBE(sram_update);
		break;

	case EFX_SRAM_CLEAR:
		DTRACE_PROBE(sram_clear);
		break;

	case EFX_SRAM_ILLEGAL_CLEAR:
		DTRACE_PROBE(sram_illegal_clear);
		break;

	default:
		ASSERT(B_FALSE);
		break;
	}

	return (B_FALSE);
}

static boolean_t
sfxge_ev_timer(void *arg, uint32_t index)
{
	_NOTE(ARGUNUSED(arg, index))

	return (B_FALSE);
}

static boolean_t
sfxge_ev_wake_up(void *arg, uint32_t index)
{
	_NOTE(ARGUNUSED(arg, index))

	return (B_FALSE);
}

static boolean_t
sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
{
	sfxge_evq_t *sep = arg;
	sfxge_t *sp = sep->se_sp;

	sfxge_mac_link_update(sp, link_mode);

	return (B_FALSE);
}

static int
sfxge_ev_kstat_update(kstat_t *ksp, int rw)
{
	sfxge_evq_t *sep = ksp->ks_private;
	kstat_named_t *knp;
	int rc;

	if (rw != KSTAT_READ) {
		rc = EACCES;
		goto fail1;
	}

	ASSERT(mutex_owned(&(sep->se_lock)));

	if (sep->se_state != SFXGE_EVQ_STARTED)
		goto done;

	efx_ev_qstats_update(sep->se_eep, sep->se_stat);

	knp = ksp->ks_data;
	knp += EV_NQSTATS;

	knp->value.ui64 = sep->se_cpu_id;

done:
	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_ev_kstat_init(sfxge_evq_t *sep)
{
	sfxge_t *sp = sep->se_sp;
	unsigned int index = sep->se_index;
	dev_info_t *dip = sp->s_dip;
	kstat_t *ksp;
	kstat_named_t *knp;
	char name[MAXNAMELEN];
	unsigned int id;
	int rc;

	/* Determine the name */
	(void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d",
	    ddi_driver_name(dip), index);

	/* Create the set */
	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
	    ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
	    EV_NQSTATS + 1, 0)) == NULL) {
		rc = ENOMEM;
		goto fail1;
	}

	sep->se_ksp = ksp;

	ksp->ks_update = sfxge_ev_kstat_update;
	ksp->ks_private = sep;
	ksp->ks_lock = &(sep->se_lock);

	/* Initialise the named stats */
	sep->se_stat = knp = ksp->ks_data;
	for (id = 0; id < EV_NQSTATS; id++) {
		kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id),
		    KSTAT_DATA_UINT64);
		knp++;
	}

	kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64);

	kstat_install(ksp);
	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static void
sfxge_ev_kstat_fini(sfxge_evq_t *sep)
{
	/* Destroy the set */
	kstat_delete(sep->se_ksp);
	sep->se_ksp = NULL;
	sep->se_stat = NULL;
}

inline unsigned pow2_ge(unsigned int n) {
	unsigned int order = 0;
	ASSERT3U(n, >, 0);
	while ((1ul << order) < n) ++order;
	return (1ul << (order));
}

static int
sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch)
{
	sfxge_evq_t *sep;
	int rc;

	ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);

	sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP);
	if (sep == NULL) {
		rc = ENOMEM;
		goto fail1;
	}
	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED);

	sep->se_index = index;

	mutex_init(&(sep->se_lock), NULL,
	    MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri));

	cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL);

	/* Initialize the statistics */
	if ((rc = sfxge_ev_kstat_init(sep)) != 0)
		goto fail2;

	sep->se_state = SFXGE_EVQ_INITIALIZED;
	sep->se_ev_batch = (uint16_t)ev_batch;
	sp->s_sep[index] = sep;

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	sep->se_index = 0;

	cv_destroy(&(sep->se_init_kv));
	mutex_destroy(&(sep->se_lock));

	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_ev_qstart(sfxge_t *sp, unsigned int index)
{
	sfxge_evq_t *sep = sp->s_sep[index];
	sfxge_intr_t *sip = &(sp->s_intr);
	efx_nic_t *enp = sp->s_enp;
	efx_ev_callbacks_t *eecp;
	efsys_mem_t *esmp;
	clock_t timeout;
	int rc;
	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;

	mutex_enter(&(sep->se_lock));
	esmp = &(sep->se_mem);

	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);

	/* Set the memory to all ones */
	(void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size));

	/* Program the buffer table */
	if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp,
	    EFX_EVQ_NBUFS(evq_size))) != 0)
		goto fail1;

	/* Set up the event callbacks */
	eecp = &(sep->se_eec);
	eecp->eec_initialized = sfxge_ev_initialized;
	eecp->eec_rx = sfxge_ev_rx;
	eecp->eec_tx = sfxge_ev_tx;
	eecp->eec_exception = sfxge_ev_exception;
	eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done;
	eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed;
	eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done;
	eecp->eec_software = sfxge_ev_software;
	eecp->eec_sram = sfxge_ev_sram;
	eecp->eec_wake_up = sfxge_ev_wake_up;
	eecp->eec_timer = sfxge_ev_timer;
	eecp->eec_link_change = sfxge_ev_link_change;

	/* Create the event queue */
	if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id,
	    &(sep->se_eep))) != 0)
		goto fail2;

	/* Set the default moderation */
	if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0)
		goto fail3;

	/* Check that interrupts are enabled at the NIC */
	if (sip->si_state != SFXGE_INTR_STARTED) {
		rc = EINVAL;
		goto fail4;
	}

	sep->se_state = SFXGE_EVQ_STARTING;

	/* Prime the event queue for interrupts */
	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
		goto fail5;

	/* Wait for the initialization event */
	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC);
	while (sep->se_state != SFXGE_EVQ_STARTED) {
		if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock),
		    timeout) < 0) {
			/* Timeout waiting for initialization */
			dev_info_t *dip = sp->s_dip;

			DTRACE_PROBE(timeout);
			dev_err(dip, CE_NOTE,
			    SFXGE_CMN_ERR "evq[%d] qstart timeout", index);

			rc = ETIMEDOUT;
			goto fail6;
		}
	}

	mutex_exit(&(sep->se_lock));
	return (0);

fail6:
	DTRACE_PROBE(fail6);

fail5:
	DTRACE_PROBE(fail5);

	sep->se_state = SFXGE_EVQ_INITIALIZED;

fail4:
	DTRACE_PROBE(fail4);

fail3:
	DTRACE_PROBE(fail3);

	/* Destroy the event queue */
	efx_ev_qdestroy(sep->se_eep);
	sep->se_eep = NULL;

fail2:
	DTRACE_PROBE(fail2);

	/* Zero out the event handlers */
	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));

	/* Clear entries from the buffer table */
	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(sep->se_lock));

	return (rc);
}

int
sfxge_ev_qpoll(sfxge_t *sp, unsigned int index)
{
	sfxge_evq_t *sep = sp->s_sep[index];
	processorid_t cpu_id;
	int rc;
	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;

	mutex_enter(&(sep->se_lock));

	if (sep->se_state != SFXGE_EVQ_STARTING &&
	    sep->se_state != SFXGE_EVQ_STARTED) {
		rc = EINVAL;
		goto fail1;
	}

	/* Make sure the CPU information is up to date */
	cpu_id = CPU->cpu_id;

	if (cpu_id != sep->se_cpu_id) {
		sep->se_cpu_id = cpu_id;

		/* sfxge_evq_t->se_lock held */
		(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
		    DDI_NOSLEEP);
	}

	/* Synchronize the DMA memory for reading */
	(void) ddi_dma_sync(sep->se_mem.esm_dma_handle,
	    0,
	    EFX_EVQ_SIZE(evq_size),
	    DDI_DMA_SYNC_FORKERNEL);

	ASSERT3U(sep->se_rx, ==, 0);
	ASSERT3U(sep->se_tx, ==, 0);
	ASSERT3P(sep->se_stp, ==, NULL);
	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));

	/* Poll the queue */
	efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec),
	    sep);

	sep->se_rx = 0;
	sep->se_tx = 0;

	/* Perform any pending completion processing */
	sfxge_ev_qcomplete(sep, B_TRUE);

	/* Re-prime the event queue for interrupts */
	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
		goto fail2;

	mutex_exit(&(sep->se_lock));

	return (0);

fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(sep->se_lock));

	return (rc);
}

int
sfxge_ev_qprime(sfxge_t *sp, unsigned int index)
{
	sfxge_evq_t *sep = sp->s_sep[index];
	int rc;

	mutex_enter(&(sep->se_lock));

	if (sep->se_state != SFXGE_EVQ_STARTING &&
	    sep->se_state != SFXGE_EVQ_STARTED) {
		rc = EINVAL;
		goto fail1;
	}

	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
		goto fail2;

	mutex_exit(&(sep->se_lock));

	return (0);

fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(sep->se_lock));

	return (rc);
}


int
sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us)
{
	sfxge_evq_t *sep = sp->s_sep[index];
	efx_evq_t *eep = sep->se_eep;

	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);

	return (efx_ev_qmoderate(eep, us));
}

static void
sfxge_ev_qstop(sfxge_t *sp, unsigned int index)
{
	sfxge_evq_t *sep = sp->s_sep[index];
	uint16_t evq_size;

	mutex_enter(&(sep->se_lock));
	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
	sep->se_state = SFXGE_EVQ_INITIALIZED;
	evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;

	/* Clear the CPU information */
	sep->se_cpu_id = 0;

	/* Clear the event count */
	sep->se_count = 0;

	/* Reset the exception flag */
	sep->se_exception = B_FALSE;

	/* Destroy the event queue */
	efx_ev_qdestroy(sep->se_eep);
	sep->se_eep = NULL;

	mutex_exit(&(sep->se_lock));

	/* Zero out the event handlers */
	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));

	/* Clear entries from the buffer table */
	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
}

static void
sfxge_ev_qfini(sfxge_t *sp, unsigned int index)
{
	sfxge_evq_t *sep = sp->s_sep[index];

	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);

	sp->s_sep[index] = NULL;
	sep->se_state = SFXGE_EVQ_UNINITIALIZED;

	/* Tear down the statistics */
	sfxge_ev_kstat_fini(sep);

	cv_destroy(&(sep->se_init_kv));
	mutex_destroy(&(sep->se_lock));

	sep->se_index = 0;

	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
}

int
sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
    unsigned int *labelp)
{
	sfxge_evq_t *sep = sp->s_sep[evq];
	sfxge_txq_t **stpp;
	unsigned int label;
	int rc;

	mutex_enter(&(sep->se_lock));

	if (stp == NULL || labelp == NULL) {
		rc = EINVAL;
		goto fail1;
	}

	stpp = NULL;
	for (label = 0; label < SFXGE_TX_NLABELS; label++) {
		if (sep->se_label_stp[label] == stp) {
			rc = EEXIST;
			goto fail2;
		}
		if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) {
			stpp = &sep->se_label_stp[label];
		}
	}
	if (stpp == NULL) {
		rc = ENOSPC;
		goto fail3;
	}
	*stpp = stp;
	label = stpp - sep->se_label_stp;

	ASSERT3U(label, <, SFXGE_TX_NLABELS);
	*labelp = label;

	mutex_exit(&(sep->se_lock));
	return (0);

fail3:
	DTRACE_PROBE(fail3);
fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(sep->se_lock));

	return (rc);
}


int
sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
    unsigned int label)
{
	sfxge_evq_t *sep = sp->s_sep[evq];
	int rc;

	mutex_enter(&(sep->se_lock));

	if (stp == NULL || label > SFXGE_TX_NLABELS) {
		rc = EINVAL;
		goto fail1;
	}

	if (sep->se_label_stp[label] != stp) {
		rc = EINVAL;
		goto fail2;
	}
	sep->se_label_stp[label] = NULL;

	mutex_exit(&(sep->se_lock));

	return (0);

fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(sep->se_lock));

	return (rc);
}


static 	kmem_cache_t *
sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname,
    int (*ctor)(void *, void *, int), void (*dtor)(void *, void *))
{
	char name[MAXNAMELEN];
	kmem_cache_t *eqc;

	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache",
	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname);

	eqc = kmem_cache_create(name, sizeof (sfxge_evq_t),
	    SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0);
	ASSERT(eqc != NULL);
	return (eqc);
}

int
sfxge_ev_init(sfxge_t *sp)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	unsigned int evq0_size;
	unsigned int evqX_size;
	unsigned int ev_batch;
	int index;
	int rc;

	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);

	/*
	 * Must account for RXQ, TXQ(s); MCDI not event completed at present
	 * Note that common code does not completely fill descriptor queues
	 */
	evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS;
	evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */
	evq0_size += SFXGE_TX_NDESCS; /* no checksums */

	ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS);
	ASSERT3U(evq0_size, >, evqX_size);

	if (evq0_size > EFX_EVQ_MAXNEVS) {
		rc = EINVAL;
		goto fail1;
	}

	sp->s_evq0_size = pow2_ge(evq0_size);
	sp->s_evqX_size = pow2_ge(evqX_size);

	/* Read driver parameters */
	sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
	    DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION);

	ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
	    DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH);

	/*
	 * It is slightly peverse to have a cache for one item. But it allows
	 * for simple alignment control without increasing the allocation size
	 */
	sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor,
	    sfxge_ev_q0dtor);
	sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor,
	    sfxge_ev_qXdtor);

	/* Initialize the event queue(s) */
	for (index = 0; index < sip->si_nalloc; index++) {
		if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0)
			goto fail2;
	}

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	while (--index >= 0)
		sfxge_ev_qfini(sp, index);
	sp->s_ev_moderation = 0;

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	kmem_cache_destroy(sp->s_eqXc);
	kmem_cache_destroy(sp->s_eq0c);
	sp->s_eqXc = NULL;
	sp->s_eq0c = NULL;

	return (rc);
}

int
sfxge_ev_start(sfxge_t *sp)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	efx_nic_t *enp = sp->s_enp;
	int index;
	int rc;

	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);

	/* Initialize the event module */
	if ((rc = efx_ev_init(enp)) != 0)
		goto fail1;

	/* Start the event queues */
	for (index = 0; index < sip->si_nalloc; index++) {
		if ((rc = sfxge_ev_qstart(sp, index)) != 0)
			goto fail2;
	}

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	/* Stop the event queue(s) */
	while (--index >= 0)
		sfxge_ev_qstop(sp, index);

	/* Tear down the event module */
	efx_ev_fini(enp);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

void
sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp)
{
	*usp = sp->s_ev_moderation;
}

int
sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	int index;
	int rc;

	if (sip->si_state != SFXGE_INTR_STARTED)
		return (ENODEV);

	for (index = 0; index < sip->si_nalloc; index++) {
		if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0)
			goto fail1;
	}

	sp->s_ev_moderation = us;
	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	/*  The only error path is if the value to set to is invalid. */
	ASSERT3U(index, ==, 0);

	return (rc);
}

void
sfxge_ev_stop(sfxge_t *sp)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	efx_nic_t *enp = sp->s_enp;
	int index;

	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);

	/* Stop the event queue(s) */
	index = sip->si_nalloc;
	while (--index >= 0)
		sfxge_ev_qstop(sp, index);

	/* Tear down the event module */
	efx_ev_fini(enp);
}

void
sfxge_ev_fini(sfxge_t *sp)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	int index;

	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);

	sp->s_ev_moderation = 0;

	/* Tear down the event queue(s) */
	index = sip->si_nalloc;
	while (--index >= 0)
		sfxge_ev_qfini(sp, index);

	kmem_cache_destroy(sp->s_eqXc);
	kmem_cache_destroy(sp->s_eq0c);
	sp->s_eqXc = NULL;
	sp->s_eq0c = NULL;
}