149ef7e0Garrett D'Amore/*
249ef7e0Garrett D'Amore * Copyright (c) 2007-2015 Solarflare Communications Inc.
349ef7e0Garrett D'Amore * All rights reserved.
449ef7e0Garrett D'Amore *
549ef7e0Garrett D'Amore * Redistribution and use in source and binary forms, with or without
649ef7e0Garrett D'Amore * modification, are permitted provided that the following conditions are met:
749ef7e0Garrett D'Amore *
849ef7e0Garrett D'Amore * 1. Redistributions of source code must retain the above copyright notice,
949ef7e0Garrett D'Amore *    this list of conditions and the following disclaimer.
1049ef7e0Garrett D'Amore * 2. Redistributions in binary form must reproduce the above copyright notice,
1149ef7e0Garrett D'Amore *    this list of conditions and the following disclaimer in the documentation
1249ef7e0Garrett D'Amore *    and/or other materials provided with the distribution.
1349ef7e0Garrett D'Amore *
1449ef7e0Garrett D'Amore * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1549ef7e0Garrett D'Amore * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
1649ef7e0Garrett D'Amore * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
1749ef7e0Garrett D'Amore * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
1849ef7e0Garrett D'Amore * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
1949ef7e0Garrett D'Amore * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
2049ef7e0Garrett D'Amore * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
2149ef7e0Garrett D'Amore * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
2249ef7e0Garrett D'Amore * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
2349ef7e0Garrett D'Amore * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
2449ef7e0Garrett D'Amore * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2549ef7e0Garrett D'Amore *
2649ef7e0Garrett D'Amore * The views and conclusions contained in the software and documentation are
2749ef7e0Garrett D'Amore * those of the authors and should not be interpreted as representing official
2849ef7e0Garrett D'Amore * policies, either expressed or implied, of the FreeBSD Project.
2949ef7e0Garrett D'Amore */
3049ef7e0Garrett D'Amore
3149ef7e0Garrett D'Amore#include "efx.h"
3249ef7e0Garrett D'Amore#include "efx_impl.h"
3349ef7e0Garrett D'Amore#if EFSYS_OPT_MON_MCDI
3449ef7e0Garrett D'Amore#include "mcdi_mon.h"
3549ef7e0Garrett D'Amore#endif
3649ef7e0Garrett D'Amore
3749ef7e0Garrett D'Amore#if EFSYS_OPT_QSTATS
3849ef7e0Garrett D'Amore#define	EFX_EV_QSTAT_INCR(_eep, _stat)					\
3949ef7e0Garrett D'Amore	do {								\
4049ef7e0Garrett D'Amore		(_eep)->ee_stat[_stat]++;				\
4149ef7e0Garrett D'Amore	_NOTE(CONSTANTCONDITION)					\
4249ef7e0Garrett D'Amore	} while (B_FALSE)
4349ef7e0Garrett D'Amore#else
4449ef7e0Garrett D'Amore#define	EFX_EV_QSTAT_INCR(_eep, _stat)
4549ef7e0Garrett D'Amore#endif
4649ef7e0Garrett D'Amore
4749ef7e0Garrett D'Amore#define	EFX_EV_PRESENT(_qword)						\
4849ef7e0Garrett D'Amore	(EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff &&	\
4949ef7e0Garrett D'Amore	EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
5049ef7e0Garrett D'Amore
5149ef7e0Garrett D'Amore
5249ef7e0Garrett D'Amore
5349ef7e0Garrett D'Amore#if EFSYS_OPT_SIENA
5449ef7e0Garrett D'Amore
5549ef7e0Garrett D'Amorestatic	__checkReturn	efx_rc_t
5649ef7e0Garrett D'Amoresiena_ev_init(
5749ef7e0Garrett D'Amore	__in		efx_nic_t *enp);
5849ef7e0Garrett D'Amore
5949ef7e0Garrett D'Amorestatic			void
6049ef7e0Garrett D'Amoresiena_ev_fini(
6149ef7e0Garrett D'Amore	__in		efx_nic_t *enp);
6249ef7e0Garrett D'Amore
6349ef7e0Garrett D'Amorestatic	__checkReturn	efx_rc_t
6449ef7e0Garrett D'Amoresiena_ev_qcreate(
6549ef7e0Garrett D'Amore	__in		efx_nic_t *enp,
6649ef7e0Garrett D'Amore	__in		unsigned int index,
6749ef7e0Garrett D'Amore	__in		efsys_mem_t *esmp,
6849ef7e0Garrett D'Amore	__in		size_t n,
6949ef7e0Garrett D'Amore	__in		uint32_t id,
7049ef7e0Garrett D'Amore	__in		efx_evq_t *eep);
7149ef7e0Garrett D'Amore
7249ef7e0Garrett D'Amorestatic			void
7349ef7e0Garrett D'Amoresiena_ev_qdestroy(
7449ef7e0Garrett D'Amore	__in		efx_evq_t *eep);
7549ef7e0Garrett D'Amore
7649ef7e0Garrett D'Amorestatic	__checkReturn	efx_rc_t
7749ef7e0Garrett D'Amoresiena_ev_qprime(
7849ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
7949ef7e0Garrett D'Amore	__in		unsigned int count);
8049ef7e0Garrett D'Amore
8149ef7e0Garrett D'Amorestatic			void
8249ef7e0Garrett D'Amoresiena_ev_qpoll(
8349ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
8449ef7e0Garrett D'Amore	__inout		unsigned int *countp,
8549ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
8649ef7e0Garrett D'Amore	__in_opt	void *arg);
8749ef7e0Garrett D'Amore
8849ef7e0Garrett D'Amorestatic			void
8949ef7e0Garrett D'Amoresiena_ev_qpost(
9049ef7e0Garrett D'Amore	__in	efx_evq_t *eep,
9149ef7e0Garrett D'Amore	__in	uint16_t data);
9249ef7e0Garrett D'Amore
9349ef7e0Garrett D'Amorestatic	__checkReturn	efx_rc_t
9449ef7e0Garrett D'Amoresiena_ev_qmoderate(
9549ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
9649ef7e0Garrett D'Amore	__in		unsigned int us);
9749ef7e0Garrett D'Amore
9849ef7e0Garrett D'Amore#if EFSYS_OPT_QSTATS
9949ef7e0Garrett D'Amorestatic			void
10049ef7e0Garrett D'Amoresiena_ev_qstats_update(
10149ef7e0Garrett D'Amore	__in				efx_evq_t *eep,
10249ef7e0Garrett D'Amore	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat);
10349ef7e0Garrett D'Amore
10449ef7e0Garrett D'Amore#endif
10549ef7e0Garrett D'Amore
10649ef7e0Garrett D'Amore#endif /* EFSYS_OPT_SIENA */
10749ef7e0Garrett D'Amore
10849ef7e0Garrett D'Amore#if EFSYS_OPT_SIENA
10949ef7e0Garrett D'Amorestatic const efx_ev_ops_t	__efx_ev_siena_ops = {
11049ef7e0Garrett D'Amore	siena_ev_init,				/* eevo_init */
11149ef7e0Garrett D'Amore	siena_ev_fini,				/* eevo_fini */
11249ef7e0Garrett D'Amore	siena_ev_qcreate,			/* eevo_qcreate */
11349ef7e0Garrett D'Amore	siena_ev_qdestroy,			/* eevo_qdestroy */
11449ef7e0Garrett D'Amore	siena_ev_qprime,			/* eevo_qprime */
11549ef7e0Garrett D'Amore	siena_ev_qpost,				/* eevo_qpost */
11649ef7e0Garrett D'Amore	siena_ev_qmoderate,			/* eevo_qmoderate */
11749ef7e0Garrett D'Amore#if EFSYS_OPT_QSTATS
11849ef7e0Garrett D'Amore	siena_ev_qstats_update,			/* eevo_qstats_update */
11949ef7e0Garrett D'Amore#endif
12049ef7e0Garrett D'Amore};
12149ef7e0Garrett D'Amore#endif /* EFSYS_OPT_SIENA */
12249ef7e0Garrett D'Amore
12349ef7e0Garrett D'Amore#if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
12449ef7e0Garrett D'Amorestatic const efx_ev_ops_t	__efx_ev_ef10_ops = {
12549ef7e0Garrett D'Amore	ef10_ev_init,				/* eevo_init */
12649ef7e0Garrett D'Amore	ef10_ev_fini,				/* eevo_fini */
12749ef7e0Garrett D'Amore	ef10_ev_qcreate,			/* eevo_qcreate */
12849ef7e0Garrett D'Amore	ef10_ev_qdestroy,			/* eevo_qdestroy */
12949ef7e0Garrett D'Amore	ef10_ev_qprime,				/* eevo_qprime */
13049ef7e0Garrett D'Amore	ef10_ev_qpost,				/* eevo_qpost */
13149ef7e0Garrett D'Amore	ef10_ev_qmoderate,			/* eevo_qmoderate */
13249ef7e0Garrett D'Amore#if EFSYS_OPT_QSTATS
13349ef7e0Garrett D'Amore	ef10_ev_qstats_update,			/* eevo_qstats_update */
13449ef7e0Garrett D'Amore#endif
13549ef7e0Garrett D'Amore};
13649ef7e0Garrett D'Amore#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
13749ef7e0Garrett D'Amore
13849ef7e0Garrett D'Amore
13949ef7e0Garrett D'Amore	__checkReturn	efx_rc_t
14049ef7e0Garrett D'Amoreefx_ev_init(
14149ef7e0Garrett D'Amore	__in		efx_nic_t *enp)
14249ef7e0Garrett D'Amore{
14349ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop;
14449ef7e0Garrett D'Amore	efx_rc_t rc;
14549ef7e0Garrett D'Amore
14649ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
14749ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
14849ef7e0Garrett D'Amore
14949ef7e0Garrett D'Amore	if (enp->en_mod_flags & EFX_MOD_EV) {
15049ef7e0Garrett D'Amore		rc = EINVAL;
15149ef7e0Garrett D'Amore		goto fail1;
15249ef7e0Garrett D'Amore	}
15349ef7e0Garrett D'Amore
15449ef7e0Garrett D'Amore	switch (enp->en_family) {
15549ef7e0Garrett D'Amore#if EFSYS_OPT_SIENA
15649ef7e0Garrett D'Amore	case EFX_FAMILY_SIENA:
15749ef7e0Garrett D'Amore		eevop = &__efx_ev_siena_ops;
15849ef7e0Garrett D'Amore		break;
15949ef7e0Garrett D'Amore#endif /* EFSYS_OPT_SIENA */
16049ef7e0Garrett D'Amore
16149ef7e0Garrett D'Amore#if EFSYS_OPT_HUNTINGTON
16249ef7e0Garrett D'Amore	case EFX_FAMILY_HUNTINGTON:
16349ef7e0Garrett D'Amore		eevop = &__efx_ev_ef10_ops;
16449ef7e0Garrett D'Amore		break;
16549ef7e0Garrett D'Amore#endif /* EFSYS_OPT_HUNTINGTON */
16649ef7e0Garrett D'Amore
16749ef7e0Garrett D'Amore#if EFSYS_OPT_MEDFORD
16849ef7e0Garrett D'Amore	case EFX_FAMILY_MEDFORD:
16949ef7e0Garrett D'Amore		eevop = &__efx_ev_ef10_ops;
17049ef7e0Garrett D'Amore		break;
17149ef7e0Garrett D'Amore#endif /* EFSYS_OPT_MEDFORD */
17249ef7e0Garrett D'Amore
17349ef7e0Garrett D'Amore	default:
17449ef7e0Garrett D'Amore		EFSYS_ASSERT(0);
17549ef7e0Garrett D'Amore		rc = ENOTSUP;
17649ef7e0Garrett D'Amore		goto fail1;
17749ef7e0Garrett D'Amore	}
17849ef7e0Garrett D'Amore
17949ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
18049ef7e0Garrett D'Amore
18149ef7e0Garrett D'Amore	if ((rc = eevop->eevo_init(enp)) != 0)
18249ef7e0Garrett D'Amore		goto fail2;
18349ef7e0Garrett D'Amore
18449ef7e0Garrett D'Amore	enp->en_eevop = eevop;
18549ef7e0Garrett D'Amore	enp->en_mod_flags |= EFX_MOD_EV;
18649ef7e0Garrett D'Amore	return (0);
18749ef7e0Garrett D'Amore
18849ef7e0Garrett D'Amorefail2:
18949ef7e0Garrett D'Amore	EFSYS_PROBE(fail2);
19049ef7e0Garrett D'Amore
19149ef7e0Garrett D'Amorefail1:
19249ef7e0Garrett D'Amore	EFSYS_PROBE1(fail1, efx_rc_t, rc);
19349ef7e0Garrett D'Amore
19449ef7e0Garrett D'Amore	enp->en_eevop = NULL;
19549ef7e0Garrett D'Amore	enp->en_mod_flags &= ~EFX_MOD_EV;
19649ef7e0Garrett D'Amore	return (rc);
19749ef7e0Garrett D'Amore}
19849ef7e0Garrett D'Amore
19949ef7e0Garrett D'Amore		void
20049ef7e0Garrett D'Amoreefx_ev_fini(
20149ef7e0Garrett D'Amore	__in	efx_nic_t *enp)
20249ef7e0Garrett D'Amore{
20349ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
20449ef7e0Garrett D'Amore
20549ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
20649ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
20749ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
20849ef7e0Garrett D'Amore	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
20949ef7e0Garrett D'Amore	EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
21049ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
21149ef7e0Garrett D'Amore
21249ef7e0Garrett D'Amore	eevop->eevo_fini(enp);
21349ef7e0Garrett D'Amore
21449ef7e0Garrett D'Amore	enp->en_eevop = NULL;
21549ef7e0Garrett D'Amore	enp->en_mod_flags &= ~EFX_MOD_EV;
21649ef7e0Garrett D'Amore}
21749ef7e0Garrett D'Amore
21849ef7e0Garrett D'Amore
21949ef7e0Garrett D'Amore	__checkReturn	efx_rc_t
22049ef7e0Garrett D'Amoreefx_ev_qcreate(
22149ef7e0Garrett D'Amore	__in		efx_nic_t *enp,
22249ef7e0Garrett D'Amore	__in		unsigned int index,
22349ef7e0Garrett D'Amore	__in		efsys_mem_t *esmp,
22449ef7e0Garrett D'Amore	__in		size_t n,
22549ef7e0Garrett D'Amore	__in		uint32_t id,
22649ef7e0Garrett D'Amore	__deref_out	efx_evq_t **eepp)
22749ef7e0Garrett D'Amore{
22849ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
22949ef7e0Garrett D'Amore	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
23049ef7e0Garrett D'Amore	efx_evq_t *eep;
23149ef7e0Garrett D'Amore	efx_rc_t rc;
23249ef7e0Garrett D'Amore
23349ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
23449ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
23549ef7e0Garrett D'Amore
23649ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
23749ef7e0Garrett D'Amore
23849ef7e0Garrett D'Amore	/* Allocate an EVQ object */
23949ef7e0Garrett D'Amore	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
24049ef7e0Garrett D'Amore	if (eep == NULL) {
24149ef7e0Garrett D'Amore		rc = ENOMEM;
24249ef7e0Garrett D'Amore		goto fail1;
24349ef7e0Garrett D'Amore	}
24449ef7e0Garrett D'Amore
24549ef7e0Garrett D'Amore	eep->ee_magic = EFX_EVQ_MAGIC;
24649ef7e0Garrett D'Amore	eep->ee_enp = enp;
24749ef7e0Garrett D'Amore	eep->ee_index = index;
24849ef7e0Garrett D'Amore	eep->ee_mask = n - 1;
24949ef7e0Garrett D'Amore	eep->ee_esmp = esmp;
25049ef7e0Garrett D'Amore
25149ef7e0Garrett D'Amore	if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, eep)) != 0)
25249ef7e0Garrett D'Amore		goto fail2;
25349ef7e0Garrett D'Amore
25449ef7e0Garrett D'Amore	enp->en_ev_qcount++;
25549ef7e0Garrett D'Amore	*eepp = eep;
25649ef7e0Garrett D'Amore
25749ef7e0Garrett D'Amore	return (0);
25849ef7e0Garrett D'Amore
25949ef7e0Garrett D'Amorefail2:
26049ef7e0Garrett D'Amore	EFSYS_PROBE(fail2);
26149ef7e0Garrett D'Amore	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
26249ef7e0Garrett D'Amorefail1:
26349ef7e0Garrett D'Amore	EFSYS_PROBE1(fail1, efx_rc_t, rc);
26449ef7e0Garrett D'Amore	return (rc);
26549ef7e0Garrett D'Amore}
26649ef7e0Garrett D'Amore
26749ef7e0Garrett D'Amore		void
26849ef7e0Garrett D'Amoreefx_ev_qdestroy(
26949ef7e0Garrett D'Amore	__in	efx_evq_t *eep)
27049ef7e0Garrett D'Amore{
27149ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
27249ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
27349ef7e0Garrett D'Amore
27449ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
27549ef7e0Garrett D'Amore
27649ef7e0Garrett D'Amore	EFSYS_ASSERT(enp->en_ev_qcount != 0);
27749ef7e0Garrett D'Amore	--enp->en_ev_qcount;
27849ef7e0Garrett D'Amore
27949ef7e0Garrett D'Amore	eevop->eevo_qdestroy(eep);
28049ef7e0Garrett D'Amore
28149ef7e0Garrett D'Amore	/* Free the EVQ object */
28249ef7e0Garrett D'Amore	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
28349ef7e0Garrett D'Amore}
28449ef7e0Garrett D'Amore
28549ef7e0Garrett D'Amore	__checkReturn	efx_rc_t
28649ef7e0Garrett D'Amoreefx_ev_qprime(
28749ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
28849ef7e0Garrett D'Amore	__in		unsigned int count)
28949ef7e0Garrett D'Amore{
29049ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
29149ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
29249ef7e0Garrett D'Amore	efx_rc_t rc;
29349ef7e0Garrett D'Amore
29449ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
29549ef7e0Garrett D'Amore
29649ef7e0Garrett D'Amore	if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
29749ef7e0Garrett D'Amore		rc = EINVAL;
29849ef7e0Garrett D'Amore		goto fail1;
29949ef7e0Garrett D'Amore	}
30049ef7e0Garrett D'Amore
30149ef7e0Garrett D'Amore	if ((rc = eevop->eevo_qprime(eep, count)) != 0)
30249ef7e0Garrett D'Amore		goto fail2;
30349ef7e0Garrett D'Amore
30449ef7e0Garrett D'Amore	return (0);
30549ef7e0Garrett D'Amore
30649ef7e0Garrett D'Amorefail2:
30749ef7e0Garrett D'Amore	EFSYS_PROBE(fail2);
30849ef7e0Garrett D'Amorefail1:
30949ef7e0Garrett D'Amore	EFSYS_PROBE1(fail1, efx_rc_t, rc);
31049ef7e0Garrett D'Amore	return (rc);
31149ef7e0Garrett D'Amore}
31249ef7e0Garrett D'Amore
31349ef7e0Garrett D'Amore	__checkReturn	boolean_t
31449ef7e0Garrett D'Amoreefx_ev_qpending(
31549ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
31649ef7e0Garrett D'Amore	__in		unsigned int count)
31749ef7e0Garrett D'Amore{
31849ef7e0Garrett D'Amore	size_t offset;
31949ef7e0Garrett D'Amore	efx_qword_t qword;
32049ef7e0Garrett D'Amore
32149ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
32249ef7e0Garrett D'Amore
32349ef7e0Garrett D'Amore	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
32449ef7e0Garrett D'Amore	EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
32549ef7e0Garrett D'Amore
32649ef7e0Garrett D'Amore	return (EFX_EV_PRESENT(qword));
32749ef7e0Garrett D'Amore}
32849ef7e0Garrett D'Amore
32949ef7e0Garrett D'Amore#if EFSYS_OPT_EV_PREFETCH
33049ef7e0Garrett D'Amore
33149ef7e0Garrett D'Amore			void
33249ef7e0Garrett D'Amoreefx_ev_qprefetch(
33349ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
33449ef7e0Garrett D'Amore	__in		unsigned int count)
33549ef7e0Garrett D'Amore{
33649ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
33749ef7e0Garrett D'Amore	unsigned int offset;
33849ef7e0Garrett D'Amore
33949ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
34049ef7e0Garrett D'Amore
34149ef7e0Garrett D'Amore	offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
34249ef7e0Garrett D'Amore	EFSYS_MEM_PREFETCH(eep->ee_esmp, offset);
34349ef7e0Garrett D'Amore}
34449ef7e0Garrett D'Amore
34549ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_EV_PREFETCH */
34649ef7e0Garrett D'Amore
34749ef7e0Garrett D'Amore			void
34849ef7e0Garrett D'Amoreefx_ev_qpoll(
34949ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
35049ef7e0Garrett D'Amore	__inout		unsigned int *countp,
35149ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
35249ef7e0Garrett D'Amore	__in_opt	void *arg)
35349ef7e0Garrett D'Amore{
35449ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
35549ef7e0Garrett D'Amore
35649ef7e0Garrett D'Amore	/*
35749ef7e0Garrett D'Amore	 * FIXME: Huntington will require support for hardware event batching
35849ef7e0Garrett D'Amore	 * and merging, which will need a different ev_qpoll implementation.
35949ef7e0Garrett D'Amore	 *
36049ef7e0Garrett D'Amore	 * Without those features the Falcon/Siena code can be used unchanged.
36149ef7e0Garrett D'Amore	 */
36249ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
36349ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
36449ef7e0Garrett D'Amore
36549ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
36649ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
36749ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
36849ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
36949ef7e0Garrett D'Amore	    FSE_AZ_EV_CODE_DRV_GEN_EV);
37049ef7e0Garrett D'Amore#if EFSYS_OPT_MCDI
37149ef7e0Garrett D'Amore	EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV ==
37249ef7e0Garrett D'Amore	    FSE_AZ_EV_CODE_MCDI_EVRESPONSE);
37349ef7e0Garrett D'Amore#endif
37449ef7e0Garrett D'Amore	siena_ev_qpoll(eep, countp, eecp, arg);
37549ef7e0Garrett D'Amore}
37649ef7e0Garrett D'Amore
37749ef7e0Garrett D'Amore			void
37849ef7e0Garrett D'Amoreefx_ev_qpost(
37949ef7e0Garrett D'Amore	__in	efx_evq_t *eep,
38049ef7e0Garrett D'Amore	__in	uint16_t data)
38149ef7e0Garrett D'Amore{
38249ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
38349ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
38449ef7e0Garrett D'Amore
38549ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
38649ef7e0Garrett D'Amore
38749ef7e0Garrett D'Amore	EFSYS_ASSERT(eevop != NULL &&
38849ef7e0Garrett D'Amore	    eevop->eevo_qpost != NULL);
38949ef7e0Garrett D'Amore
39049ef7e0Garrett D'Amore	eevop->eevo_qpost(eep, data);
39149ef7e0Garrett D'Amore}
39249ef7e0Garrett D'Amore
39349ef7e0Garrett D'Amore	__checkReturn	efx_rc_t
39449ef7e0Garrett D'Amoreefx_ev_qmoderate(
39549ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
39649ef7e0Garrett D'Amore	__in		unsigned int us)
39749ef7e0Garrett D'Amore{
39849ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
39949ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
40049ef7e0Garrett D'Amore	efx_rc_t rc;
40149ef7e0Garrett D'Amore
40249ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
40349ef7e0Garrett D'Amore
40449ef7e0Garrett D'Amore	if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
40549ef7e0Garrett D'Amore		goto fail1;
40649ef7e0Garrett D'Amore
40749ef7e0Garrett D'Amore	return (0);
40849ef7e0Garrett D'Amore
40949ef7e0Garrett D'Amorefail1:
41049ef7e0Garrett D'Amore	EFSYS_PROBE1(fail1, efx_rc_t, rc);
41149ef7e0Garrett D'Amore	return (rc);
41249ef7e0Garrett D'Amore}
41349ef7e0Garrett D'Amore
41449ef7e0Garrett D'Amore#if EFSYS_OPT_QSTATS
41549ef7e0Garrett D'Amore					void
41649ef7e0Garrett D'Amoreefx_ev_qstats_update(
41749ef7e0Garrett D'Amore	__in				efx_evq_t *eep,
41849ef7e0Garrett D'Amore	__inout_ecount(EV_NQSTATS)	efsys_stat_t *stat)
41949ef7e0Garrett D'Amore
42049ef7e0Garrett D'Amore{	efx_nic_t *enp = eep->ee_enp;
42149ef7e0Garrett D'Amore	const efx_ev_ops_t *eevop = enp->en_eevop;
42249ef7e0Garrett D'Amore
42349ef7e0Garrett D'Amore	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
42449ef7e0Garrett D'Amore
42549ef7e0Garrett D'Amore	eevop->eevo_qstats_update(eep, stat);
42649ef7e0Garrett D'Amore}
42749ef7e0Garrett D'Amore
42849ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_QSTATS */
42949ef7e0Garrett D'Amore
43049ef7e0Garrett D'Amore#if EFSYS_OPT_SIENA
43149ef7e0Garrett D'Amore
43249ef7e0Garrett D'Amorestatic	__checkReturn	efx_rc_t
43349ef7e0Garrett D'Amoresiena_ev_init(
43449ef7e0Garrett D'Amore	__in		efx_nic_t *enp)
43549ef7e0Garrett D'Amore{
43649ef7e0Garrett D'Amore	efx_oword_t oword;
43749ef7e0Garrett D'Amore
43849ef7e0Garrett D'Amore	/*
43949ef7e0Garrett D'Amore	 * Program the event queue for receive and transmit queue
44049ef7e0Garrett D'Amore	 * flush events.
44149ef7e0Garrett D'Amore	 */
44249ef7e0Garrett D'Amore	EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword);
44349ef7e0Garrett D'Amore	EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0);
44449ef7e0Garrett D'Amore	EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword);
44549ef7e0Garrett D'Amore
44649ef7e0Garrett D'Amore	return (0);
44749ef7e0Garrett D'Amore
44849ef7e0Garrett D'Amore}
44949ef7e0Garrett D'Amore
45049ef7e0Garrett D'Amorestatic  __checkReturn   boolean_t
45149ef7e0Garrett D'Amoresiena_ev_rx_not_ok(
45249ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
45349ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
45449ef7e0Garrett D'Amore	__in		uint32_t label,
45549ef7e0Garrett D'Amore	__in		uint32_t id,
45649ef7e0Garrett D'Amore	__inout		uint16_t *flagsp)
45749ef7e0Garrett D'Amore{
45849ef7e0Garrett D'Amore	boolean_t ignore = B_FALSE;
45949ef7e0Garrett D'Amore
46049ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) {
46149ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC);
46249ef7e0Garrett D'Amore		EFSYS_PROBE(tobe_disc);
46349ef7e0Garrett D'Amore		/*
46449ef7e0Garrett D'Amore		 * Assume this is a unicast address mismatch, unless below
46549ef7e0Garrett D'Amore		 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or
46649ef7e0Garrett D'Amore		 * EV_RX_PAUSE_FRM_ERR is set.
46749ef7e0Garrett D'Amore		 */
46849ef7e0Garrett D'Amore		(*flagsp) |= EFX_ADDR_MISMATCH;
46949ef7e0Garrett D'Amore	}
47049ef7e0Garrett D'Amore
47149ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) {
47249ef7e0Garrett D'Amore		EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id);
47349ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
47449ef7e0Garrett D'Amore		(*flagsp) |= EFX_DISCARD;
47549ef7e0Garrett D'Amore
47649ef7e0Garrett D'Amore#if EFSYS_OPT_RX_SCATTER
47749ef7e0Garrett D'Amore		/*
47849ef7e0Garrett D'Amore		 * Lookout for payload queue ran dry errors and ignore them.
47949ef7e0Garrett D'Amore		 *
48049ef7e0Garrett D'Amore		 * Sadly for the header/data split cases, the descriptor
48149ef7e0Garrett D'Amore		 * pointer in this event refers to the header queue and
48249ef7e0Garrett D'Amore		 * therefore cannot be easily detected as duplicate.
48349ef7e0Garrett D'Amore		 * So we drop these and rely on the receive processing seeing
48449ef7e0Garrett D'Amore		 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard
48549ef7e0Garrett D'Amore		 * the partially received packet.
48649ef7e0Garrett D'Amore		 */
48749ef7e0Garrett D'Amore		if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) &&
48849ef7e0Garrett D'Amore		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) &&
48949ef7e0Garrett D'Amore		    (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0))
49049ef7e0Garrett D'Amore			ignore = B_TRUE;
49149ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_RX_SCATTER */
49249ef7e0Garrett D'Amore	}
49349ef7e0Garrett D'Amore
49449ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) {
49549ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
49649ef7e0Garrett D'Amore		EFSYS_PROBE(crc_err);
49749ef7e0Garrett D'Amore		(*flagsp) &= ~EFX_ADDR_MISMATCH;
49849ef7e0Garrett D'Amore		(*flagsp) |= EFX_DISCARD;
49949ef7e0Garrett D'Amore	}
50049ef7e0Garrett D'Amore
50149ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) {
50249ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR);
50349ef7e0Garrett D'Amore		EFSYS_PROBE(pause_frm_err);
50449ef7e0Garrett D'Amore		(*flagsp) &= ~EFX_ADDR_MISMATCH;
50549ef7e0Garrett D'Amore		(*flagsp) |= EFX_DISCARD;
50649ef7e0Garrett D'Amore	}
50749ef7e0Garrett D'Amore
50849ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) {
50949ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR);
51049ef7e0Garrett D'Amore		EFSYS_PROBE(owner_id_err);
51149ef7e0Garrett D'Amore		(*flagsp) |= EFX_DISCARD;
51249ef7e0Garrett D'Amore	}
51349ef7e0Garrett D'Amore
51449ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) {
51549ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
51649ef7e0Garrett D'Amore		EFSYS_PROBE(ipv4_err);
51749ef7e0Garrett D'Amore		(*flagsp) &= ~EFX_CKSUM_IPV4;
51849ef7e0Garrett D'Amore	}
51949ef7e0Garrett D'Amore
52049ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) {
52149ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
52249ef7e0Garrett D'Amore		EFSYS_PROBE(udp_chk_err);
52349ef7e0Garrett D'Amore		(*flagsp) &= ~EFX_CKSUM_TCPUDP;
52449ef7e0Garrett D'Amore	}
52549ef7e0Garrett D'Amore
52649ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) {
52749ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR);
52849ef7e0Garrett D'Amore
52949ef7e0Garrett D'Amore		/*
53049ef7e0Garrett D'Amore		 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This
53149ef7e0Garrett D'Amore		 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error
53249ef7e0Garrett D'Amore		 * condition.
53349ef7e0Garrett D'Amore		 */
53449ef7e0Garrett D'Amore		(*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP);
53549ef7e0Garrett D'Amore	}
53649ef7e0Garrett D'Amore
53749ef7e0Garrett D'Amore	return (ignore);
53849ef7e0Garrett D'Amore}
53949ef7e0Garrett D'Amore
54049ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
54149ef7e0Garrett D'Amoresiena_ev_rx(
54249ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
54349ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
54449ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
54549ef7e0Garrett D'Amore	__in_opt	void *arg)
54649ef7e0Garrett D'Amore{
54749ef7e0Garrett D'Amore	uint32_t id;
54849ef7e0Garrett D'Amore	uint32_t size;
54949ef7e0Garrett D'Amore	uint32_t label;
55049ef7e0Garrett D'Amore	boolean_t ok;
55149ef7e0Garrett D'Amore#if EFSYS_OPT_RX_SCATTER
55249ef7e0Garrett D'Amore	boolean_t sop;
55349ef7e0Garrett D'Amore	boolean_t jumbo_cont;
55449ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_RX_SCATTER */
55549ef7e0Garrett D'Amore	uint32_t hdr_type;
55649ef7e0Garrett D'Amore	boolean_t is_v6;
55749ef7e0Garrett D'Amore	uint16_t flags;
55849ef7e0Garrett D'Amore	boolean_t ignore;
55949ef7e0Garrett D'Amore	boolean_t should_abort;
56049ef7e0Garrett D'Amore
56149ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_RX);
56249ef7e0Garrett D'Amore
56349ef7e0Garrett D'Amore	/* Basic packet information */
56449ef7e0Garrett D'Amore	id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR);
56549ef7e0Garrett D'Amore	size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT);
56649ef7e0Garrett D'Amore	label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL);
56749ef7e0Garrett D'Amore	ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0);
56849ef7e0Garrett D'Amore
56949ef7e0Garrett D'Amore#if EFSYS_OPT_RX_SCATTER
57049ef7e0Garrett D'Amore	sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0);
57149ef7e0Garrett D'Amore	jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0);
57249ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_RX_SCATTER */
57349ef7e0Garrett D'Amore
57449ef7e0Garrett D'Amore	hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE);
57549ef7e0Garrett D'Amore
57649ef7e0Garrett D'Amore	is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0);
57749ef7e0Garrett D'Amore
57849ef7e0Garrett D'Amore	/*
57949ef7e0Garrett D'Amore	 * If packet is marked as OK and packet type is TCP/IP or
58049ef7e0Garrett D'Amore	 * UDP/IP or other IP, then we can rely on the hardware checksums.
58149ef7e0Garrett D'Amore	 */
58249ef7e0Garrett D'Amore	switch (hdr_type) {
58349ef7e0Garrett D'Amore	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP:
58449ef7e0Garrett D'Amore		flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP;
58549ef7e0Garrett D'Amore		if (is_v6) {
58649ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6);
58749ef7e0Garrett D'Amore			flags |= EFX_PKT_IPV6;
58849ef7e0Garrett D'Amore		} else {
58949ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4);
59049ef7e0Garrett D'Amore			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
59149ef7e0Garrett D'Amore		}
59249ef7e0Garrett D'Amore		break;
59349ef7e0Garrett D'Amore
59449ef7e0Garrett D'Amore	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP:
59549ef7e0Garrett D'Amore		flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP;
59649ef7e0Garrett D'Amore		if (is_v6) {
59749ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6);
59849ef7e0Garrett D'Amore			flags |= EFX_PKT_IPV6;
59949ef7e0Garrett D'Amore		} else {
60049ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4);
60149ef7e0Garrett D'Amore			flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
60249ef7e0Garrett D'Amore		}
60349ef7e0Garrett D'Amore		break;
60449ef7e0Garrett D'Amore
60549ef7e0Garrett D'Amore	case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER:
60649ef7e0Garrett D'Amore		if (is_v6) {
60749ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6);
60849ef7e0Garrett D'Amore			flags = EFX_PKT_IPV6;
60949ef7e0Garrett D'Amore		} else {
61049ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4);
61149ef7e0Garrett D'Amore			flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4;
61249ef7e0Garrett D'Amore		}
61349ef7e0Garrett D'Amore		break;
61449ef7e0Garrett D'Amore
61549ef7e0Garrett D'Amore	case FSE_AZ_RX_EV_HDR_TYPE_OTHER:
61649ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP);
61749ef7e0Garrett D'Amore		flags = 0;
61849ef7e0Garrett D'Amore		break;
61949ef7e0Garrett D'Amore
62049ef7e0Garrett D'Amore	default:
62149ef7e0Garrett D'Amore		EFSYS_ASSERT(B_FALSE);
62249ef7e0Garrett D'Amore		flags = 0;
62349ef7e0Garrett D'Amore		break;
62449ef7e0Garrett D'Amore	}
62549ef7e0Garrett D'Amore
62649ef7e0Garrett D'Amore#if EFSYS_OPT_RX_SCATTER
62749ef7e0Garrett D'Amore	/* Report scatter and header/lookahead split buffer flags */
62849ef7e0Garrett D'Amore	if (sop)
62949ef7e0Garrett D'Amore		flags |= EFX_PKT_START;
63049ef7e0Garrett D'Amore	if (jumbo_cont)
63149ef7e0Garrett D'Amore		flags |= EFX_PKT_CONT;
63249ef7e0Garrett D'Amore#endif	/* EFSYS_OPT_RX_SCATTER */
63349ef7e0Garrett D'Amore
63449ef7e0Garrett D'Amore	/* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */
63549ef7e0Garrett D'Amore	if (!ok) {
63649ef7e0Garrett D'Amore		ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags);
63749ef7e0Garrett D'Amore		if (ignore) {
63849ef7e0Garrett D'Amore			EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
63949ef7e0Garrett D'Amore			    uint32_t, size, uint16_t, flags);
64049ef7e0Garrett D'Amore
64149ef7e0Garrett D'Amore			return (B_FALSE);
64249ef7e0Garrett D'Amore		}
64349ef7e0Garrett D'Amore	}
64449ef7e0Garrett D'Amore
64549ef7e0Garrett D'Amore	/* If we're not discarding the packet then it is ok */
64649ef7e0Garrett D'Amore	if (~flags & EFX_DISCARD)
64749ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
64849ef7e0Garrett D'Amore
64949ef7e0Garrett D'Amore	/* Detect multicast packets that didn't match the filter */
65049ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) {
65149ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT);
65249ef7e0Garrett D'Amore
65349ef7e0Garrett D'Amore		if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) {
65449ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH);
65549ef7e0Garrett D'Amore		} else {
65649ef7e0Garrett D'Amore			EFSYS_PROBE(mcast_mismatch);
65749ef7e0Garrett D'Amore			flags |= EFX_ADDR_MISMATCH;
65849ef7e0Garrett D'Amore		}
65949ef7e0Garrett D'Amore	} else {
66049ef7e0Garrett D'Amore		flags |= EFX_PKT_UNICAST;
66149ef7e0Garrett D'Amore	}
66249ef7e0Garrett D'Amore
66349ef7e0Garrett D'Amore	/*
66449ef7e0Garrett D'Amore	 * The packet parser in Siena can abort parsing packets under
66549ef7e0Garrett D'Amore	 * certain error conditions, setting the PKT_NOT_PARSED bit
66649ef7e0Garrett D'Amore	 * (which clears PKT_OK). If this is set, then don't trust
66749ef7e0Garrett D'Amore	 * the PKT_TYPE field.
66849ef7e0Garrett D'Amore	 */
66949ef7e0Garrett D'Amore	if (!ok) {
67049ef7e0Garrett D'Amore		uint32_t parse_err;
67149ef7e0Garrett D'Amore
67249ef7e0Garrett D'Amore		parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED);
67349ef7e0Garrett D'Amore		if (parse_err != 0)
67449ef7e0Garrett D'Amore			flags |= EFX_CHECK_VLAN;
67549ef7e0Garrett D'Amore	}
67649ef7e0Garrett D'Amore
67749ef7e0Garrett D'Amore	if (~flags & EFX_CHECK_VLAN) {
67849ef7e0Garrett D'Amore		uint32_t pkt_type;
67949ef7e0Garrett D'Amore
68049ef7e0Garrett D'Amore		pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE);
68149ef7e0Garrett D'Amore		if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN)
68249ef7e0Garrett D'Amore			flags |= EFX_PKT_VLAN_TAGGED;
68349ef7e0Garrett D'Amore	}
68449ef7e0Garrett D'Amore
68549ef7e0Garrett D'Amore	EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id,
68649ef7e0Garrett D'Amore	    uint32_t, size, uint16_t, flags);
68749ef7e0Garrett D'Amore
68849ef7e0Garrett D'Amore	EFSYS_ASSERT(eecp->eec_rx != NULL);
68949ef7e0Garrett D'Amore	should_abort = eecp->eec_rx(arg, label, id, size, flags);
69049ef7e0Garrett D'Amore
69149ef7e0Garrett D'Amore	return (should_abort);
69249ef7e0Garrett D'Amore}
69349ef7e0Garrett D'Amore
69449ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
69549ef7e0Garrett D'Amoresiena_ev_tx(
69649ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
69749ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
69849ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
69949ef7e0Garrett D'Amore	__in_opt	void *arg)
70049ef7e0Garrett D'Amore{
70149ef7e0Garrett D'Amore	uint32_t id;
70249ef7e0Garrett D'Amore	uint32_t label;
70349ef7e0Garrett D'Amore	boolean_t should_abort;
70449ef7e0Garrett D'Amore
70549ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_TX);
70649ef7e0Garrett D'Amore
70749ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 &&
70849ef7e0Garrett D'Amore	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 &&
70949ef7e0Garrett D'Amore	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 &&
71049ef7e0Garrett D'Amore	    EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) {
71149ef7e0Garrett D'Amore
71249ef7e0Garrett D'Amore		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR);
71349ef7e0Garrett D'Amore		label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL);
71449ef7e0Garrett D'Amore
71549ef7e0Garrett D'Amore		EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id);
71649ef7e0Garrett D'Amore
71749ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_tx != NULL);
71849ef7e0Garrett D'Amore		should_abort = eecp->eec_tx(arg, label, id);
71949ef7e0Garrett D'Amore
72049ef7e0Garrett D'Amore		return (should_abort);
72149ef7e0Garrett D'Amore	}
72249ef7e0Garrett D'Amore
72349ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0)
72449ef7e0Garrett D'Amore		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
72549ef7e0Garrett D'Amore			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
72649ef7e0Garrett D'Amore			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
72749ef7e0Garrett D'Amore
72849ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0)
72949ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR);
73049ef7e0Garrett D'Amore
73149ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0)
73249ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG);
73349ef7e0Garrett D'Amore
73449ef7e0Garrett D'Amore	if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0)
73549ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL);
73649ef7e0Garrett D'Amore
73749ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED);
73849ef7e0Garrett D'Amore	return (B_FALSE);
73949ef7e0Garrett D'Amore}
74049ef7e0Garrett D'Amore
74149ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
74249ef7e0Garrett D'Amoresiena_ev_global(
74349ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
74449ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
74549ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
74649ef7e0Garrett D'Amore	__in_opt	void *arg)
74749ef7e0Garrett D'Amore{
74849ef7e0Garrett D'Amore	_NOTE(ARGUNUSED(eqp, eecp, arg))
74949ef7e0Garrett D'Amore
75049ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_GLOBAL);
75149ef7e0Garrett D'Amore
75249ef7e0Garrett D'Amore	return (B_FALSE);
75349ef7e0Garrett D'Amore}
75449ef7e0Garrett D'Amore
75549ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
75649ef7e0Garrett D'Amoresiena_ev_driver(
75749ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
75849ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
75949ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
76049ef7e0Garrett D'Amore	__in_opt	void *arg)
76149ef7e0Garrett D'Amore{
76249ef7e0Garrett D'Amore	boolean_t should_abort;
76349ef7e0Garrett D'Amore
76449ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_DRIVER);
76549ef7e0Garrett D'Amore	should_abort = B_FALSE;
76649ef7e0Garrett D'Amore
76749ef7e0Garrett D'Amore	switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) {
76849ef7e0Garrett D'Amore	case FSE_AZ_TX_DESCQ_FLS_DONE_EV: {
76949ef7e0Garrett D'Amore		uint32_t txq_index;
77049ef7e0Garrett D'Amore
77149ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE);
77249ef7e0Garrett D'Amore
77349ef7e0Garrett D'Amore		txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
77449ef7e0Garrett D'Amore
77549ef7e0Garrett D'Amore		EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index);
77649ef7e0Garrett D'Amore
77749ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL);
77849ef7e0Garrett D'Amore		should_abort = eecp->eec_txq_flush_done(arg, txq_index);
77949ef7e0Garrett D'Amore
78049ef7e0Garrett D'Amore		break;
78149ef7e0Garrett D'Amore	}
78249ef7e0Garrett D'Amore	case FSE_AZ_RX_DESCQ_FLS_DONE_EV: {
78349ef7e0Garrett D'Amore		uint32_t rxq_index;
78449ef7e0Garrett D'Amore		uint32_t failed;
78549ef7e0Garrett D'Amore
78649ef7e0Garrett D'Amore		rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
78749ef7e0Garrett D'Amore		failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
78849ef7e0Garrett D'Amore
78949ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL);
79049ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL);
79149ef7e0Garrett D'Amore
79249ef7e0Garrett D'Amore		if (failed) {
79349ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED);
79449ef7e0Garrett D'Amore
79549ef7e0Garrett D'Amore			EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index);
79649ef7e0Garrett D'Amore
79749ef7e0Garrett D'Amore			should_abort = eecp->eec_rxq_flush_failed(arg,
79849ef7e0Garrett D'Amore								    rxq_index);
79949ef7e0Garrett D'Amore		} else {
80049ef7e0Garrett D'Amore			EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE);
80149ef7e0Garrett D'Amore
80249ef7e0Garrett D'Amore			EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index);
80349ef7e0Garrett D'Amore
80449ef7e0Garrett D'Amore			should_abort = eecp->eec_rxq_flush_done(arg, rxq_index);
80549ef7e0Garrett D'Amore		}
80649ef7e0Garrett D'Amore
80749ef7e0Garrett D'Amore		break;
80849ef7e0Garrett D'Amore	}
80949ef7e0Garrett D'Amore	case FSE_AZ_EVQ_INIT_DONE_EV:
81049ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_initialized != NULL);
81149ef7e0Garrett D'Amore		should_abort = eecp->eec_initialized(arg);
81249ef7e0Garrett D'Amore
81349ef7e0Garrett D'Amore		break;
81449ef7e0Garrett D'Amore
81549ef7e0Garrett D'Amore	case FSE_AZ_EVQ_NOT_EN_EV:
81649ef7e0Garrett D'Amore		EFSYS_PROBE(evq_not_en);
81749ef7e0Garrett D'Amore		break;
81849ef7e0Garrett D'Amore
81949ef7e0Garrett D'Amore	case FSE_AZ_SRM_UPD_DONE_EV: {
82049ef7e0Garrett D'Amore		uint32_t code;
82149ef7e0Garrett D'Amore
82249ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE);
82349ef7e0Garrett D'Amore
82449ef7e0Garrett D'Amore		code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
82549ef7e0Garrett D'Amore
82649ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_sram != NULL);
82749ef7e0Garrett D'Amore		should_abort = eecp->eec_sram(arg, code);
82849ef7e0Garrett D'Amore
82949ef7e0Garrett D'Amore		break;
83049ef7e0Garrett D'Amore	}
83149ef7e0Garrett D'Amore	case FSE_AZ_WAKE_UP_EV: {
83249ef7e0Garrett D'Amore		uint32_t id;
83349ef7e0Garrett D'Amore
83449ef7e0Garrett D'Amore		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
83549ef7e0Garrett D'Amore
83649ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_wake_up != NULL);
83749ef7e0Garrett D'Amore		should_abort = eecp->eec_wake_up(arg, id);
83849ef7e0Garrett D'Amore
83949ef7e0Garrett D'Amore		break;
84049ef7e0Garrett D'Amore	}
84149ef7e0Garrett D'Amore	case FSE_AZ_TX_PKT_NON_TCP_UDP:
84249ef7e0Garrett D'Amore		EFSYS_PROBE(tx_pkt_non_tcp_udp);
84349ef7e0Garrett D'Amore		break;
84449ef7e0Garrett D'Amore
84549ef7e0Garrett D'Amore	case FSE_AZ_TIMER_EV: {
84649ef7e0Garrett D'Amore		uint32_t id;
84749ef7e0Garrett D'Amore
84849ef7e0Garrett D'Amore		id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA);
84949ef7e0Garrett D'Amore
85049ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_timer != NULL);
85149ef7e0Garrett D'Amore		should_abort = eecp->eec_timer(arg, id);
85249ef7e0Garrett D'Amore
85349ef7e0Garrett D'Amore		break;
85449ef7e0Garrett D'Amore	}
85549ef7e0Garrett D'Amore	case FSE_AZ_RX_DSC_ERROR_EV:
85649ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR);
85749ef7e0Garrett D'Amore
85849ef7e0Garrett D'Amore		EFSYS_PROBE(rx_dsc_error);
85949ef7e0Garrett D'Amore
86049ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_exception != NULL);
86149ef7e0Garrett D'Amore		should_abort = eecp->eec_exception(arg,
86249ef7e0Garrett D'Amore			EFX_EXCEPTION_RX_DSC_ERROR, 0);
86349ef7e0Garrett D'Amore
86449ef7e0Garrett D'Amore		break;
86549ef7e0Garrett D'Amore
86649ef7e0Garrett D'Amore	case FSE_AZ_TX_DSC_ERROR_EV:
86749ef7e0Garrett D'Amore		EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR);
86849ef7e0Garrett D'Amore
86949ef7e0Garrett D'Amore		EFSYS_PROBE(tx_dsc_error);
87049ef7e0Garrett D'Amore
87149ef7e0Garrett D'Amore		EFSYS_ASSERT(eecp->eec_exception != NULL);
87249ef7e0Garrett D'Amore		should_abort = eecp->eec_exception(arg,
87349ef7e0Garrett D'Amore			EFX_EXCEPTION_TX_DSC_ERROR, 0);
87449ef7e0Garrett D'Amore
87549ef7e0Garrett D'Amore		break;
87649ef7e0Garrett D'Amore
87749ef7e0Garrett D'Amore	default:
87849ef7e0Garrett D'Amore		break;
87949ef7e0Garrett D'Amore	}
88049ef7e0Garrett D'Amore
88149ef7e0Garrett D'Amore	return (should_abort);
88249ef7e0Garrett D'Amore}
88349ef7e0Garrett D'Amore
88449ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
88549ef7e0Garrett D'Amoresiena_ev_drv_gen(
88649ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
88749ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
88849ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
88949ef7e0Garrett D'Amore	__in_opt	void *arg)
89049ef7e0Garrett D'Amore{
89149ef7e0Garrett D'Amore	uint32_t data;
89249ef7e0Garrett D'Amore	boolean_t should_abort;
89349ef7e0Garrett D'Amore
89449ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN);
89549ef7e0Garrett D'Amore
89649ef7e0Garrett D'Amore	data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0);
89749ef7e0Garrett D'Amore	if (data >= ((uint32_t)1 << 16)) {
89849ef7e0Garrett D'Amore		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
89949ef7e0Garrett D'Amore			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1),
90049ef7e0Garrett D'Amore			    uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0));
90149ef7e0Garrett D'Amore		return (B_TRUE);
90249ef7e0Garrett D'Amore	}
90349ef7e0Garrett D'Amore
90449ef7e0Garrett D'Amore	EFSYS_ASSERT(eecp->eec_software != NULL);
90549ef7e0Garrett D'Amore	should_abort = eecp->eec_software(arg, (uint16_t)data);
90649ef7e0Garrett D'Amore
90749ef7e0Garrett D'Amore	return (should_abort);
90849ef7e0Garrett D'Amore}
90949ef7e0Garrett D'Amore
91049ef7e0Garrett D'Amore#if EFSYS_OPT_MCDI
91149ef7e0Garrett D'Amore
91249ef7e0Garrett D'Amorestatic	__checkReturn	boolean_t
91349ef7e0Garrett D'Amoresiena_ev_mcdi(
91449ef7e0Garrett D'Amore	__in		efx_evq_t *eep,
91549ef7e0Garrett D'Amore	__in		efx_qword_t *eqp,
91649ef7e0Garrett D'Amore	__in		const efx_ev_callbacks_t *eecp,
91749ef7e0Garrett D'Amore	__in_opt	void *arg)
91849ef7e0Garrett D'Amore{
91949ef7e0Garrett D'Amore	efx_nic_t *enp = eep->ee_enp;
92049ef7e0Garrett D'Amore	unsigned code;
92149ef7e0Garrett D'Amore	boolean_t should_abort = B_FALSE;
92249ef7e0Garrett D'Amore
92349ef7e0Garrett D'Amore	EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA);
92449ef7e0Garrett D'Amore
92549ef7e0Garrett D'Amore	if (enp->en_family != EFX_FAMILY_SIENA)
92649ef7e0Garrett D'Amore		goto out;
92749ef7e0Garrett D'Amore
92849ef7e0Garrett D'Amore	EFSYS_ASSERT(eecp->eec_link_change != NULL);
92949ef7e0Garrett D'Amore	EFSYS_ASSERT(eecp->eec_exception != NULL);
93049ef7e0Garrett D'Amore#if EFSYS_OPT_MON_STATS
93149ef7e0Garrett D'Amore	EFSYS_ASSERT(eecp->eec_monitor != NULL);
93249ef7e0Garrett D'Amore#endif
93349ef7e0Garrett D'Amore
93449ef7e0Garrett D'Amore	EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE);
93549ef7e0Garrett D'Amore
93649ef7e0Garrett D'Amore	code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE);
93749ef7e0Garrett D'Amore	switch (code) {
93849ef7e0Garrett D'Amore	case MCDI_EVENT_CODE_BADSSERT:
93949ef7e0Garrett D'Amore		efx_mcdi_ev_death(enp, EINTR);
94049ef7e0Garrett D'Amore		break;
94149ef7e0Garrett D'Amore
94249ef7e0Garrett D'Amore	case MCDI_EVENT_CODE_CMDDONE:
94349ef7e0Garrett D'Amore		efx_mcdi_ev_cpl(enp,
94449ef7e0Garrett D'Amore		    MCDI_EV_FIELD(eqp, CMDDONE_SEQ),
94549ef7e0Garrett D'Amore		    MCDI_EV_FIELD(eqp, CMDDONE_DATALEN),
946