/*
 * Copyright (c) 2008-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#ifndef	_SYS_SFXGE_H
#define	_SYS_SFXGE_H

#ifdef	__cplusplus
extern "C" {
#endif

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/stream.h>
#include <sys/ethernet.h>
#include <sys/cpuvar.h>

#include <sys/mac.h>
#include <sys/mac_ether.h>
#include <sys/mac_provider.h>

#include "sfxge_ioc.h"
#include "sfxge_debug.h"

#include "efx.h"
#include "efx_regs.h"

#ifdef	_KERNEL

#define	SFXGE_DRIVER_NAME "sfxge"

#define	SFXGE_CPU_CACHE_SIZE	64

typedef struct sfxge_s	sfxge_t;

typedef enum sfxge_intr_state_e {
	SFXGE_INTR_UNINITIALIZED = 0,
	SFXGE_INTR_INITIALIZED,
	SFXGE_INTR_TESTING,
	SFXGE_INTR_STARTED
} sfxge_intr_state_t;

typedef struct sfxge_intr_s {
	ddi_intr_handle_t	*si_table;
	int			si_table_size;
	int			si_nalloc;
	int			si_type;
	int			si_cap;
	efsys_mem_t		si_mem;
	uint64_t		si_mask;
	sfxge_intr_state_t	si_state;
	uint32_t		si_zero_count;
	int			si_intr_pri;
} sfxge_intr_t;

typedef enum sfxge_promisc_type_e {
	SFXGE_PROMISC_OFF = 0,
	SFXGE_PROMISC_ALL_MULTI,
	SFXGE_PROMISC_ALL_PHYS
} sfxge_promisc_type_t;

typedef enum sfxge_link_duplex_e {
	SFXGE_LINK_DUPLEX_UNKNOWN = 0,
	SFXGE_LINK_DUPLEX_HALF,
	SFXGE_LINK_DUPLEX_FULL
} sfxge_link_duplex_t;

typedef enum sfxge_unicst_type_e {
	SFXGE_UNICST_BIA = 0,
	SFXGE_UNICST_LAA,
	SFXGE_UNICST_NTYPES
} sfxge_unicst_type_t;

typedef struct sfxge_phy_s {
		kstat_t			*sp_ksp;
		kstat_named_t		*sp_stat;
		uint32_t		*sp_statbuf;
		efsys_mem_t		sp_mem;
} sfxge_phy_t;

typedef enum sfxge_mac_state_e {
	SFXGE_MAC_UNINITIALIZED = 0,
	SFXGE_MAC_INITIALIZED,
	SFXGE_MAC_STARTED
} sfxge_mac_state_t;

typedef struct sfxge_mac_s {
	sfxge_t			*sm_sp;
	efsys_mem_t		sm_mem;
	kstat_t			*sm_ksp;
	kstat_named_t		*sm_stat;
	uint8_t			sm_bia[ETHERADDRL];
	uint8_t			sm_laa[ETHERADDRL];
	boolean_t		sm_laa_valid;
	unsigned int		sm_fcntl;
	sfxge_promisc_type_t	sm_promisc;
	uint8_t			sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX *
	    ETHERADDRL]; /* List of multicast addresses to filter on */
	int			sm_mcast_count;
	clock_t			sm_lbolt;
	kmutex_t		sm_lock;
	efx_link_mode_t		sm_link_mode;
	unsigned int		sm_link_speed;
	sfxge_link_duplex_t	sm_link_duplex;
	boolean_t		sm_link_up;
	boolean_t		sm_link_poll_reqd;
	kcondvar_t		sm_link_poll_kv;
	boolean_t		sm_mac_stats_timer_reqd;
	boolean_t		sm_mac_stats_pend;
	ddi_taskq_t		*sm_tqp;
	sfxge_mac_state_t	sm_state;
	sfxge_phy_t		sm_phy;
	uint32_t		sm_phy_cap_to_set;
	uint32_t		sm_phy_cap_to_unset;
} sfxge_mac_t;

typedef enum sfxge_mon_state_e {
	SFXGE_MON_UNINITIALIZED = 0,
	SFXGE_MON_INITIALIZED,
	SFXGE_MON_STARTED
} sfxge_mon_state_t;

typedef struct sfxge_mon_s {
	sfxge_t			*sm_sp;
	efx_mon_type_t		sm_type;
	unsigned int		sm_devid;
	kstat_t			*sm_ksp;
	kstat_named_t		*sm_stat;
	efx_mon_stat_value_t	*sm_statbuf;
	kmutex_t		sm_lock;
	sfxge_mon_state_t	sm_state;
	efsys_mem_t		sm_mem;
	int			sm_polling;
} sfxge_mon_t;

typedef enum sfxge_sram_state_e {
	SFXGE_SRAM_UNINITIALIZED = 0,
	SFXGE_SRAM_INITIALIZED,
	SFXGE_SRAM_STARTED
} sfxge_sram_state_t;

typedef struct sfxge_sram_s {
	sfxge_t			*ss_sp;
	kmutex_t		ss_lock;
	struct map		*ss_buf_tbl_map;
	unsigned int		ss_count;
	sfxge_sram_state_t	ss_state;
} sfxge_sram_t;

typedef enum sfxge_mcdi_state_e {
	SFXGE_MCDI_UNINITIALIZED = 0,
	SFXGE_MCDI_INITIALIZED,
	SFXGE_MCDI_BUSY,
	SFXGE_MCDI_COMPLETED
} sfxge_mcdi_state_t;

typedef struct sfxge_mcdi_s {
	sfxge_t			*sm_sp;
	kmutex_t		sm_lock;
	sfxge_mcdi_state_t	sm_state;
	efx_mcdi_transport_t	sm_emt;
	efsys_mem_t		sm_mem;
	kcondvar_t		sm_kv;		/* MCDI poll complete */
} sfxge_mcdi_t;

#define	SFXGE_NEVS			4096
#define	SFXGE_RX_NDESCS			1024
#define	SFXGE_TX_NDESCS			1024
#define	SFXGE_TX_NLABELS		EFX_EV_TX_NLABELS

#define	SFXGE_DEFAULT_RXQ_SIZE		1024
#define	SFXGE_DEFAULT_MODERATION	30

typedef enum sfxge_evq_state_e {
	SFXGE_EVQ_UNINITIALIZED = 0,
	SFXGE_EVQ_INITIALIZED,
	SFXGE_EVQ_STARTING,
	SFXGE_EVQ_STARTED
} sfxge_evq_state_t;

#define	SFXGE_EV_BATCH	(SFXGE_NEVS / 4)

typedef struct sfxge_txq_s	sfxge_txq_t;

typedef struct sfxge_evq_s {
	union {
		struct {
			sfxge_t			*__se_sp;
			unsigned int		__se_index;
			efsys_mem_t		__se_mem;
			unsigned int		__se_id;
			kstat_t			*__se_ksp;
			kstat_named_t		*__se_stat;
			efx_ev_callbacks_t	__se_eec;
			sfxge_evq_state_t	__se_state;
			boolean_t		__se_exception;
		} __se_s1;
		uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
	} __se_u1;
	union {
		struct {
			kmutex_t		__se_lock;
			kcondvar_t		__se_init_kv;
			efx_evq_t		*__se_eep;
			unsigned int		__se_count;
			unsigned int		__se_rx;
			unsigned int		__se_tx;
			sfxge_txq_t		*__se_stp;
			sfxge_txq_t		**__se_stpp;
			processorid_t		__se_cpu_id;
			uint16_t		__se_ev_batch;
		} __se_s2;
		uint8_t	__se_pad[SFXGE_CPU_CACHE_SIZE];
	} __se_u2;
	union {
		struct {
			sfxge_txq_t	*__se_label_stp[SFXGE_TX_NLABELS];
		} __se_s3;
		uint8_t	__se_pad[SFXGE_CPU_CACHE_SIZE * 4];
	} __se_u3;
} sfxge_evq_t;

#define	se_sp		__se_u1.__se_s1.__se_sp
#define	se_index	__se_u1.__se_s1.__se_index
#define	se_mem		__se_u1.__se_s1.__se_mem
#define	se_id		__se_u1.__se_s1.__se_id
#define	se_ksp		__se_u1.__se_s1.__se_ksp
#define	se_stat		__se_u1.__se_s1.__se_stat
#define	se_eec		__se_u1.__se_s1.__se_eec
#define	se_state	__se_u1.__se_s1.__se_state
#define	se_exception	__se_u1.__se_s1.__se_exception

#define	se_lock		__se_u2.__se_s2.__se_lock
#define	se_init_kv	__se_u2.__se_s2.__se_init_kv
#define	se_eep		__se_u2.__se_s2.__se_eep
#define	se_count	__se_u2.__se_s2.__se_count
#define	se_rx		__se_u2.__se_s2.__se_rx
#define	se_tx		__se_u2.__se_s2.__se_tx
#define	se_stp		__se_u2.__se_s2.__se_stp
#define	se_stpp		__se_u2.__se_s2.__se_stpp
#define	se_cpu_id	__se_u2.__se_s2.__se_cpu_id
#define	se_ev_batch	__se_u2.__se_s2.__se_ev_batch

#define	se_label_stp	__se_u3.__se_s3.__se_label_stp


#define	SFXGE_MAGIC_RESERVED	0x8000

#define	SFXGE_MAGIC_DMAQ_LABEL_WIDTH  5
#define	SFXGE_MAGIC_DMAQ_LABEL_MASK   ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)

#define	SFXGE_MAGIC_RX_QFLUSH_DONE					\
	(SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define	SFXGE_MAGIC_RX_QFLUSH_FAILED					\
	(SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define	SFXGE_MAGIC_RX_QFPP_TRIM					\
	(SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define	SFXGE_MAGIC_TX_QFLUSH_DONE					\
	(SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

typedef struct sfxge_rxq_s		sfxge_rxq_t;

#define	SFXGE_ETHERTYPE_LOOPBACK	0x9000	/* Xerox loopback */

typedef struct sfxge_rx_packet_s	sfxge_rx_packet_t;

struct sfxge_rx_packet_s {
	union {
		struct {
			frtn_t			__srp_free;
			uint16_t		__srp_flags;
			uint16_t		__srp_size;
			mblk_t			*__srp_mp;
			struct ether_header	*__srp_etherhp;
			struct ip		*__srp_iphp;
			struct tcphdr		*__srp_thp;
			size_t			__srp_off;
		} __srp_s1;
		uint8_t	__srp_pad[SFXGE_CPU_CACHE_SIZE];
	} __srp_u1;
	union {
		struct {
			sfxge_rxq_t		*__srp_srp;
			ddi_dma_handle_t	__srp_dma_handle;
			ddi_acc_handle_t	__srp_acc_handle;
			unsigned char		*__srp_base;
			size_t			__srp_mblksize;
			uint64_t		__srp_addr;
			boolean_t		__srp_recycle;
			caddr_t			__srp_putp;
		} __srp_s2;
		uint8_t	__srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
	} __srp_u2;
};

#define	srp_free	__srp_u1.__srp_s1.__srp_free
#define	srp_flags	__srp_u1.__srp_s1.__srp_flags
#define	srp_size	__srp_u1.__srp_s1.__srp_size
#define	srp_mp		__srp_u1.__srp_s1.__srp_mp
#define	srp_etherhp	__srp_u1.__srp_s1.__srp_etherhp
#define	srp_iphp	__srp_u1.__srp_s1.__srp_iphp
#define	srp_thp		__srp_u1.__srp_s1.__srp_thp
#define	srp_off		__srp_u1.__srp_s1.__srp_off

#define	srp_srp		__srp_u2.__srp_s2.__srp_srp
#define	srp_dma_handle	__srp_u2.__srp_s2.__srp_dma_handle
#define	srp_acc_handle	__srp_u2.__srp_s2.__srp_acc_handle
#define	srp_base	__srp_u2.__srp_s2.__srp_base
#define	srp_mblksize	__srp_u2.__srp_s2.__srp_mblksize
#define	srp_addr	__srp_u2.__srp_s2.__srp_addr
#define	srp_recycle	__srp_u2.__srp_s2.__srp_recycle
#define	srp_putp	__srp_u2.__srp_s2.__srp_putp

#define	SFXGE_RX_FPP_NSLOTS	8
#define	SFXGE_RX_FPP_MASK	(SFXGE_RX_FPP_NSLOTS - 1)

/* Free packet pool putlist (dynamically allocated) */
typedef struct sfxge_rx_fpp_putlist_s {
	kmutex_t		srfpl_lock;
	unsigned int		srfpl_count;
	mblk_t			*srfpl_putp;
	mblk_t			**srfpl_putpp;
} sfxge_rx_fpp_putlist_t;

/* Free packet pool */
typedef struct sfxge_rx_fpp_s {
	caddr_t		srfpp_putp;
	unsigned int	srfpp_loaned;
	mblk_t		*srfpp_get;
	unsigned int	srfpp_count;
	unsigned int	srfpp_min;
	/* Low water mark: Don't trim to below this */
	unsigned int    srfpp_lowat;
} sfxge_rx_fpp_t;

typedef struct sfxge_rx_flow_s	sfxge_rx_flow_t;

struct sfxge_rx_flow_s {
	uint32_t		srf_tag;
	/* in-order segment count */
	unsigned int		srf_count;
	uint16_t		srf_tci;
	uint32_t		srf_saddr;
	uint32_t		srf_daddr;
	uint16_t		srf_sport;
	uint16_t		srf_dport;
	/* sequence number */
	uint32_t		srf_seq;
	clock_t			srf_lbolt;
	mblk_t			*srf_mp;
	mblk_t			**srf_mpp;
	struct ether_header	*srf_etherhp;
	struct ip		*srf_iphp;
	struct tcphdr		*srf_first_thp;
	struct tcphdr		*srf_last_thp;
	size_t			srf_len;
	sfxge_rx_flow_t		*srf_next;
};

#define	SFXGE_MAX_FLOW		1024
#define	SFXGE_SLOW_START	20

typedef enum sfxge_flush_state_e {
	SFXGE_FLUSH_INACTIVE = 0,
	SFXGE_FLUSH_DONE,
	SFXGE_FLUSH_PENDING,
	SFXGE_FLUSH_FAILED
} sfxge_flush_state_t;

typedef enum sfxge_rxq_state_e {
	SFXGE_RXQ_UNINITIALIZED = 0,
	SFXGE_RXQ_INITIALIZED,
	SFXGE_RXQ_STARTED
} sfxge_rxq_state_t;


#define	SFXGE_RX_BATCH	128
#define	SFXGE_RX_NSTATS	8 /* note that *esballoc share one kstat */

struct sfxge_rxq_s {
	union {
		struct {
			sfxge_t				*__sr_sp;
			unsigned int			__sr_index;
			efsys_mem_t			__sr_mem;
			unsigned int			__sr_id;
			unsigned int			__sr_lowat;
			unsigned int			__sr_hiwat;
			volatile timeout_id_t		__sr_tid;
			sfxge_rxq_state_t		__sr_state;
		} __sr_s1;
		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
	} __sr_u1;
	union {
		struct {
			sfxge_rx_packet_t		**__sr_srpp;
			unsigned int			__sr_added;
			unsigned int			__sr_pushed;
			unsigned int			__sr_pending;
			unsigned int			__sr_completed;
			unsigned int			__sr_loopback;
			mblk_t   			*__sr_mp;
			mblk_t   			**__sr_mpp;
			sfxge_rx_flow_t			*__sr_flow;
			sfxge_rx_flow_t			*__sr_srfp;
			sfxge_rx_flow_t			**__sr_srfpp;
			clock_t				__sr_rto;
		} __sr_s2;
		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
	} __sr_u2;
	union {
		struct {
			sfxge_rx_fpp_t			__sr_fpp;
			efx_rxq_t			*__sr_erp;
			volatile sfxge_flush_state_t	__sr_flush;
			kcondvar_t			__sr_flush_kv;
			kstat_t				*__sr_ksp;
		} __sr_s3;
		uint8_t	__sr_pad[SFXGE_CPU_CACHE_SIZE];
	} __sr_u3;
	struct {
		/* NB must match SFXGE_RX_NSTATS */
		uint32_t    srk_rx_pkt_mem_limit;
		uint32_t    srk_kcache_alloc_nomem;
		uint32_t    srk_dma_alloc_nomem;
		uint32_t    srk_dma_alloc_fail;
		uint32_t    srk_dma_bind_nomem;
		uint32_t    srk_dma_bind_fail;
		uint32_t    srk_desballoc_fail;
		uint32_t    srk_rxq_empty_discard;
	} sr_kstat;
};

#define	sr_sp		__sr_u1.__sr_s1.__sr_sp
#define	sr_index	__sr_u1.__sr_s1.__sr_index
#define	sr_mem		__sr_u1.__sr_s1.__sr_mem
#define	sr_id		__sr_u1.__sr_s1.__sr_id
#define	sr_mrh		__sr_u1.__sr_s1.__sr_mrh
#define	sr_lowat	__sr_u1.__sr_s1.__sr_lowat
#define	sr_hiwat	__sr_u1.__sr_s1.__sr_hiwat
#define	sr_tid		__sr_u1.__sr_s1.__sr_tid
#define	sr_state	__sr_u1.__sr_s1.__sr_state

#define	sr_srpp		__sr_u2.__sr_s2.__sr_srpp
#define	sr_added	__sr_u2.__sr_s2.__sr_added
#define	sr_pushed	__sr_u2.__sr_s2.__sr_pushed
#define	sr_pending	__sr_u2.__sr_s2.__sr_pending
#define	sr_completed	__sr_u2.__sr_s2.__sr_completed
#define	sr_loopback	__sr_u2.__sr_s2.__sr_loopback
#define	sr_mp		__sr_u2.__sr_s2.__sr_mp
#define	sr_mpp		__sr_u2.__sr_s2.__sr_mpp
#define	sr_flow		__sr_u2.__sr_s2.__sr_flow
#define	sr_srfp		__sr_u2.__sr_s2.__sr_srfp
#define	sr_srfpp	__sr_u2.__sr_s2.__sr_srfpp
#define	sr_rto		__sr_u2.__sr_s2.__sr_rto

#define	sr_fpp		__sr_u3.__sr_s3.__sr_fpp
#define	sr_erp		__sr_u3.__sr_s3.__sr_erp
#define	sr_flush	__sr_u3.__sr_s3.__sr_flush
#define	sr_flush_kv	__sr_u3.__sr_s3.__sr_flush_kv
#define	sr_ksp		__sr_u3.__sr_s3.__sr_ksp

typedef struct sfxge_tx_packet_s	sfxge_tx_packet_t;

/* Packet type from parsing transmit packet */
typedef enum sfxge_packet_type_e {
	SFXGE_PACKET_TYPE_UNKNOWN = 0,
	SFXGE_PACKET_TYPE_IPV4_TCP,
	SFXGE_PACKET_TYPE_IPV4_UDP,
	SFXGE_PACKET_TYPE_IPV4_SCTP,
	SFXGE_PACKET_TYPE_IPV4_OTHER,
	SFXGE_PACKET_NTYPES
} sfxge_packet_type_t;

struct sfxge_tx_packet_s {
	sfxge_tx_packet_t	*stp_next;
	mblk_t			*stp_mp;
	struct ether_header	*stp_etherhp;
	struct ip 		*stp_iphp;
	struct tcphdr		*stp_thp;
	size_t			stp_off;
	size_t			stp_size;
	size_t			stp_mss;
	uint32_t		stp_dpl_put_len;
};

#define	SFXGE_TX_FPP_MAX	64

typedef struct sfxge_tx_fpp_s {
	sfxge_tx_packet_t	*stf_stpp;
	unsigned int		stf_count;
} sfxge_tx_fpp_t;

typedef struct sfxge_tx_mapping_s	sfxge_tx_mapping_t;

#define	SFXGE_TX_MAPPING_NADDR	(((1 << 16) >> 12) + 2)

struct sfxge_tx_mapping_s {
	sfxge_tx_mapping_t	*stm_next;
	sfxge_t			*stm_sp;
	mblk_t			*stm_mp;
	ddi_dma_handle_t	stm_dma_handle;
	caddr_t			stm_base;
	size_t			stm_size;
	size_t			stm_off;
	uint64_t		stm_addr[SFXGE_TX_MAPPING_NADDR];
};

typedef struct sfxge_tx_fmp_s {
	sfxge_tx_mapping_t	*stf_stmp;
	unsigned int		stf_count;
} sfxge_tx_fmp_t;

typedef struct sfxge_tx_buffer_s	sfxge_tx_buffer_t;

struct sfxge_tx_buffer_s {
	sfxge_tx_buffer_t	*stb_next;
	size_t			stb_off;
	efsys_mem_t		stb_esm;
};

#define	SFXGE_TX_BUFFER_SIZE	0x400
#define	SFXGE_TX_HEADER_SIZE	0x100
#define	SFXGE_TX_COPY_THRESHOLD	0x200

typedef struct sfxge_tx_fbp_s {
	sfxge_tx_buffer_t	*stf_stbp;
	unsigned int		stf_count;
} sfxge_tx_fbp_t;

typedef struct sfxge_tx_dpl_s {
	uintptr_t		std_put;
	sfxge_tx_packet_t	*std_get;
	sfxge_tx_packet_t	**std_getp;
	unsigned int		std_count; /* only get list count */
	unsigned int		get_pkt_limit;
	unsigned int		put_pkt_limit;
	unsigned int		get_full_count;
	unsigned int		put_full_count;
} sfxge_tx_dpl_t;

typedef enum sfxge_txq_state_e {
	SFXGE_TXQ_UNINITIALIZED = 0,
	SFXGE_TXQ_INITIALIZED,
	SFXGE_TXQ_STARTED,
	SFXGE_TXQ_FLUSH_PENDING,
	SFXGE_TXQ_FLUSH_DONE,
	SFXGE_TXQ_FLUSH_FAILED
} sfxge_txq_state_t;

typedef enum sfxge_txq_type_e {
	SFXGE_TXQ_NON_CKSUM = 0,
	SFXGE_TXQ_IP_CKSUM,
	SFXGE_TXQ_IP_TCP_UDP_CKSUM,
	SFXGE_TXQ_NTYPES
} sfxge_txq_type_t;

#define	SFXGE_TXQ_UNBLOCK_LEVEL1	(EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
#define	SFXGE_TXQ_UNBLOCK_LEVEL2	0
#define	SFXGE_TXQ_NOT_BLOCKED		-1

#define	SFXGE_TX_BATCH	64

struct sfxge_txq_s {
	union {
		struct {
			sfxge_t				*__st_sp;
			unsigned int			__st_index;
			unsigned int			__st_label;
			sfxge_txq_type_t		__st_type;
			unsigned int			__st_evq;
			efsys_mem_t			__st_mem;
			unsigned int			__st_id;
			kstat_t				*__st_ksp;
			kstat_named_t			*__st_stat;
			sfxge_txq_state_t		__st_state;
		} __st_s1;
		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE * 2];
	} __st_u1;
	union {
		struct {
			sfxge_tx_dpl_t			__st_dpl;
		} __st_s2;
		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE];
	} __st_u2;
	union {
		struct {
			kmutex_t			__st_lock;
			/* mapping pool - sfxge_tx_mapping_t */
			sfxge_tx_fmp_t			__st_fmp;
			/* buffer pool - sfxge_tx_buffer_t */
			sfxge_tx_fbp_t			__st_fbp;
			/* packet pool - sfxge_tx_packet_t */
			sfxge_tx_fpp_t			__st_fpp;
			efx_buffer_t			*__st_eb;
			unsigned int			__st_n;
			efx_txq_t			*__st_etp;
			sfxge_tx_mapping_t		**__st_stmp;
			sfxge_tx_buffer_t		**__st_stbp;
			mblk_t				**__st_mp;
			unsigned int			__st_added;
			unsigned int			__st_reaped;
			int				__st_unblock;
		} __st_s3;
		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE * 3];
	} __st_u3;
	union {
		struct {
			sfxge_txq_t			*__st_next;
			unsigned int			__st_pending;
			unsigned int			__st_completed;

		} __st_s4;
		uint8_t	__st_pad[SFXGE_CPU_CACHE_SIZE];
	} __st_u4;
};

#define	st_sp		__st_u1.__st_s1.__st_sp
#define	st_index	__st_u1.__st_s1.__st_index
#define	st_label	__st_u1.__st_s1.__st_label
#define	st_type		__st_u1.__st_s1.__st_type
#define	st_evq		__st_u1.__st_s1.__st_evq
#define	st_mem		__st_u1.__st_s1.__st_mem
#define	st_id		__st_u1.__st_s1.__st_id
#define	st_ksp		__st_u1.__st_s1.__st_ksp
#define	st_stat		__st_u1.__st_s1.__st_stat
#define	st_state	__st_u1.__st_s1.__st_state

#define	st_dpl		__st_u2.__st_s2.__st_dpl

#define	st_lock		__st_u3.__st_s3.__st_lock
#define	st_fmp		__st_u3.__st_s3.__st_fmp
#define	st_fbp		__st_u3.__st_s3.__st_fbp
#define	st_fpp		__st_u3.__st_s3.__st_fpp
#define	st_eb		__st_u3.__st_s3.__st_eb
#define	st_n		__st_u3.__st_s3.__st_n
#define	st_etp		__st_u3.__st_s3.__st_etp
#define	st_stmp		__st_u3.__st_s3.__st_stmp
#define	st_stbp		__st_u3.__st_s3.__st_stbp
#define	st_mp		__st_u3.__st_s3.__st_mp
#define	st_added	__st_u3.__st_s3.__st_added
#define	st_reaped	__st_u3.__st_s3.__st_reaped
#define	st_unblock	__st_u3.__st_s3.__st_unblock

#define	st_next		__st_u4.__st_s4.__st_next
#define	st_pending	__st_u4.__st_s4.__st_pending
#define	st_completed	__st_u4.__st_s4.__st_completed

typedef enum sfxge_rx_scale_state_e {
	SFXGE_RX_SCALE_UNINITIALIZED = 0,
	SFXGE_RX_SCALE_INITIALIZED,
	SFXGE_RX_SCALE_STARTED
} sfxge_rx_scale_state_t;

#define	SFXGE_RX_SCALE_MAX	EFX_RSS_TBL_SIZE

typedef struct sfxge_rx_scale_s {
	kmutex_t		srs_lock;
	unsigned int		*srs_cpu;
	unsigned int		srs_tbl[SFXGE_RX_SCALE_MAX];
	unsigned int		srs_count;
	kstat_t			*srs_ksp;
	sfxge_rx_scale_state_t	srs_state;
} sfxge_rx_scale_t;


typedef enum sfxge_rx_coalesce_mode_e {
	SFXGE_RX_COALESCE_OFF = 0,
	SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
	SFXGE_RX_COALESCE_ALLOW_PUSH = 2
} sfxge_rx_coalesce_mode_t;

typedef enum sfxge_vpd_type_e {
	SFXGE_VPD_ID = 0,
	SFXGE_VPD_PN = 1,
	SFXGE_VPD_SN = 2,
	SFXGE_VPD_EC = 3,
	SFXGE_VPD_MN = 4,
	SFXGE_VPD_VD = 5,
	SFXGE_VPD_VE = 6,
	SFXGE_VPD_MAX = 7,
} sfxge_vpd_type_t;

typedef struct sfxge_vpd_kstat_s {
	kstat_t		*svk_ksp;
	kstat_named_t	svk_stat[SFXGE_VPD_MAX];
	efx_vpd_value_t	*svk_vv;
} sfxge_vpd_kstat_t;

typedef struct sfxge_cfg_kstat_s {
	struct {
		kstat_named_t	sck_mac;
		kstat_named_t	sck_version;
	} kstat;
	struct {
		char		sck_mac[64 + 1];
	} buf;
} sfxge_cfg_kstat_t;

typedef enum sfxge_state_e {
	SFXGE_UNINITIALIZED = 0,
	SFXGE_INITIALIZED,
	SFXGE_REGISTERED,
	SFXGE_STARTING,
	SFXGE_STARTED,
	SFXGE_STOPPING
} sfxge_state_t;

typedef enum sfxge_hw_err_e {
	SFXGE_HW_OK = 0,
	SFXGE_HW_ERR,
} sfxge_hw_err_t;

typedef enum sfxge_action_on_hw_err_e {
	SFXGE_RECOVER = 0,
	SFXGE_INVISIBLE = 1,
	SFXGE_LEAVE_DEAD = 2,
} sfxge_action_on_hw_err_t;

typedef char *sfxge_mac_priv_prop_t;

#define	SFXGE_TOEPLITZ_KEY_LEN 40

struct sfxge_s {
	kmutex_t			s_state_lock;
	sfxge_state_t			s_state;
	dev_info_t			*s_dip;
	ddi_taskq_t			*s_tqp;
	ddi_acc_handle_t		s_pci_handle;
	uint16_t			s_pci_venid;
	uint16_t			s_pci_devid;
#if EFSYS_OPT_MCDI_LOGGING
	unsigned int			s_bus_addr;
#endif
	efx_family_t			s_family;
	unsigned int			s_pcie_nlanes;
	unsigned int 			s_pcie_linkspeed;
	kmutex_t			s_nic_lock;
	efsys_bar_t			s_bar;
	sfxge_intr_t			s_intr;
	sfxge_mac_t			s_mac;
	sfxge_mon_t			s_mon;
	sfxge_sram_t			s_sram;
	sfxge_mcdi_t			s_mcdi;
	kmem_cache_t			*s_eq0c; /* eventQ 0 */
	kmem_cache_t			*s_eqXc; /* all other eventQs */
	sfxge_evq_t			*s_sep[SFXGE_RX_SCALE_MAX];
	unsigned int			s_ev_moderation;
	kmem_cache_t			*s_rqc;
	sfxge_rxq_t			*s_srp[SFXGE_RX_SCALE_MAX];
	sfxge_rx_scale_t		s_rx_scale;
	size_t				s_rx_prefix_size;
	size_t				s_rx_buffer_size;
	size_t				s_rx_buffer_align;
	sfxge_rx_coalesce_mode_t	s_rx_coalesce_mode;
	int64_t				s_rx_pkt_mem_max;
	volatile uint64_t		s_rx_pkt_mem_alloc;
	kmem_cache_t			*s_rpc;
	kmem_cache_t			*s_tqc;
	unsigned int			s_tx_scale_base[SFXGE_TXQ_NTYPES];
	unsigned int			s_tx_scale_max[SFXGE_TXQ_NTYPES];
	int 				s_tx_qcount;
	sfxge_txq_t			*s_stp[SFXGE_RX_SCALE_MAX *
	    SFXGE_TXQ_NTYPES]; /* Sparse array */
	kmem_cache_t			*s_tpc;
	int				s_tx_flush_pending;
	kmutex_t			s_tx_flush_lock;
	kcondvar_t			s_tx_flush_kv;
	kmem_cache_t			*s_tbc;
	kmem_cache_t			*s_tmc;
	efx_nic_t			*s_enp;
	sfxge_vpd_kstat_t		s_vpd_kstat;
	sfxge_cfg_kstat_t		s_cfg_kstat;
	kstat_t				*s_cfg_ksp;
	size_t				s_mtu;
	int				s_rxq_poll_usec;
	mac_callbacks_t			s_mc;
	mac_handle_t			s_mh;
	sfxge_mac_priv_prop_t		*s_mac_priv_props;
	int				s_mac_priv_props_alloc;
	volatile uint32_t		s_nested_restarts;
	uint32_t			s_num_restarts;
	uint32_t			s_num_restarts_hw_err;
	sfxge_hw_err_t			s_hw_err;
	sfxge_action_on_hw_err_t	s_action_on_hw_err;
	uint16_t			s_rxq_size;
	uint16_t			s_evq0_size;
	uint16_t			s_evqX_size;
#if EFSYS_OPT_MCDI_LOGGING
	int				s_mcdi_logging;
#endif
	const uint32_t			*s_toeplitz_cache;
};

typedef struct sfxge_dma_buffer_attr_s {
	dev_info_t		*sdba_dip;
	ddi_dma_attr_t		*sdba_dattrp;
	int			(*sdba_callback) (caddr_t);
	size_t			sdba_length;
	uint_t			sdba_memflags;
	ddi_device_acc_attr_t	*sdba_devaccp;
	uint_t			sdba_bindflags;
	int			sdba_maxcookies;
	boolean_t		sdba_zeroinit;
} sfxge_dma_buffer_attr_t;

extern const char		sfxge_ident[];
extern uint8_t			sfxge_brdcst[];

extern kmutex_t			sfxge_global_lock;

extern unsigned int		*sfxge_cpu;

extern int			sfxge_start(sfxge_t *, boolean_t);
extern void			sfxge_stop(sfxge_t *);
extern void			sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
extern int			sfxge_restart_dispatch(sfxge_t *, uint_t,
    sfxge_hw_err_t, const char *, uint32_t);

extern void			sfxge_gld_link_update(sfxge_t *);
extern void			sfxge_gld_mtu_update(sfxge_t *);
extern void			sfxge_gld_rx_post(sfxge_t *, unsigned int,
    mblk_t *);
extern void			sfxge_gld_rx_push(sfxge_t *);
extern int			sfxge_gld_register(sfxge_t *);
extern int			sfxge_gld_unregister(sfxge_t *);

extern int			sfxge_dma_buffer_create(efsys_mem_t *,
    const sfxge_dma_buffer_attr_t *);
extern void			sfxge_dma_buffer_destroy(efsys_mem_t *);

extern int			sfxge_intr_init(sfxge_t *);
extern int			sfxge_intr_start(sfxge_t *);
extern void			sfxge_intr_stop(sfxge_t *);
extern void			sfxge_intr_fini(sfxge_t *);
extern void			sfxge_intr_fatal(sfxge_t *);

extern int			sfxge_ev_init(sfxge_t *);
extern int			sfxge_ev_start(sfxge_t *);
extern void			sfxge_ev_moderation_get(sfxge_t *,
    unsigned int *);
extern int			sfxge_ev_moderation_set(sfxge_t *,
    unsigned int);
extern int			sfxge_ev_qmoderate(sfxge_t *, unsigned int,
    unsigned int);
extern int			sfxge_ev_qpoll(sfxge_t *, unsigned int);
extern int			sfxge_ev_qprime(sfxge_t *, unsigned int);
extern void			sfxge_ev_stop(sfxge_t *);
extern void			sfxge_ev_fini(sfxge_t *);
extern int			sfxge_ev_txlabel_alloc(sfxge_t *sp,
    unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
extern int			sfxge_ev_txlabel_free(sfxge_t *sp,
    unsigned int evq, sfxge_txq_t *stp, unsigned int label);

extern int			sfxge_mon_init(sfxge_t *);
extern int			sfxge_mon_start(sfxge_t *);
extern void			sfxge_mon_stop(sfxge_t *);
extern void			sfxge_mon_fini(sfxge_t *);

extern int			sfxge_mac_init(sfxge_t *);
extern int			sfxge_mac_start(sfxge_t *, boolean_t);
extern void			sfxge_mac_stat_get(sfxge_t *, unsigned int,
    uint64_t *);
extern void			sfxge_mac_link_check(sfxge_t *, boolean_t *);
extern void			sfxge_mac_link_speed_get(sfxge_t *,
    unsigned int *);
extern void			sfxge_mac_link_duplex_get(sfxge_t *,
    sfxge_link_duplex_t *);
extern void			sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
extern int			sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
extern int			sfxge_mac_unicst_get(sfxge_t *,
    sfxge_unicst_type_t, uint8_t *);
extern int			sfxge_mac_unicst_set(sfxge_t *,
    uint8_t *);
extern int			sfxge_mac_promisc_set(sfxge_t *,
    sfxge_promisc_type_t);
extern int			sfxge_mac_multicst_add(sfxge_t *,
    uint8_t const *addr);
extern int			sfxge_mac_multicst_remove(sfxge_t *,
    uint8_t const *addr);
extern void			sfxge_mac_stop(sfxge_t *);
extern void			sfxge_mac_fini(sfxge_t *);
extern void			sfxge_mac_link_update(sfxge_t *sp,
    efx_link_mode_t mode);

extern int			sfxge_mcdi_init(sfxge_t *sp);
extern void			sfxge_mcdi_fini(sfxge_t *sp);
extern int			sfxge_mcdi_ioctl(sfxge_t *sp,
    sfxge_mcdi_ioc_t *smip);
extern int			sfxge_mcdi2_ioctl(sfxge_t *sp,
    sfxge_mcdi2_ioc_t *smip);

extern int			sfxge_phy_init(sfxge_t *);
extern void			sfxge_phy_link_mode_get(sfxge_t *,
    efx_link_mode_t *);
extern void			sfxge_phy_fini(sfxge_t *);
extern int			sfxge_phy_kstat_init(sfxge_t *sp);
extern void			sfxge_phy_kstat_fini(sfxge_t *sp);
extern uint8_t			sfxge_phy_lp_cap_test(sfxge_t *sp,
    uint32_t field);
extern int			sfxge_phy_cap_apply(sfxge_t *sp,
    boolean_t use_default);
extern uint8_t			sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
    uint32_t field, boolean_t *mutablep);
extern int			sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
    int set);

extern int			sfxge_rx_init(sfxge_t *);
extern int			sfxge_rx_start(sfxge_t *);
extern void			sfxge_rx_coalesce_mode_get(sfxge_t *,
    sfxge_rx_coalesce_mode_t *);
extern int			sfxge_rx_coalesce_mode_set(sfxge_t *,
    sfxge_rx_coalesce_mode_t);
extern unsigned int		sfxge_rx_scale_prop_get(sfxge_t *);
extern void			sfxge_rx_scale_update(void *);
extern int			sfxge_rx_scale_count_get(sfxge_t *,
    unsigned int *);
extern int			sfxge_rx_scale_count_set(sfxge_t *,
    unsigned int);
extern void			sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
extern void			sfxge_rx_qflush_done(sfxge_rxq_t *);
extern void			sfxge_rx_qflush_failed(sfxge_rxq_t *);
extern void			sfxge_rx_qfpp_trim(sfxge_rxq_t *);
extern void			sfxge_rx_stop(sfxge_t *);
extern unsigned int 		sfxge_rx_loaned(sfxge_t *);
extern void			sfxge_rx_fini(sfxge_t *);

extern int			sfxge_tx_init(sfxge_t *);
extern int			sfxge_tx_start(sfxge_t *);
extern int			sfxge_tx_packet_add(sfxge_t *, mblk_t *);
extern void			sfxge_tx_qcomplete(sfxge_txq_t *);
extern void			sfxge_tx_qflush_done(sfxge_txq_t *);
extern void			sfxge_tx_stop(sfxge_t *);
extern void			sfxge_tx_fini(sfxge_t *);
extern void			sfxge_tx_qdpl_flush(sfxge_txq_t *stp);

extern void			sfxge_sram_init(sfxge_t *);
extern int			sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
    uint32_t *);
extern int			sfxge_sram_start(sfxge_t *);
extern int			sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
    efsys_mem_t *, size_t);
extern void			sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
    size_t);
extern void			sfxge_sram_stop(sfxge_t *);
extern void			sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
    size_t);
extern void			sfxge_sram_fini(sfxge_t *);

extern sfxge_packet_type_t	sfxge_pkthdr_parse(mblk_t *,
    struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
    uint16_t *, uint16_t *);

extern int sfxge_toeplitz_hash_init(sfxge_t *);
extern void sfxge_toeplitz_hash_fini(sfxge_t *);
extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
    uint8_t *, uint16_t, uint8_t *, uint16_t);

/*
 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
 * To compute the same hash value as Siena/Huntington hardware, the inputs
 * must be in big endian (network) byte order.
 */
#define	SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)	\
	do { \
		(_hash) = sfxge_toeplitz_hash(_sp, \
					sizeof (struct in_addr), \
					(uint8_t *)(_raddr), \
					(_rport), \
					(uint8_t *)(_laddr), \
					(_lport)); \
		_NOTE(CONSTANTCONDITION) \
	} while (B_FALSE)

/*
 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
 * For other IPv4 non-TCP packets, use zero for the port numbers.
 */
#define	SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)	\
	SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))


extern int		sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);

extern int		sfxge_pci_init(sfxge_t *);
extern void		sfxge_pcie_check_link(sfxge_t *, unsigned int,
    unsigned int);
extern void		sfxge_pci_fini(sfxge_t *);

extern int		sfxge_bar_init(sfxge_t *);
extern void		sfxge_bar_fini(sfxge_t *);

extern int		sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);


#endif /* _KERNEL */

#ifdef	__cplusplus
}
#endif

#endif	/* _SYS_SFXGE_H */