xref: /illumos-gate/usr/src/uts/common/io/rge/rge_main.c (revision 0dc2366f)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "rge.h"

/*
 * This is the string displayed by modinfo, etc.
 * Make sure you keep the version ID up to date!
 */
static char rge_ident[] = "Realtek 1Gb Ethernet";

/*
 * Used for buffers allocated by ddi_dma_mem_alloc()
 */
static ddi_dma_attr_t dma_attr_buf = {
	DMA_ATTR_V0,		/* dma_attr version */
	(uint32_t)0,		/* dma_attr_addr_lo */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_addr_hi */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_count_max */
	(uint32_t)16,		/* dma_attr_align */
	0xFFFFFFFF,		/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_maxxfer */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_seg */
	1,			/* dma_attr_sgllen */
	1,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};

/*
 * Used for BDs allocated by ddi_dma_mem_alloc()
 */
static ddi_dma_attr_t dma_attr_desc = {
	DMA_ATTR_V0,		/* dma_attr version */
	(uint32_t)0,		/* dma_attr_addr_lo */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_addr_hi */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_count_max */
	(uint32_t)256,		/* dma_attr_align */
	0xFFFFFFFF,		/* dma_attr_burstsizes */
	1,			/* dma_attr_minxfer */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_maxxfer */
	(uint32_t)0xFFFFFFFF,	/* dma_attr_seg */
	1,			/* dma_attr_sgllen */
	1,			/* dma_attr_granular */
	0,			/* dma_attr_flags */
};

/*
 * PIO access attributes for registers
 */
static ddi_device_acc_attr_t rge_reg_accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_STRUCTURE_LE_ACC,
	DDI_STRICTORDER_ACC,
	DDI_DEFAULT_ACC
};

/*
 * DMA access attributes for descriptors
 */
static ddi_device_acc_attr_t rge_desc_accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC,
	DDI_DEFAULT_ACC
};

/*
 * DMA access attributes for data
 */
static ddi_device_acc_attr_t rge_buf_accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC,
	DDI_DEFAULT_ACC
};

/*
 * Property names
 */
static char debug_propname[] = "rge_debug_flags";
static char mtu_propname[] = "default_mtu";
static char msi_propname[] = "msi_enable";

static int		rge_m_start(void *);
static void		rge_m_stop(void *);
static int		rge_m_promisc(void *, boolean_t);
static int		rge_m_multicst(void *, boolean_t, const uint8_t *);
static int		rge_m_unicst(void *, const uint8_t *);
static void		rge_m_ioctl(void *, queue_t *, mblk_t *);
static boolean_t	rge_m_getcapab(void *, mac_capab_t, void *);

#define	RGE_M_CALLBACK_FLAGS	(MC_IOCTL | MC_GETCAPAB)

static mac_callbacks_t rge_m_callbacks = {
	RGE_M_CALLBACK_FLAGS,
	rge_m_stat,
	rge_m_start,
	rge_m_stop,
	rge_m_promisc,
	rge_m_multicst,
	rge_m_unicst,
	rge_m_tx,
	NULL,
	rge_m_ioctl,
	rge_m_getcapab
};

/*
 * Allocate an area of memory and a DMA handle for accessing it
 */
static int
rge_alloc_dma_mem(rge_t *rgep, size_t memsize, ddi_dma_attr_t *dma_attr_p,
	ddi_device_acc_attr_t *acc_attr_p, uint_t dma_flags, dma_area_t *dma_p)
{
	caddr_t vaddr;
	int err;

	/*
	 * Allocate handle
	 */
	err = ddi_dma_alloc_handle(rgep->devinfo, dma_attr_p,
	    DDI_DMA_SLEEP, NULL, &dma_p->dma_hdl);
	if (err != DDI_SUCCESS) {
		dma_p->dma_hdl = NULL;
		return (DDI_FAILURE);
	}

	/*
	 * Allocate memory
	 */
	err = ddi_dma_mem_alloc(dma_p->dma_hdl, memsize, acc_attr_p,
	    dma_flags & (DDI_DMA_CONSISTENT | DDI_DMA_STREAMING),
	    DDI_DMA_SLEEP, NULL, &vaddr, &dma_p->alength, &dma_p->acc_hdl);
	if (err != DDI_SUCCESS) {
		ddi_dma_free_handle(&dma_p->dma_hdl);
		dma_p->dma_hdl = NULL;
		dma_p->acc_hdl = NULL;
		return (DDI_FAILURE);
	}

	/*
	 * Bind the two together
	 */
	dma_p->mem_va = vaddr;
	err = ddi_dma_addr_bind_handle(dma_p->dma_hdl, NULL,
	    vaddr, dma_p->alength, dma_flags, DDI_DMA_SLEEP, NULL,
	    &dma_p->cookie, &dma_p->ncookies);
	if (err != DDI_DMA_MAPPED || dma_p->ncookies != 1) {
		ddi_dma_mem_free(&dma_p->acc_hdl);
		ddi_dma_free_handle(&dma_p->dma_hdl);
		dma_p->acc_hdl = NULL;
		dma_p->dma_hdl = NULL;
		return (DDI_FAILURE);
	}

	dma_p->nslots = ~0U;
	dma_p->size = ~0U;
	dma_p->token = ~0U;
	dma_p->offset = 0;
	return (DDI_SUCCESS);
}

/*
 * Free one allocated area of DMAable memory
 */
static void
rge_free_dma_mem(dma_area_t *dma_p)
{
	if (dma_p->dma_hdl != NULL) {
		if (dma_p->ncookies) {
			(void) ddi_dma_unbind_handle(dma_p->dma_hdl);
			dma_p->ncookies = 0;
		}
		ddi_dma_free_handle(&dma_p->dma_hdl);
		dma_p->dma_hdl = NULL;
	}

	if (dma_p->acc_hdl != NULL) {
		ddi_dma_mem_free(&dma_p->acc_hdl);
		dma_p->acc_hdl = NULL;
	}
}

/*
 * Utility routine to carve a slice off a chunk of allocated memory,
 * updating the chunk descriptor accordingly.  The size of the slice
 * is given by the product of the <qty> and <size> parameters.
 */
static void
rge_slice_chunk(dma_area_t *slice, dma_area_t *chunk,
	uint32_t qty, uint32_t size)
{
	static uint32_t sequence = 0xbcd5704a;
	size_t totsize;

	totsize = qty*size;
	ASSERT(totsize <= chunk->alength);

	*slice = *chunk;
	slice->nslots = qty;
	slice->size = size;
	slice->alength = totsize;
	slice->token = ++sequence;

	chunk->mem_va = (caddr_t)chunk->mem_va + totsize;
	chunk->alength -= totsize;
	chunk->offset += totsize;
	chunk->cookie.dmac_laddress += totsize;
	chunk->cookie.dmac_size -= totsize;
}

static int
rge_alloc_bufs(rge_t *rgep)
{
	size_t txdescsize;
	size_t rxdescsize;
	int err;

	/*
	 * Allocate memory & handle for packet statistics
	 */
	err = rge_alloc_dma_mem(rgep,
	    RGE_STATS_DUMP_SIZE,
	    &dma_attr_desc,
	    &rge_desc_accattr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &rgep->dma_area_stats);
	if (err != DDI_SUCCESS)
		return (DDI_FAILURE);
	rgep->hw_stats = DMA_VPTR(rgep->dma_area_stats);

	/*
	 * Allocate memory & handle for Tx descriptor ring
	 */
	txdescsize = RGE_SEND_SLOTS * sizeof (rge_bd_t);
	err = rge_alloc_dma_mem(rgep,
	    txdescsize,
	    &dma_attr_desc,
	    &rge_desc_accattr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &rgep->dma_area_txdesc);
	if (err != DDI_SUCCESS)
		return (DDI_FAILURE);

	/*
	 * Allocate memory & handle for Rx descriptor ring
	 */
	rxdescsize = RGE_RECV_SLOTS * sizeof (rge_bd_t);
	err = rge_alloc_dma_mem(rgep,
	    rxdescsize,
	    &dma_attr_desc,
	    &rge_desc_accattr,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    &rgep->dma_area_rxdesc);
	if (err != DDI_SUCCESS)
		return (DDI_FAILURE);

	return (DDI_SUCCESS);
}

/*
 * rge_free_bufs() -- free descriptors/buffers allocated for this
 * device instance.
 */
static void
rge_free_bufs(rge_t *rgep)
{
	rge_free_dma_mem(&rgep->dma_area_stats);
	rge_free_dma_mem(&rgep->dma_area_txdesc);
	rge_free_dma_mem(&rgep->dma_area_rxdesc);
}

/*
 * ========== Transmit and receive ring reinitialisation ==========
 */

/*
 * These <reinit> routines each reset the rx/tx rings to an initial
 * state, assuming that the corresponding <init> routine has already
 * been called exactly once.
 */
static void
rge_reinit_send_ring(rge_t *rgep)
{
	sw_sbd_t *ssbdp;
	rge_bd_t *bdp;
	uint32_t slot;

	/*
	 * re-init send ring
	 */
	DMA_ZERO(rgep->tx_desc);
	ssbdp = rgep->sw_sbds;
	bdp = rgep->tx_ring;
	for (slot = 0; slot < RGE_SEND_SLOTS; slot++) {
		bdp->host_buf_addr =
		    RGE_BSWAP_32(ssbdp->pbuf.cookie.dmac_laddress);
		bdp->host_buf_addr_hi =
		    RGE_BSWAP_32(ssbdp->pbuf.cookie.dmac_laddress >> 32);
		/* last BD in Tx ring */
		if (slot == (RGE_SEND_SLOTS - 1))
			bdp->flags_len = RGE_BSWAP_32(BD_FLAG_EOR);
		ssbdp++;
		bdp++;
	}
	DMA_SYNC(rgep->tx_desc, DDI_DMA_SYNC_FORDEV);
	rgep->tx_next = 0;
	rgep->tc_next = 0;
	rgep->tc_tail = 0;
	rgep->tx_flow = 0;
	rgep->tx_free = RGE_SEND_SLOTS;
}

static void
rge_reinit_recv_ring(rge_t *rgep)
{
	rge_bd_t *bdp;
	sw_rbd_t *srbdp;
	dma_area_t *pbuf;
	uint32_t slot;

	/*
	 * re-init receive ring
	 */
	DMA_ZERO(rgep->rx_desc);
	srbdp = rgep->sw_rbds;
	bdp = rgep->rx_ring;
	for (slot = 0; slot < RGE_RECV_SLOTS; slot++) {
		pbuf = &srbdp->rx_buf->pbuf;
		bdp->host_buf_addr =
		    RGE_BSWAP_32(pbuf->cookie.dmac_laddress + rgep->head_room);
		bdp->host_buf_addr_hi =
		    RGE_BSWAP_32(pbuf->cookie.dmac_laddress >> 32);
		bdp->flags_len = RGE_BSWAP_32(BD_FLAG_HW_OWN |
		    (rgep->rxbuf_size - rgep->head_room));
		/* last BD in Tx ring */
		if (slot == (RGE_RECV_SLOTS - 1))
			bdp->flags_len |= RGE_BSWAP_32(BD_FLAG_EOR);
		srbdp++;
		bdp++;
	}
	DMA_SYNC(rgep->rx_desc, DDI_DMA_SYNC_FORDEV);
	rgep->watchdog = 0;
	rgep->rx_next = 0;
}

static void
rge_reinit_buf_ring(rge_t *rgep)
{

	if (rgep->chip_flags & CHIP_FLAG_FORCE_BCOPY)
		return;

	/*
	 * If all the up-sending buffers haven't been returned to driver,
	 * use bcopy() only in rx process.
	 */
	if (rgep->rx_free != RGE_BUF_SLOTS)
		rgep->rx_bcopy = B_TRUE;
}

static void
rge_reinit_rings(rge_t *rgep)
{
	rge_reinit_send_ring(rgep);
	rge_reinit_recv_ring(rgep);
	rge_reinit_buf_ring(rgep);
}

static void
rge_fini_send_ring(rge_t *rgep)
{
	sw_sbd_t *ssbdp;
	uint32_t slot;

	ssbdp = rgep->sw_sbds;
	for (slot = 0; slot < RGE_SEND_SLOTS; ++slot) {
		rge_free_dma_mem(&ssbdp->pbuf);
		ssbdp++;
	}

	kmem_free(rgep->sw_sbds, RGE_SEND_SLOTS * sizeof (sw_sbd_t));
	rgep->sw_sbds = NULL;
}

static void
rge_fini_recv_ring(rge_t *rgep)
{
	sw_rbd_t *srbdp;
	uint32_t slot;

	srbdp = rgep->sw_rbds;
	for (slot = 0; slot < RGE_RECV_SLOTS; ++srbdp, ++slot) {
		if (srbdp->rx_buf) {
			if (srbdp->rx_buf->mp != NULL) {
				freemsg(srbdp->rx_buf->mp);
				srbdp->rx_buf->mp = NULL;
			}
			rge_free_dma_mem(&srbdp->rx_buf->pbuf);
			kmem_free(srbdp->rx_buf, sizeof (dma_buf_t));
			srbdp->rx_buf = NULL;
		}
	}

	kmem_free(rgep->sw_rbds, RGE_RECV_SLOTS * sizeof (sw_rbd_t));
	rgep->sw_rbds = NULL;
}

static void
rge_fini_buf_ring(rge_t *rgep)
{
	sw_rbd_t *srbdp;
	uint32_t slot;

	if (rgep->chip_flags & CHIP_FLAG_FORCE_BCOPY)
		return;

	ASSERT(rgep->rx_free == RGE_BUF_SLOTS);

	srbdp = rgep->free_srbds;
	for (slot = 0; slot < RGE_BUF_SLOTS; ++srbdp, ++slot) {
		if (srbdp->rx_buf != NULL) {
			if (srbdp->rx_buf->mp != NULL) {
				freemsg(srbdp->rx_buf->mp);
				srbdp->rx_buf->mp = NULL;
			}
			rge_free_dma_mem(&srbdp->rx_buf->pbuf);
			kmem_free(srbdp->rx_buf, sizeof (dma_buf_t));
			srbdp->rx_buf = NULL;
		}
	}

	kmem_free(rgep->free_srbds, RGE_BUF_SLOTS * sizeof (sw_rbd_t));
	rgep->free_srbds = NULL;
}

static void
rge_fini_rings(rge_t *rgep)
{
	rge_fini_send_ring(rgep);
	rge_fini_recv_ring(rgep);
	rge_fini_buf_ring(rgep);
}

static int
rge_init_send_ring(rge_t *rgep)
{
	uint32_t slot;
	sw_sbd_t *ssbdp;
	dma_area_t *pbuf;
	dma_area_t desc;
	int err;

	/*
	 * Allocate the array of s/w Tx Buffer Descriptors
	 */
	ssbdp = kmem_zalloc(RGE_SEND_SLOTS*sizeof (*ssbdp), KM_SLEEP);
	rgep->sw_sbds = ssbdp;

	/*
	 * Init send ring
	 */
	rgep->tx_desc = rgep->dma_area_txdesc;
	DMA_ZERO(rgep->tx_desc);
	rgep->tx_ring = rgep->tx_desc.mem_va;

	desc = rgep->tx_desc;
	for (slot = 0; slot < RGE_SEND_SLOTS; slot++) {
		rge_slice_chunk(&ssbdp->desc, &desc, 1, sizeof (rge_bd_t));

		/*
		 * Allocate memory & handle for Tx buffers
		 */
		pbuf = &ssbdp->pbuf;
		err = rge_alloc_dma_mem(rgep, rgep->txbuf_size,
		    &dma_attr_buf, &rge_buf_accattr,
		    DDI_DMA_WRITE | DDI_DMA_STREAMING, pbuf);
		if (err != DDI_SUCCESS) {
			rge_error(rgep,
			    "rge_init_send_ring: alloc tx buffer failed");
			rge_fini_send_ring(rgep);
			return (DDI_FAILURE);
		}
		ssbdp++;
	}
	ASSERT(desc.alength == 0);

	DMA_SYNC(rgep->tx_desc, DDI_DMA_SYNC_FORDEV);
	return (DDI_SUCCESS);
}

static int
rge_init_recv_ring(rge_t *rgep)
{
	uint32_t slot;
	sw_rbd_t *srbdp;
	dma_buf_t *rx_buf;
	dma_area_t *pbuf;
	int err;

	/*
	 * Allocate the array of s/w Rx Buffer Descriptors
	 */
	srbdp = kmem_zalloc(RGE_RECV_SLOTS*sizeof (*srbdp), KM_SLEEP);
	rgep->sw_rbds = srbdp;

	/*
	 * Init receive ring
	 */
	rgep->rx_next = 0;
	rgep->rx_desc = rgep->dma_area_rxdesc;
	DMA_ZERO(rgep->rx_desc);
	rgep->rx_ring = rgep->rx_desc.mem_va;

	for (slot = 0; slot < RGE_RECV_SLOTS; slot++) {
		srbdp->rx_buf = rx_buf =
		    kmem_zalloc(sizeof (dma_buf_t), KM_SLEEP);

		/*
		 * Allocate memory & handle for Rx buffers
		 */
		pbuf = &rx_buf->pbuf;
		err = rge_alloc_dma_mem(rgep, rgep->rxbuf_size,
		    &dma_attr_buf, &rge_buf_accattr,
		    DDI_DMA_READ | DDI_DMA_STREAMING, pbuf);
		if (err != DDI_SUCCESS) {
			rge_fini_recv_ring(rgep);
			rge_error(rgep,
			    "rge_init_recv_ring: alloc rx buffer failed");
			return (DDI_FAILURE);
		}

		pbuf->alength -= rgep->head_room;
		pbuf->offset += rgep->head_room;
		if (!(rgep->chip_flags & CHIP_FLAG_FORCE_BCOPY)) {
			rx_buf->rx_recycle.free_func = rge_rx_recycle;
			rx_buf->rx_recycle.free_arg = (caddr_t)rx_buf;
			rx_buf->private = (caddr_t)rgep;
			rx_buf->mp = desballoc(DMA_VPTR(rx_buf->pbuf),
			    rgep->rxbuf_size, 0, &rx_buf->rx_recycle);
			if (rx_buf->mp == NULL) {
				rge_fini_recv_ring(rgep);
				rge_problem(rgep,
				    "rge_init_recv_ring: desballoc() failed");
				return (DDI_FAILURE);
			}
		}
		srbdp++;
	}
	DMA_SYNC(rgep->rx_desc, DDI_DMA_SYNC_FORDEV);
	return (DDI_SUCCESS);
}

static int
rge_init_buf_ring(rge_t *rgep)
{
	uint32_t slot;
	sw_rbd_t *free_srbdp;
	dma_buf_t *rx_buf;
	dma_area_t *pbuf;
	int err;

	if (rgep->chip_flags & CHIP_FLAG_FORCE_BCOPY) {
		rgep->rx_bcopy = B_TRUE;
		return (DDI_SUCCESS);
	}

	/*
	 * Allocate the array of s/w free Buffer Descriptors
	 */
	free_srbdp = kmem_zalloc(RGE_BUF_SLOTS*sizeof (*free_srbdp), KM_SLEEP);
	rgep->free_srbds = free_srbdp;

	/*
	 * Init free buffer ring
	 */
	rgep->rc_next = 0;
	rgep->rf_next = 0;
	rgep->rx_bcopy = B_FALSE;
	rgep->rx_free = RGE_BUF_SLOTS;
	for (slot = 0; slot < RGE_BUF_SLOTS; slot++) {
		free_srbdp->rx_buf = rx_buf =
		    kmem_zalloc(sizeof (dma_buf_t), KM_SLEEP);

		/*
		 * Allocate memory & handle for free Rx buffers
		 */
		pbuf = &rx_buf->pbuf;
		err = rge_alloc_dma_mem(rgep, rgep->rxbuf_size,
		    &dma_attr_buf, &rge_buf_accattr,
		    DDI_DMA_READ | DDI_DMA_STREAMING, pbuf);
		if (err != DDI_SUCCESS) {
			rge_fini_buf_ring(rgep);
			rge_error(rgep,
			    "rge_init_buf_ring: alloc rx free buffer failed");
			return (DDI_FAILURE);
		}
		pbuf->alength -= rgep->head_room;
		pbuf->offset += rgep->head_room;
		rx_buf->rx_recycle.free_func = rge_rx_recycle;
		rx_buf->rx_recycle.free_arg = (caddr_t)rx_buf;
		rx_buf->private = (caddr_t)rgep;
		rx_buf->mp = desballoc(DMA_VPTR(rx_buf->pbuf),
		    rgep->rxbuf_size, 0, &rx_buf->rx_recycle);
		if (rx_buf->mp == NULL) {
			rge_fini_buf_ring(rgep);
			rge_problem(rgep,
			    "rge_init_buf_ring: desballoc() failed");
			return (DDI_FAILURE);
		}
		free_srbdp++;
	}
	return (DDI_SUCCESS);
}

static int
rge_init_rings(rge_t *rgep)
{
	int err;

	err = rge_init_send_ring(rgep);
	if (err != DDI_SUCCESS)
		return (DDI_FAILURE);

	err = rge_init_recv_ring(rgep);
	if (err != DDI_SUCCESS) {
		rge_fini_send_ring(rgep);
		return (DDI_FAILURE);
	}

	err = rge_init_buf_ring(rgep);
	if (err != DDI_SUCCESS) {
		rge_fini_send_ring(rgep);
		rge_fini_recv_ring(rgep);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

/*
 * ========== Internal state management entry points ==========
 */

#undef	RGE_DBG
#define	RGE_DBG		RGE_DBG_NEMO	/* debug flag for this code	*/

/*
 * These routines provide all the functionality required by the
 * corresponding MAC layer entry points, but don't update the
 * MAC state so they can be called internally without disturbing
 * our record of what NEMO thinks we should be doing ...
 */

/*
 *	rge_reset() -- reset h/w & rings to initial state
 */
static void
rge_reset(rge_t *rgep)
{
	ASSERT(mutex_owned(rgep->genlock));

	/*
	 * Grab all the other mutexes in the world (this should
	 * ensure no other threads are manipulating driver state)
	 */
	mutex_enter(rgep->rx_lock);
	mutex_enter(rgep->rc_lock);
	rw_enter(rgep->errlock, RW_WRITER);

	(void) rge_chip_reset(rgep);
	rge_reinit_rings(rgep);
	rge_chip_init(rgep);

	/*
	 * Free the world ...
	 */
	rw_exit(rgep->errlock);
	mutex_exit(rgep->rc_lock);
	mutex_exit(rgep->rx_lock);

	rgep->stats.rpackets = 0;
	rgep->stats.rbytes = 0;
	rgep->stats.opackets = 0;
	rgep->stats.obytes = 0;
	rgep->stats.tx_pre_ismax = B_FALSE;
	rgep->stats.tx_cur_ismax = B_FALSE;

	RGE_DEBUG(("rge_reset($%p) done", (void *)rgep));
}

/*
 *	rge_stop() -- stop processing, don't reset h/w or rings
 */
static void
rge_stop(rge_t *rgep)
{
	ASSERT(mutex_owned(rgep->genlock));

	rge_chip_stop(rgep, B_FALSE);

	RGE_DEBUG(("rge_stop($%p) done", (void *)rgep));
}

/*
 *	rge_start() -- start transmitting/receiving
 */
static void
rge_start(rge_t *rgep)
{
	ASSERT(mutex_owned(rgep->genlock));

	/*
	 * Start chip processing, including enabling interrupts
	 */
	rge_chip_start(rgep);
	rgep->watchdog = 0;
}

/*
 * rge_restart - restart transmitting/receiving after error or suspend
 */
void
rge_restart(rge_t *rgep)
{
	uint32_t i;

	ASSERT(mutex_owned(rgep->genlock));
	/*
	 * Wait for posted buffer to be freed...
	 */
	if (!rgep->rx_bcopy) {
		for (i = 0; i < RXBUFF_FREE_LOOP; i++) {
			if (rgep->rx_free == RGE_BUF_SLOTS)
				break;
			drv_usecwait(1000);
			RGE_DEBUG(("rge_restart: waiting for rx buf free..."));
		}
	}
	rge_reset(rgep);
	rgep->stats.chip_reset++;
	if (rgep->rge_mac_state == RGE_MAC_STARTED) {
		rge_start(rgep);
		rgep->resched_needed = B_TRUE;
		(void) ddi_intr_trigger_softint(rgep->resched_hdl, NULL);
	}
}


/*
 * ========== Nemo-required management entry points ==========
 */

#undef	RGE_DBG
#define	RGE_DBG		RGE_DBG_NEMO	/* debug flag for this code	*/

/*
 *	rge_m_stop() -- stop transmitting/receiving
 */
static void
rge_m_stop(void *arg)
{
	rge_t *rgep = arg;		/* private device info	*/
	uint32_t i;

	/*
	 * Just stop processing, then record new MAC state
	 */
	mutex_enter(rgep->genlock);
	if (rgep->suspended) {
		ASSERT(rgep->rge_mac_state == RGE_MAC_STOPPED);
		mutex_exit(rgep->genlock);
		return;
	}
	rge_stop(rgep);
	/*
	 * Wait for posted buffer to be freed...
	 */
	if (!rgep->rx_bcopy) {
		for (i = 0; i < RXBUFF_FREE_LOOP; i++) {
			if (rgep->rx_free == RGE_BUF_SLOTS)
				break;
			drv_usecwait(1000);
			RGE_DEBUG(("rge_m_stop: waiting for rx buf free..."));
		}
	}
	rgep->rge_mac_state = RGE_MAC_STOPPED;
	RGE_DEBUG(("rge_m_stop($%p) done", arg));
	mutex_exit(rgep->genlock);
}

/*
 *	rge_m_start() -- start transmitting/receiving
 */
static int
rge_m_start(void *arg)
{
	rge_t *rgep = arg;		/* private device info	*/

	mutex_enter(rgep->genlock);
	if (rgep->suspended) {
		mutex_exit(rgep->genlock);
		return (DDI_FAILURE);
	}
	/*
	 * Clear hw/sw statistics
	 */
	DMA_ZERO(rgep->dma_area_stats);
	bzero(&rgep->stats, sizeof (rge_stats_t));

	/*
	 * Start processing and record new MAC state
	 */
	rge_reset(rgep);
	rge_start(rgep);
	rgep->rge_mac_state = RGE_MAC_STARTED;
	RGE_DEBUG(("rge_m_start($%p) done", arg));

	mutex_exit(rgep->genlock);

	return (0);
}

/*
 *	rge_m_unicst_set() -- set the physical network address
 */
static int
rge_m_unicst(void *arg, const uint8_t *macaddr)
{
	rge_t *rgep = arg;		/* private device info	*/

	/*
	 * Remember the new current address in the driver state
	 * Sync the chip's idea of the address too ...
	 */
	mutex_enter(rgep->genlock);
	bcopy(macaddr, rgep->netaddr, ETHERADDRL);

	if (rgep->suspended) {
		mutex_exit(rgep->genlock);
		return (DDI_SUCCESS);
	}

	rge_chip_sync(rgep, RGE_SET_MAC);
	mutex_exit(rgep->genlock);

	return (0);
}

/*
 * Compute the index of the required bit in the multicast hash map.
 * This must mirror the way the hardware actually does it!
 */
static uint32_t
rge_hash_index(const uint8_t *mca)
{
	uint32_t crc = (uint32_t)RGE_HASH_CRC;
	uint32_t const POLY = RGE_HASH_POLY;
	uint32_t msb;
	int bytes;
	uchar_t currentbyte;
	uint32_t index;
	int bit;

	for (bytes = 0; bytes < ETHERADDRL; bytes++) {
		currentbyte = mca[bytes];
		for (bit = 0; bit < 8; bit++) {
			msb = crc >> 31;
			crc <<= 1;
			if (msb ^ (currentbyte & 1))
				crc ^= POLY;
			currentbyte >>= 1;
		}
	}
	index = crc >> 26;
		/* the index value is between 0 and 63(0x3f) */

	return (index);
}

/*
 *	rge_m_multicst_add() -- enable/disable a multicast address
 */
static int
rge_m_multicst(void *arg, boolean_t add, const uint8_t *mca)
{
	rge_t *rgep = arg;		/* private device info	*/
	struct ether_addr *addr;
	uint32_t index;
	uint32_t reg;
	uint8_t *hashp;

	mutex_enter(rgep->genlock);
	hashp = rgep->mcast_hash;
	addr = (struct ether_addr *)mca;
	/*
	 * Calculate the Multicast address hash index value
	 *	Normally, the position of MAR0-MAR7 is
	 *	MAR0: offset 0x08, ..., MAR7: offset 0x0F.
	 *
	 *	For pcie chipset, the position of MAR0-MAR7 is
	 *	different from others:
	 *	MAR0: offset 0x0F, ..., MAR7: offset 0x08.
	 */
	index = rge_hash_index(addr->ether_addr_octet);
	if (rgep->chipid.is_pcie)
		reg = (~(index / RGE_MCAST_NUM)) & 0x7;
	else
		reg = index / RGE_MCAST_NUM;

	if (add) {
		if (rgep->mcast_refs[index]++) {
			mutex_exit(rgep->genlock);
			return (0);
		}
		hashp[reg] |= 1 << (index % RGE_MCAST_NUM);
	} else {
		if (--rgep->mcast_refs[index]) {
			mutex_exit(rgep->genlock);
			return (0);
		}
		hashp[reg] &= ~ (1 << (index % RGE_MCAST_NUM));
	}

	if (rgep->suspended) {
		mutex_exit(rgep->genlock);
		return (DDI_SUCCESS);
	}

	/*
	 * Set multicast register
	 */
	rge_chip_sync(rgep, RGE_SET_MUL);

	mutex_exit(rgep->genlock);
	return (0);
}

/*
 * rge_m_promisc() -- set or reset promiscuous mode on the board
 *
 *	Program the hardware to enable/disable promiscuous and/or
 *	receive-all-multicast modes.
 */
static int
rge_m_promisc(void *arg, boolean_t on)
{
	rge_t *rgep = arg;

	/*
	 * Store MAC layer specified mode and pass to chip layer to update h/w
	 */
	mutex_enter(rgep->genlock);

	if (rgep->promisc == on) {
		mutex_exit(rgep->genlock);
		return (0);
	}
	rgep->promisc = on;

	if (rgep->suspended) {
		mutex_exit(rgep->genlock);
		return (DDI_SUCCESS);
	}

	rge_chip_sync(rgep, RGE_SET_PROMISC);
	RGE_DEBUG(("rge_m_promisc_set($%p) done", arg));
	mutex_exit(rgep->genlock);
	return (0);
}

/*
 * Loopback ioctl code
 */

static lb_property_t loopmodes[] = {
	{ normal,	"normal",	RGE_LOOP_NONE		},
	{ internal,	"PHY",		RGE_LOOP_INTERNAL_PHY	},
	{ internal,	"MAC",		RGE_LOOP_INTERNAL_MAC	}
};

static enum ioc_reply
rge_set_loop_mode(rge_t *rgep, uint32_t mode)
{
	/*
	 * If the mode isn't being changed, there's nothing to do ...
	 */
	if (mode == rgep->param_loop_mode)
		return (IOC_ACK);

	/*
	 * Validate the requested mode and prepare a suitable message
	 * to explain the link down/up cycle that the change will
	 * probably induce ...
	 */
	switch (mode) {
	default:
		return (IOC_INVAL);

	case RGE_LOOP_NONE:
	case RGE_LOOP_INTERNAL_PHY:
	case RGE_LOOP_INTERNAL_MAC:
		break;
	}

	/*
	 * All OK; tell the caller to reprogram
	 * the PHY and/or MAC for the new mode ...
	 */
	rgep->param_loop_mode = mode;
	return (IOC_RESTART_ACK);
}

static enum ioc_reply
rge_loop_ioctl(rge_t *rgep, queue_t *wq, mblk_t *mp, struct iocblk *iocp)
{
	lb_info_sz_t *lbsp;
	lb_property_t *lbpp;
	uint32_t *lbmp;
	int cmd;

	_NOTE(ARGUNUSED(wq))

	/*
	 * Validate format of ioctl
	 */
	if (mp->b_cont == NULL)
		return (IOC_INVAL);

	cmd = iocp->ioc_cmd;
	switch (cmd) {
	default:
		/* NOTREACHED */
		rge_error(rgep, "rge_loop_ioctl: invalid cmd 0x%x", cmd);
		return (IOC_INVAL);

	case LB_GET_INFO_SIZE:
		if (iocp->ioc_count != sizeof (lb_info_sz_t))
			return (IOC_INVAL);
		lbsp = (lb_info_sz_t *)mp->b_cont->b_rptr;
		*lbsp = sizeof (loopmodes);
		return (IOC_REPLY);

	case LB_GET_INFO:
		if (iocp->ioc_count != sizeof (loopmodes))
			return (IOC_INVAL);
		lbpp = (lb_property_t *)mp->b_cont->b_rptr;
		bcopy(loopmodes, lbpp, sizeof (loopmodes));
		return (IOC_REPLY);

	case LB_GET_MODE:
		if (iocp->ioc_count != sizeof (uint32_t))
			return (IOC_INVAL);
		lbmp = (uint32_t *)mp->b_cont->b_rptr;
		*lbmp = rgep->param_loop_mode;
		return (IOC_REPLY);

	case LB_SET_MODE:
		if (iocp->ioc_count != sizeof (uint32_t))
			return (IOC_INVAL);
		lbmp = (uint32_t *)mp->b_cont->b_rptr;
		return (rge_set_loop_mode(rgep, *lbmp));
	}
}

/*
 * Specific rge IOCTLs, the MAC layer handles the generic ones.
 */
static void
rge_m_ioctl(void *arg, queue_t *wq, mblk_t *mp)
{
	rge_t *rgep = arg;
	struct iocblk *iocp;
	enum ioc_reply status;
	boolean_t need_privilege;
	int err;
	int cmd;

	/*
	 * If suspended, we might actually be able to do some of
	 * these ioctls, but it is harder to make sure they occur
	 * without actually putting the hardware in an undesireable
	 * state.  So just NAK it.
	 */
	mutex_enter(rgep->genlock);
	if (rgep->suspended) {
		miocnak(wq, mp, 0, EINVAL);
		mutex_exit(rgep->genlock);
		return;
	}
	mutex_exit(rgep->genlock);

	/*
	 * Validate the command before bothering with the mutex ...
	 */
	iocp = (struct iocblk *)mp->b_rptr;
	iocp->ioc_error = 0;
	need_privilege = B_TRUE;
	cmd = iocp->ioc_cmd;
	switch (cmd) {
	default:
		miocnak(wq, mp, 0, EINVAL);
		return;

	case RGE_MII_READ:
	case RGE_MII_WRITE:
	case RGE_DIAG:
	case RGE_PEEK:
	case RGE_POKE:
	case RGE_PHY_RESET:
	case RGE_SOFT_RESET:
	case RGE_HARD_RESET:
		break;

	case LB_GET_INFO_SIZE:
	case LB_GET_INFO:
	case LB_GET_MODE:
		need_privilege = B_FALSE;
		/* FALLTHRU */
	case LB_SET_MODE:
		break;

	case ND_GET:
		need_privilege = B_FALSE;
		/* FALLTHRU */
	case ND_SET:
		break;
	}

	if (need_privilege) {
		/*
		 * Check for specific net_config privilege
		 */
		err = secpolicy_net_config(iocp->ioc_cr, B_FALSE);
		if (err != 0) {
			miocnak(wq, mp, 0, err);
			return;
		}
	}

	mutex_enter(rgep->genlock);

	switch (cmd) {
	default:
		_NOTE(NOTREACHED)
		status = IOC_INVAL;
		break;

	case RGE_MII_READ:
	case RGE_MII_WRITE:
	case RGE_DIAG:
	case RGE_PEEK:
	case RGE_POKE:
	case RGE_PHY_RESET:
	case RGE_SOFT_RESET:
	case RGE_HARD_RESET:
		status = rge_chip_ioctl(rgep, wq, mp, iocp);
		break;

	case LB_GET_INFO_SIZE:
	case LB_GET_INFO:
	case LB_GET_MODE:
	case LB_SET_MODE:
		status = rge_loop_ioctl(rgep, wq, mp, iocp);
		break;

	case ND_GET:
	case ND_SET:
		status = rge_nd_ioctl(rgep, wq, mp, iocp);
		break;
	}

	/*
	 * Do we need to reprogram the PHY and/or the MAC?
	 * Do it now, while we still have the mutex.
	 *
	 * Note: update the PHY first, 'cos it controls the
	 * speed/duplex parameters that the MAC code uses.
	 */
	switch (status) {
	case IOC_RESTART_REPLY:
	case IOC_RESTART_ACK:
		rge_phy_update(rgep);
		break;
	}

	mutex_exit(rgep->genlock);

	/*
	 * Finally, decide how to reply
	 */
	switch (status) {
	default:
	case IOC_INVAL:
		/*
		 * Error, reply with a NAK and EINVAL or the specified error
		 */
		miocnak(wq, mp, 0, iocp->ioc_error == 0 ?
		    EINVAL : iocp->ioc_error);
		break;

	case IOC_DONE:
		/*
		 * OK, reply already sent
		 */
		break;

	case IOC_RESTART_ACK:
	case IOC_ACK:
		/*
		 * OK, reply with an ACK
		 */
		miocack(wq, mp, 0, 0);
		break;

	case IOC_RESTART_REPLY:
	case IOC_REPLY:
		/*
		 * OK, send prepared reply as ACK or NAK
		 */
		mp->b_datap->db_type = iocp->ioc_error == 0 ?
		    M_IOCACK : M_IOCNAK;
		qreply(wq, mp);
		break;
	}
}

/* ARGSUSED */
static boolean_t
rge_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
{
	rge_t *rgep = arg;

	switch (cap) {
	case MAC_CAPAB_HCKSUM: {
		uint32_t *hcksum_txflags = cap_data;
		switch (rgep->chipid.mac_ver) {
		case MAC_VER_8169:
		case MAC_VER_8169S_D:
		case MAC_VER_8169S_E:
		case MAC_VER_8169SB:
		case MAC_VER_8169SC:
		case MAC_VER_8168:
		case MAC_VER_8168B_B:
		case MAC_VER_8168B_C:
		case MAC_VER_8101E:
			*hcksum_txflags = HCKSUM_INET_FULL_V4 |
			    HCKSUM_IPHDRCKSUM;
			break;
		case MAC_VER_8168C:
		case MAC_VER_8101E_B:
		case MAC_VER_8101E_C:
		default:
			*hcksum_txflags = 0;
			break;
		}
		break;
	}
	default:
		return (B_FALSE);
	}
	return (B_TRUE);
}

/*
 * ============ Init MSI/Fixed Interrupt routines ==============
 */

/*
 * rge_add_intrs:
 *
 * Register FIXED or MSI interrupts.
 */
static int
rge_add_intrs(rge_t *rgep, int intr_type)
{
	dev_info_t *dip = rgep->devinfo;
	int avail;
	int actual;
	int intr_size;
	int count;
	int i, j;
	int ret;

	/* Get number of interrupts */
	ret = ddi_intr_get_nintrs(dip, intr_type, &count);
	if ((ret != DDI_SUCCESS) || (count == 0)) {
		rge_error(rgep, "ddi_intr_get_nintrs() failure, ret: %d, "
		    "count: %d", ret, count);
		return (DDI_FAILURE);
	}

	/* Get number of available interrupts */
	ret = ddi_intr_get_navail(dip, intr_type, &avail);
	if ((ret != DDI_SUCCESS) || (avail == 0)) {
		rge_error(rgep, "ddi_intr_get_navail() failure, "
		    "ret: %d, avail: %d\n", ret, avail);
		return (DDI_FAILURE);
	}

	/* Allocate an array of interrupt handles */
	intr_size = count * sizeof (ddi_intr_handle_t);
	rgep->htable = kmem_alloc(intr_size, KM_SLEEP);
	rgep->intr_rqst = count;

	/* Call ddi_intr_alloc() */
	ret = ddi_intr_alloc(dip, rgep->htable, intr_type, 0,
	    count, &actual, DDI_INTR_ALLOC_NORMAL);
	if (ret != DDI_SUCCESS || actual == 0) {
		rge_error(rgep, "ddi_intr_alloc() failed %d\n", ret);
		kmem_free(rgep->htable, intr_size);
		return (DDI_FAILURE);
	}
	if (actual < count) {
		rge_log(rgep, "ddi_intr_alloc() Requested: %d, Received: %d\n",
		    count, actual);
	}
	rgep->intr_cnt = actual;

	/*
	 * Get priority for first msi, assume remaining are all the same
	 */
	if ((ret = ddi_intr_get_pri(rgep->htable[0], &rgep->intr_pri)) !=
	    DDI_SUCCESS) {
		rge_error(rgep, "ddi_intr_get_pri() failed %d\n", ret);
		/* Free already allocated intr */
		for (i = 0; i < actual; i++) {
			(void) ddi_intr_free(rgep->htable[i]);
		}
		kmem_free(rgep->htable, intr_size);
		return (DDI_FAILURE);
	}

	/* Test for high level mutex */
	if (rgep->intr_pri >= ddi_intr_get_hilevel_pri()) {
		rge_error(rgep, "rge_add_intrs:"
		    "Hi level interrupt not supported");
		for (i = 0; i < actual; i++)
			(void) ddi_intr_free(rgep->htable[i]);
		kmem_free(rgep->htable, intr_size);
		return (DDI_FAILURE);
	}

	/* Call ddi_intr_add_handler() */
	for (i = 0; i < actual; i++) {
		if ((ret = ddi_intr_add_handler(rgep->htable[i], rge_intr,
		    (caddr_t)rgep, (caddr_t)(uintptr_t)i)) != DDI_SUCCESS) {
			rge_error(rgep, "ddi_intr_add_handler() "
			    "failed %d\n", ret);
			/* Remove already added intr */
			for (j = 0; j < i; j++)
				(void) ddi_intr_remove_handler(rgep->htable[j]);
			/* Free already allocated intr */
			for (i = 0; i < actual; i++) {
				(void) ddi_intr_free(rgep->htable[i]);
			}
			kmem_free(rgep->htable, intr_size);
			return (DDI_FAILURE);
		}
	}

	if ((ret = ddi_intr_get_cap(rgep->htable[0], &rgep->intr_cap))
	    != DDI_SUCCESS) {
		rge_error(rgep, "ddi_intr_get_cap() failed %d\n", ret);
		for (i = 0; i < actual; i++) {
			(void) ddi_intr_remove_handler(rgep->htable[i]);
			(void) ddi_intr_free(rgep->htable[i]);
		}
		kmem_free(rgep->htable, intr_size);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

/*
 * rge_rem_intrs:
 *
 * Unregister FIXED or MSI interrupts
 */
static void
rge_rem_intrs(rge_t *rgep)
{
	int i;

	/* Disable all interrupts */
	if (rgep->intr_cap & DDI_INTR_FLAG_BLOCK) {
		/* Call ddi_intr_block_disable() */
		(void) ddi_intr_block_disable(rgep->htable, rgep->intr_cnt);
	} else {
		for (i = 0; i < rgep->intr_cnt; i++) {
			(void) ddi_intr_disable(rgep->htable[i]);
		}
	}

	/* Call ddi_intr_remove_handler() */
	for (i = 0; i < rgep->intr_cnt; i++) {
		(void) ddi_intr_remove_handler(rgep->htable[i]);
		(void) ddi_intr_free(rgep->htable[i]);
	}

	kmem_free(rgep->htable, rgep->intr_rqst * sizeof (ddi_intr_handle_t));
}

/*
 * ========== Per-instance setup/teardown code ==========
 */

#undef	RGE_DBG
#define	RGE_DBG		RGE_DBG_INIT	/* debug flag for this code	*/

static void
rge_unattach(rge_t *rgep)
{
	/*
	 * Flag that no more activity may be initiated
	 */
	rgep->progress &= ~PROGRESS_READY;
	rgep->rge_mac_state = RGE_MAC_UNATTACH;

	/*
	 * Quiesce the PHY and MAC (leave it reset but still powered).
	 * Clean up and free all RGE data structures
	 */
	if (rgep->periodic_id != NULL) {
		ddi_periodic_delete(rgep->periodic_id);
		rgep->periodic_id = NULL;
	}

	if (rgep->progress & PROGRESS_KSTATS)
		rge_fini_kstats(rgep);

	if (rgep->progress & PROGRESS_PHY)
		(void) rge_phy_reset(rgep);

	if (rgep->progress & PROGRESS_INIT) {
		mutex_enter(rgep->genlock);
		(void) rge_chip_reset(rgep);
		mutex_exit(rgep->genlock);
		rge_fini_rings(rgep);
	}

	if (rgep->progress & PROGRESS_INTR) {
		rge_rem_intrs(rgep);
		mutex_destroy(rgep->rc_lock);
		mutex_destroy(rgep->rx_lock);
		mutex_destroy(rgep->tc_lock);
		mutex_destroy(rgep->tx_lock);
		rw_destroy(rgep->errlock);
		mutex_destroy(rgep->genlock);
	}

	if (rgep->progress & PROGRESS_FACTOTUM)
		(void) ddi_intr_remove_softint(rgep->factotum_hdl);

	if (rgep->progress & PROGRESS_RESCHED)
		(void) ddi_intr_remove_softint(rgep->resched_hdl);

	if (rgep->progress & PROGRESS_NDD)
		rge_nd_cleanup(rgep);

	rge_free_bufs(rgep);

	if (rgep->progress & PROGRESS_REGS)
		ddi_regs_map_free(&rgep->io_handle);

	if (rgep->progress & PROGRESS_CFG)
		pci_config_teardown(&rgep->cfg_handle);

	ddi_remove_minor_node(rgep->devinfo, NULL);
	kmem_free(rgep, sizeof (*rgep));
}

static int
rge_resume(dev_info_t *devinfo)
{
	rge_t *rgep;			/* Our private data	*/
	chip_id_t *cidp;
	chip_id_t chipid;

	rgep = ddi_get_driver_private(devinfo);

	/*
	 * If there are state inconsistancies, this is bad.  Returning
	 * DDI_FAILURE here will eventually cause the machine to panic,
	 * so it is best done here so that there is a possibility of
	 * debugging the problem.
	 */
	if (rgep == NULL)
		cmn_err(CE_PANIC,
		    "rge: ngep returned from ddi_get_driver_private was NULL");

	/*
	 * Refuse to resume if the data structures aren't consistent
	 */
	if (rgep->devinfo != devinfo)
		cmn_err(CE_PANIC,
		    "rge: passed devinfo not the same as saved devinfo");

	/*
	 * Read chip ID & set up config space command register(s)
	 * Refuse to resume if the chip has changed its identity!
	 */
	cidp = &rgep->chipid;
	rge_chip_cfg_init(rgep, &chipid);
	if (chipid.vendor != cidp->vendor)
		return (DDI_FAILURE);
	if (chipid.device != cidp->device)
		return (DDI_FAILURE);
	if (chipid.revision != cidp->revision)
		return (DDI_FAILURE);

	mutex_enter(rgep->genlock);

	/*
	 * Only in one case, this conditional branch can be executed: the port
	 * hasn't been plumbed.
	 */
	if (rgep->suspended == B_FALSE) {
		mutex_exit(rgep->genlock);
		return (DDI_SUCCESS);
	}
	rgep->rge_mac_state = RGE_MAC_STARTED;
	/*
	 * All OK, reinitialise h/w & kick off NEMO scheduling
	 */
	rge_restart(rgep);
	rgep->suspended = B_FALSE;

	mutex_exit(rgep->genlock);

	return (DDI_SUCCESS);
}


/*
 * attach(9E) -- Attach a device to the system
 *
 * Called once for each board successfully probed.
 */
static int
rge_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
{
	rge_t *rgep;			/* Our private data	*/
	mac_register_t *macp;
	chip_id_t *cidp;
	int intr_types;
	caddr_t regs;
	int instance;
	int i;
	int err;

	/*
	 * we don't support high level interrupts in the driver
	 */
	if (ddi_intr_hilevel(devinfo, 0) != 0) {
		cmn_err(CE_WARN,
		    "rge_attach -- unsupported high level interrupt");
		return (DDI_FAILURE);
	}

	instance = ddi_get_instance(devinfo);
	RGE_GTRACE(("rge_attach($%p, %d) instance %d",
	    (void *)devinfo, cmd, instance));
	RGE_BRKPT(NULL, "rge_attach");

	switch (cmd) {
	default:
		return (DDI_FAILURE);

	case DDI_RESUME:
		return (rge_resume(devinfo));

	case DDI_ATTACH:
		break;
	}

	rgep = kmem_zalloc(sizeof (*rgep), KM_SLEEP);
	ddi_set_driver_private(devinfo, rgep);
	rgep->devinfo = devinfo;

	/*
	 * Initialize more fields in RGE private data
	 */
	rgep->rge_mac_state = RGE_MAC_ATTACH;
	rgep->debug = ddi_prop_get_int(DDI_DEV_T_ANY, devinfo,
	    DDI_PROP_DONTPASS, debug_propname, rge_debug);
	rgep->default_mtu = ddi_prop_get_int(DDI_DEV_T_ANY, devinfo,
	    DDI_PROP_DONTPASS, mtu_propname, ETHERMTU);
	rgep->msi_enable = ddi_prop_get_int(DDI_DEV_T_ANY, devinfo,
	    DDI_PROP_DONTPASS, msi_propname, B_TRUE);
	(void) snprintf(rgep->ifname, sizeof (rgep->ifname), "%s%d",
	    RGE_DRIVER_NAME, instance);

	/*
	 * Map config space registers
	 * Read chip ID & set up config space command register(s)
	 *
	 * Note: this leaves the chip accessible by Memory Space
	 * accesses, but with interrupts and Bus Mastering off.
	 * This should ensure that nothing untoward will happen
	 * if it has been left active by the (net-)bootloader.
	 * We'll re-enable Bus Mastering once we've reset the chip,
	 * and allow interrupts only when everything else is set up.
	 */
	err = pci_config_setup(devinfo, &rgep->cfg_handle);
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "pci_config_setup() failed");
		goto attach_fail;
	}
	rgep->progress |= PROGRESS_CFG;
	cidp = &rgep->chipid;
	bzero(cidp, sizeof (*cidp));
	rge_chip_cfg_init(rgep, cidp);

	/*
	 * Map operating registers
	 */
	err = ddi_regs_map_setup(devinfo, 2, &regs,
	    0, 0, &rge_reg_accattr, &rgep->io_handle);

	/*
	 * MMIO map will fail if the assigned address is bigger than 4G
	 * then choose I/O map
	 */
	if (err != DDI_SUCCESS) {
		err = ddi_regs_map_setup(devinfo, 1, &regs,
		    0, 0, &rge_reg_accattr, &rgep->io_handle);
	}
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "ddi_regs_map_setup() failed");
		goto attach_fail;
	}
	rgep->io_regs = regs;
	rgep->progress |= PROGRESS_REGS;

	/*
	 * Characterise the device, so we know its requirements.
	 * Then allocate the appropriate TX and RX descriptors & buffers.
	 */
	rge_chip_ident(rgep);
	err = rge_alloc_bufs(rgep);
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "DMA buffer allocation failed");
		goto attach_fail;
	}

	/*
	 * Register NDD-tweakable parameters
	 */
	if (rge_nd_init(rgep)) {
		rge_problem(rgep, "rge_nd_init() failed");
		goto attach_fail;
	}
	rgep->progress |= PROGRESS_NDD;

	/*
	 * Add the softint handlers:
	 *
	 * Both of these handlers are used to avoid restrictions on the
	 * context and/or mutexes required for some operations.  In
	 * particular, the hardware interrupt handler and its subfunctions
	 * can detect a number of conditions that we don't want to handle
	 * in that context or with that set of mutexes held.  So, these
	 * softints are triggered instead:
	 *
	 * the <resched> softint is triggered if if we have previously
	 * had to refuse to send a packet because of resource shortage
	 * (we've run out of transmit buffers), but the send completion
	 * interrupt handler has now detected that more buffers have
	 * become available.
	 *
	 * the <factotum> is triggered if the h/w interrupt handler
	 * sees the <link state changed> or <error> bits in the status
	 * block.  It's also triggered periodically to poll the link
	 * state, just in case we aren't getting link status change
	 * interrupts ...
	 */
	err = ddi_intr_add_softint(devinfo, &rgep->resched_hdl,
	    DDI_INTR_SOFTPRI_MIN, rge_reschedule, (caddr_t)rgep);
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "ddi_intr_add_softint() failed");
		goto attach_fail;
	}
	rgep->progress |= PROGRESS_RESCHED;
	err = ddi_intr_add_softint(devinfo, &rgep->factotum_hdl,
	    DDI_INTR_SOFTPRI_MIN, rge_chip_factotum, (caddr_t)rgep);
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "ddi_intr_add_softint() failed");
		goto attach_fail;
	}
	rgep->progress |= PROGRESS_FACTOTUM;

	/*
	 * Get supported interrupt types
	 */
	if (ddi_intr_get_supported_types(devinfo, &intr_types)
	    != DDI_SUCCESS) {
		rge_error(rgep, "ddi_intr_get_supported_types failed\n");
		goto attach_fail;
	}

	/*
	 * Add the h/w interrupt handler and initialise mutexes
	 * RTL8101E is observed to have MSI invalidation issue after S/R.
	 * So the FIXED interrupt is used instead.
	 */
	if (rgep->chipid.mac_ver == MAC_VER_8101E)
		rgep->msi_enable = B_FALSE;
	if ((intr_types & DDI_INTR_TYPE_MSI) && rgep->msi_enable) {
		if (rge_add_intrs(rgep, DDI_INTR_TYPE_MSI) != DDI_SUCCESS) {
			rge_error(rgep, "MSI registration failed, "
			    "trying FIXED interrupt type\n");
		} else {
			rge_log(rgep, "Using MSI interrupt type\n");
			rgep->intr_type = DDI_INTR_TYPE_MSI;
			rgep->progress |= PROGRESS_INTR;
		}
	}
	if (!(rgep->progress & PROGRESS_INTR) &&
	    (intr_types & DDI_INTR_TYPE_FIXED)) {
		if (rge_add_intrs(rgep, DDI_INTR_TYPE_FIXED) != DDI_SUCCESS) {
			rge_error(rgep, "FIXED interrupt "
			    "registration failed\n");
			goto attach_fail;
		}
		rge_log(rgep, "Using FIXED interrupt type\n");
		rgep->intr_type = DDI_INTR_TYPE_FIXED;
		rgep->progress |= PROGRESS_INTR;
	}
	if (!(rgep->progress & PROGRESS_INTR)) {
		rge_error(rgep, "No interrupts registered\n");
		goto attach_fail;
	}
	mutex_init(rgep->genlock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));
	rw_init(rgep->errlock, NULL, RW_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));
	mutex_init(rgep->tx_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));
	mutex_init(rgep->tc_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));
	mutex_init(rgep->rx_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));
	mutex_init(rgep->rc_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(rgep->intr_pri));

	/*
	 * Initialize rings
	 */
	err = rge_init_rings(rgep);
	if (err != DDI_SUCCESS) {
		rge_problem(rgep, "rge_init_rings() failed");
		goto attach_fail;
	}
	rgep->progress |= PROGRESS_INIT;

	/*
	 * Now that mutex locks are initialized, enable interrupts.
	 */
	if (rgep->intr_cap & DDI_INTR_FLAG_BLOCK) {
		/* Call ddi_intr_block_enable() for MSI interrupts */
		(void) ddi_intr_block_enable(rgep->htable, rgep->intr_cnt);
	} else {
		/* Call ddi_intr_enable for MSI or FIXED interrupts */
		for (i = 0; i < rgep->intr_cnt; i++) {
			(void) ddi_intr_enable(rgep->htable[i]);
		}
	}

	/*
	 * Initialise link state variables
	 * Stop, reset & reinitialise the chip.
	 * Initialise the (internal) PHY.
	 */
	rgep->param_link_up = LINK_STATE_UNKNOWN;

	/*
	 * Reset chip & rings to initial state; also reset address
	 * filtering, promiscuity, loopback mode.
	 */
	mutex_enter(rgep->genlock);
	(void) rge_chip_reset(rgep);
	rge_chip_sync(rgep, RGE_GET_MAC);
	bzero(rgep->mcast_hash, sizeof (rgep->mcast_hash));
	bzero(rgep->mcast_refs, sizeof (rgep->mcast_refs));
	rgep->promisc = B_FALSE;
	rgep->param_loop_mode = RGE_LOOP_NONE;
	mutex_exit(rgep->genlock);
	rge_phy_init(rgep);
	rgep->progress |= PROGRESS_PHY;

	/*
	 * Create & initialise named kstats
	 */
	rge_init_kstats(rgep, instance);
	rgep->progress |= PROGRESS_KSTATS;

	if ((macp = mac_alloc(MAC_VERSION)) == NULL)
		goto attach_fail;
	macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
	macp->m_driver = rgep;
	macp->m_dip = devinfo;
	macp->m_src_addr = rgep->netaddr;
	macp->m_callbacks = &rge_m_callbacks;
	macp->m_min_sdu = 0;
	macp->m_max_sdu = rgep->default_mtu;
	macp->m_margin = VLAN_TAGSZ;

	/*
	 * Finally, we're ready to register ourselves with the MAC layer
	 * interface; if this succeeds, we're all ready to start()
	 */
	err = mac_register(macp, &rgep->mh);
	mac_free(macp);
	if (err != 0)
		goto attach_fail;

	/*
	 * Register a periodical handler.
	 * reg_chip_cyclic() is invoked in kernel context.
	 */
	rgep->periodic_id = ddi_periodic_add(rge_chip_cyclic, rgep,
	    RGE_CYCLIC_PERIOD, DDI_IPL_0);

	rgep->progress |= PROGRESS_READY;
	return (DDI_SUCCESS);

attach_fail:
	rge_unattach(rgep);
	return (DDI_FAILURE);
}

/*
 *	rge_suspend() -- suspend transmit/receive for powerdown
 */
static int
rge_suspend(rge_t *rgep)
{
	/*
	 * Stop processing and idle (powerdown) the PHY ...
	 */
	mutex_enter(rgep->genlock);
	rw_enter(rgep->errlock, RW_WRITER);

	if (rgep->rge_mac_state != RGE_MAC_STARTED) {
		rw_exit(rgep->errlock);
		mutex_exit(rgep->genlock);
		return (DDI_SUCCESS);
	}

	rgep->suspended = B_TRUE;
	rge_stop(rgep);
	rgep->rge_mac_state = RGE_MAC_STOPPED;

	rw_exit(rgep->errlock);
	mutex_exit(rgep->genlock);

	return (DDI_SUCCESS);
}

/*
 * quiesce(9E) entry point.
 *
 * This function is called when the system is single-threaded at high
 * PIL with preemption disabled. Therefore, this function must not be
 * blocked.
 *
 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 * DDI_FAILURE indicates an error condition and should almost never happen.
 */
static int
rge_quiesce(dev_info_t *devinfo)
{
	rge_t *rgep = ddi_get_driver_private(devinfo);

	if (rgep == NULL)
		return (DDI_FAILURE);

	/*
	 * Turn off debugging
	 */
	rge_debug = 0;
	rgep->debug = 0;

	/* Stop the chip */
	rge_chip_stop(rgep, B_FALSE);

	return (DDI_SUCCESS);
}

/*
 * detach(9E) -- Detach a device from the system
 */
static int
rge_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
{
	rge_t *rgep;

	RGE_GTRACE(("rge_detach($%p, %d)", (void *)devinfo, cmd));

	rgep = ddi_get_driver_private(devinfo);

	switch (cmd) {
	default:
		return (DDI_FAILURE);

	case DDI_SUSPEND:
		return (rge_suspend(rgep));

	case DDI_DETACH:
		break;
	}

	/*
	 * If there is any posted buffer, the driver should reject to be
	 * detached. Need notice upper layer to release them.
	 */
	if (!(rgep->chip_flags & CHIP_FLAG_FORCE_BCOPY) &&
	    rgep->rx_free != RGE_BUF_SLOTS)
		return (DDI_FAILURE);

	/*
	 * Unregister from the MAC layer subsystem.  This can fail, in
	 * particular if there are DLPI style-2 streams still open -
	 * in which case we just return failure without shutting
	 * down chip operations.
	 */
	if (mac_unregister(rgep->mh) != 0)
		return (DDI_FAILURE);

	/*
	 * All activity stopped, so we can clean up & exit
	 */
	rge_unattach(rgep);
	return (DDI_SUCCESS);
}


/*
 * ========== Module Loading Data & Entry Points ==========
 */

#undef	RGE_DBG
#define	RGE_DBG		RGE_DBG_INIT	/* debug flag for this code	*/
DDI_DEFINE_STREAM_OPS(rge_dev_ops, nulldev, nulldev, rge_attach, rge_detach,
    nodev, NULL, D_MP, NULL, rge_quiesce);

static struct modldrv rge_modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	rge_ident,		/* short description */
	&rge_dev_ops		/* driver specific ops */
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&rge_modldrv, NULL
};


int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_init(void)
{
	int status;

	mac_init_ops(&rge_dev_ops, "rge");
	status = mod_install(&modlinkage);
	if (status == DDI_SUCCESS)
		mutex_init(rge_log_mutex, NULL, MUTEX_DRIVER, NULL);
	else
		mac_fini_ops(&rge_dev_ops);

	return (status);
}

int
_fini(void)
{
	int status;

	status = mod_remove(&modlinkage);
	if (status == DDI_SUCCESS) {
		mac_fini_ops(&rge_dev_ops);
		mutex_destroy(rge_log_mutex);
	}
	return (status);
}