144961713Sgirish /*
244961713Sgirish  * CDDL HEADER START
344961713Sgirish  *
444961713Sgirish  * The contents of this file are subject to the terms of the
544961713Sgirish  * Common Development and Distribution License (the "License").
644961713Sgirish  * You may not use this file except in compliance with the License.
744961713Sgirish  *
844961713Sgirish  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
944961713Sgirish  * or http://www.opensolaris.org/os/licensing.
1044961713Sgirish  * See the License for the specific language governing permissions
1144961713Sgirish  * and limitations under the License.
1244961713Sgirish  *
1344961713Sgirish  * When distributing Covered Code, include this CDDL HEADER in each
1444961713Sgirish  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1544961713Sgirish  * If applicable, add the following below this CDDL HEADER, with the
1644961713Sgirish  * fields enclosed by brackets "[]" replaced with your own identifying
1744961713Sgirish  * information: Portions Copyright [yyyy] [name of copyright owner]
1844961713Sgirish  *
1944961713Sgirish  * CDDL HEADER END
2044961713Sgirish  */
21952a2464SMichael Speer 
2244961713Sgirish /*
230dc2366fSVenugopal Iyer  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
2444961713Sgirish  * Use is subject to license terms.
2544961713Sgirish  */
2644961713Sgirish 
2744961713Sgirish #include <sys/nxge/nxge_impl.h>
2844961713Sgirish #include <sys/nxge/nxge_txdma.h>
29678453a8Sspeer #include <sys/nxge/nxge_hio.h>
30678453a8Sspeer #include <npi_tx_rd64.h>
31678453a8Sspeer #include <npi_tx_wr64.h>
3244961713Sgirish #include <sys/llc1.h>
3344961713Sgirish 
34*86ef0a63SRichard Lowe uint32_t	nxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
35da14cebeSEric Cheng uint32_t	nxge_tx_minfree = 64;
3644961713Sgirish uint32_t	nxge_tx_intr_thres = 0;
3744961713Sgirish uint32_t	nxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
3844961713Sgirish uint32_t	nxge_tx_tiny_pack = 1;
3944961713Sgirish uint32_t	nxge_tx_use_bcopy = 1;
4044961713Sgirish 
41*86ef0a63SRichard Lowe extern uint32_t		nxge_tx_ring_size;
42*86ef0a63SRichard Lowe extern uint32_t		nxge_bcopy_thresh;
43*86ef0a63SRichard Lowe extern uint32_t		nxge_dvma_thresh;
44*86ef0a63SRichard Lowe extern uint32_t		nxge_dma_stream_thresh;
45*86ef0a63SRichard Lowe extern dma_method_t	nxge_force_dma;
46b4d05839Sml extern uint32_t		nxge_cksum_offload;
4744961713Sgirish 
4844961713Sgirish /* Device register access attributes for PIO.  */
4944961713Sgirish extern ddi_device_acc_attr_t nxge_dev_reg_acc_attr;
50*86ef0a63SRichard Lowe /* Device descriptor access attributes for DMA.	 */
5144961713Sgirish extern ddi_device_acc_attr_t nxge_dev_desc_dma_acc_attr;
5244961713Sgirish /* Device buffer access attributes for DMA.  */
5344961713Sgirish extern ddi_device_acc_attr_t nxge_dev_buf_dma_acc_attr;
5444961713Sgirish extern ddi_dma_attr_t nxge_desc_dma_attr;
5544961713Sgirish extern ddi_dma_attr_t nxge_tx_dma_attr;
5644961713Sgirish 
57da14cebeSEric Cheng extern void nxge_tx_ring_task(void *arg);
582d99c5d4SMichael Speer 
59678453a8Sspeer static nxge_status_t nxge_map_txdma(p_nxge_t, int);
6044961713Sgirish 
61678453a8Sspeer static nxge_status_t nxge_txdma_hw_start(p_nxge_t, int);
6244961713Sgirish 
6344961713Sgirish static nxge_status_t nxge_map_txdma_channel(p_nxge_t, uint16_t,
6444961713Sgirish 	p_nxge_dma_common_t *, p_tx_ring_t *,
6544961713Sgirish 	uint32_t, p_nxge_dma_common_t *,
6644961713Sgirish 	p_tx_mbox_t *);
67678453a8Sspeer static void nxge_unmap_txdma_channel(p_nxge_t, uint16_t);
6844961713Sgirish 
6944961713Sgirish static nxge_status_t nxge_map_txdma_channel_buf_ring(p_nxge_t, uint16_t,
7044961713Sgirish 	p_nxge_dma_common_t *, p_tx_ring_t *, uint32_t);
7144961713Sgirish static void nxge_unmap_txdma_channel_buf_ring(p_nxge_t, p_tx_ring_t);
7244961713Sgirish 
7344961713Sgirish static void nxge_map_txdma_channel_cfg_ring(p_nxge_t, uint16_t,
7444961713Sgirish 	p_nxge_dma_common_t *, p_tx_ring_t,
7544961713Sgirish 	p_tx_mbox_t *);
7644961713Sgirish static void nxge_unmap_txdma_channel_cfg_ring(p_nxge_t,
7744961713Sgirish 	p_tx_ring_t, p_tx_mbox_t);
7844961713Sgirish 
7944961713Sgirish static nxge_status_t nxge_txdma_start_channel(p_nxge_t, uint16_t,
8044961713Sgirish     p_tx_ring_t, p_tx_mbox_t);
81678453a8Sspeer static nxge_status_t nxge_txdma_stop_channel(p_nxge_t, uint16_t);
8244961713Sgirish 
8344961713Sgirish static p_tx_ring_t nxge_txdma_get_ring(p_nxge_t, uint16_t);
8444961713Sgirish static nxge_status_t nxge_tx_err_evnts(p_nxge_t, uint_t,
8544961713Sgirish 	p_nxge_ldv_t, tx_cs_t);
8644961713Sgirish static p_tx_mbox_t nxge_txdma_get_mbox(p_nxge_t, uint16_t);
8744961713Sgirish static nxge_status_t nxge_txdma_fatal_err_recover(p_nxge_t,
8844961713Sgirish 	uint16_t, p_tx_ring_t);
8944961713Sgirish 
90678453a8Sspeer static void nxge_txdma_fixup_hung_channel(p_nxge_t nxgep,
91678453a8Sspeer     p_tx_ring_t ring_p, uint16_t channel);
92678453a8Sspeer 
9344961713Sgirish nxge_status_t
nxge_init_txdma_channels(p_nxge_t nxgep)9444961713Sgirish nxge_init_txdma_channels(p_nxge_t nxgep)
9544961713Sgirish {
96e11f0814SMichael Speer 	nxge_grp_set_t	*set = &nxgep->tx_set;
97e11f0814SMichael Speer 	int		i, tdc, count;
98e11f0814SMichael Speer 	nxge_grp_t	*group;
99da14cebeSEric Cheng 	dc_map_t	map;
100da14cebeSEric Cheng 	int		dev_gindex;
101678453a8Sspeer 
102678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_txdma_channels"));
103678453a8Sspeer 
104678453a8Sspeer 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
105678453a8Sspeer 		if ((1 << i) & set->lg.map) {
106e11f0814SMichael Speer 			group = set->group[i];
107da14cebeSEric Cheng 			dev_gindex =
108da14cebeSEric Cheng 			    nxgep->pt_config.hw_config.def_mac_txdma_grpid + i;
109da14cebeSEric Cheng 			map = nxgep->pt_config.tdc_grps[dev_gindex].map;
110678453a8Sspeer 			for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
111da14cebeSEric Cheng 				if ((1 << tdc) & map) {
112da14cebeSEric Cheng 					if ((nxge_grp_dc_add(nxgep,
113da14cebeSEric Cheng 					    group, VP_BOUND_TX, tdc)))
114e11f0814SMichael Speer 						goto init_txdma_channels_exit;
115678453a8Sspeer 				}
116678453a8Sspeer 			}
117678453a8Sspeer 		}
118678453a8Sspeer 		if (++count == set->lg.count)
119678453a8Sspeer 			break;
120678453a8Sspeer 	}
121678453a8Sspeer 
122678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_txdma_channels"));
123678453a8Sspeer 	return (NXGE_OK);
124e11f0814SMichael Speer 
125e11f0814SMichael Speer init_txdma_channels_exit:
126e11f0814SMichael Speer 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
127e11f0814SMichael Speer 		if ((1 << i) & set->lg.map) {
128e11f0814SMichael Speer 			group = set->group[i];
129da14cebeSEric Cheng 			dev_gindex =
130da14cebeSEric Cheng 			    nxgep->pt_config.hw_config.def_mac_txdma_grpid + i;
131da14cebeSEric Cheng 			map = nxgep->pt_config.tdc_grps[dev_gindex].map;
132e11f0814SMichael Speer 			for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
133da14cebeSEric Cheng 				if ((1 << tdc) & map) {
134e11f0814SMichael Speer 					nxge_grp_dc_remove(nxgep,
135e11f0814SMichael Speer 					    VP_BOUND_TX, tdc);
136e11f0814SMichael Speer 				}
137e11f0814SMichael Speer 			}
138e11f0814SMichael Speer 		}
139e11f0814SMichael Speer 		if (++count == set->lg.count)
140e11f0814SMichael Speer 			break;
141e11f0814SMichael Speer 	}
142e11f0814SMichael Speer 
143e11f0814SMichael Speer 	return (NXGE_ERROR);
144da14cebeSEric Cheng 
145678453a8Sspeer }
146678453a8Sspeer 
147678453a8Sspeer nxge_status_t
nxge_init_txdma_channel(p_nxge_t nxge,int channel)148678453a8Sspeer nxge_init_txdma_channel(
149678453a8Sspeer 	p_nxge_t nxge,
150678453a8Sspeer 	int channel)
151678453a8Sspeer {
152678453a8Sspeer 	nxge_status_t status;
15344961713Sgirish 
154678453a8Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_txdma_channel"));
15544961713Sgirish 
156678453a8Sspeer 	status = nxge_map_txdma(nxge, channel);
15744961713Sgirish 	if (status != NXGE_OK) {
158678453a8Sspeer 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
159678453a8Sspeer 		    "<== nxge_init_txdma_channel: status 0x%x", status));
160678453a8Sspeer 		(void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel);
16144961713Sgirish 		return (status);
16244961713Sgirish 	}
16344961713Sgirish 
164678453a8Sspeer 	status = nxge_txdma_hw_start(nxge, channel);
16544961713Sgirish 	if (status != NXGE_OK) {
166678453a8Sspeer 		(void) nxge_unmap_txdma_channel(nxge, channel);
167678453a8Sspeer 		(void) npi_txdma_dump_tdc_regs(nxge->npi_handle, channel);
16844961713Sgirish 		return (status);
16944961713Sgirish 	}
17044961713Sgirish 
171678453a8Sspeer 	if (!nxge->statsp->tdc_ksp[channel])
172678453a8Sspeer 		nxge_setup_tdc_kstats(nxge, channel);
17344961713Sgirish 
174678453a8Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL, "<== nxge_init_txdma_channel"));
175678453a8Sspeer 
176678453a8Sspeer 	return (status);
17744961713Sgirish }
17844961713Sgirish 
17944961713Sgirish void
nxge_uninit_txdma_channels(p_nxge_t nxgep)18044961713Sgirish nxge_uninit_txdma_channels(p_nxge_t nxgep)
18144961713Sgirish {
182678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
183678453a8Sspeer 	int tdc;
184678453a8Sspeer 
185678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_txdma_channels"));
186678453a8Sspeer 
187678453a8Sspeer 	if (set->owned.map == 0) {
188678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
189678453a8Sspeer 		    "nxge_uninit_txdma_channels: no channels"));
190678453a8Sspeer 		return;
191678453a8Sspeer 	}
192678453a8Sspeer 
193678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
194678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
195678453a8Sspeer 			nxge_grp_dc_remove(nxgep, VP_BOUND_TX, tdc);
196678453a8Sspeer 		}
197678453a8Sspeer 	}
198678453a8Sspeer 
199678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_txdma_channels"));
200678453a8Sspeer }
201678453a8Sspeer 
202678453a8Sspeer void
nxge_uninit_txdma_channel(p_nxge_t nxgep,int channel)203678453a8Sspeer nxge_uninit_txdma_channel(p_nxge_t nxgep, int channel)
204678453a8Sspeer {
205678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_uninit_txdma_channel"));
206678453a8Sspeer 
207678453a8Sspeer 	if (nxgep->statsp->tdc_ksp[channel]) {
208678453a8Sspeer 		kstat_delete(nxgep->statsp->tdc_ksp[channel]);
209678453a8Sspeer 		nxgep->statsp->tdc_ksp[channel] = 0;
210678453a8Sspeer 	}
21144961713Sgirish 
212ef523517SMichael Speer 	if (nxge_txdma_stop_channel(nxgep, channel) != NXGE_OK)
213ef523517SMichael Speer 		goto nxge_uninit_txdma_channel_exit;
214ef523517SMichael Speer 
215678453a8Sspeer 	nxge_unmap_txdma_channel(nxgep, channel);
21644961713Sgirish 
217ef523517SMichael Speer nxge_uninit_txdma_channel_exit:
218ef523517SMichael Speer 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_uninit_txdma_channel"));
21944961713Sgirish }
22044961713Sgirish 
22144961713Sgirish void
nxge_setup_dma_common(p_nxge_dma_common_t dest_p,p_nxge_dma_common_t src_p,uint32_t entries,uint32_t size)22244961713Sgirish nxge_setup_dma_common(p_nxge_dma_common_t dest_p, p_nxge_dma_common_t src_p,
223*86ef0a63SRichard Lowe     uint32_t entries, uint32_t size)
22444961713Sgirish {
22544961713Sgirish 	size_t		tsize;
22644961713Sgirish 	*dest_p = *src_p;
22744961713Sgirish 	tsize = size * entries;
22844961713Sgirish 	dest_p->alength = tsize;
22944961713Sgirish 	dest_p->nblocks = entries;
23044961713Sgirish 	dest_p->block_size = size;
23144961713Sgirish 	dest_p->offset += tsize;
23244961713Sgirish 
23344961713Sgirish 	src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
23444961713Sgirish 	src_p->alength -= tsize;
23544961713Sgirish 	src_p->dma_cookie.dmac_laddress += tsize;
23644961713Sgirish 	src_p->dma_cookie.dmac_size -= tsize;
23744961713Sgirish }
23844961713Sgirish 
239678453a8Sspeer /*
240678453a8Sspeer  * nxge_reset_txdma_channel
241678453a8Sspeer  *
242678453a8Sspeer  *	Reset a TDC.
243678453a8Sspeer  *
244678453a8Sspeer  * Arguments:
245*86ef0a63SRichard Lowe  *	nxgep
246*86ef0a63SRichard Lowe  *	channel		The channel to reset.
247*86ef0a63SRichard Lowe  *	reg_data	The current TX_CS.
248678453a8Sspeer  *
249678453a8Sspeer  * Notes:
250678453a8Sspeer  *
251678453a8Sspeer  * NPI/NXGE function calls:
252678453a8Sspeer  *	npi_txdma_channel_reset()
253678453a8Sspeer  *	npi_txdma_channel_control()
254678453a8Sspeer  *
255678453a8Sspeer  * Registers accessed:
256678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
257678453a8Sspeer  *	TX_RING_KICK	DMC+0x40018 Transmit Ring Kick
258678453a8Sspeer  *
259678453a8Sspeer  * Context:
260678453a8Sspeer  *	Any domain
261678453a8Sspeer  */
26244961713Sgirish nxge_status_t
nxge_reset_txdma_channel(p_nxge_t nxgep,uint16_t channel,uint64_t reg_data)26344961713Sgirish nxge_reset_txdma_channel(p_nxge_t nxgep, uint16_t channel, uint64_t reg_data)
26444961713Sgirish {
26544961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
26644961713Sgirish 	nxge_status_t		status = NXGE_OK;
26744961713Sgirish 	npi_handle_t		handle;
26844961713Sgirish 
26944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, " ==> nxge_reset_txdma_channel"));
27044961713Sgirish 
27144961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
27244961713Sgirish 	if ((reg_data & TX_CS_RST_MASK) == TX_CS_RST_MASK) {
27344961713Sgirish 		rs = npi_txdma_channel_reset(handle, channel);
27444961713Sgirish 	} else {
27544961713Sgirish 		rs = npi_txdma_channel_control(handle, TXDMA_RESET,
27652ccf843Smisaki 		    channel);
27744961713Sgirish 	}
27844961713Sgirish 
27944961713Sgirish 	if (rs != NPI_SUCCESS) {
28044961713Sgirish 		status = NXGE_ERROR | rs;
28144961713Sgirish 	}
28244961713Sgirish 
28344961713Sgirish 	/*
28444961713Sgirish 	 * Reset the tail (kick) register to 0.
28544961713Sgirish 	 * (Hardware will not reset it. Tx overflow fatal
28644961713Sgirish 	 * error if tail is not set to 0 after reset!
28744961713Sgirish 	 */
28844961713Sgirish 	TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0);
28944961713Sgirish 
29044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, " <== nxge_reset_txdma_channel"));
29144961713Sgirish 	return (status);
29244961713Sgirish }
29344961713Sgirish 
294678453a8Sspeer /*
295678453a8Sspeer  * nxge_init_txdma_channel_event_mask
296678453a8Sspeer  *
297678453a8Sspeer  *	Enable interrupts for a set of events.
298678453a8Sspeer  *
299678453a8Sspeer  * Arguments:
300*86ef0a63SRichard Lowe  *	nxgep
301*86ef0a63SRichard Lowe  *	channel	The channel to map.
302*86ef0a63SRichard Lowe  *	mask_p	The events to enable.
303678453a8Sspeer  *
304678453a8Sspeer  * Notes:
305678453a8Sspeer  *
306678453a8Sspeer  * NPI/NXGE function calls:
307678453a8Sspeer  *	npi_txdma_event_mask()
308678453a8Sspeer  *
309678453a8Sspeer  * Registers accessed:
310678453a8Sspeer  *	TX_ENT_MSK	DMC+0x40020 Transmit Event Mask
311678453a8Sspeer  *
312678453a8Sspeer  * Context:
313678453a8Sspeer  *	Any domain
314678453a8Sspeer  */
31544961713Sgirish nxge_status_t
nxge_init_txdma_channel_event_mask(p_nxge_t nxgep,uint16_t channel,p_tx_dma_ent_msk_t mask_p)31644961713Sgirish nxge_init_txdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel,
317*86ef0a63SRichard Lowe     p_tx_dma_ent_msk_t mask_p)
31844961713Sgirish {
31944961713Sgirish 	npi_handle_t		handle;
32044961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
32144961713Sgirish 	nxge_status_t		status = NXGE_OK;
32244961713Sgirish 
32344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
32452ccf843Smisaki 	    "<== nxge_init_txdma_channel_event_mask"));
32544961713Sgirish 
32644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
32744961713Sgirish 	rs = npi_txdma_event_mask(handle, OP_SET, channel, mask_p);
32844961713Sgirish 	if (rs != NPI_SUCCESS) {
32944961713Sgirish 		status = NXGE_ERROR | rs;
33044961713Sgirish 	}
33144961713Sgirish 
33244961713Sgirish 	return (status);
33344961713Sgirish }
33444961713Sgirish 
335678453a8Sspeer /*
336678453a8Sspeer  * nxge_init_txdma_channel_cntl_stat
337678453a8Sspeer  *
338678453a8Sspeer  *	Stop a TDC.  If at first we don't succeed, inject an error.
339678453a8Sspeer  *
340678453a8Sspeer  * Arguments:
341*86ef0a63SRichard Lowe  *	nxgep
342*86ef0a63SRichard Lowe  *	channel		The channel to stop.
343678453a8Sspeer  *
344678453a8Sspeer  * Notes:
345678453a8Sspeer  *
346678453a8Sspeer  * NPI/NXGE function calls:
347678453a8Sspeer  *	npi_txdma_control_status()
348678453a8Sspeer  *
349678453a8Sspeer  * Registers accessed:
350678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
351678453a8Sspeer  *
352678453a8Sspeer  * Context:
353678453a8Sspeer  *	Any domain
354678453a8Sspeer  */
35544961713Sgirish nxge_status_t
nxge_init_txdma_channel_cntl_stat(p_nxge_t nxgep,uint16_t channel,uint64_t reg_data)35644961713Sgirish nxge_init_txdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel,
357*86ef0a63SRichard Lowe     uint64_t reg_data)
35844961713Sgirish {
35944961713Sgirish 	npi_handle_t		handle;
36044961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
36144961713Sgirish 	nxge_status_t		status = NXGE_OK;
36244961713Sgirish 
36344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
36452ccf843Smisaki 	    "<== nxge_init_txdma_channel_cntl_stat"));
36544961713Sgirish 
36644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
36744961713Sgirish 	rs = npi_txdma_control_status(handle, OP_SET, channel,
36852ccf843Smisaki 	    (p_tx_cs_t)&reg_data);
36944961713Sgirish 
37044961713Sgirish 	if (rs != NPI_SUCCESS) {
37144961713Sgirish 		status = NXGE_ERROR | rs;
37244961713Sgirish 	}
37344961713Sgirish 
37444961713Sgirish 	return (status);
37544961713Sgirish }
37644961713Sgirish 
377678453a8Sspeer /*
378678453a8Sspeer  * nxge_enable_txdma_channel
379678453a8Sspeer  *
380678453a8Sspeer  *	Enable a TDC.
381678453a8Sspeer  *
382678453a8Sspeer  * Arguments:
383*86ef0a63SRichard Lowe  *	nxgep
384*86ef0a63SRichard Lowe  *	channel		The channel to enable.
385*86ef0a63SRichard Lowe  *	tx_desc_p	channel's transmit descriptor ring.
386*86ef0a63SRichard Lowe  *	mbox_p		channel's mailbox,
387678453a8Sspeer  *
388678453a8Sspeer  * Notes:
389678453a8Sspeer  *
390678453a8Sspeer  * NPI/NXGE function calls:
391678453a8Sspeer  *	npi_txdma_ring_config()
392678453a8Sspeer  *	npi_txdma_mbox_config()
393678453a8Sspeer  *	npi_txdma_channel_init_enable()
394678453a8Sspeer  *
395678453a8Sspeer  * Registers accessed:
396678453a8Sspeer  *	TX_RNG_CFIG	DMC+0x40000 Transmit Ring Configuration
397678453a8Sspeer  *	TXDMA_MBH	DMC+0x40030 TXDMA Mailbox High
398678453a8Sspeer  *	TXDMA_MBL	DMC+0x40038 TXDMA Mailbox Low
399678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
400678453a8Sspeer  *
401678453a8Sspeer  * Context:
402678453a8Sspeer  *	Any domain
403678453a8Sspeer  */
40444961713Sgirish nxge_status_t
nxge_enable_txdma_channel(p_nxge_t nxgep,uint16_t channel,p_tx_ring_t tx_desc_p,p_tx_mbox_t mbox_p)40544961713Sgirish nxge_enable_txdma_channel(p_nxge_t nxgep,
406*86ef0a63SRichard Lowe     uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
40744961713Sgirish {
40844961713Sgirish 	npi_handle_t		handle;
40944961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
41044961713Sgirish 	nxge_status_t		status = NXGE_OK;
41144961713Sgirish 
41244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_enable_txdma_channel"));
41344961713Sgirish 
41444961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
41544961713Sgirish 	/*
41644961713Sgirish 	 * Use configuration data composed at init time.
41744961713Sgirish 	 * Write to hardware the transmit ring configurations.
41844961713Sgirish 	 */
41944961713Sgirish 	rs = npi_txdma_ring_config(handle, OP_SET, channel,
420678453a8Sspeer 	    (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
42144961713Sgirish 
42244961713Sgirish 	if (rs != NPI_SUCCESS) {
42344961713Sgirish 		return (NXGE_ERROR | rs);
42444961713Sgirish 	}
42544961713Sgirish 
426678453a8Sspeer 	if (isLDOMguest(nxgep)) {
427678453a8Sspeer 		/* Add interrupt handler for this channel. */
428678453a8Sspeer 		if (nxge_hio_intr_add(nxgep, VP_BOUND_TX, channel) != NXGE_OK)
429678453a8Sspeer 			return (NXGE_ERROR);
430678453a8Sspeer 	}
431678453a8Sspeer 
43244961713Sgirish 	/* Write to hardware the mailbox */
43344961713Sgirish 	rs = npi_txdma_mbox_config(handle, OP_SET, channel,
43452ccf843Smisaki 	    (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
43544961713Sgirish 
43644961713Sgirish 	if (rs != NPI_SUCCESS) {
43744961713Sgirish 		return (NXGE_ERROR | rs);
43844961713Sgirish 	}
43944961713Sgirish 
44044961713Sgirish 	/* Start the DMA engine. */
44144961713Sgirish 	rs = npi_txdma_channel_init_enable(handle, channel);
44244961713Sgirish 
44344961713Sgirish 	if (rs != NPI_SUCCESS) {
44444961713Sgirish 		return (NXGE_ERROR | rs);
44544961713Sgirish 	}
44644961713Sgirish 
44744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_enable_txdma_channel"));
44844961713Sgirish 
44944961713Sgirish 	return (status);
45044961713Sgirish }
45144961713Sgirish 
45244961713Sgirish void
nxge_fill_tx_hdr(p_mblk_t mp,boolean_t fill_len,boolean_t l4_cksum,int pkt_len,uint8_t npads,p_tx_pkt_hdr_all_t pkthdrp,t_uscalar_t start_offset,t_uscalar_t stuff_offset)45344961713Sgirish nxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len,
454*86ef0a63SRichard Lowe     boolean_t l4_cksum, int pkt_len, uint8_t npads,
455*86ef0a63SRichard Lowe     p_tx_pkt_hdr_all_t pkthdrp,
456*86ef0a63SRichard Lowe     t_uscalar_t start_offset,
457*86ef0a63SRichard Lowe     t_uscalar_t stuff_offset)
45844961713Sgirish {
45944961713Sgirish 	p_tx_pkt_header_t	hdrp;
460*86ef0a63SRichard Lowe 	p_mblk_t		nmp;
46144961713Sgirish 	uint64_t		tmp;
462*86ef0a63SRichard Lowe 	size_t			mblk_len;
463*86ef0a63SRichard Lowe 	size_t			iph_len;
464*86ef0a63SRichard Lowe 	size_t			hdrs_size;
46544961713Sgirish 	uint8_t			hdrs_buf[sizeof (struct ether_header) +
46652ccf843Smisaki 	    64 + sizeof (uint32_t)];
467ae2d3f74Smisaki 	uint8_t			*cursor;
468*86ef0a63SRichard Lowe 	uint8_t			*ip_buf;
46944961713Sgirish 	uint16_t		eth_type;
47044961713Sgirish 	uint8_t			ipproto;
47144961713Sgirish 	boolean_t		is_vlan = B_FALSE;
47244961713Sgirish 	size_t			eth_hdr_size;
47344961713Sgirish 
47444961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: mp $%p", mp));
47544961713Sgirish 
47644961713Sgirish 	/*
47744961713Sgirish 	 * Caller should zero out the headers first.
47844961713Sgirish 	 */
47944961713Sgirish 	hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
48044961713Sgirish 
48144961713Sgirish 	if (fill_len) {
48244961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
48352ccf843Smisaki 		    "==> nxge_fill_tx_hdr: pkt_len %d "
48452ccf843Smisaki 		    "npads %d", pkt_len, npads));
48544961713Sgirish 		tmp = (uint64_t)pkt_len;
48644961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
48744961713Sgirish 		goto fill_tx_header_done;
48844961713Sgirish 	}
48944961713Sgirish 
490b4d05839Sml 	hdrp->value |= (((uint64_t)npads) << TX_PKT_HEADER_PAD_SHIFT);
49144961713Sgirish 
49244961713Sgirish 	/*
49344961713Sgirish 	 * mp is the original data packet (does not include the
49444961713Sgirish 	 * Neptune transmit header).
49544961713Sgirish 	 */
49644961713Sgirish 	nmp = mp;
49744961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: "
49852ccf843Smisaki 	    "mp $%p b_rptr $%p len %d",
49952ccf843Smisaki 	    mp, nmp->b_rptr, MBLKL(nmp)));
500ae2d3f74Smisaki 	/* copy ether_header from mblk to hdrs_buf */
501ae2d3f74Smisaki 	cursor = &hdrs_buf[0];
502ae2d3f74Smisaki 	tmp = sizeof (struct ether_vlan_header);
503ae2d3f74Smisaki 	while ((nmp != NULL) && (tmp > 0)) {
504ae2d3f74Smisaki 		size_t buflen;
505ae2d3f74Smisaki 		mblk_len = MBLKL(nmp);
5067c29db66Smisaki 		buflen = min((size_t)tmp, mblk_len);
507ae2d3f74Smisaki 		bcopy(nmp->b_rptr, cursor, buflen);
508ae2d3f74Smisaki 		cursor += buflen;
509ae2d3f74Smisaki 		tmp -= buflen;
510ae2d3f74Smisaki 		nmp = nmp->b_cont;
511ae2d3f74Smisaki 	}
512ae2d3f74Smisaki 
513ae2d3f74Smisaki 	nmp = mp;
514ae2d3f74Smisaki 	mblk_len = MBLKL(nmp);
51544961713Sgirish 	ip_buf = NULL;
51644961713Sgirish 	eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
51744961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> : nxge_fill_tx_hdr: (value 0x%llx) "
51852ccf843Smisaki 	    "ether type 0x%x", eth_type, hdrp->value));
51944961713Sgirish 
52044961713Sgirish 	if (eth_type < ETHERMTU) {
52144961713Sgirish 		tmp = 1ull;
52244961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
52344961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: LLC "
52452ccf843Smisaki 		    "value 0x%llx", hdrp->value));
52544961713Sgirish 		if (*(hdrs_buf + sizeof (struct ether_header))
52652ccf843Smisaki 		    == LLC_SNAP_SAP) {
52744961713Sgirish 			eth_type = ntohs(*((uint16_t *)(hdrs_buf +
52852ccf843Smisaki 			    sizeof (struct ether_header) + 6)));
52944961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
53052ccf843Smisaki 			    "==> nxge_tx_pkt_hdr_init: LLC ether type 0x%x",
53152ccf843Smisaki 			    eth_type));
53244961713Sgirish 		} else {
53344961713Sgirish 			goto fill_tx_header_done;
53444961713Sgirish 		}
53544961713Sgirish 	} else if (eth_type == VLAN_ETHERTYPE) {
53644961713Sgirish 		tmp = 1ull;
53744961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
53844961713Sgirish 
53944961713Sgirish 		eth_type = ntohs(((struct ether_vlan_header *)
54052ccf843Smisaki 		    hdrs_buf)->ether_type);
54144961713Sgirish 		is_vlan = B_TRUE;
54244961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: VLAN "
54352ccf843Smisaki 		    "value 0x%llx", hdrp->value));
54444961713Sgirish 	}
54544961713Sgirish 
54644961713Sgirish 	if (!is_vlan) {
54744961713Sgirish 		eth_hdr_size = sizeof (struct ether_header);
54844961713Sgirish 	} else {
54944961713Sgirish 		eth_hdr_size = sizeof (struct ether_vlan_header);
55044961713Sgirish 	}
55144961713Sgirish 
55244961713Sgirish 	switch (eth_type) {
55344961713Sgirish 	case ETHERTYPE_IP:
55444961713Sgirish 		if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
55544961713Sgirish 			ip_buf = nmp->b_rptr + eth_hdr_size;
55644961713Sgirish 			mblk_len -= eth_hdr_size;
55744961713Sgirish 			iph_len = ((*ip_buf) & 0x0f);
55844961713Sgirish 			if (mblk_len > (iph_len + sizeof (uint32_t))) {
55944961713Sgirish 				ip_buf = nmp->b_rptr;
56044961713Sgirish 				ip_buf += eth_hdr_size;
56144961713Sgirish 			} else {
56244961713Sgirish 				ip_buf = NULL;
56344961713Sgirish 			}
56444961713Sgirish 
56544961713Sgirish 		}
56644961713Sgirish 		if (ip_buf == NULL) {
56744961713Sgirish 			hdrs_size = 0;
56844961713Sgirish 			((p_ether_header_t)hdrs_buf)->ether_type = 0;
56944961713Sgirish 			while ((nmp) && (hdrs_size <
57052ccf843Smisaki 			    sizeof (hdrs_buf))) {
57144961713Sgirish 				mblk_len = (size_t)nmp->b_wptr -
57252ccf843Smisaki 				    (size_t)nmp->b_rptr;
57344961713Sgirish 				if (mblk_len >=
57452ccf843Smisaki 				    (sizeof (hdrs_buf) - hdrs_size))
57544961713Sgirish 					mblk_len = sizeof (hdrs_buf) -
57652ccf843Smisaki 					    hdrs_size;
57744961713Sgirish 				bcopy(nmp->b_rptr,
57852ccf843Smisaki 				    &hdrs_buf[hdrs_size], mblk_len);
57944961713Sgirish 				hdrs_size += mblk_len;
58044961713Sgirish 				nmp = nmp->b_cont;
58144961713Sgirish 			}
58244961713Sgirish 			ip_buf = hdrs_buf;
58344961713Sgirish 			ip_buf += eth_hdr_size;
58444961713Sgirish 			iph_len = ((*ip_buf) & 0x0f);
58544961713Sgirish 		}
58644961713Sgirish 
58744961713Sgirish 		ipproto = ip_buf[9];
58844961713Sgirish 
58944961713Sgirish 		tmp = (uint64_t)iph_len;
59044961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
59144961713Sgirish 		tmp = (uint64_t)(eth_hdr_size >> 1);
59244961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
59344961713Sgirish 
59444961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv4 "
59552ccf843Smisaki 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
59652ccf843Smisaki 		    "tmp 0x%x",
59752ccf843Smisaki 		    iph_len, hdrp->bits.hdw.l3start, eth_hdr_size,
59852ccf843Smisaki 		    ipproto, tmp));
59944961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IP "
60052ccf843Smisaki 		    "value 0x%llx", hdrp->value));
60144961713Sgirish 
60244961713Sgirish 		break;
60344961713Sgirish 
60444961713Sgirish 	case ETHERTYPE_IPV6:
60544961713Sgirish 		hdrs_size = 0;
60644961713Sgirish 		((p_ether_header_t)hdrs_buf)->ether_type = 0;
60744961713Sgirish 		while ((nmp) && (hdrs_size <
60852ccf843Smisaki 		    sizeof (hdrs_buf))) {
60944961713Sgirish 			mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
61044961713Sgirish 			if (mblk_len >=
61152ccf843Smisaki 			    (sizeof (hdrs_buf) - hdrs_size))
61244961713Sgirish 				mblk_len = sizeof (hdrs_buf) -
61352ccf843Smisaki 				    hdrs_size;
61444961713Sgirish 			bcopy(nmp->b_rptr,
61552ccf843Smisaki 			    &hdrs_buf[hdrs_size], mblk_len);
61644961713Sgirish 			hdrs_size += mblk_len;
61744961713Sgirish 			nmp = nmp->b_cont;
61844961713Sgirish 		}
61944961713Sgirish 		ip_buf = hdrs_buf;
62044961713Sgirish 		ip_buf += eth_hdr_size;
62144961713Sgirish 
62244961713Sgirish 		tmp = 1ull;
62344961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
62444961713Sgirish 
62544961713Sgirish 		tmp = (eth_hdr_size >> 1);
62644961713Sgirish 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
62744961713Sgirish 
62844961713Sgirish 		/* byte 6 is the next header protocol */
62944961713Sgirish 		ipproto = ip_buf[6];
63044961713Sgirish 
63144961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: IPv6 "
63252ccf843Smisaki 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
63352ccf843Smisaki 		    iph_len, hdrp->bits.hdw.l3start, eth_hdr_size,
63452ccf843Smisaki 		    ipproto));
63544961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: IPv6 "
63652ccf843Smisaki 		    "value 0x%llx", hdrp->value));
63744961713Sgirish 
63844961713Sgirish 		break;
63944961713Sgirish 
64044961713Sgirish 	default:
64144961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: non-IP"));
64244961713Sgirish 		goto fill_tx_header_done;
64344961713Sgirish 	}
64444961713Sgirish 
64544961713Sgirish 	switch (ipproto) {
64644961713Sgirish 	case IPPROTO_TCP:
64744961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
648b4d05839Sml 		    "==> nxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
64944961713Sgirish 		if (l4_cksum) {
650b4d05839Sml 			hdrp->value |= TX_CKSUM_EN_PKT_TYPE_TCP;
651b4d05839Sml 			hdrp->value |=
652b4d05839Sml 			    (((uint64_t)(start_offset >> 1)) <<
653b4d05839Sml 			    TX_PKT_HEADER_L4START_SHIFT);
654b4d05839Sml 			hdrp->value |=
655b4d05839Sml 			    (((uint64_t)(stuff_offset >> 1)) <<
656b4d05839Sml 			    TX_PKT_HEADER_L4STUFF_SHIFT);
657b4d05839Sml 
65844961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
659b4d05839Sml 			    "==> nxge_tx_pkt_hdr_init: TCP CKSUM "
660b4d05839Sml 			    "value 0x%llx", hdrp->value));
66144961713Sgirish 		}
66244961713Sgirish 
66344961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_hdr_init: TCP "
664b4d05839Sml 		    "value 0x%llx", hdrp->value));
66544961713Sgirish 		break;
66644961713Sgirish 
66744961713Sgirish 	case IPPROTO_UDP:
66844961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_fill_tx_hdr: UDP"));
66944961713Sgirish 		if (l4_cksum) {
670b4d05839Sml 			if (!nxge_cksum_offload) {
671b4d05839Sml 				uint16_t	*up;
672b4d05839Sml 				uint16_t	cksum;
673b4d05839Sml 				t_uscalar_t	stuff_len;
674b4d05839Sml 
675b4d05839Sml 				/*
676b4d05839Sml 				 * The checksum field has the
677b4d05839Sml 				 * partial checksum.
678b4d05839Sml 				 * IP_CSUM() macro calls ip_cksum() which
679b4d05839Sml 				 * can add in the partial checksum.
680b4d05839Sml 				 */
681b4d05839Sml 				cksum = IP_CSUM(mp, start_offset, 0);
682b4d05839Sml 				stuff_len = stuff_offset;
683b4d05839Sml 				nmp = mp;
684b4d05839Sml 				mblk_len = MBLKL(nmp);
685b4d05839Sml 				while ((nmp != NULL) &&
686b4d05839Sml 				    (mblk_len < stuff_len)) {
687b4d05839Sml 					stuff_len -= mblk_len;
688b4d05839Sml 					nmp = nmp->b_cont;
689ef523517SMichael Speer 					if (nmp)
690ef523517SMichael Speer 						mblk_len = MBLKL(nmp);
691b4d05839Sml 				}
692b4d05839Sml 				ASSERT(nmp);
693b4d05839Sml 				up = (uint16_t *)(nmp->b_rptr + stuff_len);
694b4d05839Sml 
695b4d05839Sml 				*up = cksum;
696b4d05839Sml 				hdrp->value &= ~TX_CKSUM_EN_PKT_TYPE_UDP;
697b4d05839Sml 				NXGE_DEBUG_MSG((NULL, TX_CTL,
698b4d05839Sml 				    "==> nxge_tx_pkt_hdr_init: UDP offset %d "
699b4d05839Sml 				    "use sw cksum "
700b4d05839Sml 				    "write to $%p cksum 0x%x content up 0x%x",
701b4d05839Sml 				    stuff_len,
702b4d05839Sml 				    up,
703b4d05839Sml 				    cksum,
704b4d05839Sml 				    *up));
705b4d05839Sml 			} else {
706b4d05839Sml 				/* Hardware will compute the full checksum */
707b4d05839Sml 				hdrp->value |= TX_CKSUM_EN_PKT_TYPE_UDP;
708b4d05839Sml 				hdrp->value |=
709b4d05839Sml 				    (((uint64_t)(start_offset >> 1)) <<
710b4d05839Sml 				    TX_PKT_HEADER_L4START_SHIFT);
711b4d05839Sml 				hdrp->value |=
712b4d05839Sml 				    (((uint64_t)(stuff_offset >> 1)) <<
713b4d05839Sml 				    TX_PKT_HEADER_L4STUFF_SHIFT);
714b4d05839Sml 
715b4d05839Sml 				NXGE_DEBUG_MSG((NULL, TX_CTL,
716b4d05839Sml 				    "==> nxge_tx_pkt_hdr_init: UDP offset %d "
717b4d05839Sml 				    " use partial checksum "
718b4d05839Sml 				    "cksum 0x%x ",
719b4d05839Sml 				    "value 0x%llx",
720b4d05839Sml 				    stuff_offset,
721b4d05839Sml 				    IP_CSUM(mp, start_offset, 0),
722b4d05839Sml 				    hdrp->value));
723b4d05839Sml 			}
72444961713Sgirish 		}
725b4d05839Sml 
72644961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
72752ccf843Smisaki 		    "==> nxge_tx_pkt_hdr_init: UDP"
72852ccf843Smisaki 		    "value 0x%llx", hdrp->value));
72944961713Sgirish 		break;
73044961713Sgirish 
73144961713Sgirish 	default:
73244961713Sgirish 		goto fill_tx_header_done;
73344961713Sgirish 	}
73444961713Sgirish 
73544961713Sgirish fill_tx_header_done:
73644961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL,
73752ccf843Smisaki 	    "==> nxge_fill_tx_hdr: pkt_len %d  "
73852ccf843Smisaki 	    "npads %d value 0x%llx", pkt_len, npads, hdrp->value));
73944961713Sgirish 
74044961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_fill_tx_hdr"));
74144961713Sgirish }
74244961713Sgirish 
74344961713Sgirish /*ARGSUSED*/
74444961713Sgirish p_mblk_t
nxge_tx_pkt_header_reserve(p_mblk_t mp,uint8_t * npads)74544961713Sgirish nxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
74644961713Sgirish {
747*86ef0a63SRichard Lowe 	p_mblk_t		newmp = NULL;
74844961713Sgirish 
74944961713Sgirish 	if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
75044961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
75152ccf843Smisaki 		    "<== nxge_tx_pkt_header_reserve: allocb failed"));
75244961713Sgirish 		return (NULL);
75344961713Sgirish 	}
75444961713Sgirish 
75544961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL,
75652ccf843Smisaki 	    "==> nxge_tx_pkt_header_reserve: get new mp"));
75744961713Sgirish 	DB_TYPE(newmp) = M_DATA;
75844961713Sgirish 	newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
75944961713Sgirish 	linkb(newmp, mp);
76044961713Sgirish 	newmp->b_rptr -= TX_PKT_HEADER_SIZE;
76144961713Sgirish 
76244961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==>nxge_tx_pkt_header_reserve: "
76352ccf843Smisaki 	    "b_rptr $%p b_wptr $%p",
76452ccf843Smisaki 	    newmp->b_rptr, newmp->b_wptr));
76544961713Sgirish 
76644961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL,
76752ccf843Smisaki 	    "<== nxge_tx_pkt_header_reserve: use new mp"));
76844961713Sgirish 
76944961713Sgirish 	return (newmp);
77044961713Sgirish }
77144961713Sgirish 
77244961713Sgirish int
nxge_tx_pkt_nmblocks(p_mblk_t mp,int * tot_xfer_len_p)77344961713Sgirish nxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
77444961713Sgirish {
775*86ef0a63SRichard Lowe 	uint_t			nmblks;
77644961713Sgirish 	ssize_t			len;
777*86ef0a63SRichard Lowe 	uint_t			pkt_len;
778*86ef0a63SRichard Lowe 	p_mblk_t		nmp, bmp, tmp;
779*86ef0a63SRichard Lowe 	uint8_t			*b_wptr;
78044961713Sgirish 
78144961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL,
78252ccf843Smisaki 	    "==> nxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p "
78352ccf843Smisaki 	    "len %d", mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
78444961713Sgirish 
78544961713Sgirish 	nmp = mp;
78644961713Sgirish 	bmp = mp;
78744961713Sgirish 	nmblks = 0;
78844961713Sgirish 	pkt_len = 0;
78944961713Sgirish 	*tot_xfer_len_p = 0;
79044961713Sgirish 
79144961713Sgirish 	while (nmp) {
79244961713Sgirish 		len = MBLKL(nmp);
79344961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: "
79452ccf843Smisaki 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
79552ccf843Smisaki 		    len, pkt_len, nmblks,
79652ccf843Smisaki 		    *tot_xfer_len_p));
79744961713Sgirish 
79844961713Sgirish 		if (len <= 0) {
79944961713Sgirish 			bmp = nmp;
80044961713Sgirish 			nmp = nmp->b_cont;
80144961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
80252ccf843Smisaki 			    "==> nxge_tx_pkt_nmblocks: "
80352ccf843Smisaki 			    "len (0) pkt_len %d nmblks %d",
80452ccf843Smisaki 			    pkt_len, nmblks));
80544961713Sgirish 			continue;
80644961713Sgirish 		}
80744961713Sgirish 
80844961713Sgirish 		*tot_xfer_len_p += len;
80944961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_pkt_nmblocks: "
81052ccf843Smisaki 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
81152ccf843Smisaki 		    len, pkt_len, nmblks,
81252ccf843Smisaki 		    *tot_xfer_len_p));
81344961713Sgirish 
81444961713Sgirish 		if (len < nxge_bcopy_thresh) {
81544961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
81652ccf843Smisaki 			    "==> nxge_tx_pkt_nmblocks: "
81752ccf843Smisaki 			    "len %d (< thresh) pkt_len %d nmblks %d",
81852ccf843Smisaki 			    len, pkt_len, nmblks));
81944961713Sgirish 			if (pkt_len == 0)
82044961713Sgirish 				nmblks++;
82144961713Sgirish 			pkt_len += len;
82244961713Sgirish 			if (pkt_len >= nxge_bcopy_thresh) {
82344961713Sgirish 				pkt_len = 0;
82444961713Sgirish 				len = 0;
82544961713Sgirish 				nmp = bmp;
82644961713Sgirish 			}
82744961713Sgirish 		} else {
82844961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
82952ccf843Smisaki 			    "==> nxge_tx_pkt_nmblocks: "
83052ccf843Smisaki 			    "len %d (> thresh) pkt_len %d nmblks %d",
83152ccf843Smisaki 			    len, pkt_len, nmblks));
83244961713Sgirish 			pkt_len = 0;
83344961713Sgirish 			nmblks++;
83444961713Sgirish 			/*
83544961713Sgirish 			 * Hardware limits the transfer length to 4K.
83644961713Sgirish 			 * If len is more than 4K, we need to break
83744961713Sgirish 			 * it up to at most 2 more blocks.
83844961713Sgirish 			 */
83944961713Sgirish 			if (len > TX_MAX_TRANSFER_LENGTH) {
84044961713Sgirish 				uint32_t	nsegs;
84144961713Sgirish 
842678453a8Sspeer 				nsegs = 1;
84344961713Sgirish 				NXGE_DEBUG_MSG((NULL, TX_CTL,
84452ccf843Smisaki 				    "==> nxge_tx_pkt_nmblocks: "
84552ccf843Smisaki 				    "len %d pkt_len %d nmblks %d nsegs %d",
84652ccf843Smisaki 				    len, pkt_len, nmblks, nsegs));
84744961713Sgirish 				if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
84844961713Sgirish 					++nsegs;
84944961713Sgirish 				}
85044961713Sgirish 				do {
85144961713Sgirish 					b_wptr = nmp->b_rptr +
85252ccf843Smisaki 					    TX_MAX_TRANSFER_LENGTH;
85344961713Sgirish 					nmp->b_wptr = b_wptr;
85444961713Sgirish 					if ((tmp = dupb(nmp)) == NULL) {
85544961713Sgirish 						return (0);
85644961713Sgirish 					}
85744961713Sgirish 					tmp->b_rptr = b_wptr;
85844961713Sgirish 					tmp->b_wptr = nmp->b_wptr;
85944961713Sgirish 					tmp->b_cont = nmp->b_cont;
86044961713Sgirish 					nmp->b_cont = tmp;
86144961713Sgirish 					nmblks++;
86244961713Sgirish 					if (--nsegs) {
86344961713Sgirish 						nmp = tmp;
86444961713Sgirish 					}
86544961713Sgirish 				} while (nsegs);
86644961713Sgirish 				nmp = tmp;
86744961713Sgirish 			}
86844961713Sgirish 		}
86944961713Sgirish 
87044961713Sgirish 		/*
87144961713Sgirish 		 * Hardware limits the transmit gather pointers to 15.
87244961713Sgirish 		 */
87344961713Sgirish 		if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
87452ccf843Smisaki 		    TX_MAX_GATHER_POINTERS) {
87544961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
87652ccf843Smisaki 			    "==> nxge_tx_pkt_nmblocks: pull msg - "
87752ccf843Smisaki 			    "len %d pkt_len %d nmblks %d",
87852ccf843Smisaki 			    len, pkt_len, nmblks));
87944961713Sgirish 			/* Pull all message blocks from b_cont */
88044961713Sgirish 			if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
88144961713Sgirish 				return (0);
88244961713Sgirish 			}
88344961713Sgirish 			freemsg(nmp->b_cont);
88444961713Sgirish 			nmp->b_cont = tmp;
88544961713Sgirish 			pkt_len = 0;
88644961713Sgirish 		}
88744961713Sgirish 		bmp = nmp;
88844961713Sgirish 		nmp = nmp->b_cont;
88944961713Sgirish 	}
89044961713Sgirish 
89144961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL,
89252ccf843Smisaki 	    "<== nxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
89352ccf843Smisaki 	    "nmblks %d len %d tot_xfer_len %d",
89452ccf843Smisaki 	    mp->b_rptr, mp->b_wptr, nmblks,
89552ccf843Smisaki 	    MBLKL(mp), *tot_xfer_len_p));
89644961713Sgirish 
89744961713Sgirish 	return (nmblks);
89844961713Sgirish }
89944961713Sgirish 
90044961713Sgirish boolean_t
nxge_txdma_reclaim(p_nxge_t nxgep,p_tx_ring_t tx_ring_p,int nmblks)90144961713Sgirish nxge_txdma_reclaim(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, int nmblks)
90244961713Sgirish {
903*86ef0a63SRichard Lowe 	boolean_t		status = B_TRUE;
90444961713Sgirish 	p_nxge_dma_common_t	tx_desc_dma_p;
90544961713Sgirish 	nxge_dma_common_t	desc_area;
906*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_ring_vp;
907*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_p;
908*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_pp;
909*86ef0a63SRichard Lowe 	tx_desc_t		r_tx_desc;
910*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_ring;
911*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_p;
91244961713Sgirish 	npi_handle_t		handle;
91344961713Sgirish 	tx_ring_hdl_t		tx_head;
914*86ef0a63SRichard Lowe 	uint32_t		pkt_len;
91544961713Sgirish 	uint_t			tx_rd_index;
91644961713Sgirish 	uint16_t		head_index, tail_index;
91744961713Sgirish 	uint8_t			tdc;
91844961713Sgirish 	boolean_t		head_wrap, tail_wrap;
919da14cebeSEric Cheng 	p_nxge_tx_ring_stats_t tdc_stats;
92044961713Sgirish 	int			rc;
92144961713Sgirish 
92244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_reclaim"));
92344961713Sgirish 
92444961713Sgirish 	status = ((tx_ring_p->descs_pending < nxge_reclaim_pending) &&
92552ccf843Smisaki 	    (nmblks != 0));
92644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
927*86ef0a63SRichard Lowe 	    "==> nxge_txdma_reclaim: pending %d	 reclaim %d nmblks %d",
92852ccf843Smisaki 	    tx_ring_p->descs_pending, nxge_reclaim_pending,
92952ccf843Smisaki 	    nmblks));
93044961713Sgirish 	if (!status) {
93144961713Sgirish 		tx_desc_dma_p = &tx_ring_p->tdc_desc;
93244961713Sgirish 		desc_area = tx_ring_p->tdc_desc;
93344961713Sgirish 		handle = NXGE_DEV_NPI_HANDLE(nxgep);
93444961713Sgirish 		tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
93544961713Sgirish 		tx_desc_ring_vp =
93652ccf843Smisaki 		    (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
93744961713Sgirish 		tx_rd_index = tx_ring_p->rd_index;
93844961713Sgirish 		tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
93944961713Sgirish 		tx_msg_ring = tx_ring_p->tx_msg_ring;
94044961713Sgirish 		tx_msg_p = &tx_msg_ring[tx_rd_index];
94144961713Sgirish 		tdc = tx_ring_p->tdc;
94244961713Sgirish 		tdc_stats = tx_ring_p->tdc_stats;
94344961713Sgirish 		if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
94444961713Sgirish 			tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
94544961713Sgirish 		}
94644961713Sgirish 
94744961713Sgirish 		tail_index = tx_ring_p->wr_index;
94844961713Sgirish 		tail_wrap = tx_ring_p->wr_index_wrap;
94944961713Sgirish 
95044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
95152ccf843Smisaki 		    "==> nxge_txdma_reclaim: tdc %d tx_rd_index %d "
95252ccf843Smisaki 		    "tail_index %d tail_wrap %d "
95352ccf843Smisaki 		    "tx_desc_p $%p ($%p) ",
95452ccf843Smisaki 		    tdc, tx_rd_index, tail_index, tail_wrap,
95552ccf843Smisaki 		    tx_desc_p, (*(uint64_t *)tx_desc_p)));
95644961713Sgirish 		/*
95744961713Sgirish 		 * Read the hardware maintained transmit head
95844961713Sgirish 		 * and wrap around bit.
95944961713Sgirish 		 */
96044961713Sgirish 		TXDMA_REG_READ64(handle, TX_RING_HDL_REG, tdc, &tx_head.value);
96144961713Sgirish 		head_index =  tx_head.bits.ldw.head;
96244961713Sgirish 		head_wrap = tx_head.bits.ldw.wrap;
96344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
96452ccf843Smisaki 		    "==> nxge_txdma_reclaim: "
96552ccf843Smisaki 		    "tx_rd_index %d tail %d tail_wrap %d "
96652ccf843Smisaki 		    "head %d wrap %d",
96752ccf843Smisaki 		    tx_rd_index, tail_index, tail_wrap,
96852ccf843Smisaki 		    head_index, head_wrap));
96944961713Sgirish 
97044961713Sgirish 		if (head_index == tail_index) {
97144961713Sgirish 			if (TXDMA_RING_EMPTY(head_index, head_wrap,
97252ccf843Smisaki 			    tail_index, tail_wrap) &&
97352ccf843Smisaki 			    (head_index == tx_rd_index)) {
97444961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
97552ccf843Smisaki 				    "==> nxge_txdma_reclaim: EMPTY"));
97644961713Sgirish 				return (B_TRUE);
97744961713Sgirish 			}
97844961713Sgirish 
97944961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
98052ccf843Smisaki 			    "==> nxge_txdma_reclaim: Checking "
98152ccf843Smisaki 			    "if ring full"));
98244961713Sgirish 			if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
98352ccf843Smisaki 			    tail_wrap)) {
98444961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
98552ccf843Smisaki 				    "==> nxge_txdma_reclaim: full"));
98644961713Sgirish 				return (B_FALSE);
98744961713Sgirish 			}
98844961713Sgirish 		}
98944961713Sgirish 
99044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
99152ccf843Smisaki 		    "==> nxge_txdma_reclaim: tx_rd_index and head_index"));
99244961713Sgirish 
99344961713Sgirish 		tx_desc_pp = &r_tx_desc;
99444961713Sgirish 		while ((tx_rd_index != head_index) &&
99552ccf843Smisaki 		    (tx_ring_p->descs_pending != 0)) {
99644961713Sgirish 
99744961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
99852ccf843Smisaki 			    "==> nxge_txdma_reclaim: Checking if pending"));
99944961713Sgirish 
100044961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
100152ccf843Smisaki 			    "==> nxge_txdma_reclaim: "
100252ccf843Smisaki 			    "descs_pending %d ",
100352ccf843Smisaki 			    tx_ring_p->descs_pending));
100444961713Sgirish 
100544961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
100652ccf843Smisaki 			    "==> nxge_txdma_reclaim: "
100752ccf843Smisaki 			    "(tx_rd_index %d head_index %d "
100852ccf843Smisaki 			    "(tx_desc_p $%p)",
100952ccf843Smisaki 			    tx_rd_index, head_index,
101052ccf843Smisaki 			    tx_desc_p));
101144961713Sgirish 
101244961713Sgirish 			tx_desc_pp->value = tx_desc_p->value;
101344961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
101452ccf843Smisaki 			    "==> nxge_txdma_reclaim: "
101552ccf843Smisaki 			    "(tx_rd_index %d head_index %d "
101652ccf843Smisaki 			    "tx_desc_p $%p (desc value 0x%llx) ",
101752ccf843Smisaki 			    tx_rd_index, head_index,
101852ccf843Smisaki 			    tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
101944961713Sgirish 
102044961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
102152ccf843Smisaki 			    "==> nxge_txdma_reclaim: dump desc:"));
102244961713Sgirish 
102344961713Sgirish 			pkt_len = tx_desc_pp->bits.hdw.tr_len;
10240dc2366fSVenugopal Iyer 			tdc_stats->obytes += (pkt_len - TX_PKT_HEADER_SIZE);
102544961713Sgirish 			tdc_stats->opackets += tx_desc_pp->bits.hdw.sop;
102644961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
102752ccf843Smisaki 			    "==> nxge_txdma_reclaim: pkt_len %d "
102852ccf843Smisaki 			    "tdc channel %d opackets %d",
102952ccf843Smisaki 			    pkt_len,
103052ccf843Smisaki 			    tdc,
103152ccf843Smisaki 			    tdc_stats->opackets));
103244961713Sgirish 
103344961713Sgirish 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
103444961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
103552ccf843Smisaki 				    "tx_desc_p = $%p "
103652ccf843Smisaki 				    "tx_desc_pp = $%p "
103752ccf843Smisaki 				    "index = %d",
103852ccf843Smisaki 				    tx_desc_p,
103952ccf843Smisaki 				    tx_desc_pp,
104052ccf843Smisaki 				    tx_ring_p->rd_index));
104144961713Sgirish 				(void) dvma_unload(tx_msg_p->dvma_handle,
104252ccf843Smisaki 				    0, -1);
104344961713Sgirish 				tx_msg_p->dvma_handle = NULL;
104444961713Sgirish 				if (tx_ring_p->dvma_wr_index ==
104552ccf843Smisaki 				    tx_ring_p->dvma_wrap_mask) {
104644961713Sgirish 					tx_ring_p->dvma_wr_index = 0;
104744961713Sgirish 				} else {
104844961713Sgirish 					tx_ring_p->dvma_wr_index++;
104944961713Sgirish 				}
105044961713Sgirish 				tx_ring_p->dvma_pending--;
105144961713Sgirish 			} else if (tx_msg_p->flags.dma_type ==
105252ccf843Smisaki 			    USE_DMA) {
105344961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
105452ccf843Smisaki 				    "==> nxge_txdma_reclaim: "
105552ccf843Smisaki 				    "USE DMA"));
105644961713Sgirish 				if (rc = ddi_dma_unbind_handle
105752ccf843Smisaki 				    (tx_msg_p->dma_handle)) {
105844961713Sgirish 					cmn_err(CE_WARN, "!nxge_reclaim: "
105952ccf843Smisaki 					    "ddi_dma_unbind_handle "
106052ccf843Smisaki 					    "failed. status %d", rc);
106144961713Sgirish 				}
106244961713Sgirish 			}
106344961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
106452ccf843Smisaki 			    "==> nxge_txdma_reclaim: count packets"));
106544961713Sgirish 			/*
106644961713Sgirish 			 * count a chained packet only once.
106744961713Sgirish 			 */
106844961713Sgirish 			if (tx_msg_p->tx_message != NULL) {
1069da14cebeSEric Cheng 				freemsg(tx_msg_p->tx_message);
1070da14cebeSEric Cheng 				tx_msg_p->tx_message = NULL;
107144961713Sgirish 			}
107244961713Sgirish 
107344961713Sgirish 			tx_msg_p->flags.dma_type = USE_NONE;
107444961713Sgirish 			tx_rd_index = tx_ring_p->rd_index;
107544961713Sgirish 			tx_rd_index = (tx_rd_index + 1) &
107652ccf843Smisaki 			    tx_ring_p->tx_wrap_mask;
107744961713Sgirish 			tx_ring_p->rd_index = tx_rd_index;
107844961713Sgirish 			tx_ring_p->descs_pending--;
107944961713Sgirish 			tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
108044961713Sgirish 			tx_msg_p = &tx_msg_ring[tx_rd_index];
108144961713Sgirish 		}
108244961713Sgirish 
1083257bdc55SMichael Speer 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
1084257bdc55SMichael Speer 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
108544961713Sgirish 		if (status) {
108675d94465SJosef 'Jeff' Sipek 			(void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing,
108775d94465SJosef 'Jeff' Sipek 			    1, 0);
108844961713Sgirish 		}
108944961713Sgirish 	} else {
1090257bdc55SMichael Speer 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
1091257bdc55SMichael Speer 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
109244961713Sgirish 	}
109344961713Sgirish 
109444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
109552ccf843Smisaki 	    "<== nxge_txdma_reclaim status = 0x%08x", status));
109644961713Sgirish 
109744961713Sgirish 	return (status);
109844961713Sgirish }
109944961713Sgirish 
1100678453a8Sspeer /*
1101678453a8Sspeer  * nxge_tx_intr
1102678453a8Sspeer  *
1103678453a8Sspeer  *	Process a TDC interrupt
1104678453a8Sspeer  *
1105678453a8Sspeer  * Arguments:
1106*86ef0a63SRichard Lowe  *	arg1	A Logical Device state Vector (LSV) data structure.
1107*86ef0a63SRichard Lowe  *	arg2	nxge_t *
1108678453a8Sspeer  *
1109678453a8Sspeer  * Notes:
1110678453a8Sspeer  *
1111678453a8Sspeer  * NPI/NXGE function calls:
1112678453a8Sspeer  *	npi_txdma_control_status()
1113678453a8Sspeer  *	npi_intr_ldg_mgmt_set()
1114678453a8Sspeer  *
1115678453a8Sspeer  *	nxge_tx_err_evnts()
1116678453a8Sspeer  *	nxge_txdma_reclaim()
1117678453a8Sspeer  *
1118678453a8Sspeer  * Registers accessed:
1119678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
1120678453a8Sspeer  *	PIO_LDSV
1121678453a8Sspeer  *
1122678453a8Sspeer  * Context:
1123678453a8Sspeer  *	Any domain
1124678453a8Sspeer  */
112544961713Sgirish uint_t
nxge_tx_intr(char * arg1,char * arg2)1126e3d11eeeSToomas Soome nxge_tx_intr(char *arg1, char *arg2)
112744961713Sgirish {
112844961713Sgirish 	p_nxge_ldv_t		ldvp = (p_nxge_ldv_t)arg1;
112944961713Sgirish 	p_nxge_t		nxgep = (p_nxge_t)arg2;
113044961713Sgirish 	p_nxge_ldg_t		ldgp;
113144961713Sgirish 	uint8_t			channel;
113244961713Sgirish 	uint32_t		vindex;
113344961713Sgirish 	npi_handle_t		handle;
113444961713Sgirish 	tx_cs_t			cs;
1135*86ef0a63SRichard Lowe 	p_tx_ring_t		*tx_rings;
1136*86ef0a63SRichard Lowe 	p_tx_ring_t		tx_ring_p;
113744961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
1138*86ef0a63SRichard Lowe 	uint_t			serviced = DDI_INTR_UNCLAIMED;
1139*86ef0a63SRichard Lowe 	nxge_status_t		status = NXGE_OK;
114044961713Sgirish 
114144961713Sgirish 	if (ldvp == NULL) {
114244961713Sgirish 		NXGE_DEBUG_MSG((NULL, INT_CTL,
114352ccf843Smisaki 		    "<== nxge_tx_intr: nxgep $%p ldvp $%p",
114452ccf843Smisaki 		    nxgep, ldvp));
114544961713Sgirish 		return (DDI_INTR_UNCLAIMED);
114644961713Sgirish 	}
114744961713Sgirish 
114844961713Sgirish 	if (arg2 == NULL || (void *)ldvp->nxgep != arg2) {
114944961713Sgirish 		nxgep = ldvp->nxgep;
115044961713Sgirish 	}
115144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, INT_CTL,
115252ccf843Smisaki 	    "==> nxge_tx_intr: nxgep(arg2) $%p ldvp(arg1) $%p",
115352ccf843Smisaki 	    nxgep, ldvp));
115422c0d73aSspeer 
115522c0d73aSspeer 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
115622c0d73aSspeer 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
115722c0d73aSspeer 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
115822c0d73aSspeer 		    "<== nxge_tx_intr: interface not started or intialized"));
115922c0d73aSspeer 		return (DDI_INTR_CLAIMED);
116022c0d73aSspeer 	}
116122c0d73aSspeer 
116244961713Sgirish 	/*
116344961713Sgirish 	 * This interrupt handler is for a specific
116444961713Sgirish 	 * transmit dma channel.
116544961713Sgirish 	 */
116644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
116744961713Sgirish 	/* Get the control and status for this channel. */
116844961713Sgirish 	channel = ldvp->channel;
116944961713Sgirish 	ldgp = ldvp->ldgp;
117044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, INT_CTL,
117152ccf843Smisaki 	    "==> nxge_tx_intr: nxgep $%p ldvp (ldvp) $%p "
117252ccf843Smisaki 	    "channel %d",
117352ccf843Smisaki 	    nxgep, ldvp, channel));
117444961713Sgirish 
117544961713Sgirish 	rs = npi_txdma_control_status(handle, OP_GET, channel, &cs);
117644961713Sgirish 	vindex = ldvp->vdma_index;
117744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, INT_CTL,
117852ccf843Smisaki 	    "==> nxge_tx_intr:channel %d ring index %d status 0x%08x",
117952ccf843Smisaki 	    channel, vindex, rs));
118044961713Sgirish 	if (!rs && cs.bits.ldw.mk) {
118144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
118252ccf843Smisaki 		    "==> nxge_tx_intr:channel %d ring index %d "
118352ccf843Smisaki 		    "status 0x%08x (mk bit set)",
118452ccf843Smisaki 		    channel, vindex, rs));
118544961713Sgirish 		tx_rings = nxgep->tx_rings->rings;
118644961713Sgirish 		tx_ring_p = tx_rings[vindex];
118744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
118852ccf843Smisaki 		    "==> nxge_tx_intr:channel %d ring index %d "
118952ccf843Smisaki 		    "status 0x%08x (mk bit set, calling reclaim)",
119052ccf843Smisaki 		    channel, vindex, rs));
119144961713Sgirish 
1192da14cebeSEric Cheng 		nxge_tx_ring_task((void *)tx_ring_p);
119344961713Sgirish 	}
119444961713Sgirish 
119544961713Sgirish 	/*
119644961713Sgirish 	 * Process other transmit control and status.
119744961713Sgirish 	 * Check the ldv state.
119844961713Sgirish 	 */
119944961713Sgirish 	status = nxge_tx_err_evnts(nxgep, ldvp->vdma_index, ldvp, cs);
120044961713Sgirish 	/*
120144961713Sgirish 	 * Rearm this logical group if this is a single device
120244961713Sgirish 	 * group.
120344961713Sgirish 	 */
120444961713Sgirish 	if (ldgp->nldvs == 1) {
120544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
120652ccf843Smisaki 		    "==> nxge_tx_intr: rearm"));
120744961713Sgirish 		if (status == NXGE_OK) {
1208678453a8Sspeer 			if (isLDOMguest(nxgep)) {
1209678453a8Sspeer 				nxge_hio_ldgimgn(nxgep, ldgp);
1210678453a8Sspeer 			} else {
1211678453a8Sspeer 				(void) npi_intr_ldg_mgmt_set(handle, ldgp->ldg,
1212678453a8Sspeer 				    B_TRUE, ldgp->ldg_timer);
1213678453a8Sspeer 			}
121444961713Sgirish 		}
121544961713Sgirish 	}
121644961713Sgirish 
121744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, INT_CTL, "<== nxge_tx_intr"));
121844961713Sgirish 	serviced = DDI_INTR_CLAIMED;
121944961713Sgirish 	return (serviced);
122044961713Sgirish }
122144961713Sgirish 
122244961713Sgirish void
nxge_txdma_stop(p_nxge_t nxgep)1223678453a8Sspeer nxge_txdma_stop(p_nxge_t nxgep)	/* Dead */
122444961713Sgirish {
122544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop"));
122644961713Sgirish 
122744961713Sgirish 	(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
122844961713Sgirish 
122944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop"));
123044961713Sgirish }
123144961713Sgirish 
123244961713Sgirish void
nxge_txdma_stop_start(p_nxge_t nxgep)1233678453a8Sspeer nxge_txdma_stop_start(p_nxge_t nxgep) /* Dead */
123444961713Sgirish {
123544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_start"));
123644961713Sgirish 
123744961713Sgirish 	(void) nxge_txdma_stop(nxgep);
123844961713Sgirish 
123944961713Sgirish 	(void) nxge_fixup_txdma_rings(nxgep);
124044961713Sgirish 	(void) nxge_txdma_hw_mode(nxgep, NXGE_DMA_START);
124144961713Sgirish 	(void) nxge_tx_mac_enable(nxgep);
124244961713Sgirish 	(void) nxge_txdma_hw_kick(nxgep);
124344961713Sgirish 
124444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_start"));
124544961713Sgirish }
124644961713Sgirish 
1247678453a8Sspeer npi_status_t
nxge_txdma_channel_disable(nxge_t * nxge,int channel)1248678453a8Sspeer nxge_txdma_channel_disable(
1249678453a8Sspeer 	nxge_t *nxge,
1250678453a8Sspeer 	int channel)
1251678453a8Sspeer {
1252678453a8Sspeer 	npi_handle_t	handle = NXGE_DEV_NPI_HANDLE(nxge);
1253678453a8Sspeer 	npi_status_t	rs;
1254678453a8Sspeer 	tdmc_intr_dbg_t	intr_dbg;
1255678453a8Sspeer 
1256678453a8Sspeer 	/*
1257678453a8Sspeer 	 * Stop the dma channel and wait for the stop-done.
1258678453a8Sspeer 	 * If the stop-done bit is not present, then force
1259678453a8Sspeer 	 * an error so TXC will stop.
1260678453a8Sspeer 	 * All channels bound to this port need to be stopped
1261678453a8Sspeer 	 * and reset after injecting an interrupt error.
1262678453a8Sspeer 	 */
1263678453a8Sspeer 	rs = npi_txdma_channel_disable(handle, channel);
1264678453a8Sspeer 	NXGE_DEBUG_MSG((nxge, MEM3_CTL,
126552ccf843Smisaki 	    "==> nxge_txdma_channel_disable(%d) "
126652ccf843Smisaki 	    "rs 0x%x", channel, rs));
1267678453a8Sspeer 	if (rs != NPI_SUCCESS) {
1268678453a8Sspeer 		/* Inject any error */
1269678453a8Sspeer 		intr_dbg.value = 0;
1270678453a8Sspeer 		intr_dbg.bits.ldw.nack_pref = 1;
1271678453a8Sspeer 		NXGE_DEBUG_MSG((nxge, MEM3_CTL,
127252ccf843Smisaki 		    "==> nxge_txdma_hw_mode: "
127352ccf843Smisaki 		    "channel %d (stop failed 0x%x) "
127452ccf843Smisaki 		    "(inject err)", rs, channel));
1275678453a8Sspeer 		(void) npi_txdma_inj_int_error_set(
127652ccf843Smisaki 		    handle, channel, &intr_dbg);
1277678453a8Sspeer 		rs = npi_txdma_channel_disable(handle, channel);
1278678453a8Sspeer 		NXGE_DEBUG_MSG((nxge, MEM3_CTL,
127952ccf843Smisaki 		    "==> nxge_txdma_hw_mode: "
128052ccf843Smisaki 		    "channel %d (stop again 0x%x) "
128152ccf843Smisaki 		    "(after inject err)",
128252ccf843Smisaki 		    rs, channel));
1283678453a8Sspeer 	}
1284678453a8Sspeer 
1285678453a8Sspeer 	return (rs);
1286678453a8Sspeer }
1287678453a8Sspeer 
1288678453a8Sspeer /*
1289678453a8Sspeer  * nxge_txdma_hw_mode
1290678453a8Sspeer  *
1291678453a8Sspeer  *	Toggle all TDCs on (enable) or off (disable).
1292678453a8Sspeer  *
1293678453a8Sspeer  * Arguments:
1294*86ef0a63SRichard Lowe  *	nxgep
1295*86ef0a63SRichard Lowe  *	enable	Enable or disable a TDC.
1296678453a8Sspeer  *
1297678453a8Sspeer  * Notes:
1298678453a8Sspeer  *
1299678453a8Sspeer  * NPI/NXGE function calls:
1300678453a8Sspeer  *	npi_txdma_channel_enable(TX_CS)
1301678453a8Sspeer  *	npi_txdma_channel_disable(TX_CS)
1302678453a8Sspeer  *	npi_txdma_inj_int_error_set(TDMC_INTR_DBG)
1303678453a8Sspeer  *
1304678453a8Sspeer  * Registers accessed:
1305678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
1306678453a8Sspeer  *	TDMC_INTR_DBG	DMC + 0x40060 Transmit DMA Interrupt Debug
1307678453a8Sspeer  *
1308678453a8Sspeer  * Context:
1309678453a8Sspeer  *	Any domain
1310678453a8Sspeer  */
131144961713Sgirish nxge_status_t
nxge_txdma_hw_mode(p_nxge_t nxgep,boolean_t enable)131244961713Sgirish nxge_txdma_hw_mode(p_nxge_t nxgep, boolean_t enable)
131344961713Sgirish {
1314678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
1315678453a8Sspeer 
1316678453a8Sspeer 	npi_handle_t	handle;
1317678453a8Sspeer 	nxge_status_t	status;
1318678453a8Sspeer 	npi_status_t	rs;
1319678453a8Sspeer 	int		tdc;
132044961713Sgirish 
132144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
132252ccf843Smisaki 	    "==> nxge_txdma_hw_mode: enable mode %d", enable));
132344961713Sgirish 
132444961713Sgirish 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
132544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
132652ccf843Smisaki 		    "<== nxge_txdma_mode: not initialized"));
132744961713Sgirish 		return (NXGE_ERROR);
132844961713Sgirish 	}
132944961713Sgirish 
1330678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
133144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1332678453a8Sspeer 		    "<== nxge_txdma_hw_mode: NULL ring pointer(s)"));
133344961713Sgirish 		return (NXGE_ERROR);
133444961713Sgirish 	}
133544961713Sgirish 
1336678453a8Sspeer 	/* Enable or disable all of the TDCs owned by us. */
1337e3d11eeeSToomas Soome 	rs = 0;
133844961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
1339678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1340678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
1341678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1342678453a8Sspeer 			if (ring) {
1343678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1344678453a8Sspeer 				    "==> nxge_txdma_hw_mode: channel %d", tdc));
1345678453a8Sspeer 				if (enable) {
1346678453a8Sspeer 					rs = npi_txdma_channel_enable
1347678453a8Sspeer 					    (handle, tdc);
134844961713Sgirish 					NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1349678453a8Sspeer 					    "==> nxge_txdma_hw_mode: "
1350678453a8Sspeer 					    "channel %d (enable) rs 0x%x",
1351678453a8Sspeer 					    tdc, rs));
1352678453a8Sspeer 				} else {
1353678453a8Sspeer 					rs = nxge_txdma_channel_disable
1354678453a8Sspeer 					    (nxgep, tdc);
135544961713Sgirish 				}
135644961713Sgirish 			}
135744961713Sgirish 		}
135844961713Sgirish 	}
135944961713Sgirish 
136044961713Sgirish 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
136144961713Sgirish 
136244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
136352ccf843Smisaki 	    "<== nxge_txdma_hw_mode: status 0x%x", status));
136444961713Sgirish 
136544961713Sgirish 	return (status);
136644961713Sgirish }
136744961713Sgirish 
136844961713Sgirish void
nxge_txdma_enable_channel(p_nxge_t nxgep,uint16_t channel)136944961713Sgirish nxge_txdma_enable_channel(p_nxge_t nxgep, uint16_t channel)
137044961713Sgirish {
137144961713Sgirish 	npi_handle_t		handle;
137244961713Sgirish 
137344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
137452ccf843Smisaki 	    "==> nxge_txdma_enable_channel: channel %d", channel));
137544961713Sgirish 
137644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
137744961713Sgirish 	/* enable the transmit dma channels */
137844961713Sgirish 	(void) npi_txdma_channel_enable(handle, channel);
137944961713Sgirish 
138044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_txdma_enable_channel"));
138144961713Sgirish }
138244961713Sgirish 
138344961713Sgirish void
nxge_txdma_disable_channel(p_nxge_t nxgep,uint16_t channel)138444961713Sgirish nxge_txdma_disable_channel(p_nxge_t nxgep, uint16_t channel)
138544961713Sgirish {
138644961713Sgirish 	npi_handle_t		handle;
138744961713Sgirish 
138844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
138952ccf843Smisaki 	    "==> nxge_txdma_disable_channel: channel %d", channel));
139044961713Sgirish 
139144961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
139244961713Sgirish 	/* stop the transmit dma channels */
139344961713Sgirish 	(void) npi_txdma_channel_disable(handle, channel);
139444961713Sgirish 
139544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_disable_channel"));
139644961713Sgirish }
139744961713Sgirish 
1398678453a8Sspeer /*
1399678453a8Sspeer  * nxge_txdma_stop_inj_err
1400678453a8Sspeer  *
1401678453a8Sspeer  *	Stop a TDC.  If at first we don't succeed, inject an error.
1402678453a8Sspeer  *
1403678453a8Sspeer  * Arguments:
1404*86ef0a63SRichard Lowe  *	nxgep
1405*86ef0a63SRichard Lowe  *	channel		The channel to stop.
1406678453a8Sspeer  *
1407678453a8Sspeer  * Notes:
1408678453a8Sspeer  *
1409678453a8Sspeer  * NPI/NXGE function calls:
1410678453a8Sspeer  *	npi_txdma_channel_disable()
1411678453a8Sspeer  *	npi_txdma_inj_int_error_set()
1412678453a8Sspeer  * #if defined(NXGE_DEBUG)
1413678453a8Sspeer  *	nxge_txdma_regs_dump_channels(nxgep);
1414678453a8Sspeer  * #endif
1415678453a8Sspeer  *
1416678453a8Sspeer  * Registers accessed:
1417678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
1418678453a8Sspeer  *	TDMC_INTR_DBG	DMC + 0x40060 Transmit DMA Interrupt Debug
1419678453a8Sspeer  *
1420678453a8Sspeer  * Context:
1421678453a8Sspeer  *	Any domain
1422678453a8Sspeer  */
142344961713Sgirish int
nxge_txdma_stop_inj_err(p_nxge_t nxgep,int channel)142444961713Sgirish nxge_txdma_stop_inj_err(p_nxge_t nxgep, int channel)
142544961713Sgirish {
142644961713Sgirish 	npi_handle_t		handle;
142744961713Sgirish 	tdmc_intr_dbg_t		intr_dbg;
142844961713Sgirish 	int			status;
142944961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
143044961713Sgirish 
143144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_stop_inj_err"));
143244961713Sgirish 	/*
143344961713Sgirish 	 * Stop the dma channel waits for the stop done.
143444961713Sgirish 	 * If the stop done bit is not set, then create
143544961713Sgirish 	 * an error.
143644961713Sgirish 	 */
143744961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
143844961713Sgirish 	rs = npi_txdma_channel_disable(handle, channel);
143944961713Sgirish 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
144044961713Sgirish 	if (status == NXGE_OK) {
144144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
144252ccf843Smisaki 		    "<== nxge_txdma_stop_inj_err (channel %d): "
144352ccf843Smisaki 		    "stopped OK", channel));
144444961713Sgirish 		return (status);
144544961713Sgirish 	}
144644961713Sgirish 
144744961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
144852ccf843Smisaki 	    "==> nxge_txdma_stop_inj_err (channel %d): stop failed (0x%x) "
144952ccf843Smisaki 	    "injecting error", channel, rs));
145044961713Sgirish 	/* Inject any error */
145144961713Sgirish 	intr_dbg.value = 0;
145244961713Sgirish 	intr_dbg.bits.ldw.nack_pref = 1;
145344961713Sgirish 	(void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg);
145444961713Sgirish 
145544961713Sgirish 	/* Stop done bit will be set as a result of error injection */
145644961713Sgirish 	rs = npi_txdma_channel_disable(handle, channel);
145744961713Sgirish 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
145844961713Sgirish 	if (!(rs & NPI_TXDMA_STOP_FAILED)) {
145944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
146052ccf843Smisaki 		    "<== nxge_txdma_stop_inj_err (channel %d): "
146152ccf843Smisaki 		    "stopped OK ", channel));
146244961713Sgirish 		return (status);
146344961713Sgirish 	}
146444961713Sgirish 
146544961713Sgirish #if	defined(NXGE_DEBUG)
146644961713Sgirish 	nxge_txdma_regs_dump_channels(nxgep);
146744961713Sgirish #endif
146844961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
146952ccf843Smisaki 	    "==> nxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
147052ccf843Smisaki 	    " (injected error but still not stopped)", channel, rs));
147144961713Sgirish 
147244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_stop_inj_err"));
147344961713Sgirish 	return (status);
147444961713Sgirish }
147544961713Sgirish 
147644961713Sgirish /*ARGSUSED*/
147744961713Sgirish void
nxge_fixup_txdma_rings(p_nxge_t nxgep)147844961713Sgirish nxge_fixup_txdma_rings(p_nxge_t nxgep)
147944961713Sgirish {
1480678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
1481678453a8Sspeer 	int tdc;
148244961713Sgirish 
148344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_txdma_rings"));
148444961713Sgirish 
1485678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
1486678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1487678453a8Sspeer 		    "<== nxge_fixup_txdma_rings: NULL ring pointer(s)"));
148844961713Sgirish 		return;
148944961713Sgirish 	}
149044961713Sgirish 
1491678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1492678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
1493678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1494678453a8Sspeer 			if (ring) {
1495678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1496678453a8Sspeer 				    "==> nxge_fixup_txdma_rings: channel %d",
1497678453a8Sspeer 				    tdc));
1498678453a8Sspeer 				nxge_txdma_fixup_channel(nxgep, ring, tdc);
1499678453a8Sspeer 			}
1500678453a8Sspeer 		}
150144961713Sgirish 	}
150244961713Sgirish 
150344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_txdma_rings"));
150444961713Sgirish }
150544961713Sgirish 
150644961713Sgirish /*ARGSUSED*/
150744961713Sgirish void
nxge_txdma_fix_channel(p_nxge_t nxgep,uint16_t channel)150844961713Sgirish nxge_txdma_fix_channel(p_nxge_t nxgep, uint16_t channel)
150944961713Sgirish {
151044961713Sgirish 	p_tx_ring_t	ring_p;
151144961713Sgirish 
151244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_channel"));
151344961713Sgirish 	ring_p = nxge_txdma_get_ring(nxgep, channel);
151444961713Sgirish 	if (ring_p == NULL) {
151544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel"));
151644961713Sgirish 		return;
151744961713Sgirish 	}
151844961713Sgirish 
151944961713Sgirish 	if (ring_p->tdc != channel) {
152044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
152152ccf843Smisaki 		    "<== nxge_txdma_fix_channel: channel not matched "
152252ccf843Smisaki 		    "ring tdc %d passed channel",
152352ccf843Smisaki 		    ring_p->tdc, channel));
152444961713Sgirish 		return;
152544961713Sgirish 	}
152644961713Sgirish 
152744961713Sgirish 	nxge_txdma_fixup_channel(nxgep, ring_p, channel);
152844961713Sgirish 
152944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_channel"));
153044961713Sgirish }
153144961713Sgirish 
153244961713Sgirish /*ARGSUSED*/
153344961713Sgirish void
nxge_txdma_fixup_channel(p_nxge_t nxgep,p_tx_ring_t ring_p,uint16_t channel)153444961713Sgirish nxge_txdma_fixup_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel)
153544961713Sgirish {
153644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_channel"));
153744961713Sgirish 
153844961713Sgirish 	if (ring_p == NULL) {
153944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
154052ccf843Smisaki 		    "<== nxge_txdma_fixup_channel: NULL ring pointer"));
154144961713Sgirish 		return;
154244961713Sgirish 	}
154344961713Sgirish 
154444961713Sgirish 	if (ring_p->tdc != channel) {
154544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
154652ccf843Smisaki 		    "<== nxge_txdma_fixup_channel: channel not matched "
154752ccf843Smisaki 		    "ring tdc %d passed channel",
154852ccf843Smisaki 		    ring_p->tdc, channel));
154944961713Sgirish 		return;
155044961713Sgirish 	}
155144961713Sgirish 
155244961713Sgirish 	MUTEX_ENTER(&ring_p->lock);
155344961713Sgirish 	(void) nxge_txdma_reclaim(nxgep, ring_p, 0);
155444961713Sgirish 	ring_p->rd_index = 0;
155544961713Sgirish 	ring_p->wr_index = 0;
155644961713Sgirish 	ring_p->ring_head.value = 0;
155744961713Sgirish 	ring_p->ring_kick_tail.value = 0;
155844961713Sgirish 	ring_p->descs_pending = 0;
155944961713Sgirish 	MUTEX_EXIT(&ring_p->lock);
156044961713Sgirish 
156144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_channel"));
156244961713Sgirish }
156344961713Sgirish 
156444961713Sgirish /*ARGSUSED*/
156544961713Sgirish void
nxge_txdma_hw_kick(p_nxge_t nxgep)156644961713Sgirish nxge_txdma_hw_kick(p_nxge_t nxgep)
156744961713Sgirish {
1568678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
1569678453a8Sspeer 	int tdc;
157044961713Sgirish 
157144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick"));
157244961713Sgirish 
1573678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
157444961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1575678453a8Sspeer 		    "<== nxge_txdma_hw_kick: NULL ring pointer(s)"));
157644961713Sgirish 		return;
157744961713Sgirish 	}
157844961713Sgirish 
1579678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1580678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
1581678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1582678453a8Sspeer 			if (ring) {
1583678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
1584678453a8Sspeer 				    "==> nxge_txdma_hw_kick: channel %d", tdc));
1585678453a8Sspeer 				nxge_txdma_hw_kick_channel(nxgep, ring, tdc);
1586678453a8Sspeer 			}
1587678453a8Sspeer 		}
158844961713Sgirish 	}
158944961713Sgirish 
159044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick"));
159144961713Sgirish }
159244961713Sgirish 
159344961713Sgirish /*ARGSUSED*/
159444961713Sgirish void
nxge_txdma_kick_channel(p_nxge_t nxgep,uint16_t channel)159544961713Sgirish nxge_txdma_kick_channel(p_nxge_t nxgep, uint16_t channel)
159644961713Sgirish {
159744961713Sgirish 	p_tx_ring_t	ring_p;
159844961713Sgirish 
159944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_kick_channel"));
160044961713Sgirish 
160144961713Sgirish 	ring_p = nxge_txdma_get_ring(nxgep, channel);
160244961713Sgirish 	if (ring_p == NULL) {
160344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
160452ccf843Smisaki 		    " nxge_txdma_kick_channel"));
160544961713Sgirish 		return;
160644961713Sgirish 	}
160744961713Sgirish 
160844961713Sgirish 	if (ring_p->tdc != channel) {
160944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
161052ccf843Smisaki 		    "<== nxge_txdma_kick_channel: channel not matched "
161152ccf843Smisaki 		    "ring tdc %d passed channel",
161252ccf843Smisaki 		    ring_p->tdc, channel));
161344961713Sgirish 		return;
161444961713Sgirish 	}
161544961713Sgirish 
161644961713Sgirish 	nxge_txdma_hw_kick_channel(nxgep, ring_p, channel);
161744961713Sgirish 
161844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_kick_channel"));
161944961713Sgirish }
1620a3c5bd6dSspeer 
162144961713Sgirish /*ARGSUSED*/
162244961713Sgirish void
nxge_txdma_hw_kick_channel(p_nxge_t nxgep,p_tx_ring_t ring_p,uint16_t channel)162344961713Sgirish nxge_txdma_hw_kick_channel(p_nxge_t nxgep, p_tx_ring_t ring_p, uint16_t channel)
162444961713Sgirish {
162544961713Sgirish 
162644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hw_kick_channel"));
162744961713Sgirish 
162844961713Sgirish 	if (ring_p == NULL) {
162944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
163052ccf843Smisaki 		    "<== nxge_txdma_hw_kick_channel: NULL ring pointer"));
163144961713Sgirish 		return;
163244961713Sgirish 	}
163344961713Sgirish 
163444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hw_kick_channel"));
163544961713Sgirish }
163644961713Sgirish 
1637678453a8Sspeer /*
1638678453a8Sspeer  * nxge_check_tx_hang
1639678453a8Sspeer  *
1640678453a8Sspeer  *	Check the state of all TDCs belonging to nxgep.
1641678453a8Sspeer  *
1642678453a8Sspeer  * Arguments:
1643*86ef0a63SRichard Lowe  *	nxgep
1644678453a8Sspeer  *
1645678453a8Sspeer  * Notes:
1646678453a8Sspeer  *	Called by nxge_hw.c:nxge_check_hw_state().
1647678453a8Sspeer  *
1648678453a8Sspeer  * NPI/NXGE function calls:
1649678453a8Sspeer  *
1650678453a8Sspeer  * Registers accessed:
1651678453a8Sspeer  *
1652678453a8Sspeer  * Context:
1653678453a8Sspeer  *	Any domain
1654678453a8Sspeer  */
165544961713Sgirish /*ARGSUSED*/
165644961713Sgirish void
nxge_check_tx_hang(p_nxge_t nxgep)165744961713Sgirish nxge_check_tx_hang(p_nxge_t nxgep)
165844961713Sgirish {
165944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_check_tx_hang"));
166044961713Sgirish 
166122c0d73aSspeer 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
166222c0d73aSspeer 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
166322c0d73aSspeer 		goto nxge_check_tx_hang_exit;
166422c0d73aSspeer 	}
166522c0d73aSspeer 
166644961713Sgirish 	/*
166744961713Sgirish 	 * Needs inputs from hardware for regs:
166844961713Sgirish 	 *	head index had not moved since last timeout.
166944961713Sgirish 	 *	packets not transmitted or stuffed registers.
167044961713Sgirish 	 */
167144961713Sgirish 	if (nxge_txdma_hung(nxgep)) {
167244961713Sgirish 		nxge_fixup_hung_txdma_rings(nxgep);
167344961713Sgirish 	}
167422c0d73aSspeer 
167522c0d73aSspeer nxge_check_tx_hang_exit:
167644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_check_tx_hang"));
167744961713Sgirish }
167844961713Sgirish 
1679678453a8Sspeer /*
1680678453a8Sspeer  * nxge_txdma_hung
1681678453a8Sspeer  *
1682678453a8Sspeer  *	Reset a TDC.
1683678453a8Sspeer  *
1684678453a8Sspeer  * Arguments:
1685*86ef0a63SRichard Lowe  *	nxgep
1686*86ef0a63SRichard Lowe  *	channel		The channel to reset.
1687*86ef0a63SRichard Lowe  *	reg_data	The current TX_CS.
1688678453a8Sspeer  *
1689678453a8Sspeer  * Notes:
1690678453a8Sspeer  *	Called by nxge_check_tx_hang()
1691678453a8Sspeer  *
1692678453a8Sspeer  * NPI/NXGE function calls:
1693678453a8Sspeer  *	nxge_txdma_channel_hung()
1694678453a8Sspeer  *
1695678453a8Sspeer  * Registers accessed:
1696678453a8Sspeer  *
1697678453a8Sspeer  * Context:
1698678453a8Sspeer  *	Any domain
1699678453a8Sspeer  */
170044961713Sgirish int
nxge_txdma_hung(p_nxge_t nxgep)170144961713Sgirish nxge_txdma_hung(p_nxge_t nxgep)
170244961713Sgirish {
1703330cd344SMichael Speer 	nxge_grp_set_t	*set = &nxgep->tx_set;
1704330cd344SMichael Speer 	int		tdc;
1705330cd344SMichael Speer 	boolean_t	shared;
170644961713Sgirish 
170744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_hung"));
170844961713Sgirish 
1709678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
171044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1711678453a8Sspeer 		    "<== nxge_txdma_hung: NULL ring pointer(s)"));
171244961713Sgirish 		return (B_FALSE);
171344961713Sgirish 	}
171444961713Sgirish 
1715678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1716330cd344SMichael Speer 		/*
1717330cd344SMichael Speer 		 * Grab the shared state of the TDC.
1718330cd344SMichael Speer 		 */
1719330cd344SMichael Speer 		if (isLDOMservice(nxgep)) {
1720330cd344SMichael Speer 			nxge_hio_data_t *nhd =
1721330cd344SMichael Speer 			    (nxge_hio_data_t *)nxgep->nxge_hw_p->hio;
1722330cd344SMichael Speer 
1723330cd344SMichael Speer 			MUTEX_ENTER(&nhd->lock);
1724330cd344SMichael Speer 			shared = nxgep->tdc_is_shared[tdc];
1725330cd344SMichael Speer 			MUTEX_EXIT(&nhd->lock);
1726330cd344SMichael Speer 		} else {
1727330cd344SMichael Speer 			shared = B_FALSE;
1728330cd344SMichael Speer 		}
1729330cd344SMichael Speer 
1730330cd344SMichael Speer 		/*
1731330cd344SMichael Speer 		 * Now, process continue to process.
1732330cd344SMichael Speer 		 */
1733330cd344SMichael Speer 		if (((1 << tdc) & set->owned.map) && !shared) {
1734678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1735678453a8Sspeer 			if (ring) {
1736678453a8Sspeer 				if (nxge_txdma_channel_hung(nxgep, ring, tdc)) {
1737678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
1738678453a8Sspeer 					    "==> nxge_txdma_hung: TDC %d hung",
1739678453a8Sspeer 					    tdc));
1740678453a8Sspeer 					return (B_TRUE);
1741678453a8Sspeer 				}
1742678453a8Sspeer 			}
174344961713Sgirish 		}
174444961713Sgirish 	}
174544961713Sgirish 
174644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_hung"));
174744961713Sgirish 
174844961713Sgirish 	return (B_FALSE);
174944961713Sgirish }
175044961713Sgirish 
1751678453a8Sspeer /*
1752678453a8Sspeer  * nxge_txdma_channel_hung
1753678453a8Sspeer  *
1754678453a8Sspeer  *	Reset a TDC.
1755678453a8Sspeer  *
1756678453a8Sspeer  * Arguments:
1757*86ef0a63SRichard Lowe  *	nxgep
1758*86ef0a63SRichard Lowe  *	ring		<channel>'s ring.
1759*86ef0a63SRichard Lowe  *	channel		The channel to reset.
1760678453a8Sspeer  *
1761678453a8Sspeer  * Notes:
1762678453a8Sspeer  *	Called by nxge_txdma.c:nxge_txdma_hung()
1763678453a8Sspeer  *
1764678453a8Sspeer  * NPI/NXGE function calls:
1765678453a8Sspeer  *	npi_txdma_ring_head_get()
1766678453a8Sspeer  *
1767678453a8Sspeer  * Registers accessed:
1768678453a8Sspeer  *	TX_RING_HDL	DMC+0x40010 Transmit Ring Head Low
1769678453a8Sspeer  *
1770678453a8Sspeer  * Context:
1771678453a8Sspeer  *	Any domain
1772678453a8Sspeer  */
177344961713Sgirish int
nxge_txdma_channel_hung(p_nxge_t nxgep,p_tx_ring_t tx_ring_p,uint16_t channel)177444961713Sgirish nxge_txdma_channel_hung(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
177544961713Sgirish {
177644961713Sgirish 	uint16_t		head_index, tail_index;
177744961713Sgirish 	boolean_t		head_wrap, tail_wrap;
177844961713Sgirish 	npi_handle_t		handle;
177944961713Sgirish 	tx_ring_hdl_t		tx_head;
178044961713Sgirish 	uint_t			tx_rd_index;
178144961713Sgirish 
178244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_channel_hung"));
178344961713Sgirish 
178444961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
178544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
178652ccf843Smisaki 	    "==> nxge_txdma_channel_hung: channel %d", channel));
178744961713Sgirish 	MUTEX_ENTER(&tx_ring_p->lock);
178844961713Sgirish 	(void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
178944961713Sgirish 
179044961713Sgirish 	tail_index = tx_ring_p->wr_index;
179144961713Sgirish 	tail_wrap = tx_ring_p->wr_index_wrap;
179244961713Sgirish 	tx_rd_index = tx_ring_p->rd_index;
179344961713Sgirish 	MUTEX_EXIT(&tx_ring_p->lock);
179444961713Sgirish 
179544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
179652ccf843Smisaki 	    "==> nxge_txdma_channel_hung: tdc %d tx_rd_index %d "
179752ccf843Smisaki 	    "tail_index %d tail_wrap %d ",
179852ccf843Smisaki 	    channel, tx_rd_index, tail_index, tail_wrap));
179944961713Sgirish 	/*
180044961713Sgirish 	 * Read the hardware maintained transmit head
180144961713Sgirish 	 * and wrap around bit.
180244961713Sgirish 	 */
180344961713Sgirish 	(void) npi_txdma_ring_head_get(handle, channel, &tx_head);
180444961713Sgirish 	head_index =  tx_head.bits.ldw.head;
180544961713Sgirish 	head_wrap = tx_head.bits.ldw.wrap;
180644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
180752ccf843Smisaki 	    "==> nxge_txdma_channel_hung: "
180852ccf843Smisaki 	    "tx_rd_index %d tail %d tail_wrap %d "
180952ccf843Smisaki 	    "head %d wrap %d",
181052ccf843Smisaki 	    tx_rd_index, tail_index, tail_wrap,
181152ccf843Smisaki 	    head_index, head_wrap));
181244961713Sgirish 
181344961713Sgirish 	if (TXDMA_RING_EMPTY(head_index, head_wrap,
181452ccf843Smisaki 	    tail_index, tail_wrap) &&
181552ccf843Smisaki 	    (head_index == tx_rd_index)) {
181644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
181752ccf843Smisaki 		    "==> nxge_txdma_channel_hung: EMPTY"));
181844961713Sgirish 		return (B_FALSE);
181944961713Sgirish 	}
182044961713Sgirish 
182144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
182252ccf843Smisaki 	    "==> nxge_txdma_channel_hung: Checking if ring full"));
182344961713Sgirish 	if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
182452ccf843Smisaki 	    tail_wrap)) {
182544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
182652ccf843Smisaki 		    "==> nxge_txdma_channel_hung: full"));
182744961713Sgirish 		return (B_TRUE);
182844961713Sgirish 	}
182944961713Sgirish 
183044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_channel_hung"));
183144961713Sgirish 
183244961713Sgirish 	return (B_FALSE);
183344961713Sgirish }
183444961713Sgirish 
1835678453a8Sspeer /*
1836678453a8Sspeer  * nxge_fixup_hung_txdma_rings
1837678453a8Sspeer  *
1838678453a8Sspeer  *	Disable a TDC.
1839678453a8Sspeer  *
1840678453a8Sspeer  * Arguments:
1841*86ef0a63SRichard Lowe  *	nxgep
1842*86ef0a63SRichard Lowe  *	channel		The channel to reset.
1843*86ef0a63SRichard Lowe  *	reg_data	The current TX_CS.
1844678453a8Sspeer  *
1845678453a8Sspeer  * Notes:
1846678453a8Sspeer  *	Called by nxge_check_tx_hang()
1847678453a8Sspeer  *
1848678453a8Sspeer  * NPI/NXGE function calls:
1849678453a8Sspeer  *	npi_txdma_ring_head_get()
1850678453a8Sspeer  *
1851678453a8Sspeer  * Registers accessed:
1852678453a8Sspeer  *	TX_RING_HDL	DMC+0x40010 Transmit Ring Head Low
1853678453a8Sspeer  *
1854678453a8Sspeer  * Context:
1855678453a8Sspeer  *	Any domain
1856678453a8Sspeer  */
185744961713Sgirish /*ARGSUSED*/
185844961713Sgirish void
nxge_fixup_hung_txdma_rings(p_nxge_t nxgep)185944961713Sgirish nxge_fixup_hung_txdma_rings(p_nxge_t nxgep)
186044961713Sgirish {
1861678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
1862678453a8Sspeer 	int tdc;
186344961713Sgirish 
186444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_fixup_hung_txdma_rings"));
186544961713Sgirish 
1866678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
186744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1868678453a8Sspeer 		    "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
186944961713Sgirish 		return;
187044961713Sgirish 	}
187144961713Sgirish 
1872678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
1873678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
1874678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
1875678453a8Sspeer 			if (ring) {
1876678453a8Sspeer 				nxge_txdma_fixup_hung_channel(nxgep, ring, tdc);
1877678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
1878678453a8Sspeer 				    "==> nxge_fixup_hung_txdma_rings: TDC %d",
1879678453a8Sspeer 				    tdc));
1880678453a8Sspeer 			}
1881678453a8Sspeer 		}
188244961713Sgirish 	}
188344961713Sgirish 
188444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_fixup_hung_txdma_rings"));
188544961713Sgirish }
188644961713Sgirish 
1887678453a8Sspeer /*
1888678453a8Sspeer  * nxge_txdma_fixup_hung_channel
1889678453a8Sspeer  *
1890678453a8Sspeer  *	'Fix' a hung TDC.
1891678453a8Sspeer  *
1892678453a8Sspeer  * Arguments:
1893*86ef0a63SRichard Lowe  *	nxgep
1894*86ef0a63SRichard Lowe  *	channel		The channel to fix.
1895678453a8Sspeer  *
1896678453a8Sspeer  * Notes:
1897678453a8Sspeer  *	Called by nxge_fixup_hung_txdma_rings()
1898678453a8Sspeer  *
1899678453a8Sspeer  *	1. Reclaim the TDC.
1900678453a8Sspeer  *	2. Disable the TDC.
1901678453a8Sspeer  *
1902678453a8Sspeer  * NPI/NXGE function calls:
1903678453a8Sspeer  *	nxge_txdma_reclaim()
1904678453a8Sspeer  *	npi_txdma_channel_disable(TX_CS)
1905678453a8Sspeer  *	npi_txdma_inj_int_error_set(TDMC_INTR_DBG)
1906678453a8Sspeer  *
1907678453a8Sspeer  * Registers accessed:
1908678453a8Sspeer  *	TX_CS		DMC+0x40028 Transmit Control And Status
1909678453a8Sspeer  *	TDMC_INTR_DBG	DMC + 0x40060 Transmit DMA Interrupt Debug
1910678453a8Sspeer  *
1911678453a8Sspeer  * Context:
1912678453a8Sspeer  *	Any domain
1913678453a8Sspeer  */
191444961713Sgirish /*ARGSUSED*/
191544961713Sgirish void
nxge_txdma_fix_hung_channel(p_nxge_t nxgep,uint16_t channel)191644961713Sgirish nxge_txdma_fix_hung_channel(p_nxge_t nxgep, uint16_t channel)
191744961713Sgirish {
191844961713Sgirish 	p_tx_ring_t	ring_p;
191944961713Sgirish 
192044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fix_hung_channel"));
192144961713Sgirish 	ring_p = nxge_txdma_get_ring(nxgep, channel);
192244961713Sgirish 	if (ring_p == NULL) {
192344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
192452ccf843Smisaki 		    "<== nxge_txdma_fix_hung_channel"));
192544961713Sgirish 		return;
192644961713Sgirish 	}
192744961713Sgirish 
192844961713Sgirish 	if (ring_p->tdc != channel) {
192944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
193052ccf843Smisaki 		    "<== nxge_txdma_fix_hung_channel: channel not matched "
193152ccf843Smisaki 		    "ring tdc %d passed channel",
193252ccf843Smisaki 		    ring_p->tdc, channel));
193344961713Sgirish 		return;
193444961713Sgirish 	}
193544961713Sgirish 
193644961713Sgirish 	nxge_txdma_fixup_channel(nxgep, ring_p, channel);
193744961713Sgirish 
193844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fix_hung_channel"));
193944961713Sgirish }
194044961713Sgirish 
194144961713Sgirish /*ARGSUSED*/
194244961713Sgirish void
nxge_txdma_fixup_hung_channel(p_nxge_t nxgep,p_tx_ring_t ring_p,uint16_t channel)194344961713Sgirish nxge_txdma_fixup_hung_channel(p_nxge_t nxgep, p_tx_ring_t ring_p,
1944*86ef0a63SRichard Lowe     uint16_t channel)
194544961713Sgirish {
194644961713Sgirish 	npi_handle_t		handle;
194744961713Sgirish 	tdmc_intr_dbg_t		intr_dbg;
194844961713Sgirish 	int			status = NXGE_OK;
194944961713Sgirish 
195044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fixup_hung_channel"));
195144961713Sgirish 
195244961713Sgirish 	if (ring_p == NULL) {
195344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
195452ccf843Smisaki 		    "<== nxge_txdma_fixup_channel: NULL ring pointer"));
195544961713Sgirish 		return;
195644961713Sgirish 	}
195744961713Sgirish 
195844961713Sgirish 	if (ring_p->tdc != channel) {
195944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
196052ccf843Smisaki 		    "<== nxge_txdma_fixup_hung_channel: channel "
196152ccf843Smisaki 		    "not matched "
196252ccf843Smisaki 		    "ring tdc %d passed channel",
196352ccf843Smisaki 		    ring_p->tdc, channel));
196444961713Sgirish 		return;
196544961713Sgirish 	}
196644961713Sgirish 
196744961713Sgirish 	/* Reclaim descriptors */
196844961713Sgirish 	MUTEX_ENTER(&ring_p->lock);
196944961713Sgirish 	(void) nxge_txdma_reclaim(nxgep, ring_p, 0);
197044961713Sgirish 	MUTEX_EXIT(&ring_p->lock);
197144961713Sgirish 
197244961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
197344961713Sgirish 	/*
197444961713Sgirish 	 * Stop the dma channel waits for the stop done.
197544961713Sgirish 	 * If the stop done bit is not set, then force
197644961713Sgirish 	 * an error.
197744961713Sgirish 	 */
197844961713Sgirish 	status = npi_txdma_channel_disable(handle, channel);
197944961713Sgirish 	if (!(status & NPI_TXDMA_STOP_FAILED)) {
198044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
198152ccf843Smisaki 		    "<== nxge_txdma_fixup_hung_channel: stopped OK "
198252ccf843Smisaki 		    "ring tdc %d passed channel %d",
198352ccf843Smisaki 		    ring_p->tdc, channel));
198444961713Sgirish 		return;
198544961713Sgirish 	}
198644961713Sgirish 
198744961713Sgirish 	/* Inject any error */
198844961713Sgirish 	intr_dbg.value = 0;
198944961713Sgirish 	intr_dbg.bits.ldw.nack_pref = 1;
199044961713Sgirish 	(void) npi_txdma_inj_int_error_set(handle, channel, &intr_dbg);
199144961713Sgirish 
199244961713Sgirish 	/* Stop done bit will be set as a result of error injection */
199344961713Sgirish 	status = npi_txdma_channel_disable(handle, channel);
199444961713Sgirish 	if (!(status & NPI_TXDMA_STOP_FAILED)) {
199544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
199652ccf843Smisaki 		    "<== nxge_txdma_fixup_hung_channel: stopped again"
199752ccf843Smisaki 		    "ring tdc %d passed channel",
199852ccf843Smisaki 		    ring_p->tdc, channel));
199944961713Sgirish 		return;
200044961713Sgirish 	}
200144961713Sgirish 
200244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
200352ccf843Smisaki 	    "<== nxge_txdma_fixup_hung_channel: stop done still not set!! "
200452ccf843Smisaki 	    "ring tdc %d passed channel",
200552ccf843Smisaki 	    ring_p->tdc, channel));
200644961713Sgirish 
200744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fixup_hung_channel"));
200844961713Sgirish }
200944961713Sgirish 
201044961713Sgirish /*ARGSUSED*/
201144961713Sgirish void
nxge_reclaim_rings(p_nxge_t nxgep)201244961713Sgirish nxge_reclaim_rings(p_nxge_t nxgep)
201344961713Sgirish {
2014678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
2015678453a8Sspeer 	int tdc;
201644961713Sgirish 
2017678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reclaim_rings"));
201844961713Sgirish 
2019678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
202044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
2021678453a8Sspeer 		    "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
202244961713Sgirish 		return;
202344961713Sgirish 	}
202444961713Sgirish 
2025678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
2026678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
2027678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
2028678453a8Sspeer 			if (ring) {
2029678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
2030678453a8Sspeer 				    "==> nxge_reclaim_rings: TDC %d", tdc));
2031678453a8Sspeer 				MUTEX_ENTER(&ring->lock);
2032da14cebeSEric Cheng 				(void) nxge_txdma_reclaim(nxgep, ring, 0);
2033678453a8Sspeer 				MUTEX_EXIT(&ring->lock);
2034678453a8Sspeer 			}
2035678453a8Sspeer 		}
203644961713Sgirish 	}
203744961713Sgirish 
203844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_reclaim_rings"));
203944961713Sgirish }
204044961713Sgirish 
204144961713Sgirish void
nxge_txdma_regs_dump_channels(p_nxge_t nxgep)204244961713Sgirish nxge_txdma_regs_dump_channels(p_nxge_t nxgep)
204344961713Sgirish {
2044678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
2045678453a8Sspeer 	npi_handle_t handle;
2046678453a8Sspeer 	int tdc;
204744961713Sgirish 
2048678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_regs_dump_channels"));
204944961713Sgirish 
205044961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
205144961713Sgirish 
2052678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
2053678453a8Sspeer 		(void) npi_txdma_dump_fzc_regs(handle);
205444961713Sgirish 
2055678453a8Sspeer 		/* Dump TXC registers. */
2056678453a8Sspeer 		(void) npi_txc_dump_fzc_regs(handle);
2057678453a8Sspeer 		(void) npi_txc_dump_port_fzc_regs(handle, nxgep->function_num);
205844961713Sgirish 	}
205944961713Sgirish 
2060678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
206144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
2062678453a8Sspeer 		    "<== nxge_fixup_hung_txdma_rings: NULL ring pointer(s)"));
206344961713Sgirish 		return;
206444961713Sgirish 	}
206544961713Sgirish 
2066678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
2067678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
2068678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
2069678453a8Sspeer 			if (ring) {
2070678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
2071678453a8Sspeer 				    "==> nxge_txdma_regs_dump_channels: "
2072678453a8Sspeer 				    "TDC %d", tdc));
2073678453a8Sspeer 				(void) npi_txdma_dump_tdc_regs(handle, tdc);
2074678453a8Sspeer 
2075678453a8Sspeer 				/* Dump TXC registers, if able to. */
2076678453a8Sspeer 				if (!isLDOMguest(nxgep)) {
2077678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
2078678453a8Sspeer 					    "==> nxge_txdma_regs_dump_channels:"
2079678453a8Sspeer 					    " FZC TDC %d", tdc));
2080678453a8Sspeer 					(void) npi_txc_dump_tdc_fzc_regs
2081678453a8Sspeer 					    (handle, tdc);
2082678453a8Sspeer 				}
2083678453a8Sspeer 				nxge_txdma_regs_dump(nxgep, tdc);
2084678453a8Sspeer 			}
2085678453a8Sspeer 		}
208644961713Sgirish 	}
208744961713Sgirish 
208844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_regs_dump"));
208944961713Sgirish }
209044961713Sgirish 
209144961713Sgirish void
nxge_txdma_regs_dump(p_nxge_t nxgep,int channel)209244961713Sgirish nxge_txdma_regs_dump(p_nxge_t nxgep, int channel)
209344961713Sgirish {
209444961713Sgirish 	npi_handle_t		handle;
2095*86ef0a63SRichard Lowe 	tx_ring_hdl_t		hdl;
2096*86ef0a63SRichard Lowe 	tx_ring_kick_t		kick;
2097*86ef0a63SRichard Lowe 	tx_cs_t			cs;
209844961713Sgirish 	txc_control_t		control;
209944961713Sgirish 	uint32_t		bitmap = 0;
210044961713Sgirish 	uint32_t		burst = 0;
210144961713Sgirish 	uint32_t		bytes = 0;
210244961713Sgirish 	dma_log_page_t		cfg;
210344961713Sgirish 
210444961713Sgirish 	printf("\n\tfunc # %d tdc %d ",
210552ccf843Smisaki 	    nxgep->function_num, channel);
210644961713Sgirish 	cfg.page_num = 0;
210744961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
210844961713Sgirish 	(void) npi_txdma_log_page_get(handle, channel, &cfg);
210944961713Sgirish 	printf("\n\tlog page func %d valid page 0 %d",
211052ccf843Smisaki 	    cfg.func_num, cfg.valid);
211144961713Sgirish 	cfg.page_num = 1;
211244961713Sgirish 	(void) npi_txdma_log_page_get(handle, channel, &cfg);
211344961713Sgirish 	printf("\n\tlog page func %d valid page 1 %d",
211452ccf843Smisaki 	    cfg.func_num, cfg.valid);
211544961713Sgirish 
211644961713Sgirish 	(void) npi_txdma_ring_head_get(handle, channel, &hdl);
211744961713Sgirish 	(void) npi_txdma_desc_kick_reg_get(handle, channel, &kick);
211844961713Sgirish 	printf("\n\thead value is 0x%0llx",
211952ccf843Smisaki 	    (long long)hdl.value);
212044961713Sgirish 	printf("\n\thead index %d", hdl.bits.ldw.head);
212144961713Sgirish 	printf("\n\tkick value is 0x%0llx",
212252ccf843Smisaki 	    (long long)kick.value);
212344961713Sgirish 	printf("\n\ttail index %d\n", kick.bits.ldw.tail);
212444961713Sgirish 
212544961713Sgirish 	(void) npi_txdma_control_status(handle, OP_GET, channel, &cs);
212644961713Sgirish 	printf("\n\tControl statue is 0x%0llx", (long long)cs.value);
212744961713Sgirish 	printf("\n\tControl status RST state %d", cs.bits.ldw.rst);
212844961713Sgirish 
212944961713Sgirish 	(void) npi_txc_control(handle, OP_GET, &control);
213044961713Sgirish 	(void) npi_txc_port_dma_list_get(handle, nxgep->function_num, &bitmap);
213144961713Sgirish 	(void) npi_txc_dma_max_burst(handle, OP_GET, channel, &burst);
213244961713Sgirish 	(void) npi_txc_dma_bytes_transmitted(handle, channel, &bytes);
213344961713Sgirish 
213444961713Sgirish 	printf("\n\tTXC port control 0x%0llx",
213552ccf843Smisaki 	    (long long)control.value);
213644961713Sgirish 	printf("\n\tTXC port bitmap 0x%x", bitmap);
213744961713Sgirish 	printf("\n\tTXC max burst %d", burst);
213844961713Sgirish 	printf("\n\tTXC bytes xmt %d\n", bytes);
213944961713Sgirish 
214044961713Sgirish 	{
214144961713Sgirish 		ipp_status_t status;
214244961713Sgirish 
214344961713Sgirish 		(void) npi_ipp_get_status(handle, nxgep->function_num, &status);
214444961713Sgirish 		printf("\n\tIPP status 0x%lux\n", (uint64_t)status.value);
214544961713Sgirish 	}
214644961713Sgirish }
214744961713Sgirish 
214844961713Sgirish /*
2149678453a8Sspeer  * nxge_tdc_hvio_setup
2150678453a8Sspeer  *
2151678453a8Sspeer  *	I'm not exactly sure what this code does.
2152678453a8Sspeer  *
2153678453a8Sspeer  * Arguments:
2154*86ef0a63SRichard Lowe  *	nxgep
2155*86ef0a63SRichard Lowe  *	channel	The channel to map.
2156678453a8Sspeer  *
2157678453a8Sspeer  * Notes:
2158678453a8Sspeer  *
2159678453a8Sspeer  * NPI/NXGE function calls:
2160678453a8Sspeer  *	na
2161678453a8Sspeer  *
2162678453a8Sspeer  * Context:
2163678453a8Sspeer  *	Service domain?
216444961713Sgirish  */
2165678453a8Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
2166678453a8Sspeer static void
nxge_tdc_hvio_setup(nxge_t * nxgep,int channel)2167678453a8Sspeer nxge_tdc_hvio_setup(
2168678453a8Sspeer 	nxge_t *nxgep, int channel)
216944961713Sgirish {
2170678453a8Sspeer 	nxge_dma_common_t	*data;
2171678453a8Sspeer 	nxge_dma_common_t	*control;
2172*86ef0a63SRichard Lowe 	tx_ring_t		*ring;
2173678453a8Sspeer 
2174678453a8Sspeer 	ring = nxgep->tx_rings->rings[channel];
2175678453a8Sspeer 	data = nxgep->tx_buf_pool_p->dma_buf_pool_p[channel];
2176678453a8Sspeer 
2177678453a8Sspeer 	ring->hv_set = B_FALSE;
2178678453a8Sspeer 
2179678453a8Sspeer 	ring->hv_tx_buf_base_ioaddr_pp =
2180678453a8Sspeer 	    (uint64_t)data->orig_ioaddr_pp;
2181678453a8Sspeer 	ring->hv_tx_buf_ioaddr_size =
2182678453a8Sspeer 	    (uint64_t)data->orig_alength;
2183678453a8Sspeer 
2184678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: "
218552ccf843Smisaki 	    "hv data buf base io $%p size 0x%llx (%d) buf base io $%p "
218652ccf843Smisaki 	    "orig vatopa base io $%p orig_len 0x%llx (%d)",
218752ccf843Smisaki 	    ring->hv_tx_buf_base_ioaddr_pp,
218852ccf843Smisaki 	    ring->hv_tx_buf_ioaddr_size, ring->hv_tx_buf_ioaddr_size,
218952ccf843Smisaki 	    data->ioaddr_pp, data->orig_vatopa,
219052ccf843Smisaki 	    data->orig_alength, data->orig_alength));
2191678453a8Sspeer 
2192678453a8Sspeer 	control = nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel];
2193678453a8Sspeer 
2194678453a8Sspeer 	ring->hv_tx_cntl_base_ioaddr_pp =
2195678453a8Sspeer 	    (uint64_t)control->orig_ioaddr_pp;
2196678453a8Sspeer 	ring->hv_tx_cntl_ioaddr_size =
2197678453a8Sspeer 	    (uint64_t)control->orig_alength;
2198678453a8Sspeer 
2199678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel: "
220052ccf843Smisaki 	    "hv cntl base io $%p orig ioaddr_pp ($%p) "
220152ccf843Smisaki 	    "orig vatopa ($%p) size 0x%llx (%d 0x%x)",
220252ccf843Smisaki 	    ring->hv_tx_cntl_base_ioaddr_pp,
220352ccf843Smisaki 	    control->orig_ioaddr_pp, control->orig_vatopa,
220452ccf843Smisaki 	    ring->hv_tx_cntl_ioaddr_size,
220552ccf843Smisaki 	    control->orig_alength, control->orig_alength));
2206678453a8Sspeer }
220744961713Sgirish #endif
220844961713Sgirish 
2209678453a8Sspeer static nxge_status_t
nxge_map_txdma(p_nxge_t nxgep,int channel)2210678453a8Sspeer nxge_map_txdma(p_nxge_t nxgep, int channel)
2211678453a8Sspeer {
2212678453a8Sspeer 	nxge_dma_common_t	**pData;
2213678453a8Sspeer 	nxge_dma_common_t	**pControl;
2214*86ef0a63SRichard Lowe 	tx_ring_t		**pRing, *ring;
2215678453a8Sspeer 	tx_mbox_t		**mailbox;
2216678453a8Sspeer 	uint32_t		num_chunks;
2217678453a8Sspeer 
2218678453a8Sspeer 	nxge_status_t		status = NXGE_OK;
221944961713Sgirish 
2220678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma"));
222144961713Sgirish 
2222678453a8Sspeer 	if (!nxgep->tx_cntl_pool_p->buf_allocated) {
2223678453a8Sspeer 		if (nxge_alloc_tx_mem_pool(nxgep) != NXGE_OK) {
2224678453a8Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2225678453a8Sspeer 			    "<== nxge_map_txdma: buf not allocated"));
2226678453a8Sspeer 			return (NXGE_ERROR);
2227678453a8Sspeer 		}
222844961713Sgirish 	}
222944961713Sgirish 
2230678453a8Sspeer 	if (nxge_alloc_txb(nxgep, channel) != NXGE_OK)
223144961713Sgirish 		return (NXGE_ERROR);
223244961713Sgirish 
2233678453a8Sspeer 	num_chunks = nxgep->tx_buf_pool_p->num_chunks[channel];
2234678453a8Sspeer 	pData = &nxgep->tx_buf_pool_p->dma_buf_pool_p[channel];
2235678453a8Sspeer 	pControl = &nxgep->tx_cntl_pool_p->dma_buf_pool_p[channel];
2236678453a8Sspeer 	pRing = &nxgep->tx_rings->rings[channel];
2237678453a8Sspeer 	mailbox = &nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
223844961713Sgirish 
2239678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: "
224052ccf843Smisaki 	    "tx_rings $%p tx_desc_rings $%p",
224152ccf843Smisaki 	    nxgep->tx_rings, nxgep->tx_rings->rings));
224244961713Sgirish 
224344961713Sgirish 	/*
2244678453a8Sspeer 	 * Map descriptors from the buffer pools for <channel>.
224544961713Sgirish 	 */
224644961713Sgirish 
2247678453a8Sspeer 	/*
2248678453a8Sspeer 	 * Set up and prepare buffer blocks, descriptors
2249678453a8Sspeer 	 * and mailbox.
2250678453a8Sspeer 	 */
2251678453a8Sspeer 	status = nxge_map_txdma_channel(nxgep, channel,
2252678453a8Sspeer 	    pData, pRing, num_chunks, pControl, mailbox);
2253678453a8Sspeer 	if (status != NXGE_OK) {
2254678453a8Sspeer 		NXGE_ERROR_MSG((nxgep, MEM3_CTL,
225552ccf843Smisaki 		    "==> nxge_map_txdma(%d): nxge_map_txdma_channel() "
225652ccf843Smisaki 		    "returned 0x%x",
225752ccf843Smisaki 		    nxgep, channel, status));
2258678453a8Sspeer 		return (status);
225944961713Sgirish 	}
226044961713Sgirish 
2261678453a8Sspeer 	ring = *pRing;
226244961713Sgirish 
2263678453a8Sspeer 	ring->index = (uint16_t)channel;
2264678453a8Sspeer 	ring->tdc_stats = &nxgep->statsp->tdc_stats[channel];
226544961713Sgirish 
2266678453a8Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
2267678453a8Sspeer 	if (isLDOMguest(nxgep)) {
2268678453a8Sspeer 		(void) nxge_tdc_lp_conf(nxgep, channel);
2269678453a8Sspeer 	} else {
2270678453a8Sspeer 		nxge_tdc_hvio_setup(nxgep, channel);
227144961713Sgirish 	}
2272678453a8Sspeer #endif
227344961713Sgirish 
2274678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma: "
2275678453a8Sspeer 	    "(status 0x%x channel %d)", status, channel));
227644961713Sgirish 
2277678453a8Sspeer 	return (status);
227844961713Sgirish }
227944961713Sgirish 
228044961713Sgirish static nxge_status_t
nxge_map_txdma_channel(p_nxge_t nxgep,uint16_t channel,p_nxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks,p_nxge_dma_common_t * dma_cntl_p,p_tx_mbox_t * tx_mbox_p)228144961713Sgirish nxge_map_txdma_channel(p_nxge_t nxgep, uint16_t channel,
2282*86ef0a63SRichard Lowe     p_nxge_dma_common_t *dma_buf_p,
2283*86ef0a63SRichard Lowe     p_tx_ring_t *tx_desc_p,
2284*86ef0a63SRichard Lowe     uint32_t num_chunks,
2285*86ef0a63SRichard Lowe     p_nxge_dma_common_t *dma_cntl_p,
2286*86ef0a63SRichard Lowe     p_tx_mbox_t *tx_mbox_p)
228744961713Sgirish {
228844961713Sgirish 	int	status = NXGE_OK;
228944961713Sgirish 
229044961713Sgirish 	/*
229144961713Sgirish 	 * Set up and prepare buffer blocks, descriptors
229244961713Sgirish 	 * and mailbox.
229344961713Sgirish 	 */
2294678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL,
229552ccf843Smisaki 	    "==> nxge_map_txdma_channel (channel %d)", channel));
229644961713Sgirish 	/*
229744961713Sgirish 	 * Transmit buffer blocks
229844961713Sgirish 	 */
229944961713Sgirish 	status = nxge_map_txdma_channel_buf_ring(nxgep, channel,
230052ccf843Smisaki 	    dma_buf_p, tx_desc_p, num_chunks);
230144961713Sgirish 	if (status != NXGE_OK) {
230244961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
230352ccf843Smisaki 		    "==> nxge_map_txdma_channel (channel %d): "
230452ccf843Smisaki 		    "map buffer failed 0x%x", channel, status));
230544961713Sgirish 		goto nxge_map_txdma_channel_exit;
230644961713Sgirish 	}
230744961713Sgirish 
230844961713Sgirish 	/*
230944961713Sgirish 	 * Transmit block ring, and mailbox.
231044961713Sgirish 	 */
231144961713Sgirish 	nxge_map_txdma_channel_cfg_ring(nxgep, channel, dma_cntl_p, *tx_desc_p,
231252ccf843Smisaki 	    tx_mbox_p);
231344961713Sgirish 
231444961713Sgirish 	goto nxge_map_txdma_channel_exit;
231544961713Sgirish 
231644961713Sgirish nxge_map_txdma_channel_fail1:
2317678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL,
231852ccf843Smisaki 	    "==> nxge_map_txdma_channel: unmap buf"
231952ccf843Smisaki 	    "(status 0x%x channel %d)",
232052ccf843Smisaki 	    status, channel));
232144961713Sgirish 	nxge_unmap_txdma_channel_buf_ring(nxgep, *tx_desc_p);
232244961713Sgirish 
232344961713Sgirish nxge_map_txdma_channel_exit:
2324678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL,
232552ccf843Smisaki 	    "<== nxge_map_txdma_channel: "
232652ccf843Smisaki 	    "(status 0x%x channel %d)",
232752ccf843Smisaki 	    status, channel));
232844961713Sgirish 
232944961713Sgirish 	return (status);
233044961713Sgirish }
233144961713Sgirish 
233244961713Sgirish /*ARGSUSED*/
233344961713Sgirish static void
nxge_unmap_txdma_channel(p_nxge_t nxgep,uint16_t channel)2334678453a8Sspeer nxge_unmap_txdma_channel(p_nxge_t nxgep, uint16_t channel)
233544961713Sgirish {
2336678453a8Sspeer 	tx_ring_t *ring;
2337678453a8Sspeer 	tx_mbox_t *mailbox;
2338678453a8Sspeer 
233944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
234052ccf843Smisaki 	    "==> nxge_unmap_txdma_channel (channel %d)", channel));
234144961713Sgirish 	/*
234244961713Sgirish 	 * unmap tx block ring, and mailbox.
234344961713Sgirish 	 */
2344678453a8Sspeer 	ring = nxgep->tx_rings->rings[channel];
2345678453a8Sspeer 	mailbox = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
2346678453a8Sspeer 
2347678453a8Sspeer 	(void) nxge_unmap_txdma_channel_cfg_ring(nxgep, ring, mailbox);
234844961713Sgirish 
234944961713Sgirish 	/* unmap buffer blocks */
2350678453a8Sspeer 	(void) nxge_unmap_txdma_channel_buf_ring(nxgep, ring);
2351678453a8Sspeer 
2352678453a8Sspeer 	nxge_free_txb(nxgep, channel);
235344961713Sgirish 
235448056c53SMichael Speer 	/*
235548056c53SMichael Speer 	 * Cleanup the reference to the ring now that it does not exist.
235648056c53SMichael Speer 	 */
235748056c53SMichael Speer 	nxgep->tx_rings->rings[channel] = NULL;
235848056c53SMichael Speer 
235944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_unmap_txdma_channel"));
236044961713Sgirish }
236144961713Sgirish 
2362678453a8Sspeer /*
2363678453a8Sspeer  * nxge_map_txdma_channel_cfg_ring
2364678453a8Sspeer  *
2365678453a8Sspeer  *	Map a TDC into our kernel space.
2366678453a8Sspeer  *	This function allocates all of the per-channel data structures.
2367678453a8Sspeer  *
2368678453a8Sspeer  * Arguments:
2369*86ef0a63SRichard Lowe  *	nxgep
2370*86ef0a63SRichard Lowe  *	dma_channel	The channel to map.
2371678453a8Sspeer  *	dma_cntl_p
2372678453a8Sspeer  *	tx_ring_p	dma_channel's transmit ring
2373678453a8Sspeer  *	tx_mbox_p	dma_channel's mailbox
2374678453a8Sspeer  *
2375678453a8Sspeer  * Notes:
2376678453a8Sspeer  *
2377678453a8Sspeer  * NPI/NXGE function calls:
2378678453a8Sspeer  *	nxge_setup_dma_common()
2379678453a8Sspeer  *
2380678453a8Sspeer  * Registers accessed:
2381678453a8Sspeer  *	none.
2382678453a8Sspeer  *
2383678453a8Sspeer  * Context:
2384678453a8Sspeer  *	Any domain
2385678453a8Sspeer  */
238644961713Sgirish /*ARGSUSED*/
238744961713Sgirish static void
nxge_map_txdma_channel_cfg_ring(p_nxge_t nxgep,uint16_t dma_channel,p_nxge_dma_common_t * dma_cntl_p,p_tx_ring_t tx_ring_p,p_tx_mbox_t * tx_mbox_p)238844961713Sgirish nxge_map_txdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel,
2389*86ef0a63SRichard Lowe     p_nxge_dma_common_t *dma_cntl_p,
2390*86ef0a63SRichard Lowe     p_tx_ring_t tx_ring_p,
2391*86ef0a63SRichard Lowe     p_tx_mbox_t *tx_mbox_p)
239244961713Sgirish {
2393*86ef0a63SRichard Lowe 	p_tx_mbox_t		mboxp;
2394*86ef0a63SRichard Lowe 	p_nxge_dma_common_t	cntl_dmap;
2395*86ef0a63SRichard Lowe 	p_nxge_dma_common_t	dmap;
239644961713Sgirish 	p_tx_rng_cfig_t		tx_ring_cfig_p;
239744961713Sgirish 	p_tx_ring_kick_t	tx_ring_kick_p;
239844961713Sgirish 	p_tx_cs_t		tx_cs_p;
239944961713Sgirish 	p_tx_dma_ent_msk_t	tx_evmask_p;
240044961713Sgirish 	p_txdma_mbh_t		mboxh_p;
240144961713Sgirish 	p_txdma_mbl_t		mboxl_p;
240244961713Sgirish 	uint64_t		tx_desc_len;
240344961713Sgirish 
240444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
240552ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring"));
240644961713Sgirish 
240744961713Sgirish 	cntl_dmap = *dma_cntl_p;
240844961713Sgirish 
240944961713Sgirish 	dmap = (p_nxge_dma_common_t)&tx_ring_p->tdc_desc;
241044961713Sgirish 	nxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
241152ccf843Smisaki 	    sizeof (tx_desc_t));
241244961713Sgirish 	/*
241344961713Sgirish 	 * Zero out transmit ring descriptors.
241444961713Sgirish 	 */
241544961713Sgirish 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
241644961713Sgirish 	tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
241744961713Sgirish 	tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
241844961713Sgirish 	tx_cs_p = &(tx_ring_p->tx_cs);
241944961713Sgirish 	tx_evmask_p = &(tx_ring_p->tx_evmask);
242044961713Sgirish 	tx_ring_cfig_p->value = 0;
242144961713Sgirish 	tx_ring_kick_p->value = 0;
242244961713Sgirish 	tx_cs_p->value = 0;
242344961713Sgirish 	tx_evmask_p->value = 0;
242444961713Sgirish 
242544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
242652ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring: channel %d des $%p",
242752ccf843Smisaki 	    dma_channel,
242852ccf843Smisaki 	    dmap->dma_cookie.dmac_laddress));
242944961713Sgirish 
243044961713Sgirish 	tx_ring_cfig_p->value = 0;
243144961713Sgirish 	tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 3);
243244961713Sgirish 	tx_ring_cfig_p->value =
243352ccf843Smisaki 	    (dmap->dma_cookie.dmac_laddress & TX_RNG_CFIG_ADDR_MASK) |
243452ccf843Smisaki 	    (tx_desc_len << TX_RNG_CFIG_LEN_SHIFT);
243544961713Sgirish 
243644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
243752ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
243852ccf843Smisaki 	    dma_channel,
243952ccf843Smisaki 	    tx_ring_cfig_p->value));
244044961713Sgirish 
244144961713Sgirish 	tx_cs_p->bits.ldw.rst = 1;
244244961713Sgirish 
244344961713Sgirish 	/* Map in mailbox */
244444961713Sgirish 	mboxp = (p_tx_mbox_t)
244552ccf843Smisaki 	    KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
244644961713Sgirish 	dmap = (p_nxge_dma_common_t)&mboxp->tx_mbox;
244744961713Sgirish 	nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
244844961713Sgirish 	mboxh_p = (p_txdma_mbh_t)&tx_ring_p->tx_mbox_mbh;
244944961713Sgirish 	mboxl_p = (p_txdma_mbl_t)&tx_ring_p->tx_mbox_mbl;
245044961713Sgirish 	mboxh_p->value = mboxl_p->value = 0;
245144961713Sgirish 
245244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
245352ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
245452ccf843Smisaki 	    dmap->dma_cookie.dmac_laddress));
245544961713Sgirish 
245644961713Sgirish 	mboxh_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
245752ccf843Smisaki 	    TXDMA_MBH_ADDR_SHIFT) & TXDMA_MBH_MASK);
245844961713Sgirish 
245944961713Sgirish 	mboxl_p->bits.ldw.mbaddr = ((dmap->dma_cookie.dmac_laddress &
246052ccf843Smisaki 	    TXDMA_MBL_MASK) >> TXDMA_MBL_SHIFT);
246144961713Sgirish 
246244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
246352ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
246452ccf843Smisaki 	    dmap->dma_cookie.dmac_laddress));
246544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
246652ccf843Smisaki 	    "==> nxge_map_txdma_channel_cfg_ring: hmbox $%p "
246752ccf843Smisaki 	    "mbox $%p",
246852ccf843Smisaki 	    mboxh_p->bits.ldw.mbaddr, mboxl_p->bits.ldw.mbaddr));
246944961713Sgirish 	tx_ring_p->page_valid.value = 0;
247044961713Sgirish 	tx_ring_p->page_mask_1.value = tx_ring_p->page_mask_2.value = 0;
247144961713Sgirish 	tx_ring_p->page_value_1.value = tx_ring_p->page_value_2.value = 0;
247244961713Sgirish 	tx_ring_p->page_reloc_1.value = tx_ring_p->page_reloc_2.value = 0;
247344961713Sgirish 	tx_ring_p->page_hdl.value = 0;
247444961713Sgirish 
247544961713Sgirish 	tx_ring_p->page_valid.bits.ldw.page0 = 1;
247644961713Sgirish 	tx_ring_p->page_valid.bits.ldw.page1 = 1;
247744961713Sgirish 
247844961713Sgirish 	tx_ring_p->max_burst.value = 0;
247944961713Sgirish 	tx_ring_p->max_burst.bits.ldw.dma_max_burst = TXC_DMA_MAX_BURST_DEFAULT;
248044961713Sgirish 
248144961713Sgirish 	*tx_mbox_p = mboxp;
248244961713Sgirish 
248344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
248452ccf843Smisaki 	    "<== nxge_map_txdma_channel_cfg_ring"));
248544961713Sgirish }
248644961713Sgirish 
248744961713Sgirish /*ARGSUSED*/
248844961713Sgirish static void
nxge_unmap_txdma_channel_cfg_ring(p_nxge_t nxgep,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)248944961713Sgirish nxge_unmap_txdma_channel_cfg_ring(p_nxge_t nxgep,
2490*86ef0a63SRichard Lowe     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
249144961713Sgirish {
249244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
249352ccf843Smisaki 	    "==> nxge_unmap_txdma_channel_cfg_ring: channel %d",
249452ccf843Smisaki 	    tx_ring_p->tdc));
249544961713Sgirish 
249644961713Sgirish 	KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
249744961713Sgirish 
249844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
249952ccf843Smisaki 	    "<== nxge_unmap_txdma_channel_cfg_ring"));
250044961713Sgirish }
250144961713Sgirish 
2502678453a8Sspeer /*
2503678453a8Sspeer  * nxge_map_txdma_channel_buf_ring
2504678453a8Sspeer  *
2505678453a8Sspeer  *
2506678453a8Sspeer  * Arguments:
2507*86ef0a63SRichard Lowe  *	nxgep
2508*86ef0a63SRichard Lowe  *	channel		The channel to map.
2509678453a8Sspeer  *	dma_buf_p
2510678453a8Sspeer  *	tx_desc_p	channel's descriptor ring
2511678453a8Sspeer  *	num_chunks
2512678453a8Sspeer  *
2513678453a8Sspeer  * Notes:
2514678453a8Sspeer  *
2515678453a8Sspeer  * NPI/NXGE function calls:
2516678453a8Sspeer  *	nxge_setup_dma_common()
2517678453a8Sspeer  *
2518678453a8Sspeer  * Registers accessed:
2519678453a8Sspeer  *	none.
2520678453a8Sspeer  *
2521678453a8Sspeer  * Context:
2522678453a8Sspeer  *	Any domain
2523678453a8Sspeer  */
252444961713Sgirish static nxge_status_t
nxge_map_txdma_channel_buf_ring(p_nxge_t nxgep,uint16_t channel,p_nxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks)252544961713Sgirish nxge_map_txdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel,
2526*86ef0a63SRichard Lowe     p_nxge_dma_common_t *dma_buf_p,
2527*86ef0a63SRichard Lowe     p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
252844961713Sgirish {
2529*86ef0a63SRichard Lowe 	p_nxge_dma_common_t	dma_bufp, tmp_bufp;
2530*86ef0a63SRichard Lowe 	p_nxge_dma_common_t	dmap;
253144961713Sgirish 	nxge_os_dma_handle_t	tx_buf_dma_handle;
2532*86ef0a63SRichard Lowe 	p_tx_ring_t		tx_ring_p;
2533*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_ring = NULL;
253444961713Sgirish 	nxge_status_t		status = NXGE_OK;
253544961713Sgirish 	int			ddi_status = DDI_SUCCESS;
2536e3d11eeeSToomas Soome 	int			i, j, index = 0;
2537e3d11eeeSToomas Soome 	uint32_t		size = 0, bsize;
2538*86ef0a63SRichard Lowe 	uint32_t		nblocks, nmsgs;
2539da14cebeSEric Cheng 	char			qname[TASKQ_NAMELEN];
254044961713Sgirish 
254144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
254252ccf843Smisaki 	    "==> nxge_map_txdma_channel_buf_ring"));
254344961713Sgirish 
254444961713Sgirish 	dma_bufp = tmp_bufp = *dma_buf_p;
254544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
254644961713Sgirish 		" nxge_map_txdma_channel_buf_ring: channel %d to map %d "
254744961713Sgirish 		"chunks bufp $%p",
254852ccf843Smisaki 		    channel, num_chunks, dma_bufp));
254944961713Sgirish 
255044961713Sgirish 	nmsgs = 0;
255144961713Sgirish 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
255244961713Sgirish 		nmsgs += tmp_bufp->nblocks;
255344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
255452ccf843Smisaki 		    "==> nxge_map_txdma_channel_buf_ring: channel %d "
255552ccf843Smisaki 		    "bufp $%p nblocks %d nmsgs %d",
255652ccf843Smisaki 		    channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
255744961713Sgirish 	}
255844961713Sgirish 	if (!nmsgs) {
255944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
256052ccf843Smisaki 		    "<== nxge_map_txdma_channel_buf_ring: channel %d "
256152ccf843Smisaki 		    "no msg blocks",
256252ccf843Smisaki 		    channel));
256344961713Sgirish 		status = NXGE_ERROR;
256444961713Sgirish 		goto nxge_map_txdma_channel_buf_ring_exit;
256544961713Sgirish 	}
256644961713Sgirish 
256744961713Sgirish 	tx_ring_p = (p_tx_ring_t)
256852ccf843Smisaki 	    KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
256944961713Sgirish 	MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
257052ccf843Smisaki 	    (void *)nxgep->interrupt_cookie);
25711f8914d5Sml 
257222c0d73aSspeer 	(void) atomic_swap_32(&tx_ring_p->tx_ring_offline, NXGE_TX_RING_ONLINE);
25736895688eSspeer 	tx_ring_p->tx_ring_busy = B_FALSE;
25741f8914d5Sml 	tx_ring_p->nxgep = nxgep;
2575da14cebeSEric Cheng 	tx_ring_p->tx_ring_handle = (mac_ring_handle_t)NULL;
2576da14cebeSEric Cheng 	(void) snprintf(qname, TASKQ_NAMELEN, "tx_%d_%d",
2577da14cebeSEric Cheng 	    nxgep->instance, channel);
2578da14cebeSEric Cheng 	tx_ring_p->taskq = ddi_taskq_create(nxgep->dip, qname, 1,
2579da14cebeSEric Cheng 	    TASKQ_DEFAULTPRI, 0);
2580da14cebeSEric Cheng 	if (tx_ring_p->taskq == NULL) {
2581da14cebeSEric Cheng 		goto nxge_map_txdma_channel_buf_ring_fail1;
2582da14cebeSEric Cheng 	}
2583da14cebeSEric Cheng 
258444961713Sgirish 	/*
258544961713Sgirish 	 * Allocate transmit message rings and handles for packets
258644961713Sgirish 	 * not to be copied to premapped buffers.
258744961713Sgirish 	 */
258844961713Sgirish 	size = nmsgs * sizeof (tx_msg_t);
258944961713Sgirish 	tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
259044961713Sgirish 	for (i = 0; i < nmsgs; i++) {
259144961713Sgirish 		ddi_status = ddi_dma_alloc_handle(nxgep->dip, &nxge_tx_dma_attr,
259252ccf843Smisaki 		    DDI_DMA_DONTWAIT, 0,
259352ccf843Smisaki 		    &tx_msg_ring[i].dma_handle);
259444961713Sgirish 		if (ddi_status != DDI_SUCCESS) {
259544961713Sgirish 			status |= NXGE_DDI_FAILED;
259644961713Sgirish 			break;
259744961713Sgirish 		}
259844961713Sgirish 	}
259944961713Sgirish 	if (i < nmsgs) {
260056d930aeSspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
260156d930aeSspeer 		    "Allocate handles failed."));
260244961713Sgirish 		goto nxge_map_txdma_channel_buf_ring_fail1;
260344961713Sgirish 	}
260444961713Sgirish 
260544961713Sgirish 	tx_ring_p->tdc = channel;
260644961713Sgirish 	tx_ring_p->tx_msg_ring = tx_msg_ring;
260744961713Sgirish 	tx_ring_p->tx_ring_size = nmsgs;
260844961713Sgirish 	tx_ring_p->num_chunks = num_chunks;
260944961713Sgirish 	if (!nxge_tx_intr_thres) {
261044961713Sgirish 		nxge_tx_intr_thres = tx_ring_p->tx_ring_size/4;
261144961713Sgirish 	}
261244961713Sgirish 	tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
261344961713Sgirish 	tx_ring_p->rd_index = 0;
261444961713Sgirish 	tx_ring_p->wr_index = 0;
261544961713Sgirish 	tx_ring_p->ring_head.value = 0;
261644961713Sgirish 	tx_ring_p->ring_kick_tail.value = 0;
261744961713Sgirish 	tx_ring_p->descs_pending = 0;
261844961713Sgirish 
261944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
262052ccf843Smisaki 	    "==> nxge_map_txdma_channel_buf_ring: channel %d "
262152ccf843Smisaki 	    "actual tx desc max %d nmsgs %d "
262252ccf843Smisaki 	    "(config nxge_tx_ring_size %d)",
262352ccf843Smisaki 	    channel, tx_ring_p->tx_ring_size, nmsgs,
262452ccf843Smisaki 	    nxge_tx_ring_size));
262544961713Sgirish 
262644961713Sgirish 	/*
262744961713Sgirish 	 * Map in buffers from the buffer pool.
262844961713Sgirish 	 */
262944961713Sgirish 	index = 0;
263044961713Sgirish 	bsize = dma_bufp->block_size;
263144961713Sgirish 
263244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_map_txdma_channel_buf_ring: "
263352ccf843Smisaki 	    "dma_bufp $%p tx_rng_p $%p "
263452ccf843Smisaki 	    "tx_msg_rng_p $%p bsize %d",
263552ccf843Smisaki 	    dma_bufp, tx_ring_p, tx_msg_ring, bsize));
263644961713Sgirish 
263744961713Sgirish 	tx_buf_dma_handle = dma_bufp->dma_handle;
263844961713Sgirish 	for (i = 0; i < num_chunks; i++, dma_bufp++) {
263944961713Sgirish 		bsize = dma_bufp->block_size;
264044961713Sgirish 		nblocks = dma_bufp->nblocks;
264144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
264252ccf843Smisaki 		    "==> nxge_map_txdma_channel_buf_ring: dma chunk %d "
264352ccf843Smisaki 		    "size %d dma_bufp $%p",
264452ccf843Smisaki 		    i, sizeof (nxge_dma_common_t), dma_bufp));
264544961713Sgirish 
264644961713Sgirish 		for (j = 0; j < nblocks; j++) {
264744961713Sgirish 			tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
264844961713Sgirish 			dmap = &tx_msg_ring[index++].buf_dma;
264944961713Sgirish #ifdef TX_MEM_DEBUG
265044961713Sgirish 			NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
265152ccf843Smisaki 			    "==> nxge_map_txdma_channel_buf_ring: j %d"
265252ccf843Smisaki 			    "dmap $%p", i, dmap));
265344961713Sgirish #endif
265444961713Sgirish 			nxge_setup_dma_common(dmap, dma_bufp, 1,
265552ccf843Smisaki 			    bsize);
265644961713Sgirish 		}
265744961713Sgirish 	}
265844961713Sgirish 
265944961713Sgirish 	if (i < num_chunks) {
266056d930aeSspeer 		status = NXGE_ERROR;
266144961713Sgirish 		goto nxge_map_txdma_channel_buf_ring_fail1;
266244961713Sgirish 	}
266344961713Sgirish 
266444961713Sgirish 	*tx_desc_p = tx_ring_p;
266544961713Sgirish 
266644961713Sgirish 	goto nxge_map_txdma_channel_buf_ring_exit;
266744961713Sgirish 
266844961713Sgirish nxge_map_txdma_channel_buf_ring_fail1:
2669da14cebeSEric Cheng 	if (tx_ring_p->taskq) {
2670da14cebeSEric Cheng 		ddi_taskq_destroy(tx_ring_p->taskq);
2671da14cebeSEric Cheng 		tx_ring_p->taskq = NULL;
26721f8914d5Sml 	}
26731f8914d5Sml 
267444961713Sgirish 	index--;
267544961713Sgirish 	for (; index >= 0; index--) {
267656d930aeSspeer 		if (tx_msg_ring[index].dma_handle != NULL) {
267756d930aeSspeer 			ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
267844961713Sgirish 		}
267944961713Sgirish 	}
268044961713Sgirish 	MUTEX_DESTROY(&tx_ring_p->lock);
268156d930aeSspeer 	KMEM_FREE(tx_msg_ring, size);
268244961713Sgirish 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
268344961713Sgirish 
268456d930aeSspeer 	status = NXGE_ERROR;
268556d930aeSspeer 
268644961713Sgirish nxge_map_txdma_channel_buf_ring_exit:
268744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
268852ccf843Smisaki 	    "<== nxge_map_txdma_channel_buf_ring status 0x%x", status));
268944961713Sgirish 
269044961713Sgirish 	return (status);
269144961713Sgirish }
269244961713Sgirish 
269344961713Sgirish /*ARGSUSED*/
269444961713Sgirish static void
nxge_unmap_txdma_channel_buf_ring(p_nxge_t nxgep,p_tx_ring_t tx_ring_p)269544961713Sgirish nxge_unmap_txdma_channel_buf_ring(p_nxge_t nxgep, p_tx_ring_t tx_ring_p)
269644961713Sgirish {
2697*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_ring;
2698*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_p;
269944961713Sgirish 	int			i;
270044961713Sgirish 
270144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
270252ccf843Smisaki 	    "==> nxge_unmap_txdma_channel_buf_ring"));
270344961713Sgirish 	if (tx_ring_p == NULL) {
270444961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
270552ccf843Smisaki 		    "<== nxge_unmap_txdma_channel_buf_ring: NULL ringp"));
270644961713Sgirish 		return;
270744961713Sgirish 	}
270844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
270952ccf843Smisaki 	    "==> nxge_unmap_txdma_channel_buf_ring: channel %d",
271052ccf843Smisaki 	    tx_ring_p->tdc));
271144961713Sgirish 
271244961713Sgirish 	tx_msg_ring = tx_ring_p->tx_msg_ring;
2713678453a8Sspeer 
2714678453a8Sspeer 	/*
2715678453a8Sspeer 	 * Since the serialization thread, timer thread and
2716678453a8Sspeer 	 * interrupt thread can all call the transmit reclaim,
2717678453a8Sspeer 	 * the unmapping function needs to acquire the lock
2718678453a8Sspeer 	 * to free those buffers which were transmitted
2719678453a8Sspeer 	 * by the hardware already.
2720678453a8Sspeer 	 */
2721678453a8Sspeer 	MUTEX_ENTER(&tx_ring_p->lock);
2722678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
2723678453a8Sspeer 	    "==> nxge_unmap_txdma_channel_buf_ring (reclaim): "
2724678453a8Sspeer 	    "channel %d",
2725678453a8Sspeer 	    tx_ring_p->tdc));
2726678453a8Sspeer 	(void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
2727678453a8Sspeer 
272844961713Sgirish 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
272944961713Sgirish 		tx_msg_p = &tx_msg_ring[i];
273044961713Sgirish 		if (tx_msg_p->tx_message != NULL) {
273144961713Sgirish 			freemsg(tx_msg_p->tx_message);
273244961713Sgirish 			tx_msg_p->tx_message = NULL;
273344961713Sgirish 		}
273444961713Sgirish 	}
273544961713Sgirish 
273644961713Sgirish 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
273744961713Sgirish 		if (tx_msg_ring[i].dma_handle != NULL) {
273844961713Sgirish 			ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
273944961713Sgirish 		}
2740678453a8Sspeer 		tx_msg_ring[i].dma_handle = NULL;
274144961713Sgirish 	}
274244961713Sgirish 
2743678453a8Sspeer 	MUTEX_EXIT(&tx_ring_p->lock);
2744678453a8Sspeer 
2745da14cebeSEric Cheng 	if (tx_ring_p->taskq) {
2746da14cebeSEric Cheng 		ddi_taskq_destroy(tx_ring_p->taskq);
2747da14cebeSEric Cheng 		tx_ring_p->taskq = NULL;
27481f8914d5Sml 	}
27491f8914d5Sml 
275044961713Sgirish 	MUTEX_DESTROY(&tx_ring_p->lock);
275144961713Sgirish 	KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
275244961713Sgirish 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
275344961713Sgirish 
275444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
275552ccf843Smisaki 	    "<== nxge_unmap_txdma_channel_buf_ring"));
275644961713Sgirish }
275744961713Sgirish 
275844961713Sgirish static nxge_status_t
nxge_txdma_hw_start(p_nxge_t nxgep,int channel)2759678453a8Sspeer nxge_txdma_hw_start(p_nxge_t nxgep, int channel)
276044961713Sgirish {
2761*86ef0a63SRichard Lowe 	p_tx_rings_t		tx_rings;
2762*86ef0a63SRichard Lowe 	p_tx_ring_t		*tx_desc_rings;
2763*86ef0a63SRichard Lowe 	p_tx_mbox_areas_t	tx_mbox_areas_p;
276444961713Sgirish 	p_tx_mbox_t		*tx_mbox_p;
276544961713Sgirish 	nxge_status_t		status = NXGE_OK;
276644961713Sgirish 
276744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start"));
276844961713Sgirish 
276944961713Sgirish 	tx_rings = nxgep->tx_rings;
277044961713Sgirish 	if (tx_rings == NULL) {
277144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
277252ccf843Smisaki 		    "<== nxge_txdma_hw_start: NULL ring pointer"));
277344961713Sgirish 		return (NXGE_ERROR);
277444961713Sgirish 	}
277544961713Sgirish 	tx_desc_rings = tx_rings->rings;
277644961713Sgirish 	if (tx_desc_rings == NULL) {
277744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
277852ccf843Smisaki 		    "<== nxge_txdma_hw_start: NULL ring pointers"));
277944961713Sgirish 		return (NXGE_ERROR);
278044961713Sgirish 	}
278144961713Sgirish 
2782678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
2783678453a8Sspeer 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
278444961713Sgirish 
278544961713Sgirish 	tx_mbox_areas_p = nxgep->tx_mbox_areas_p;
278644961713Sgirish 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
278744961713Sgirish 
2788678453a8Sspeer 	status = nxge_txdma_start_channel(nxgep, channel,
2789678453a8Sspeer 	    (p_tx_ring_t)tx_desc_rings[channel],
2790678453a8Sspeer 	    (p_tx_mbox_t)tx_mbox_p[channel]);
2791678453a8Sspeer 	if (status != NXGE_OK) {
2792678453a8Sspeer 		goto nxge_txdma_hw_start_fail1;
279344961713Sgirish 	}
279444961713Sgirish 
279544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
279652ccf843Smisaki 	    "tx_rings $%p rings $%p",
279752ccf843Smisaki 	    nxgep->tx_rings, nxgep->tx_rings->rings));
279844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "==> nxge_txdma_hw_start: "
279952ccf843Smisaki 	    "tx_rings $%p tx_desc_rings $%p",
280052ccf843Smisaki 	    nxgep->tx_rings, tx_desc_rings));
280144961713Sgirish 
280244961713Sgirish 	goto nxge_txdma_hw_start_exit;
280344961713Sgirish 
280444961713Sgirish nxge_txdma_hw_start_fail1:
280544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
280652ccf843Smisaki 	    "==> nxge_txdma_hw_start: disable "
280752ccf843Smisaki 	    "(status 0x%x channel %d)", status, channel));
280844961713Sgirish 
280944961713Sgirish nxge_txdma_hw_start_exit:
281044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
281152ccf843Smisaki 	    "==> nxge_txdma_hw_start: (status 0x%x)", status));
281244961713Sgirish 
281344961713Sgirish 	return (status);
281444961713Sgirish }
281544961713Sgirish 
2816678453a8Sspeer /*
2817678453a8Sspeer  * nxge_txdma_start_channel
2818678453a8Sspeer  *
2819678453a8Sspeer  *	Start a TDC.
2820678453a8Sspeer  *
2821678453a8Sspeer  * Arguments:
2822*86ef0a63SRichard Lowe  *	nxgep
2823*86ef0a63SRichard Lowe  *	channel		The channel to start.
2824*86ef0a63SRichard Lowe  *	tx_ring_p	channel's transmit descriptor ring.
2825*86ef0a63SRichard Lowe  *	tx_mbox_p	channel' smailbox.
2826678453a8Sspeer  *
2827678453a8Sspeer  * Notes:
2828678453a8Sspeer  *
2829678453a8Sspeer  * NPI/NXGE function calls:
2830678453a8Sspeer  *	nxge_reset_txdma_channel()
2831678453a8Sspeer  *	nxge_init_txdma_channel_event_mask()
2832678453a8Sspeer  *	nxge_enable_txdma_channel()
2833678453a8Sspeer  *
2834678453a8Sspeer  * Registers accessed:
2835678453a8Sspeer  *	none directly (see functions above).
2836678453a8Sspeer  *
2837678453a8Sspeer  * Context:
2838678453a8Sspeer  *	Any domain
2839678453a8Sspeer  */
284044961713Sgirish static nxge_status_t
nxge_txdma_start_channel(p_nxge_t nxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)284144961713Sgirish nxge_txdma_start_channel(p_nxge_t nxgep, uint16_t channel,
284244961713Sgirish     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
284344961713Sgirish {
284444961713Sgirish 	nxge_status_t		status = NXGE_OK;
284544961713Sgirish 
284644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
2847*86ef0a63SRichard Lowe 	    "==> nxge_txdma_start_channel (channel %d)", channel));
284844961713Sgirish 	/*
284944961713Sgirish 	 * TXDMA/TXC must be in stopped state.
285044961713Sgirish 	 */
285144961713Sgirish 	(void) nxge_txdma_stop_inj_err(nxgep, channel);
285244961713Sgirish 
285344961713Sgirish 	/*
285444961713Sgirish 	 * Reset TXDMA channel
285544961713Sgirish 	 */
285644961713Sgirish 	tx_ring_p->tx_cs.value = 0;
285744961713Sgirish 	tx_ring_p->tx_cs.bits.ldw.rst = 1;
285844961713Sgirish 	status = nxge_reset_txdma_channel(nxgep, channel,
2859*86ef0a63SRichard Lowe 	    tx_ring_p->tx_cs.value);
286044961713Sgirish 	if (status != NXGE_OK) {
286144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2862*86ef0a63SRichard Lowe 		    "==> nxge_txdma_start_channel (channel %d)"
2863*86ef0a63SRichard Lowe 		    " reset channel failed 0x%x", channel, status));
286444961713Sgirish 		goto nxge_txdma_start_channel_exit;
286544961713Sgirish 	}
286644961713Sgirish 
286744961713Sgirish 	/*
286844961713Sgirish 	 * Initialize the TXDMA channel specific FZC control
286944961713Sgirish 	 * configurations. These FZC registers are pertaining
287044961713Sgirish 	 * to each TX channel (i.e. logical pages).
287144961713Sgirish 	 */
2872678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
2873678453a8Sspeer 		status = nxge_init_fzc_txdma_channel(nxgep, channel,
2874678453a8Sspeer 		    tx_ring_p, tx_mbox_p);
2875678453a8Sspeer 		if (status != NXGE_OK) {
2876678453a8Sspeer 			goto nxge_txdma_start_channel_exit;
2877678453a8Sspeer 		}
287844961713Sgirish 	}
287944961713Sgirish 
288044961713Sgirish 	/*
288144961713Sgirish 	 * Initialize the event masks.
288244961713Sgirish 	 */
288344961713Sgirish 	tx_ring_p->tx_evmask.value = 0;
288444961713Sgirish 	status = nxge_init_txdma_channel_event_mask(nxgep,
2885678453a8Sspeer 	    channel, &tx_ring_p->tx_evmask);
288644961713Sgirish 	if (status != NXGE_OK) {
288744961713Sgirish 		goto nxge_txdma_start_channel_exit;
288844961713Sgirish 	}
288944961713Sgirish 
289044961713Sgirish 	/*
289144961713Sgirish 	 * Load TXDMA descriptors, buffers, mailbox,
289244961713Sgirish 	 * initialise the DMA channels and
289344961713Sgirish 	 * enable each DMA channel.
289444961713Sgirish 	 */
289544961713Sgirish 	status = nxge_enable_txdma_channel(nxgep, channel,
2896*86ef0a63SRichard Lowe 	    tx_ring_p, tx_mbox_p);
289744961713Sgirish 	if (status != NXGE_OK) {
289844961713Sgirish 		goto nxge_txdma_start_channel_exit;
289944961713Sgirish 	}
290044961713Sgirish 
290144961713Sgirish nxge_txdma_start_channel_exit:
290244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_start_channel"));
290344961713Sgirish 
290444961713Sgirish 	return (status);
290544961713Sgirish }
290644961713Sgirish 
2907678453a8Sspeer /*
2908678453a8Sspeer  * nxge_txdma_stop_channel
2909678453a8Sspeer  *
2910678453a8Sspeer  *	Stop a TDC.
2911678453a8Sspeer  *
2912678453a8Sspeer  * Arguments:
2913*86ef0a63SRichard Lowe  *	nxgep
2914*86ef0a63SRichard Lowe  *	channel		The channel to stop.
2915*86ef0a63SRichard Lowe  *	tx_ring_p	channel's transmit descriptor ring.
2916*86ef0a63SRichard Lowe  *	tx_mbox_p	channel' smailbox.
2917678453a8Sspeer  *
2918678453a8Sspeer  * Notes:
2919678453a8Sspeer  *
2920678453a8Sspeer  * NPI/NXGE function calls:
2921678453a8Sspeer  *	nxge_txdma_stop_inj_err()
2922678453a8Sspeer  *	nxge_reset_txdma_channel()
2923678453a8Sspeer  *	nxge_init_txdma_channel_event_mask()
2924678453a8Sspeer  *	nxge_init_txdma_channel_cntl_stat()
2925678453a8Sspeer  *	nxge_disable_txdma_channel()
2926678453a8Sspeer  *
2927678453a8Sspeer  * Registers accessed:
2928678453a8Sspeer  *	none directly (see functions above).
2929678453a8Sspeer  *
2930678453a8Sspeer  * Context:
2931678453a8Sspeer  *	Any domain
2932678453a8Sspeer  */
293344961713Sgirish /*ARGSUSED*/
293444961713Sgirish static nxge_status_t
nxge_txdma_stop_channel(p_nxge_t nxgep,uint16_t channel)2935678453a8Sspeer nxge_txdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
293644961713Sgirish {
2937678453a8Sspeer 	p_tx_ring_t tx_ring_p;
2938678453a8Sspeer 	int status = NXGE_OK;
293944961713Sgirish 
294044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
294152ccf843Smisaki 	    "==> nxge_txdma_stop_channel: channel %d", channel));
294244961713Sgirish 
294344961713Sgirish 	/*
294444961713Sgirish 	 * Stop (disable) TXDMA and TXC (if stop bit is set
294544961713Sgirish 	 * and STOP_N_GO bit not set, the TXDMA reset state will
294644961713Sgirish 	 * not be set if reset TXDMA.
294744961713Sgirish 	 */
294844961713Sgirish 	(void) nxge_txdma_stop_inj_err(nxgep, channel);
294944961713Sgirish 
2950ef523517SMichael Speer 	if (nxgep->tx_rings == NULL) {
2951ef523517SMichael Speer 		status = NXGE_ERROR;
2952ef523517SMichael Speer 		goto nxge_txdma_stop_channel_exit;
2953ef523517SMichael Speer 	}
2954ef523517SMichael Speer 
2955678453a8Sspeer 	tx_ring_p = nxgep->tx_rings->rings[channel];
2956ef523517SMichael Speer 	if (tx_ring_p == NULL) {
2957ef523517SMichael Speer 		status = NXGE_ERROR;
2958ef523517SMichael Speer 		goto nxge_txdma_stop_channel_exit;
2959ef523517SMichael Speer 	}
2960678453a8Sspeer 
296144961713Sgirish 	/*
296244961713Sgirish 	 * Reset TXDMA channel
296344961713Sgirish 	 */
296444961713Sgirish 	tx_ring_p->tx_cs.value = 0;
296544961713Sgirish 	tx_ring_p->tx_cs.bits.ldw.rst = 1;
296644961713Sgirish 	status = nxge_reset_txdma_channel(nxgep, channel,
296752ccf843Smisaki 	    tx_ring_p->tx_cs.value);
296844961713Sgirish 	if (status != NXGE_OK) {
296944961713Sgirish 		goto nxge_txdma_stop_channel_exit;
297044961713Sgirish 	}
297144961713Sgirish 
297244961713Sgirish #ifdef HARDWARE_REQUIRED
297344961713Sgirish 	/* Set up the interrupt event masks. */
297444961713Sgirish 	tx_ring_p->tx_evmask.value = 0;
297544961713Sgirish 	status = nxge_init_txdma_channel_event_mask(nxgep,
297652ccf843Smisaki 	    channel, &tx_ring_p->tx_evmask);
297744961713Sgirish 	if (status != NXGE_OK) {
297844961713Sgirish 		goto nxge_txdma_stop_channel_exit;
297944961713Sgirish 	}
298044961713Sgirish 
298144961713Sgirish 	/* Initialize the DMA control and status register */
298244961713Sgirish 	tx_ring_p->tx_cs.value = TX_ENT_MSK_MK_ALL;
298344961713Sgirish 	status = nxge_init_txdma_channel_cntl_stat(nxgep, channel,
298452ccf843Smisaki 	    tx_ring_p->tx_cs.value);
298544961713Sgirish 	if (status != NXGE_OK) {
298644961713Sgirish 		goto nxge_txdma_stop_channel_exit;
298744961713Sgirish 	}
298844961713Sgirish 
2989678453a8Sspeer 	tx_mbox_p = nxgep->tx_mbox_areas_p->txmbox_areas_p[channel];
2990678453a8Sspeer 
299144961713Sgirish 	/* Disable channel */
299244961713Sgirish 	status = nxge_disable_txdma_channel(nxgep, channel,
2993678453a8Sspeer 	    tx_ring_p, tx_mbox_p);
299444961713Sgirish 	if (status != NXGE_OK) {
299544961713Sgirish 		goto nxge_txdma_start_channel_exit;
299644961713Sgirish 	}
299744961713Sgirish 
299844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL,
299952ccf843Smisaki 	    "==> nxge_txdma_stop_channel: event done"));
300044961713Sgirish 
300144961713Sgirish #endif
300244961713Sgirish 
300344961713Sgirish nxge_txdma_stop_channel_exit:
300444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM3_CTL, "<== nxge_txdma_stop_channel"));
300544961713Sgirish 	return (status);
300644961713Sgirish }
300744961713Sgirish 
3008678453a8Sspeer /*
3009678453a8Sspeer  * nxge_txdma_get_ring
3010678453a8Sspeer  *
3011678453a8Sspeer  *	Get the ring for a TDC.
3012678453a8Sspeer  *
3013678453a8Sspeer  * Arguments:
3014*86ef0a63SRichard Lowe  *	nxgep
3015*86ef0a63SRichard Lowe  *	channel
3016678453a8Sspeer  *
3017678453a8Sspeer  * Notes:
3018678453a8Sspeer  *
3019678453a8Sspeer  * NPI/NXGE function calls:
3020678453a8Sspeer  *
3021678453a8Sspeer  * Registers accessed:
3022678453a8Sspeer  *
3023678453a8Sspeer  * Context:
3024678453a8Sspeer  *	Any domain
3025678453a8Sspeer  */
302644961713Sgirish static p_tx_ring_t
nxge_txdma_get_ring(p_nxge_t nxgep,uint16_t channel)302744961713Sgirish nxge_txdma_get_ring(p_nxge_t nxgep, uint16_t channel)
302844961713Sgirish {
3029678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
3030678453a8Sspeer 	int tdc;
303144961713Sgirish 
303244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_ring"));
303344961713Sgirish 
3034678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
303544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
3036678453a8Sspeer 		    "<== nxge_txdma_get_ring: NULL ring pointer(s)"));
3037678453a8Sspeer 		goto return_null;
3038678453a8Sspeer 	}
3039678453a8Sspeer 
3040678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3041678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3042678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3043678453a8Sspeer 			if (ring) {
3044678453a8Sspeer 				if (channel == ring->tdc) {
3045678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
3046678453a8Sspeer 					    "<== nxge_txdma_get_ring: "
3047678453a8Sspeer 					    "tdc %d ring $%p", tdc, ring));
3048678453a8Sspeer 					return (ring);
3049678453a8Sspeer 				}
3050678453a8Sspeer 			}
305144961713Sgirish 		}
305244961713Sgirish 	}
305344961713Sgirish 
3054678453a8Sspeer return_null:
3055678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_ring: "
305652ccf843Smisaki 	    "ring not found"));
3057678453a8Sspeer 
305844961713Sgirish 	return (NULL);
305944961713Sgirish }
306044961713Sgirish 
3061678453a8Sspeer /*
3062678453a8Sspeer  * nxge_txdma_get_mbox
3063678453a8Sspeer  *
3064678453a8Sspeer  *	Get the mailbox for a TDC.
3065678453a8Sspeer  *
3066678453a8Sspeer  * Arguments:
3067*86ef0a63SRichard Lowe  *	nxgep
3068*86ef0a63SRichard Lowe  *	channel
3069678453a8Sspeer  *
3070678453a8Sspeer  * Notes:
3071678453a8Sspeer  *
3072678453a8Sspeer  * NPI/NXGE function calls:
3073678453a8Sspeer  *
3074678453a8Sspeer  * Registers accessed:
3075678453a8Sspeer  *
3076678453a8Sspeer  * Context:
3077678453a8Sspeer  *	Any domain
3078678453a8Sspeer  */
307944961713Sgirish static p_tx_mbox_t
nxge_txdma_get_mbox(p_nxge_t nxgep,uint16_t channel)308044961713Sgirish nxge_txdma_get_mbox(p_nxge_t nxgep, uint16_t channel)
308144961713Sgirish {
3082678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
3083678453a8Sspeer 	int tdc;
308444961713Sgirish 
308544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_get_mbox"));
308644961713Sgirish 
3087678453a8Sspeer 	if (nxgep->tx_mbox_areas_p == 0 ||
3088678453a8Sspeer 	    nxgep->tx_mbox_areas_p->txmbox_areas_p == 0) {
3089678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
3090678453a8Sspeer 		    "<== nxge_txdma_get_mbox: NULL mailbox pointer(s)"));
3091678453a8Sspeer 		goto return_null;
309244961713Sgirish 	}
309344961713Sgirish 
3094678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
3095678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
3096678453a8Sspeer 		    "<== nxge_txdma_get_mbox: NULL ring pointer(s)"));
3097678453a8Sspeer 		goto return_null;
3098678453a8Sspeer 	}
3099678453a8Sspeer 
3100678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3101678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3102678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3103678453a8Sspeer 			if (ring) {
3104678453a8Sspeer 				if (channel == ring->tdc) {
3105678453a8Sspeer 					tx_mbox_t *mailbox = nxgep->
3106678453a8Sspeer 					    tx_mbox_areas_p->
3107678453a8Sspeer 					    txmbox_areas_p[tdc];
3108678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
3109678453a8Sspeer 					    "<== nxge_txdma_get_mbox: tdc %d "
3110678453a8Sspeer 					    "ring $%p", tdc, mailbox));
3111678453a8Sspeer 					return (mailbox);
3112678453a8Sspeer 				}
3113678453a8Sspeer 			}
311444961713Sgirish 		}
311544961713Sgirish 	}
311644961713Sgirish 
3117678453a8Sspeer return_null:
3118678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_get_mbox: "
311952ccf843Smisaki 	    "mailbox not found"));
3120678453a8Sspeer 
312144961713Sgirish 	return (NULL);
312244961713Sgirish }
312344961713Sgirish 
3124678453a8Sspeer /*
3125678453a8Sspeer  * nxge_tx_err_evnts
3126678453a8Sspeer  *
3127678453a8Sspeer  *	Recover a TDC.
3128678453a8Sspeer  *
3129678453a8Sspeer  * Arguments:
3130*86ef0a63SRichard Lowe  *	nxgep
3131*86ef0a63SRichard Lowe  *	index	The index to the TDC ring.
3132*86ef0a63SRichard Lowe  *	ldvp	Used to get the channel number ONLY.
3133*86ef0a63SRichard Lowe  *	cs	A copy of the bits from TX_CS.
3134678453a8Sspeer  *
3135678453a8Sspeer  * Notes:
3136678453a8Sspeer  *	Calling tree:
3137678453a8Sspeer  *	 nxge_tx_intr()
3138678453a8Sspeer  *
3139678453a8Sspeer  * NPI/NXGE function calls:
3140678453a8Sspeer  *	npi_txdma_ring_error_get()
3141678453a8Sspeer  *	npi_txdma_inj_par_error_get()
3142678453a8Sspeer  *	nxge_txdma_fatal_err_recover()
3143678453a8Sspeer  *
3144678453a8Sspeer  * Registers accessed:
3145678453a8Sspeer  *	TX_RNG_ERR_LOGH	DMC+0x40048 Transmit Ring Error Log High
3146678453a8Sspeer  *	TX_RNG_ERR_LOGL DMC+0x40050 Transmit Ring Error Log Low
3147678453a8Sspeer  *	TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error
3148678453a8Sspeer  *
3149678453a8Sspeer  * Context:
3150678453a8Sspeer  *	Any domain	XXX Remove code which accesses TDMC_INJ_PAR_ERR.
3151678453a8Sspeer  */
315244961713Sgirish /*ARGSUSED*/
315344961713Sgirish static nxge_status_t
nxge_tx_err_evnts(p_nxge_t nxgep,uint_t index,p_nxge_ldv_t ldvp,tx_cs_t cs)315444961713Sgirish nxge_tx_err_evnts(p_nxge_t nxgep, uint_t index, p_nxge_ldv_t ldvp, tx_cs_t cs)
315544961713Sgirish {
315644961713Sgirish 	npi_handle_t		handle;
315744961713Sgirish 	npi_status_t		rs;
315844961713Sgirish 	uint8_t			channel;
3159*86ef0a63SRichard Lowe 	p_tx_ring_t		*tx_rings;
3160*86ef0a63SRichard Lowe 	p_tx_ring_t		tx_ring_p;
316144961713Sgirish 	p_nxge_tx_ring_stats_t	tdc_stats;
316244961713Sgirish 	boolean_t		txchan_fatal = B_FALSE;
316344961713Sgirish 	nxge_status_t		status = NXGE_OK;
316444961713Sgirish 	tdmc_inj_par_err_t	par_err;
316544961713Sgirish 	uint32_t		value;
316644961713Sgirish 
3167678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX2_CTL, "==> nxge_tx_err_evnts"));
316844961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
316944961713Sgirish 	channel = ldvp->channel;
317044961713Sgirish 
317144961713Sgirish 	tx_rings = nxgep->tx_rings->rings;
317244961713Sgirish 	tx_ring_p = tx_rings[index];
317344961713Sgirish 	tdc_stats = tx_ring_p->tdc_stats;
317444961713Sgirish 	if ((cs.bits.ldw.pkt_size_err) || (cs.bits.ldw.pref_buf_par_err) ||
317552ccf843Smisaki 	    (cs.bits.ldw.nack_pref) || (cs.bits.ldw.nack_pkt_rd) ||
317652ccf843Smisaki 	    (cs.bits.ldw.conf_part_err) || (cs.bits.ldw.pkt_prt_err)) {
317744961713Sgirish 		if ((rs = npi_txdma_ring_error_get(handle, channel,
317852ccf843Smisaki 		    &tdc_stats->errlog)) != NPI_SUCCESS)
317944961713Sgirish 			return (NXGE_ERROR | rs);
318044961713Sgirish 	}
318144961713Sgirish 
318244961713Sgirish 	if (cs.bits.ldw.mbox_err) {
318344961713Sgirish 		tdc_stats->mbox_err++;
318444961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
318552ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_MBOX_ERR);
318644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
318752ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
318852ccf843Smisaki 		    "fatal error: mailbox", channel));
318944961713Sgirish 		txchan_fatal = B_TRUE;
319044961713Sgirish 	}
319144961713Sgirish 	if (cs.bits.ldw.pkt_size_err) {
319244961713Sgirish 		tdc_stats->pkt_size_err++;
319344961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
319452ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
319544961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
319652ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
319752ccf843Smisaki 		    "fatal error: pkt_size_err", channel));
319844961713Sgirish 		txchan_fatal = B_TRUE;
319944961713Sgirish 	}
320044961713Sgirish 	if (cs.bits.ldw.tx_ring_oflow) {
320144961713Sgirish 		tdc_stats->tx_ring_oflow++;
320244961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
320352ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW);
320444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
320552ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
320652ccf843Smisaki 		    "fatal error: tx_ring_oflow", channel));
320744961713Sgirish 		txchan_fatal = B_TRUE;
320844961713Sgirish 	}
320944961713Sgirish 	if (cs.bits.ldw.pref_buf_par_err) {
321044961713Sgirish 		tdc_stats->pre_buf_par_err++;
321144961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
321252ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR);
321344961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
321452ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
321552ccf843Smisaki 		    "fatal error: pre_buf_par_err", channel));
321644961713Sgirish 		/* Clear error injection source for parity error */
321744961713Sgirish 		(void) npi_txdma_inj_par_error_get(handle, &value);
321844961713Sgirish 		par_err.value = value;
321944961713Sgirish 		par_err.bits.ldw.inject_parity_error &= ~(1 << channel);
322044961713Sgirish 		(void) npi_txdma_inj_par_error_set(handle, par_err.value);
322144961713Sgirish 		txchan_fatal = B_TRUE;
322244961713Sgirish 	}
322344961713Sgirish 	if (cs.bits.ldw.nack_pref) {
322444961713Sgirish 		tdc_stats->nack_pref++;
322544961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
322652ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_NACK_PREF);
322744961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
322852ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
322952ccf843Smisaki 		    "fatal error: nack_pref", channel));
323044961713Sgirish 		txchan_fatal = B_TRUE;
323144961713Sgirish 	}
323244961713Sgirish 	if (cs.bits.ldw.nack_pkt_rd) {
323344961713Sgirish 		tdc_stats->nack_pkt_rd++;
323444961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
323552ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_NACK_PKT_RD);
323644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
323752ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
323852ccf843Smisaki 		    "fatal error: nack_pkt_rd", channel));
323944961713Sgirish 		txchan_fatal = B_TRUE;
324044961713Sgirish 	}
324144961713Sgirish 	if (cs.bits.ldw.conf_part_err) {
324244961713Sgirish 		tdc_stats->conf_part_err++;
324344961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
324452ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_CONF_PART_ERR);
324544961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
324652ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
324752ccf843Smisaki 		    "fatal error: config_partition_err", channel));
324844961713Sgirish 		txchan_fatal = B_TRUE;
324944961713Sgirish 	}
325044961713Sgirish 	if (cs.bits.ldw.pkt_prt_err) {
325144961713Sgirish 		tdc_stats->pkt_part_err++;
325244961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, channel,
325352ccf843Smisaki 		    NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR);
325444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
325552ccf843Smisaki 		    "==> nxge_tx_err_evnts(channel %d): "
325652ccf843Smisaki 		    "fatal error: pkt_prt_err", channel));
325744961713Sgirish 		txchan_fatal = B_TRUE;
325844961713Sgirish 	}
325944961713Sgirish 
326044961713Sgirish 	/* Clear error injection source in case this is an injected error */
326144961713Sgirish 	TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG, channel, 0);
326244961713Sgirish 
326344961713Sgirish 	if (txchan_fatal) {
326444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
326552ccf843Smisaki 		    " nxge_tx_err_evnts: "
326652ccf843Smisaki 		    " fatal error on channel %d cs 0x%llx\n",
326752ccf843Smisaki 		    channel, cs.value));
326844961713Sgirish 		status = nxge_txdma_fatal_err_recover(nxgep, channel,
326952ccf843Smisaki 		    tx_ring_p);
327044961713Sgirish 		if (status == NXGE_OK) {
327144961713Sgirish 			FM_SERVICE_RESTORED(nxgep);
327244961713Sgirish 		}
327344961713Sgirish 	}
327444961713Sgirish 
3275678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX2_CTL, "<== nxge_tx_err_evnts"));
327644961713Sgirish 
327744961713Sgirish 	return (status);
327844961713Sgirish }
327944961713Sgirish 
328044961713Sgirish static nxge_status_t
nxge_txdma_fatal_err_recover(p_nxge_t nxgep,uint16_t channel,p_tx_ring_t tx_ring_p)3281678453a8Sspeer nxge_txdma_fatal_err_recover(
3282678453a8Sspeer 	p_nxge_t nxgep,
3283678453a8Sspeer 	uint16_t channel,
3284678453a8Sspeer 	p_tx_ring_t tx_ring_p)
328544961713Sgirish {
328644961713Sgirish 	npi_handle_t	handle;
328744961713Sgirish 	npi_status_t	rs = NPI_SUCCESS;
328844961713Sgirish 	p_tx_mbox_t	tx_mbox_p;
328944961713Sgirish 	nxge_status_t	status = NXGE_OK;
329044961713Sgirish 
329144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_txdma_fatal_err_recover"));
329244961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
329352ccf843Smisaki 	    "Recovering from TxDMAChannel#%d error...", channel));
329444961713Sgirish 
329544961713Sgirish 	/*
329644961713Sgirish 	 * Stop the dma channel waits for the stop done.
329744961713Sgirish 	 * If the stop done bit is not set, then create
329844961713Sgirish 	 * an error.
329944961713Sgirish 	 */
330044961713Sgirish 
330144961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
330244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel stop..."));
330344961713Sgirish 	MUTEX_ENTER(&tx_ring_p->lock);
330444961713Sgirish 	rs = npi_txdma_channel_control(handle, TXDMA_STOP, channel);
330544961713Sgirish 	if (rs != NPI_SUCCESS) {
330644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
330752ccf843Smisaki 		    "==> nxge_txdma_fatal_err_recover (channel %d): "
330852ccf843Smisaki 		    "stop failed ", channel));
330944961713Sgirish 		goto fail;
331044961713Sgirish 	}
331144961713Sgirish 
331244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reclaim..."));
331344961713Sgirish 	(void) nxge_txdma_reclaim(nxgep, tx_ring_p, 0);
331444961713Sgirish 
331544961713Sgirish 	/*
331644961713Sgirish 	 * Reset TXDMA channel
331744961713Sgirish 	 */
331844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel reset..."));
331944961713Sgirish 	if ((rs = npi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
332052ccf843Smisaki 	    NPI_SUCCESS) {
332144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
332252ccf843Smisaki 		    "==> nxge_txdma_fatal_err_recover (channel %d)"
332352ccf843Smisaki 		    " reset channel failed 0x%x", channel, rs));
332444961713Sgirish 		goto fail;
332544961713Sgirish 	}
332644961713Sgirish 
332744961713Sgirish 	/*
332844961713Sgirish 	 * Reset the tail (kick) register to 0.
332944961713Sgirish 	 * (Hardware will not reset it. Tx overflow fatal
333044961713Sgirish 	 * error if tail is not set to 0 after reset!
333144961713Sgirish 	 */
333244961713Sgirish 	TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, channel, 0);
333344961713Sgirish 
333444961713Sgirish 	/* Restart TXDMA channel */
333544961713Sgirish 
3336e3d11eeeSToomas Soome 	tx_mbox_p = NULL;
3337678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
3338678453a8Sspeer 		tx_mbox_p = nxge_txdma_get_mbox(nxgep, channel);
333944961713Sgirish 
3340678453a8Sspeer 		// XXX This is a problem in HIO!
3341678453a8Sspeer 		/*
3342678453a8Sspeer 		 * Initialize the TXDMA channel specific FZC control
3343678453a8Sspeer 		 * configurations. These FZC registers are pertaining
3344678453a8Sspeer 		 * to each TX channel (i.e. logical pages).
3345678453a8Sspeer 		 */
3346678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel restart..."));
3347678453a8Sspeer 		status = nxge_init_fzc_txdma_channel(nxgep, channel,
3348678453a8Sspeer 		    tx_ring_p, tx_mbox_p);
3349678453a8Sspeer 		if (status != NXGE_OK)
3350678453a8Sspeer 			goto fail;
3351678453a8Sspeer 	}
335244961713Sgirish 
335344961713Sgirish 	/*
335444961713Sgirish 	 * Initialize the event masks.
335544961713Sgirish 	 */
335644961713Sgirish 	tx_ring_p->tx_evmask.value = 0;
335744961713Sgirish 	status = nxge_init_txdma_channel_event_mask(nxgep, channel,
335852ccf843Smisaki 	    &tx_ring_p->tx_evmask);
335944961713Sgirish 	if (status != NXGE_OK)
336044961713Sgirish 		goto fail;
336144961713Sgirish 
336244961713Sgirish 	tx_ring_p->wr_index_wrap = B_FALSE;
336344961713Sgirish 	tx_ring_p->wr_index = 0;
336444961713Sgirish 	tx_ring_p->rd_index = 0;
336544961713Sgirish 
336644961713Sgirish 	/*
336744961713Sgirish 	 * Load TXDMA descriptors, buffers, mailbox,
336844961713Sgirish 	 * initialise the DMA channels and
336944961713Sgirish 	 * enable each DMA channel.
337044961713Sgirish 	 */
337144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "TxDMA channel enable..."));
337244961713Sgirish 	status = nxge_enable_txdma_channel(nxgep, channel,
337352ccf843Smisaki 	    tx_ring_p, tx_mbox_p);
337444961713Sgirish 	MUTEX_EXIT(&tx_ring_p->lock);
337544961713Sgirish 	if (status != NXGE_OK)
337644961713Sgirish 		goto fail;
337744961713Sgirish 
337844961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
337952ccf843Smisaki 	    "Recovery Successful, TxDMAChannel#%d Restored",
338052ccf843Smisaki 	    channel));
338144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_txdma_fatal_err_recover"));
338244961713Sgirish 
338344961713Sgirish 	return (NXGE_OK);
338444961713Sgirish 
338544961713Sgirish fail:
338644961713Sgirish 	MUTEX_EXIT(&tx_ring_p->lock);
33872d99c5d4SMichael Speer 
338844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
338952ccf843Smisaki 	    "nxge_txdma_fatal_err_recover (channel %d): "
339052ccf843Smisaki 	    "failed to recover this txdma channel", channel));
339144961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
339244961713Sgirish 
339344961713Sgirish 	return (status);
339444961713Sgirish }
339544961713Sgirish 
3396678453a8Sspeer /*
3397678453a8Sspeer  * nxge_tx_port_fatal_err_recover
3398678453a8Sspeer  *
3399678453a8Sspeer  *	Attempt to recover from a fatal port error.
3400678453a8Sspeer  *
3401678453a8Sspeer  * Arguments:
3402*86ef0a63SRichard Lowe  *	nxgep
3403678453a8Sspeer  *
3404678453a8Sspeer  * Notes:
3405678453a8Sspeer  *	How would a guest do this?
3406678453a8Sspeer  *
3407678453a8Sspeer  * NPI/NXGE function calls:
3408678453a8Sspeer  *
3409678453a8Sspeer  * Registers accessed:
3410678453a8Sspeer  *
3411678453a8Sspeer  * Context:
3412678453a8Sspeer  *	Service domain
3413678453a8Sspeer  */
341444961713Sgirish nxge_status_t
nxge_tx_port_fatal_err_recover(p_nxge_t nxgep)341544961713Sgirish nxge_tx_port_fatal_err_recover(p_nxge_t nxgep)
341644961713Sgirish {
3417678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->tx_set;
3418678453a8Sspeer 	nxge_channel_t tdc;
3419678453a8Sspeer 
3420678453a8Sspeer 	tx_ring_t	*ring;
3421678453a8Sspeer 	tx_mbox_t	*mailbox;
3422678453a8Sspeer 
342344961713Sgirish 	npi_handle_t	handle;
3424e3d11eeeSToomas Soome 	nxge_status_t	status = NXGE_OK;
3425678453a8Sspeer 	npi_status_t	rs;
342644961713Sgirish 
342744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_tx_port_fatal_err_recover"));
342844961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3429678453a8Sspeer 	    "Recovering from TxPort error..."));
343044961713Sgirish 
3431678453a8Sspeer 	if (isLDOMguest(nxgep)) {
3432678453a8Sspeer 		return (NXGE_OK);
343344961713Sgirish 	}
343444961713Sgirish 
3435678453a8Sspeer 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
3436678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
3437678453a8Sspeer 		    "<== nxge_tx_port_fatal_err_recover: not initialized"));
3438678453a8Sspeer 		return (NXGE_ERROR);
343944961713Sgirish 	}
344044961713Sgirish 
3441678453a8Sspeer 	if (nxgep->tx_rings == 0 || nxgep->tx_rings->rings == 0) {
3442678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
3443678453a8Sspeer 		    "<== nxge_tx_port_fatal_err_recover: "
3444678453a8Sspeer 		    "NULL ring pointer(s)"));
3445678453a8Sspeer 		return (NXGE_ERROR);
3446678453a8Sspeer 	}
344744961713Sgirish 
3448678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3449678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3450678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3451678453a8Sspeer 			if (ring)
3452678453a8Sspeer 				MUTEX_ENTER(&ring->lock);
345344961713Sgirish 		}
345444961713Sgirish 	}
345544961713Sgirish 
3456678453a8Sspeer 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
3457678453a8Sspeer 
345844961713Sgirish 	/*
3459678453a8Sspeer 	 * Stop all the TDCs owned by us.
3460678453a8Sspeer 	 * (The shared TDCs will have been stopped by their owners.)
346144961713Sgirish 	 */
3462678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3463678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3464678453a8Sspeer 			ring = nxgep->tx_rings->rings[tdc];
3465678453a8Sspeer 			if (ring) {
3466678453a8Sspeer 				rs = npi_txdma_channel_control
3467678453a8Sspeer 				    (handle, TXDMA_STOP, tdc);
3468678453a8Sspeer 				if (rs != NPI_SUCCESS) {
3469678453a8Sspeer 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3470678453a8Sspeer 					    "nxge_tx_port_fatal_err_recover "
3471678453a8Sspeer 					    "(channel %d): stop failed ", tdc));
3472678453a8Sspeer 					goto fail;
3473678453a8Sspeer 				}
3474678453a8Sspeer 			}
347544961713Sgirish 		}
3476678453a8Sspeer 	}
347744961713Sgirish 
3478678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "Reclaiming all TDCs..."));
347944961713Sgirish 
3480678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3481678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3482678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
34832d99c5d4SMichael Speer 			if (ring) {
3484678453a8Sspeer 				(void) nxge_txdma_reclaim(nxgep, ring, 0);
34852d99c5d4SMichael Speer 			}
3486678453a8Sspeer 		}
348744961713Sgirish 	}
348844961713Sgirish 
348944961713Sgirish 	/*
3490678453a8Sspeer 	 * Reset all the TDCs.
349144961713Sgirish 	 */
3492678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "Resetting all TDCs..."));
3493678453a8Sspeer 
3494678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3495678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3496678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3497678453a8Sspeer 			if (ring) {
3498678453a8Sspeer 				if ((rs = npi_txdma_channel_control
349952ccf843Smisaki 				    (handle, TXDMA_RESET, tdc))
3500678453a8Sspeer 				    != NPI_SUCCESS) {
3501678453a8Sspeer 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3502678453a8Sspeer 					    "nxge_tx_port_fatal_err_recover "
3503678453a8Sspeer 					    "(channel %d) reset channel "
3504678453a8Sspeer 					    "failed 0x%x", tdc, rs));
3505678453a8Sspeer 					goto fail;
3506678453a8Sspeer 				}
3507678453a8Sspeer 			}
3508678453a8Sspeer 			/*
3509678453a8Sspeer 			 * Reset the tail (kick) register to 0.
3510678453a8Sspeer 			 * (Hardware will not reset it. Tx overflow fatal
3511678453a8Sspeer 			 * error if tail is not set to 0 after reset!
3512678453a8Sspeer 			 */
3513678453a8Sspeer 			TXDMA_REG_WRITE64(handle, TX_RING_KICK_REG, tdc, 0);
351444961713Sgirish 		}
3515678453a8Sspeer 	}
351644961713Sgirish 
3517678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "Restarting all TDCs..."));
3518678453a8Sspeer 
3519678453a8Sspeer 	/* Restart all the TDCs */
3520678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3521678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3522678453a8Sspeer 			ring = nxgep->tx_rings->rings[tdc];
3523678453a8Sspeer 			if (ring) {
3524678453a8Sspeer 				mailbox = nxge_txdma_get_mbox(nxgep, tdc);
3525678453a8Sspeer 				status = nxge_init_fzc_txdma_channel(nxgep, tdc,
3526678453a8Sspeer 				    ring, mailbox);
3527678453a8Sspeer 				ring->tx_evmask.value = 0;
3528678453a8Sspeer 				/*
3529678453a8Sspeer 				 * Initialize the event masks.
3530678453a8Sspeer 				 */
3531678453a8Sspeer 				status = nxge_init_txdma_channel_event_mask
3532678453a8Sspeer 				    (nxgep, tdc, &ring->tx_evmask);
3533678453a8Sspeer 
3534678453a8Sspeer 				ring->wr_index_wrap = B_FALSE;
3535678453a8Sspeer 				ring->wr_index = 0;
3536678453a8Sspeer 				ring->rd_index = 0;
3537678453a8Sspeer 
3538678453a8Sspeer 				if (status != NXGE_OK)
3539678453a8Sspeer 					goto fail;
3540678453a8Sspeer 				if (status != NXGE_OK)
3541678453a8Sspeer 					goto fail;
3542678453a8Sspeer 			}
3543678453a8Sspeer 		}
354444961713Sgirish 	}
354544961713Sgirish 
3546678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "Re-enabling all TDCs..."));
354744961713Sgirish 
3548678453a8Sspeer 	/* Re-enable all the TDCs */
3549678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3550678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3551678453a8Sspeer 			ring = nxgep->tx_rings->rings[tdc];
3552678453a8Sspeer 			if (ring) {
3553678453a8Sspeer 				mailbox = nxge_txdma_get_mbox(nxgep, tdc);
3554678453a8Sspeer 				status = nxge_enable_txdma_channel(nxgep, tdc,
3555678453a8Sspeer 				    ring, mailbox);
3556678453a8Sspeer 				if (status != NXGE_OK)
3557678453a8Sspeer 					goto fail;
3558678453a8Sspeer 			}
355944961713Sgirish 		}
356044961713Sgirish 	}
356144961713Sgirish 
3562678453a8Sspeer 	/*
3563678453a8Sspeer 	 * Unlock all the TDCs.
3564678453a8Sspeer 	 */
3565678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3566678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3567678453a8Sspeer 			tx_ring_t *ring = nxgep->tx_rings->rings[tdc];
3568678453a8Sspeer 			if (ring)
3569678453a8Sspeer 				MUTEX_EXIT(&ring->lock);
357044961713Sgirish 		}
357144961713Sgirish 	}
357244961713Sgirish 
3573678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery succeeded"));
357444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover"));
357544961713Sgirish 
357644961713Sgirish 	return (NXGE_OK);
357744961713Sgirish 
357844961713Sgirish fail:
3579678453a8Sspeer 	for (tdc = 0; tdc < NXGE_MAX_TDCS; tdc++) {
3580678453a8Sspeer 		if ((1 << tdc) & set->owned.map) {
3581678453a8Sspeer 			ring = nxgep->tx_rings->rings[tdc];
3582678453a8Sspeer 			if (ring)
3583678453a8Sspeer 				MUTEX_EXIT(&ring->lock);
358444961713Sgirish 		}
358544961713Sgirish 	}
358644961713Sgirish 
3587678453a8Sspeer 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Tx port recovery failed"));
3588678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_port_fatal_err_recover"));
358944961713Sgirish 
359044961713Sgirish 	return (status);
359144961713Sgirish }
359244961713Sgirish 
3593678453a8Sspeer /*
3594678453a8Sspeer  * nxge_txdma_inject_err
3595678453a8Sspeer  *
3596678453a8Sspeer  *	Inject an error into a TDC.
3597678453a8Sspeer  *
3598678453a8Sspeer  * Arguments:
3599*86ef0a63SRichard Lowe  *	nxgep
3600*86ef0a63SRichard Lowe  *	err_id	The error to inject.
3601*86ef0a63SRichard Lowe  *	chan	The channel to inject into.
3602678453a8Sspeer  *
3603678453a8Sspeer  * Notes:
3604678453a8Sspeer  *	This is called from nxge_main.c:nxge_err_inject()
3605678453a8Sspeer  *	Has this ioctl ever been used?
3606678453a8Sspeer  *
3607678453a8Sspeer  * NPI/NXGE function calls:
3608678453a8Sspeer  *	npi_txdma_inj_par_error_get()
3609678453a8Sspeer  *	npi_txdma_inj_par_error_set()
3610678453a8Sspeer  *
3611678453a8Sspeer  * Registers accessed:
3612678453a8Sspeer  *	TDMC_INJ_PAR_ERR (FZC_DMC + 0x45040) TDMC Inject Parity Error
3613678453a8Sspeer  *	TDMC_INTR_DBG	DMC + 0x40060 Transmit DMA Interrupt Debug
3614678453a8Sspeer  *	TDMC_INTR_DBG	DMC + 0x40060 Transmit DMA Interrupt Debug
3615678453a8Sspeer  *
3616678453a8Sspeer  * Context:
3617678453a8Sspeer  *	Service domain
3618678453a8Sspeer  */
361944961713Sgirish void
nxge_txdma_inject_err(p_nxge_t nxgep,uint32_t err_id,uint8_t chan)362044961713Sgirish nxge_txdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan)
362144961713Sgirish {
362244961713Sgirish 	tdmc_intr_dbg_t		tdi;
362344961713Sgirish 	tdmc_inj_par_err_t	par_err;
362444961713Sgirish 	uint32_t		value;
362544961713Sgirish 	npi_handle_t		handle;
362644961713Sgirish 
362744961713Sgirish 	switch (err_id) {
362844961713Sgirish 
362944961713Sgirish 	case NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR:
363044961713Sgirish 		handle = NXGE_DEV_NPI_HANDLE(nxgep);
363144961713Sgirish 		/* Clear error injection source for parity error */
363244961713Sgirish 		(void) npi_txdma_inj_par_error_get(handle, &value);
363344961713Sgirish 		par_err.value = value;
363444961713Sgirish 		par_err.bits.ldw.inject_parity_error &= ~(1 << chan);
363544961713Sgirish 		(void) npi_txdma_inj_par_error_set(handle, par_err.value);
363644961713Sgirish 
363744961713Sgirish 		par_err.bits.ldw.inject_parity_error = (1 << chan);
363844961713Sgirish 		(void) npi_txdma_inj_par_error_get(handle, &value);
363944961713Sgirish 		par_err.value = value;
364044961713Sgirish 		par_err.bits.ldw.inject_parity_error |= (1 << chan);
364144961713Sgirish 		cmn_err(CE_NOTE, "!Write 0x%llx to TDMC_INJ_PAR_ERR_REG\n",
364252ccf843Smisaki 		    (unsigned long long)par_err.value);
364344961713Sgirish 		(void) npi_txdma_inj_par_error_set(handle, par_err.value);
364444961713Sgirish 		break;
364544961713Sgirish 
364644961713Sgirish 	case NXGE_FM_EREPORT_TDMC_MBOX_ERR:
364744961713Sgirish 	case NXGE_FM_EREPORT_TDMC_NACK_PREF:
364844961713Sgirish 	case NXGE_FM_EREPORT_TDMC_NACK_PKT_RD:
364944961713Sgirish 	case NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR:
365044961713Sgirish 	case NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW:
365144961713Sgirish 	case NXGE_FM_EREPORT_TDMC_CONF_PART_ERR:
365244961713Sgirish 	case NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR:
365344961713Sgirish 		TXDMA_REG_READ64(nxgep->npi_handle, TDMC_INTR_DBG_REG,
365452ccf843Smisaki 		    chan, &tdi.value);
365544961713Sgirish 		if (err_id == NXGE_FM_EREPORT_TDMC_PREF_BUF_PAR_ERR)
365644961713Sgirish 			tdi.bits.ldw.pref_buf_par_err = 1;
365744961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_MBOX_ERR)
365844961713Sgirish 			tdi.bits.ldw.mbox_err = 1;
365944961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PREF)
366044961713Sgirish 			tdi.bits.ldw.nack_pref = 1;
366144961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_NACK_PKT_RD)
366244961713Sgirish 			tdi.bits.ldw.nack_pkt_rd = 1;
366344961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR)
366444961713Sgirish 			tdi.bits.ldw.pkt_size_err = 1;
366544961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_TX_RING_OFLOW)
366644961713Sgirish 			tdi.bits.ldw.tx_ring_oflow = 1;
366744961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_CONF_PART_ERR)
366844961713Sgirish 			tdi.bits.ldw.conf_part_err = 1;
366944961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_TDMC_PKT_PRT_ERR)
367044961713Sgirish 			tdi.bits.ldw.pkt_part_err = 1;
367144961713Sgirish 		cmn_err(CE_NOTE, "!Write 0x%lx to TDMC_INTR_DBG_REG\n",
367252ccf843Smisaki 		    tdi.value);
367344961713Sgirish 		TXDMA_REG_WRITE64(nxgep->npi_handle, TDMC_INTR_DBG_REG,
367452ccf843Smisaki 		    chan, tdi.value);
367544961713Sgirish 
367644961713Sgirish 		break;
367744961713Sgirish 	}
367844961713Sgirish }
3679