xref: /illumos-gate/usr/src/uts/common/io/nxge/nxge_send.c (revision 86ef0a63)
144961713Sgirish /*
244961713Sgirish  * CDDL HEADER START
344961713Sgirish  *
444961713Sgirish  * The contents of this file are subject to the terms of the
544961713Sgirish  * Common Development and Distribution License (the "License").
644961713Sgirish  * You may not use this file except in compliance with the License.
744961713Sgirish  *
844961713Sgirish  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
944961713Sgirish  * or http://www.opensolaris.org/os/licensing.
1044961713Sgirish  * See the License for the specific language governing permissions
1144961713Sgirish  * and limitations under the License.
1244961713Sgirish  *
1344961713Sgirish  * When distributing Covered Code, include this CDDL HEADER in each
1444961713Sgirish  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1544961713Sgirish  * If applicable, add the following below this CDDL HEADER, with the
1644961713Sgirish  * fields enclosed by brackets "[]" replaced with your own identifying
1744961713Sgirish  * information: Portions Copyright [yyyy] [name of copyright owner]
1844961713Sgirish  *
1944961713Sgirish  * CDDL HEADER END
2044961713Sgirish  */
2144961713Sgirish /*
220dc2366fSVenugopal Iyer  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
2344961713Sgirish  * Use is subject to license terms.
2444961713Sgirish  */
26ae6aa22aSVenugopal Iyer #include <sys/mac_provider.h>
27a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h>
28678453a8Sspeer #include <sys/nxge/nxge_hio.h>
29678453a8Sspeer #include <npi_tx_wr64.h>
3130ac2e7bSml /* Software LSO required header files */
3230ac2e7bSml #include <netinet/tcp.h>
3330ac2e7bSml #include <inet/ip_impl.h>
3430ac2e7bSml #include <inet/tcp.h>
36ae6aa22aSVenugopal Iyer extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy,
37ae6aa22aSVenugopal Iyer     boolean_t is_outbound);
38ae6aa22aSVenugopal Iyer 
3930ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *);
4030ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
4130ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
4230ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *,
4330ac2e7bSml     uint32_t *, uint32_t *, uint32_t *,
4430ac2e7bSml     uint32_t *, uint32_t *);
4530ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int);
4744961713Sgirish extern uint32_t		nxge_reclaim_pending;
48*86ef0a63SRichard Lowe extern uint32_t		nxge_bcopy_thresh;
49*86ef0a63SRichard Lowe extern uint32_t		nxge_dvma_thresh;
50*86ef0a63SRichard Lowe extern uint32_t		nxge_dma_stream_thresh;
5144961713Sgirish extern uint32_t		nxge_tx_minfree;
5244961713Sgirish extern uint32_t		nxge_tx_intr_thres;
5344961713Sgirish extern uint32_t		nxge_tx_max_gathers;
5444961713Sgirish extern uint32_t		nxge_tx_tiny_pack;
5544961713Sgirish extern uint32_t		nxge_tx_use_bcopy;
561f8914d5Sml extern nxge_tx_mode_t	nxge_tx_scheme;
5730ac2e7bSml uint32_t		nxge_lso_kick_cnt = 2;
60da14cebeSEric Cheng void
nxge_tx_ring_task(void * arg)61da14cebeSEric Cheng nxge_tx_ring_task(void *arg)
62da14cebeSEric Cheng {
63da14cebeSEric Cheng 	p_tx_ring_t	ring = (p_tx_ring_t)arg;
64da14cebeSEric Cheng 
650dc2366fSVenugopal Iyer 	ASSERT(ring->tx_ring_handle != NULL);
660dc2366fSVenugopal Iyer 
67da14cebeSEric Cheng 	MUTEX_ENTER(&ring->lock);
68da14cebeSEric Cheng 	(void) nxge_txdma_reclaim(ring->nxgep, ring, 0);
69da14cebeSEric Cheng 	MUTEX_EXIT(&ring->lock);
70da14cebeSEric Cheng 
7163f531d1SSriharsha Basavapatna 	if (!ring->tx_ring_offline) {
72da14cebeSEric Cheng 		mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle);
73da14cebeSEric Cheng 	}
74da14cebeSEric Cheng }
75da14cebeSEric Cheng 
76da14cebeSEric Cheng static void
nxge_tx_ring_dispatch(p_tx_ring_t ring)77da14cebeSEric Cheng nxge_tx_ring_dispatch(p_tx_ring_t ring)
78da14cebeSEric Cheng {
79da14cebeSEric Cheng 	/*
80da14cebeSEric Cheng 	 * Kick the ring task to reclaim some buffers.
81da14cebeSEric Cheng 	 */
82da14cebeSEric Cheng 	(void) ddi_taskq_dispatch(ring->taskq,
83da14cebeSEric Cheng 	    nxge_tx_ring_task, (void *)ring, DDI_SLEEP);
84da14cebeSEric Cheng }
85da14cebeSEric Cheng 
86da14cebeSEric Cheng mblk_t *
nxge_tx_ring_send(void * arg,mblk_t * mp)87da14cebeSEric Cheng nxge_tx_ring_send(void *arg, mblk_t *mp)
88da14cebeSEric Cheng {
89da14cebeSEric Cheng 	p_nxge_ring_handle_t	nrhp = (p_nxge_ring_handle_t)arg;
90da14cebeSEric Cheng 	p_nxge_t		nxgep;
91da14cebeSEric Cheng 	p_tx_ring_t		tx_ring_p;
92da14cebeSEric Cheng 	int			status, channel;
93da14cebeSEric Cheng 
94da14cebeSEric Cheng 	ASSERT(nrhp != NULL);
95da14cebeSEric Cheng 	nxgep = nrhp->nxgep;
96da14cebeSEric Cheng 	channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index;
97da14cebeSEric Cheng 	tx_ring_p = nxgep->tx_rings->rings[channel];
98da14cebeSEric Cheng 
9948056c53SMichael Speer 	/*
10048056c53SMichael Speer 	 * We may be in a transition from offlined DMA to onlined
10148056c53SMichael Speer 	 * DMA.
10248056c53SMichael Speer 	 */
10348056c53SMichael Speer 	if (tx_ring_p == NULL) {
10448056c53SMichael Speer 		ASSERT(tx_ring_p != NULL);
10548056c53SMichael Speer 		freemsg(mp);
10648056c53SMichael Speer 		return ((mblk_t *)NULL);
10748056c53SMichael Speer 	}
10848056c53SMichael Speer 
10948056c53SMichael Speer 	/*
11048056c53SMichael Speer 	 * Valid DMA?
11148056c53SMichael Speer 	 */
112da14cebeSEric Cheng 	ASSERT(nxgep == tx_ring_p->nxgep);
113da14cebeSEric Cheng 
11448056c53SMichael Speer 	/*
11548056c53SMichael Speer 	 * Make sure DMA is not offlined.
11648056c53SMichael Speer 	 */
11748056c53SMichael Speer 	if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) {
118da14cebeSEric Cheng 		ASSERT(!tx_ring_p->tx_ring_offline);
11948056c53SMichael Speer 		freemsg(mp);
12048056c53SMichael Speer 		return ((mblk_t *)NULL);
121da14cebeSEric Cheng 	}
122da14cebeSEric Cheng 
12348056c53SMichael Speer 	/*
12448056c53SMichael Speer 	 * Transmit the packet.
12548056c53SMichael Speer 	 */
126da14cebeSEric Cheng 	status = nxge_start(nxgep, tx_ring_p, mp);
127da14cebeSEric Cheng 	if (status) {
128da14cebeSEric Cheng 		nxge_tx_ring_dispatch(tx_ring_p);
129da14cebeSEric Cheng 		return (mp);
130da14cebeSEric Cheng 	}
131da14cebeSEric Cheng 
132da14cebeSEric Cheng 	return ((mblk_t *)NULL);
133da14cebeSEric Cheng }
134da14cebeSEric Cheng 
13544961713Sgirish int
nxge_start(p_nxge_t nxgep,p_tx_ring_t tx_ring_p,p_mblk_t mp)13644961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
13744961713Sgirish {
138*86ef0a63SRichard Lowe 	int			dma_status, status = 0;
139*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_ring_vp;
14044961713Sgirish 	npi_handle_t		npi_desc_handle;
141*86ef0a63SRichard Lowe 	nxge_os_dma_handle_t	tx_desc_dma_handle;
142*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_p;
143*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_ring;
144*86ef0a63SRichard Lowe 	p_tx_msg_t		tx_msg_p = NULL;
14544961713Sgirish 	tx_desc_t		tx_desc, *tmp_desc_p;
14644961713Sgirish 	tx_desc_t		sop_tx_desc, *sop_tx_desc_p;
147e3d11eeeSToomas Soome 	p_tx_pkt_header_t	hdrp = NULL;
148ef755e7aStc 	tx_pkt_hdr_all_t	tmp_hdrp;
14944961713Sgirish 	p_tx_pkt_hdr_all_t	pkthdrp;
15044961713Sgirish 	uint8_t			npads = 0;
151*86ef0a63SRichard Lowe 	uint64_t		dma_ioaddr;
15244961713Sgirish 	uint32_t		dma_flags;
15344961713Sgirish 	int			last_bidx;
154*86ef0a63SRichard Lowe 	uint8_t			*b_rptr;
155*86ef0a63SRichard Lowe 	caddr_t			kaddr;
15644961713Sgirish 	uint32_t		nmblks;
15744961713Sgirish 	uint32_t		ngathers;
15844961713Sgirish 	uint32_t		clen;
159*86ef0a63SRichard Lowe 	int			len;
16044961713Sgirish 	uint32_t		pkt_len, pack_len, min_len;
16144961713Sgirish 	uint32_t		bcopy_thresh;
162*86ef0a63SRichard Lowe 	int			i, cur_index, sop_index;
16344961713Sgirish 	uint16_t		tail_index;
16444961713Sgirish 	boolean_t		tail_wrap = B_FALSE;
16544961713Sgirish 	nxge_dma_common_t	desc_area;
166*86ef0a63SRichard Lowe 	nxge_os_dma_handle_t	dma_handle;
167*86ef0a63SRichard Lowe 	ddi_dma_cookie_t	dma_cookie;
16844961713Sgirish 	npi_handle_t		npi_handle;
169*86ef0a63SRichard Lowe 	p_mblk_t		nmp;
17044961713Sgirish 	p_mblk_t		t_mp;
171*86ef0a63SRichard Lowe 	uint32_t		ncookies;
172*86ef0a63SRichard Lowe 	boolean_t		good_packet;
173*86ef0a63SRichard Lowe 	boolean_t		mark_mode = B_FALSE;
174*86ef0a63SRichard Lowe 	p_nxge_stats_t		statsp;
17544961713Sgirish 	p_nxge_tx_ring_stats_t tdc_stats;
176*86ef0a63SRichard Lowe 	t_uscalar_t		start_offset = 0;
177*86ef0a63SRichard Lowe 	t_uscalar_t		stuff_offset = 0;
178*86ef0a63SRichard Lowe 	t_uscalar_t		end_offset = 0;
179*86ef0a63SRichard Lowe 	t_uscalar_t		value = 0;
180*86ef0a63SRichard Lowe 	t_uscalar_t		cksum_flags = 0;
18144961713Sgirish 	boolean_t		cksum_on = B_FALSE;
18244961713Sgirish 	uint32_t		boff = 0;
183b4d05839Sml 	uint64_t		tot_xfer_len = 0;
18444961713Sgirish 	boolean_t		header_set = B_FALSE;
18544961713Sgirish #ifdef NXGE_DEBUG
186*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_ring_pp;
187*86ef0a63SRichard Lowe 	p_tx_desc_t		tx_desc_pp;
18844961713Sgirish 	tx_desc_t		*save_desc_p;
18944961713Sgirish 	int			dump_len;
19044961713Sgirish 	int			sad_len;
19144961713Sgirish 	uint64_t		sad;
19244961713Sgirish 	int			xfer_len;
19344961713Sgirish 	uint32_t		msgsize;
19444961713Sgirish #endif
195*86ef0a63SRichard Lowe 	p_mblk_t		mp_chain = NULL;
19630ac2e7bSml 	boolean_t		is_lso = B_FALSE;
19730ac2e7bSml 	boolean_t		lso_again;
19830ac2e7bSml 	int			cur_index_lso;
199*86ef0a63SRichard Lowe 	p_mblk_t		nmp_lso_save = NULL;
20030ac2e7bSml 	uint32_t		lso_ngathers;
20130ac2e7bSml 	boolean_t		lso_tail_wrap = B_FALSE;
20344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
204678453a8Sspeer 	    "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
20544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
206678453a8Sspeer 	    "==> nxge_start: Starting tdc %d desc pending %d",
207678453a8Sspeer 	    tx_ring_p->tdc, tx_ring_p->descs_pending));
20944961713Sgirish 	statsp = nxgep->statsp;
211678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
212678453a8Sspeer 		switch (nxgep->mac.portmode) {
213678453a8Sspeer 		default:
214678453a8Sspeer 			if (nxgep->statsp->port_stats.lb_mode ==
215678453a8Sspeer 			    nxge_lb_normal) {
216678453a8Sspeer 				if (!statsp->mac_stats.link_up) {
217678453a8Sspeer 					freemsg(mp);
218678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
219678453a8Sspeer 					    "==> nxge_start: "
220678453a8Sspeer 					    "link not up"));
221678453a8Sspeer 					goto nxge_start_fail1;
222678453a8Sspeer 				}
223321febdeSsbehera 			}
224678453a8Sspeer 			break;
225678453a8Sspeer 		case PORT_10G_FIBER:
226678453a8Sspeer 			/*
227678453a8Sspeer 			 * For the following modes, check the link status
228678453a8Sspeer 			 * before sending the packet out:
229eb1db165Stc 			 * nxge_lb_normal,
230eb1db165Stc 			 * nxge_lb_ext10g,
231eb1db165Stc 			 * nxge_lb_ext1000,
232eb1db165Stc 			 * nxge_lb_ext100,
233eb1db165Stc 			 * nxge_lb_ext10.
234678453a8Sspeer 			 */
235678453a8Sspeer 			if (nxgep->statsp->port_stats.lb_mode <
236eb1db165Stc 			    nxge_lb_phy10g) {
237678453a8Sspeer 				if (!statsp->mac_stats.link_up) {
238678453a8Sspeer 					freemsg(mp);
239678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
240678453a8Sspeer 					    "==> nxge_start: "
241678453a8Sspeer 					    "link not up"));
242678453a8Sspeer 					goto nxge_start_fail1;
243678453a8Sspeer 				}
244321febdeSsbehera 			}
245678453a8Sspeer 			break;
24644961713Sgirish 		}
247678453a8Sspeer 	}
249678453a8Sspeer 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
250678453a8Sspeer 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
251678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
252678453a8Sspeer 		    "==> nxge_start: hardware not initialized or stopped"));
253678453a8Sspeer 		freemsg(mp);
254678453a8Sspeer 		goto nxge_start_fail1;
25544961713Sgirish 	}
2573d16f8e7Sml 	if (nxgep->soft_lso_enable) {
25830ac2e7bSml 		mp_chain = nxge_lso_eliminate(mp);
25930ac2e7bSml 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
26030ac2e7bSml 		    "==> nxge_start(0): LSO mp $%p mp_chain $%p",
26130ac2e7bSml 		    mp, mp_chain));
26230ac2e7bSml 		if (mp_chain == NULL) {
26330ac2e7bSml 			NXGE_ERROR_MSG((nxgep, TX_CTL,
26430ac2e7bSml 			    "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
26530ac2e7bSml 			    mp_chain, mp));
26630ac2e7bSml 			goto nxge_start_fail1;
26730ac2e7bSml 		}
26830ac2e7bSml 		if (mp_chain != mp) {
26930ac2e7bSml 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
27030ac2e7bSml 			    "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
27130ac2e7bSml 			    mp_chain, mp));
27230ac2e7bSml 			is_lso = B_TRUE;
27330ac2e7bSml 			mp = mp_chain;
27430ac2e7bSml 			mp_chain = mp_chain->b_next;
27530ac2e7bSml 			mp->b_next = NULL;
27630ac2e7bSml 		}
27730ac2e7bSml 	}
2790dc2366fSVenugopal Iyer 	mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset,
2800dc2366fSVenugopal Iyer 	    &value, &cksum_flags);
28144961713Sgirish 	if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
28244961713Sgirish 		start_offset += sizeof (ether_header_t);
28344961713Sgirish 		stuff_offset += sizeof (ether_header_t);
28444961713Sgirish 	} else {
28544961713Sgirish 		start_offset += sizeof (struct ether_vlan_header);
28644961713Sgirish 		stuff_offset += sizeof (struct ether_vlan_header);
28744961713Sgirish 	}
28944961713Sgirish 	if (cksum_flags & HCK_PARTIALCKSUM) {
29044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
29152ccf843Smisaki 		    "==> nxge_start: mp $%p len %d "
29252ccf843Smisaki 		    "cksum_flags 0x%x (partial checksum) ",
29352ccf843Smisaki 		    mp, MBLKL(mp), cksum_flags));
29444961713Sgirish 		cksum_on = B_TRUE;
29544961713Sgirish 	}
297b4d05839Sml 	pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
298b4d05839Sml 	pkthdrp->reserved = 0;
299ef755e7aStc 	tmp_hdrp.pkthdr.value = 0;
300b4d05839Sml 	nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
301b4d05839Sml 	    0, 0, pkthdrp,
302b4d05839Sml 	    start_offset, stuff_offset);
30430ac2e7bSml 	lso_again = B_FALSE;
30530ac2e7bSml 	lso_ngathers = 0;
30730ac2e7bSml 	MUTEX_ENTER(&tx_ring_p->lock);
30922c0d73aSspeer 	if (isLDOMservice(nxgep)) {
3106895688eSspeer 		tx_ring_p->tx_ring_busy = B_TRUE;
31122c0d73aSspeer 		if (tx_ring_p->tx_ring_offline) {
31222c0d73aSspeer 			freemsg(mp);
3136895688eSspeer 			tx_ring_p->tx_ring_busy = B_FALSE;
3146895688eSspeer 			(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
3156895688eSspeer 			    NXGE_TX_RING_OFFLINED);
31622c0d73aSspeer 			MUTEX_EXIT(&tx_ring_p->lock);
31722c0d73aSspeer 			return (status);
31822c0d73aSspeer 		}
31922c0d73aSspeer 	}
32130ac2e7bSml 	cur_index_lso = tx_ring_p->wr_index;
32230ac2e7bSml 	lso_tail_wrap = tx_ring_p->wr_index_wrap;
32330ac2e7bSml start_again:
32430ac2e7bSml 	ngathers = 0;
32530ac2e7bSml 	sop_index = tx_ring_p->wr_index;
32644961713Sgirish #ifdef	NXGE_DEBUG
32744961713Sgirish 	if (tx_ring_p->descs_pending) {
32844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
32952ccf843Smisaki 		    "desc pending %d ", tx_ring_p->descs_pending));
33044961713Sgirish 	}
33244961713Sgirish 	dump_len = (int)(MBLKL(mp));
33344961713Sgirish 	dump_len = (dump_len > 128) ? 128: dump_len;
33544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
33652ccf843Smisaki 	    "==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
33752ccf843Smisaki 	    "(Before header reserve: ORIGINAL LEN %d)",
33852ccf843Smisaki 	    tx_ring_p->tdc,
33952ccf843Smisaki 	    mp->b_rptr,
34052ccf843Smisaki 	    dump_len));
34244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
34352ccf843Smisaki 	    "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
34452ccf843Smisaki 	    nxge_dump_packet((char *)mp->b_rptr, dump_len)));
34544961713Sgirish #endif
34744961713Sgirish 	tdc_stats = tx_ring_p->tdc_stats;
34844961713Sgirish 	mark_mode = (tx_ring_p->descs_pending &&
349257bdc55SMichael Speer 	    (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) <
350257bdc55SMichael Speer 	    (int)nxge_tx_minfree));
35244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
35352ccf843Smisaki 	    "TX Descriptor ring is channel %d mark mode %d",
35452ccf843Smisaki 	    tx_ring_p->tdc, mark_mode));
3567127d9f6Sml 	if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) {
3577127d9f6Sml 		if (!nxge_txdma_reclaim(nxgep, tx_ring_p,
3587127d9f6Sml 		    (nxge_tx_minfree + lso_ngathers))) {
3597127d9f6Sml 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
3607127d9f6Sml 			    "TX Descriptor ring is full: channel %d",
3617127d9f6Sml 			    tx_ring_p->tdc));
36230ac2e7bSml 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
3637127d9f6Sml 			    "TX Descriptor ring is full: channel %d",
36430ac2e7bSml 			    tx_ring_p->tdc));
3657127d9f6Sml 			if (is_lso) {
3667127d9f6Sml 				/*
3677127d9f6Sml 				 * free the current mp and mp_chain if not FULL.
3687127d9f6Sml 				 */
3697127d9f6Sml 				tdc_stats->tx_no_desc++;
3707127d9f6Sml 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
3717127d9f6Sml 				    "LSO packet: TX Descriptor ring is full: "
3727127d9f6Sml 				    "channel %d",
3737127d9f6Sml 				    tx_ring_p->tdc));
3747127d9f6Sml 				goto nxge_start_fail_lso;
3757127d9f6Sml 			} else {
37675d94465SJosef 'Jeff' Sipek 				(void) atomic_cas_32(
37775d94465SJosef 'Jeff' Sipek 				    (uint32_t *)&tx_ring_p->queueing, 0, 1);
3787127d9f6Sml 				tdc_stats->tx_no_desc++;
3807127d9f6Sml 				if (isLDOMservice(nxgep)) {
3817127d9f6Sml 					tx_ring_p->tx_ring_busy = B_FALSE;
3827127d9f6Sml 					if (tx_ring_p->tx_ring_offline) {
3837127d9f6Sml 						(void) atomic_swap_32(
3847127d9f6Sml 						    &tx_ring_p->tx_ring_offline,
3857127d9f6Sml 						    NXGE_TX_RING_OFFLINED);
3867127d9f6Sml 					}
3876895688eSspeer 				}
3897127d9f6Sml 				MUTEX_EXIT(&tx_ring_p->lock);
3907127d9f6Sml 				status = 1;
3917127d9f6Sml 				goto nxge_start_fail1;
39230ac2e7bSml 			}
39344961713Sgirish 		}
39444961713Sgirish 	}
39644961713Sgirish 	nmp = mp;
39744961713Sgirish 	i = sop_index = tx_ring_p->wr_index;
39844961713Sgirish 	nmblks = 0;
39944961713Sgirish 	ngathers = 0;
40044961713Sgirish 	pkt_len = 0;
40144961713Sgirish 	pack_len = 0;
40244961713Sgirish 	clen = 0;
40344961713Sgirish 	last_bidx = -1;
40444961713Sgirish 	good_packet = B_TRUE;
40644961713Sgirish 	desc_area = tx_ring_p->tdc_desc;
40744961713Sgirish 	npi_handle = desc_area.npi_handle;
40844961713Sgirish 	npi_desc_handle.regh = (nxge_os_acc_handle_t)
40952ccf843Smisaki 	    DMA_COMMON_ACC_HANDLE(desc_area);
41044961713Sgirish 	tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
41144961713Sgirish 	tx_desc_dma_handle = (nxge_os_dma_handle_t)
41252ccf843Smisaki 	    DMA_COMMON_HANDLE(desc_area);
41344961713Sgirish 	tx_msg_ring = tx_ring_p->tx_msg_ring;
41544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
41652ccf843Smisaki 	    sop_index, i));
41844961713Sgirish #ifdef	NXGE_DEBUG
41944961713Sgirish 	msgsize = msgdsize(nmp);
42044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
42152ccf843Smisaki 	    "==> nxge_start(1): wr_index %d i %d msgdsize %d",
42252ccf843Smisaki 	    sop_index, i, msgsize));
42344961713Sgirish #endif
42444961713Sgirish 	/*
42544961713Sgirish 	 * The first 16 bytes of the premapped buffer are reserved
42644961713Sgirish 	 * for header. No padding will be used.
42744961713Sgirish 	 */
42844961713Sgirish 	pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
4291f8914d5Sml 	if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
43044961713Sgirish 		bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
43144961713Sgirish 	} else {
43244961713Sgirish 		bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
43344961713Sgirish 	}
43444961713Sgirish 	while (nmp) {
43544961713Sgirish 		good_packet = B_TRUE;
43644961713Sgirish 		b_rptr = nmp->b_rptr;
43744961713Sgirish 		len = MBLKL(nmp);
43844961713Sgirish 		if (len <= 0) {
43944961713Sgirish 			nmp = nmp->b_cont;
44044961713Sgirish 			continue;
44144961713Sgirish 		}
44244961713Sgirish 		nmblks++;
44444961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
44552ccf843Smisaki 		    "len %d pkt_len %d pack_len %d",
44652ccf843Smisaki 		    nmblks, len, pkt_len, pack_len));
44744961713Sgirish 		/*
44814ea4bb7Ssd 		 * Hardware limits the transfer length to 4K for NIU and
44914ea4bb7Ssd 		 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
45014ea4bb7Ssd 		 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
45114ea4bb7Ssd 		 * If len is longer than the limit, then we break nmp into
45214ea4bb7Ssd 		 * two chunks: Make the first chunk equal to the limit and
45314ea4bb7Ssd 		 * the second chunk for the remaining data. If the second
45414ea4bb7Ssd 		 * chunk is still larger than the limit, then it will be
45514ea4bb7Ssd 		 * broken into two in the next pass.
45644961713Sgirish 		 */
45714ea4bb7Ssd 		if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
45853f3d8ecSyc 			if ((t_mp = dupb(nmp)) != NULL) {
45953f3d8ecSyc 				nmp->b_wptr = nmp->b_rptr +
46053f3d8ecSyc 				    (TX_MAX_TRANSFER_LENGTH
46153f3d8ecSyc 				    - TX_PKT_HEADER_SIZE);
46253f3d8ecSyc 				t_mp->b_rptr = nmp->b_wptr;
46353f3d8ecSyc 				t_mp->b_cont = nmp->b_cont;
46453f3d8ecSyc 				nmp->b_cont = t_mp;
46553f3d8ecSyc 				len = MBLKL(nmp);
46653f3d8ecSyc 			} else {
46730ac2e7bSml 				if (is_lso) {
46830ac2e7bSml 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
46930ac2e7bSml 					    "LSO packet: dupb failed: "
47030ac2e7bSml 					    "channel %d",
47130ac2e7bSml 					    tx_ring_p->tdc));
47230ac2e7bSml 					mp = nmp;
47330ac2e7bSml 					goto nxge_start_fail_lso;
47430ac2e7bSml 				} else {
47530ac2e7bSml 					good_packet = B_FALSE;
47630ac2e7bSml 					goto nxge_start_fail2;
47730ac2e7bSml 				}
47853f3d8ecSyc 			}
47944961713Sgirish 		}
48044961713Sgirish 		tx_desc.value = 0;
48144961713Sgirish 		tx_desc_p = &tx_desc_ring_vp[i];
48244961713Sgirish #ifdef	NXGE_DEBUG
48344961713Sgirish 		tx_desc_pp = &tx_desc_ring_pp[i];
48444961713Sgirish #endif
48544961713Sgirish 		tx_msg_p = &tx_msg_ring[i];
48644961713Sgirish 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
48744961713Sgirish 		if (!header_set &&
48852ccf843Smisaki 		    ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
48952ccf843Smisaki 		    (len >= bcopy_thresh))) {
49044961713Sgirish 			header_set = B_TRUE;
49144961713Sgirish 			bcopy_thresh += TX_PKT_HEADER_SIZE;
49244961713Sgirish 			boff = 0;
49344961713Sgirish 			pack_len = 0;
49444961713Sgirish 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
49544961713Sgirish 			hdrp = (p_tx_pkt_header_t)kaddr;
49644961713Sgirish 			clen = pkt_len;
49744961713Sgirish 			dma_handle = tx_msg_p->buf_dma_handle;
49844961713Sgirish 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
49944961713Sgirish 			(void) ddi_dma_sync(dma_handle,
50052ccf843Smisaki 			    i * nxge_bcopy_thresh, nxge_bcopy_thresh,
50152ccf843Smisaki 			    DDI_DMA_SYNC_FORDEV);
50344961713Sgirish 			tx_msg_p->flags.dma_type = USE_BCOPY;
50444961713Sgirish 			goto nxge_start_control_header_only;
50544961713Sgirish 		}
50744961713Sgirish 		pkt_len += len;
50844961713Sgirish 		pack_len += len;
51044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
51152ccf843Smisaki 		    "desc entry %d "
51252ccf843Smisaki 		    "DESC IOADDR $%p "
51352ccf843Smisaki 		    "desc_vp $%p tx_desc_p $%p "
51452ccf843Smisaki 		    "desc_pp $%p tx_desc_pp $%p "
51552ccf843Smisaki 		    "len %d pkt_len %d pack_len %d",
51652ccf843Smisaki 		    i,
51752ccf843Smisaki 		    DMA_COMMON_IOADDR(desc_area),
51852ccf843Smisaki 		    tx_desc_ring_vp, tx_desc_p,
51952ccf843Smisaki 		    tx_desc_ring_pp, tx_desc_pp,
52052ccf843Smisaki 		    len, pkt_len, pack_len));
52244961713Sgirish 		if (len < bcopy_thresh) {
52344961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
52452ccf843Smisaki 			    "USE BCOPY: "));
52544961713Sgirish 			if (nxge_tx_tiny_pack) {
52644961713Sgirish 				uint32_t blst =
52752ccf843Smisaki 				    TXDMA_DESC_NEXT_INDEX(i, -1,
52852ccf843Smisaki 				    tx_ring_p->tx_wrap_mask);
52944961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
53052ccf843Smisaki 				    "==> nxge_start(5): pack"));
53144961713Sgirish 				if ((pack_len <= bcopy_thresh) &&
53252ccf843Smisaki 				    (last_bidx == blst)) {
53344961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
53452ccf843Smisaki 					    "==> nxge_start: pack(6) "
53552ccf843Smisaki 					    "(pkt_len %d pack_len %d)",
53652ccf843Smisaki 					    pkt_len, pack_len));
53744961713Sgirish 					i = blst;
53844961713Sgirish 					tx_desc_p = &tx_desc_ring_vp[i];
53944961713Sgirish #ifdef	NXGE_DEBUG
54044961713Sgirish 					tx_desc_pp = &tx_desc_ring_pp[i];
54144961713Sgirish #endif
54244961713Sgirish 					tx_msg_p = &tx_msg_ring[i];
54344961713Sgirish 					boff = pack_len - len;
54444961713Sgirish 					ngathers--;
545a3c5bd6dSspeer 				} else if (pack_len > bcopy_thresh &&
54652ccf843Smisaki 				    header_set) {
54744961713Sgirish 					pack_len = len;
54844961713Sgirish 					boff = 0;
54944961713Sgirish 					bcopy_thresh = nxge_bcopy_thresh;
55044961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
55152ccf843Smisaki 					    "==> nxge_start(7): > max NEW "
55252ccf843Smisaki 					    "bcopy thresh %d "
55352ccf843Smisaki 					    "pkt_len %d pack_len %d(next)",
55452ccf843Smisaki 					    bcopy_thresh,
55552ccf843Smisaki 					    pkt_len, pack_len));
55644961713Sgirish 				}
55744961713Sgirish 				last_bidx = i;
55844961713Sgirish 			}
55944961713Sgirish 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
56044961713Sgirish 			if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
56144961713Sgirish 				hdrp = (p_tx_pkt_header_t)kaddr;
56244961713Sgirish 				header_set = B_TRUE;
56344961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
56452ccf843Smisaki 				    "==> nxge_start(7_x2): "
56552ccf843Smisaki 				    "pkt_len %d pack_len %d (new hdrp $%p)",
56652ccf843Smisaki 				    pkt_len, pack_len, hdrp));
56744961713Sgirish 			}
56844961713Sgirish 			tx_msg_p->flags.dma_type = USE_BCOPY;
56944961713Sgirish 			kaddr += boff;
57044961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
57152ccf843Smisaki 			    "USE BCOPY: before bcopy "
57252ccf843Smisaki 			    "DESC IOADDR $%p entry %d "
57352ccf843Smisaki 			    "bcopy packets %d "
57452ccf843Smisaki 			    "bcopy kaddr $%p "
57552ccf843Smisaki 			    "bcopy ioaddr (SAD) $%p "
57652ccf843Smisaki 			    "bcopy clen %d "
57752ccf843Smisaki 			    "bcopy boff %d",
57852ccf843Smisaki 			    DMA_COMMON_IOADDR(desc_area), i,
57952ccf843Smisaki 			    tdc_stats->tx_hdr_pkts,
58052ccf843Smisaki 			    kaddr,
58152ccf843Smisaki 			    dma_ioaddr,
58252ccf843Smisaki 			    clen,
58352ccf843Smisaki 			    boff));
58444961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
58552ccf843Smisaki 			    "1USE BCOPY: "));
58644961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
58752ccf843Smisaki 			    "2USE BCOPY: "));
58844961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
58952ccf843Smisaki 			    "last USE BCOPY: copy from b_rptr $%p "
59052ccf843Smisaki 			    "to KADDR $%p (len %d offset %d",
59152ccf843Smisaki 			    b_rptr, kaddr, len, boff));
59344961713Sgirish 			bcopy(b_rptr, kaddr, len);
59544961713Sgirish #ifdef	NXGE_DEBUG
59644961713Sgirish 			dump_len = (len > 128) ? 128: len;
59744961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
59852ccf843Smisaki 			    "==> nxge_start: dump packets "
59952ccf843Smisaki 			    "(After BCOPY len %d)"
60052ccf843Smisaki 			    "(b_rptr $%p): %s", len, nmp->b_rptr,
60152ccf843Smisaki 			    nxge_dump_packet((char *)nmp->b_rptr,
60252ccf843Smisaki 			    dump_len)));
60344961713Sgirish #endif
60544961713Sgirish 			dma_handle = tx_msg_p->buf_dma_handle;
60644961713Sgirish 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
60744961713Sgirish 			(void) ddi_dma_sync(dma_handle,
60852ccf843Smisaki 			    i * nxge_bcopy_thresh, nxge_bcopy_thresh,
60952ccf843Smisaki 			    DDI_DMA_SYNC_FORDEV);
61044961713Sgirish 			clen = len + boff;
61144961713Sgirish 			tdc_stats->tx_hdr_pkts++;
61244961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
61352ccf843Smisaki 			    "USE BCOPY: "
61452ccf843Smisaki 			    "DESC IOADDR $%p entry %d "
61552ccf843Smisaki 			    "bcopy packets %d "
61652ccf843Smisaki 			    "bcopy kaddr $%p "
61752ccf843Smisaki 			    "bcopy ioaddr (SAD) $%p "
61852ccf843Smisaki 			    "bcopy clen %d "
61952ccf843Smisaki 			    "bcopy boff %d",
62052ccf843Smisaki 			    DMA_COMMON_IOADDR(desc_area),
62152ccf843Smisaki 			    i,
62252ccf843Smisaki 			    tdc_stats->tx_hdr_pkts,
62352ccf843Smisaki 			    kaddr,
62452ccf843Smisaki 			    dma_ioaddr,
62552ccf843Smisaki 			    clen,
62652ccf843Smisaki 			    boff));
62744961713Sgirish 		} else {
62844961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
62952ccf843Smisaki 			    "USE DVMA: len %d", len));
63044961713Sgirish 			tx_msg_p->flags.dma_type = USE_DMA;
63144961713Sgirish 			dma_flags = DDI_DMA_WRITE;
63244961713Sgirish 			if (len < nxge_dma_stream_thresh) {
63344961713Sgirish 				dma_flags |= DDI_DMA_CONSISTENT;
63444961713Sgirish 			} else {
63544961713Sgirish 				dma_flags |= DDI_DMA_STREAMING;
63644961713Sgirish 			}
63844961713Sgirish 			dma_handle = tx_msg_p->dma_handle;
63931f519f9SMichael Speer 			dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
64052ccf843Smisaki 			    (caddr_t)b_rptr, len, dma_flags,
64152ccf843Smisaki 			    DDI_DMA_DONTWAIT, NULL,
64252ccf843Smisaki 			    &dma_cookie, &ncookies);
64331f519f9SMichael Speer 			if (dma_status == DDI_DMA_MAPPED) {
64444961713Sgirish 				dma_ioaddr = dma_cookie.dmac_laddress;
64544961713Sgirish 				len = (int)dma_cookie.dmac_size;
64644961713Sgirish 				clen = (uint32_t)dma_cookie.dmac_size;
64744961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
64852ccf843Smisaki 				    "==> nxge_start(12_1): "
64952ccf843Smisaki 				    "USE DVMA: len %d clen %d "
65052ccf843Smisaki 				    "ngathers %d",
65152ccf843Smisaki 				    len, clen,
65252ccf843Smisaki 				    ngathers));
65344961713Sgirish 				npi_desc_handle.regp = (uint64_t)tx_desc_p;
65444961713Sgirish 				while (ncookies > 1) {
65544961713Sgirish 					ngathers++;
65644961713Sgirish 					/*
65744961713Sgirish 					 * this is the fix for multiple
65830ac2e7bSml 					 * cookies, which are basically
65944961713Sgirish 					 * a descriptor entry, we don't set
66044961713Sgirish 					 * SOP bit as well as related fields
66144961713Sgirish 					 */
66344961713Sgirish 					(void) npi_txdma_desc_gather_set(
66452ccf843Smisaki 					    npi_desc_handle,
66552ccf843Smisaki 					    &tx_desc,
66652ccf843Smisaki 					    (ngathers -1),
66752ccf843Smisaki 					    mark_mode,
66852ccf843Smisaki 					    ngathers,
66952ccf843Smisaki 					    dma_ioaddr,
67052ccf843Smisaki 					    clen);
67244961713Sgirish 					tx_msg_p->tx_msg_size = clen;
67344961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
67452ccf843Smisaki 					    "==> nxge_start:  DMA "
67552ccf843Smisaki 					    "ncookie %d "
67652ccf843Smisaki 					    "ngathers %d "
67752ccf843Smisaki 					    "dma_ioaddr $%p len %d"
67852ccf843Smisaki 					    "desc $%p descp $%p (%d)",
67952ccf843Smisaki 					    ncookies,
68052ccf843Smisaki 					    ngathers,
68152ccf843Smisaki 					    dma_ioaddr, clen,
68252ccf843Smisaki 					    *tx_desc_p, tx_desc_p, i));
68444961713Sgirish 					ddi_dma_nextcookie(dma_handle,
68552ccf843Smisaki 					    &dma_cookie);
68644961713Sgirish 					dma_ioaddr =
68752ccf843Smisaki 					    dma_cookie.dmac_laddress;
68944961713Sgirish 					len = (int)dma_cookie.dmac_size;
69044961713Sgirish 					clen = (uint32_t)dma_cookie.dmac_size;
69144961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
69252ccf843Smisaki 					    "==> nxge_start(12_2): "
69352ccf843Smisaki 					    "USE DVMA: len %d clen %d ",
69452ccf843Smisaki 					    len, clen));
69644961713Sgirish 					i = TXDMA_DESC_NEXT_INDEX(i, 1,
69752ccf843Smisaki 					    tx_ring_p->tx_wrap_mask);
69844961713Sgirish 					tx_desc_p = &tx_desc_ring_vp[i];
70052ccf843Smisaki 					npi_desc_handle.regp =
70152ccf843Smisaki 					    (uint64_t)tx_desc_p;
70244961713Sgirish 					tx_msg_p = &tx_msg_ring[i];
70344961713Sgirish 					tx_msg_p->flags.dma_type = USE_NONE;
70444961713Sgirish 					tx_desc.value = 0;
70644961713Sgirish 					ncookies--;
70744961713Sgirish 				}
70844961713Sgirish 				tdc_stats->tx_ddi_pkts++;
70944961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
71052ccf843Smisaki 				    "DMA: ddi packets %d",
71152ccf843Smisaki 				    tdc_stats->tx_ddi_pkts));
71244961713Sgirish 			} else {
71344961713Sgirish 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
71444961713Sgirish 				    "dma mapping failed for %d "
71544961713Sgirish 				    "bytes addr $%p flags %x (%d)",
71644961713Sgirish 				    len, b_rptr, status, status));
71744961713Sgirish 				good_packet = B_FALSE;
71844961713Sgirish 				tdc_stats->tx_dma_bind_fail++;
71944961713Sgirish 				tx_msg_p->flags.dma_type = USE_NONE;
72030ac2e7bSml 				if (is_lso) {
72130ac2e7bSml 					mp = nmp;
72230ac2e7bSml 					goto nxge_start_fail_lso;
72330ac2e7bSml 				} else {
72431f519f9SMichael Speer 					status = 1;
72530ac2e7bSml 					goto nxge_start_fail2;
72630ac2e7bSml 				}
72744961713Sgirish 			}
72844961713Sgirish 		} /* ddi dvma */
73030ac2e7bSml 		if (is_lso) {
73130ac2e7bSml 			nmp_lso_save = nmp;
73230ac2e7bSml 		}
73344961713Sgirish 		nmp = nmp->b_cont;
73444961713Sgirish nxge_start_control_header_only:
73544961713Sgirish 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
73644961713Sgirish 		ngathers++;
73844961713Sgirish 		if (ngathers == 1) {
73944961713Sgirish #ifdef	NXGE_DEBUG
74044961713Sgirish 			save_desc_p = &sop_tx_desc;
74144961713Sgirish #endif
74244961713Sgirish 			sop_tx_desc_p = &sop_tx_desc;
74344961713Sgirish 			sop_tx_desc_p->value = 0;
74444961713Sgirish 			sop_tx_desc_p->bits.hdw.tr_len = clen;
74544961713Sgirish 			sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
74644961713Sgirish 			sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
74744961713Sgirish 		} else {
74844961713Sgirish #ifdef	NXGE_DEBUG
74944961713Sgirish 			save_desc_p = &tx_desc;
75044961713Sgirish #endif
75144961713Sgirish 			tmp_desc_p = &tx_desc;
75244961713Sgirish 			tmp_desc_p->value = 0;
75344961713Sgirish 			tmp_desc_p->bits.hdw.tr_len = clen;
75444961713Sgirish 			tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
75544961713Sgirish 			tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
75744961713Sgirish 			tx_desc_p->value = tmp_desc_p->value;
75844961713Sgirish 		}
76044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
76152ccf843Smisaki 		    "Desc_entry %d ngathers %d "
76252ccf843Smisaki 		    "desc_vp $%p tx_desc_p $%p "
76352ccf843Smisaki 		    "len %d clen %d pkt_len %d pack_len %d nmblks %d "
76452ccf843Smisaki 		    "dma_ioaddr (SAD) $%p mark %d",
76552ccf843Smisaki 		    i, ngathers,
76652ccf843Smisaki 		    tx_desc_ring_vp, tx_desc_p,
76752ccf843Smisaki 		    len, clen, pkt_len, pack_len, nmblks,
76852ccf843Smisaki 		    dma_ioaddr, mark_mode));
77044961713Sgirish #ifdef NXGE_DEBUG
77144961713Sgirish 		npi_desc_handle.nxgep = nxgep;
77244961713Sgirish 		npi_desc_handle.function.function = nxgep->function_num;
77344961713Sgirish 		npi_desc_handle.function.instance = nxgep->instance;
77444961713Sgirish 		sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
77544961713Sgirish 		xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
77652ccf843Smisaki 		    TX_PKT_DESC_TR_LEN_SHIFT);
77944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
78052ccf843Smisaki 		    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
78152ccf843Smisaki 		    "mark %d sop %d\n",
78252ccf843Smisaki 		    save_desc_p->value,
78352ccf843Smisaki 		    sad,
78452ccf843Smisaki 		    save_desc_p->bits.hdw.tr_len,
78552ccf843Smisaki 		    xfer_len,
78652ccf843Smisaki 		    save_desc_p->bits.hdw.num_ptr,
78752ccf843Smisaki 		    save_desc_p->bits.hdw.mark,
78852ccf843Smisaki 		    save_desc_p->bits.hdw.sop));
79044961713Sgirish 		npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
79144961713Sgirish #endif
79344961713Sgirish 		tx_msg_p->tx_msg_size = clen;
79444961713Sgirish 		i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
79544961713Sgirish 		if (ngathers > nxge_tx_max_gathers) {
79644961713Sgirish 			good_packet = B_FALSE;
7970dc2366fSVenugopal Iyer 			mac_hcksum_get(mp, &start_offset,
79852ccf843Smisaki 			    &stuff_offset, &end_offset, &value,
79952ccf843Smisaki 			    &cksum_flags);
80144961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
80252ccf843Smisaki 			    "==> nxge_start(14): pull msg - "
80352ccf843Smisaki 			    "len %d pkt_len %d ngathers %d",
80452ccf843Smisaki 			    len, pkt_len, ngathers));
80531f519f9SMichael Speer 
80631f519f9SMichael Speer 			/*
80731f519f9SMichael Speer 			 * Just give up on this packet.
80831f519f9SMichael Speer 			 */
80930ac2e7bSml 			if (is_lso) {
81030ac2e7bSml 				mp = nmp_lso_save;
81130ac2e7bSml 				goto nxge_start_fail_lso;
81230ac2e7bSml 			}
81331f519f9SMichael Speer 			status = 0;
81444961713Sgirish 			goto nxge_start_fail2;
81544961713Sgirish 		}
81644961713Sgirish 	} /* while (nmp) */
81844961713Sgirish 	tx_msg_p->tx_message = mp;
81944961713Sgirish 	tx_desc_p = &tx_desc_ring_vp[sop_index];
82044961713Sgirish 	npi_desc_handle.regp = (uint64_t)tx_desc_p;
82244961713Sgirish 	pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
82344961713Sgirish 	pkthdrp->reserved = 0;
82444961713Sgirish 	hdrp->value = 0;
825b4d05839Sml 	bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));
82744961713Sgirish 	if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
82844961713Sgirish 		tdc_stats->tx_jumbo_pkts++;
82944961713Sgirish 	}
831678453a8Sspeer 	min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
83244961713Sgirish 	if (pkt_len < min_len) {
83344961713Sgirish 		/* Assume we use bcopy to premapped buffers */
83444961713Sgirish 		kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
83544961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
83652ccf843Smisaki 		    "==> nxge_start(14-1): < (msg_min + 16)"
83752ccf843Smisaki 		    "len %d pkt_len %d min_len %d bzero %d ngathers %d",
83852ccf843Smisaki 		    len, pkt_len, min_len, (min_len - pkt_len), ngathers));
83944961713Sgirish 		bzero((kaddr + pkt_len), (min_len - pkt_len));
84044961713Sgirish 		pkt_len = tx_msg_p->tx_msg_size = min_len;
84244961713Sgirish 		sop_tx_desc_p->bits.hdw.tr_len = min_len;
84444961713Sgirish 		NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
84544961713Sgirish 		tx_desc_p->value = sop_tx_desc_p->value;
84744961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
84852ccf843Smisaki 		    "==> nxge_start(14-2): < msg_min - "
84952ccf843Smisaki 		    "len %d pkt_len %d min_len %d ngathers %d",
85052ccf843Smisaki 		    len, pkt_len, min_len, ngathers));
85144961713Sgirish 	}
85344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
85452ccf843Smisaki 	    cksum_flags));
85544961713Sgirish 	{
85644961713Sgirish 		uint64_t	tmp_len;
85844961713Sgirish 		/* pkt_len already includes 16 + paddings!! */
85944961713Sgirish 		/* Update the control header length */
86044961713Sgirish 		tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
86144961713Sgirish 		tmp_len = hdrp->value |
86252ccf843Smisaki 		    (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
86444961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
86552ccf843Smisaki 		    "==> nxge_start(15_x1): setting SOP "
86652ccf843Smisaki 		    "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
86752ccf843Smisaki 		    "0x%llx hdrp->value 0x%llx",
86852ccf843Smisaki 		    tot_xfer_len, tot_xfer_len, pkt_len,
86952ccf843Smisaki 		    tmp_len, hdrp->value));
87044961713Sgirish #if defined(_BIG_ENDIAN)
87144961713Sgirish 		hdrp->value = ddi_swap64(tmp_len);
87244961713Sgirish #else
87344961713Sgirish 		hdrp->value = tmp_len;
87444961713Sgirish #endif
87544961713Sgirish 		NXGE_DEBUG_MSG((nxgep,
87652ccf843Smisaki 		    TX_CTL, "==> nxge_start(15_x2): setting SOP "
87752ccf843Smisaki 		    "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
87852ccf843Smisaki 		    "tmp_len 0x%llx hdrp->value 0x%llx",
87952ccf843Smisaki 		    tot_xfer_len, pkt_len,
88052ccf843Smisaki 		    tmp_len, hdrp->value));
88144961713Sgirish 	}
88344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
88452ccf843Smisaki 	    "wr_index %d "
88552ccf843Smisaki 	    "tot_xfer_len (%d) pkt_len %d npads %d",
88652ccf843Smisaki 	    sop_index,
88752ccf843Smisaki 	    tot_xfer_len, pkt_len,
88852ccf843Smisaki 	    npads));
89044961713Sgirish 	sop_tx_desc_p->bits.hdw.sop = 1;
89144961713Sgirish 	sop_tx_desc_p->bits.hdw.mark = mark_mode;
89244961713Sgirish 	sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
89444961713Sgirish 	NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
89644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
89844961713Sgirish #ifdef NXGE_DEBUG
89944961713Sgirish 	npi_desc_handle.nxgep = nxgep;
90044961713Sgirish 	npi_desc_handle.function.function = nxgep->function_num;
90144961713Sgirish 	npi_desc_handle.function.instance = nxgep->instance;
90344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
90452ccf843Smisaki 	    "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
90552ccf843Smisaki 	    save_desc_p->value,
90652ccf843Smisaki 	    sad,
90752ccf843Smisaki 	    save_desc_p->bits.hdw.tr_len,
90852ccf843Smisaki 	    xfer_len,
90952ccf843Smisaki 	    save_desc_p->bits.hdw.num_ptr,
91052ccf843Smisaki 	    save_desc_p->bits.hdw.mark,
91152ccf843Smisaki 	    save_desc_p->bits.hdw.sop));
91244961713Sgirish 	(void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
91444961713Sgirish 	dump_len = (pkt_len > 128) ? 128: pkt_len;
91544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
91652ccf843Smisaki 	    "==> nxge_start: dump packets(17) (after sop set, len "
91752ccf843Smisaki 	    " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
91852ccf843Smisaki 	    "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
91952ccf843Smisaki 	    (char *)hdrp,
92052ccf843Smisaki 	    nxge_dump_packet((char *)hdrp, dump_len)));
92144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
92252ccf843Smisaki 	    "==> nxge_start(18): TX desc sync: sop_index %d",
92352ccf843Smisaki 	    sop_index));
92444961713Sgirish #endif
92644961713Sgirish 	if ((ngathers == 1) || tx_ring_p->wr_index < i) {
92744961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
92852ccf843Smisaki 		    sop_index * sizeof (tx_desc_t),
92952ccf843Smisaki 		    ngathers * sizeof (tx_desc_t),
93052ccf843Smisaki 		    DDI_DMA_SYNC_FORDEV);
93244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
93352ccf843Smisaki 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
93452ccf843Smisaki 		    "pkt_len %d ngathers %d sop_index %d\n",
93552ccf843Smisaki 		    stuff_offset, start_offset,
93652ccf843Smisaki 		    pkt_len, ngathers, sop_index));
93744961713Sgirish 	} else { /* more than one descriptor and wrap around */
93844961713Sgirish 		uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
93944961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
94052ccf843Smisaki 		    sop_index * sizeof (tx_desc_t),
94152ccf843Smisaki 		    nsdescs * sizeof (tx_desc_t),
94252ccf843Smisaki 		    DDI_DMA_SYNC_FORDEV);
94344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
94452ccf843Smisaki 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
94552ccf843Smisaki 		    "pkt_len %d ngathers %d sop_index %d\n",
94652ccf843Smisaki 		    stuff_offset, start_offset,
94752ccf843Smisaki 		    pkt_len, ngathers, sop_index));
94944961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
95052ccf843Smisaki 		    0,
95152ccf843Smisaki 		    (ngathers - nsdescs) * sizeof (tx_desc_t),
95252ccf843Smisaki 		    DDI_DMA_SYNC_FORDEV);
95344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
95452ccf843Smisaki 		    "cs_off = 0x%02X cs_s_off = 0x%02X "
95552ccf843Smisaki 		    "pkt_len %d ngathers %d sop_index %d\n",
95652ccf843Smisaki 		    stuff_offset, start_offset,
95752ccf843Smisaki 		    pkt_len, ngathers, sop_index));
95844961713Sgirish 	}
96044961713Sgirish 	tail_index = tx_ring_p->wr_index;
96144961713Sgirish 	tail_wrap =