xref: /illumos-gate/usr/src/uts/common/io/nxge/nxge_send.c (revision 22c0d73a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/nxge/nxge_impl.h>
29 #include <sys/nxge/nxge_hio.h>
30 #include <npi_tx_wr64.h>
31 
32 /* Software LSO required header files */
33 #include <netinet/tcp.h>
34 #include <inet/ip_impl.h>
35 #include <inet/tcp.h>
36 
37 static mblk_t *nxge_lso_eliminate(mblk_t *);
38 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
39 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
40 static void nxge_hcksum_retrieve(mblk_t *,
41     uint32_t *, uint32_t *, uint32_t *,
42     uint32_t *, uint32_t *);
43 static uint32_t nxge_csgen(uint16_t *, int);
44 
45 extern uint32_t		nxge_reclaim_pending;
46 extern uint32_t 	nxge_bcopy_thresh;
47 extern uint32_t 	nxge_dvma_thresh;
48 extern uint32_t 	nxge_dma_stream_thresh;
49 extern uint32_t		nxge_tx_minfree;
50 extern uint32_t		nxge_tx_intr_thres;
51 extern uint32_t		nxge_tx_max_gathers;
52 extern uint32_t		nxge_tx_tiny_pack;
53 extern uint32_t		nxge_tx_use_bcopy;
54 extern uint32_t		nxge_tx_lb_policy;
55 extern uint32_t		nxge_no_tx_lb;
56 extern nxge_tx_mode_t	nxge_tx_scheme;
57 uint32_t		nxge_lso_kick_cnt = 2;
58 
59 typedef struct _mac_tx_hint {
60 	uint16_t	sap;
61 	uint16_t	vid;
62 	void		*hash;
63 } mac_tx_hint_t, *p_mac_tx_hint_t;
64 
65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t);
66 
67 int
68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
69 {
70 	int 			status = 0;
71 	p_tx_desc_t 		tx_desc_ring_vp;
72 	npi_handle_t		npi_desc_handle;
73 	nxge_os_dma_handle_t 	tx_desc_dma_handle;
74 	p_tx_desc_t 		tx_desc_p;
75 	p_tx_msg_t 		tx_msg_ring;
76 	p_tx_msg_t 		tx_msg_p;
77 	tx_desc_t		tx_desc, *tmp_desc_p;
78 	tx_desc_t		sop_tx_desc, *sop_tx_desc_p;
79 	p_tx_pkt_header_t	hdrp;
80 	tx_pkt_header_t		tmp_hdrp;
81 	p_tx_pkt_hdr_all_t	pkthdrp;
82 	uint8_t			npads = 0;
83 	uint64_t 		dma_ioaddr;
84 	uint32_t		dma_flags;
85 	int			last_bidx;
86 	uint8_t 		*b_rptr;
87 	caddr_t 		kaddr;
88 	uint32_t		nmblks;
89 	uint32_t		ngathers;
90 	uint32_t		clen;
91 	int 			len;
92 	uint32_t		pkt_len, pack_len, min_len;
93 	uint32_t		bcopy_thresh;
94 	int 			i, cur_index, sop_index;
95 	uint16_t		tail_index;
96 	boolean_t		tail_wrap = B_FALSE;
97 	nxge_dma_common_t	desc_area;
98 	nxge_os_dma_handle_t 	dma_handle;
99 	ddi_dma_cookie_t 	dma_cookie;
100 	npi_handle_t		npi_handle;
101 	p_mblk_t 		nmp;
102 	p_mblk_t		t_mp;
103 	uint32_t 		ncookies;
104 	boolean_t 		good_packet;
105 	boolean_t 		mark_mode = B_FALSE;
106 	p_nxge_stats_t 		statsp;
107 	p_nxge_tx_ring_stats_t tdc_stats;
108 	t_uscalar_t 		start_offset = 0;
109 	t_uscalar_t 		stuff_offset = 0;
110 	t_uscalar_t 		end_offset = 0;
111 	t_uscalar_t 		value = 0;
112 	t_uscalar_t 		cksum_flags = 0;
113 	boolean_t		cksum_on = B_FALSE;
114 	uint32_t		boff = 0;
115 	uint64_t		tot_xfer_len = 0;
116 	boolean_t		header_set = B_FALSE;
117 #ifdef NXGE_DEBUG
118 	p_tx_desc_t 		tx_desc_ring_pp;
119 	p_tx_desc_t 		tx_desc_pp;
120 	tx_desc_t		*save_desc_p;
121 	int			dump_len;
122 	int			sad_len;
123 	uint64_t		sad;
124 	int			xfer_len;
125 	uint32_t		msgsize;
126 #endif
127 	p_mblk_t 		mp_chain = NULL;
128 	boolean_t		is_lso = B_FALSE;
129 	boolean_t		lso_again;
130 	int			cur_index_lso;
131 	p_mblk_t 		nmp_lso_save;
132 	uint32_t		lso_ngathers;
133 	boolean_t		lso_tail_wrap = B_FALSE;
134 
135 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
136 	    "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
137 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
138 	    "==> nxge_start: Starting tdc %d desc pending %d",
139 	    tx_ring_p->tdc, tx_ring_p->descs_pending));
140 
141 	statsp = nxgep->statsp;
142 
143 	if (!isLDOMguest(nxgep)) {
144 		switch (nxgep->mac.portmode) {
145 		default:
146 			if (nxgep->statsp->port_stats.lb_mode ==
147 			    nxge_lb_normal) {
148 				if (!statsp->mac_stats.link_up) {
149 					freemsg(mp);
150 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
151 					    "==> nxge_start: "
152 					    "link not up"));
153 					goto nxge_start_fail1;
154 				}
155 			}
156 			break;
157 		case PORT_10G_FIBER:
158 			/*
159 			 * For the following modes, check the link status
160 			 * before sending the packet out:
161 			 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g
162 			 */
163 			if (nxgep->statsp->port_stats.lb_mode <
164 			    nxge_lb_serdes10g) {
165 				if (!statsp->mac_stats.link_up) {
166 					freemsg(mp);
167 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
168 					    "==> nxge_start: "
169 					    "link not up"));
170 					goto nxge_start_fail1;
171 				}
172 			}
173 			break;
174 		}
175 	}
176 
177 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
178 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
179 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
180 		    "==> nxge_start: hardware not initialized or stopped"));
181 		freemsg(mp);
182 		goto nxge_start_fail1;
183 	}
184 
185 	if (nxgep->soft_lso_enable) {
186 		mp_chain = nxge_lso_eliminate(mp);
187 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
188 		    "==> nxge_start(0): LSO mp $%p mp_chain $%p",
189 		    mp, mp_chain));
190 		if (mp_chain == NULL) {
191 			NXGE_ERROR_MSG((nxgep, TX_CTL,
192 			    "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
193 			    mp_chain, mp));
194 			goto nxge_start_fail1;
195 		}
196 		if (mp_chain != mp) {
197 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
198 			    "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
199 			    mp_chain, mp));
200 			is_lso = B_TRUE;
201 			mp = mp_chain;
202 			mp_chain = mp_chain->b_next;
203 			mp->b_next = NULL;
204 		}
205 	}
206 
207 	hcksum_retrieve(mp, NULL, NULL, &start_offset,
208 		&stuff_offset, &end_offset, &value, &cksum_flags);
209 	if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
210 		start_offset += sizeof (ether_header_t);
211 		stuff_offset += sizeof (ether_header_t);
212 	} else {
213 		start_offset += sizeof (struct ether_vlan_header);
214 		stuff_offset += sizeof (struct ether_vlan_header);
215 	}
216 
217 	if (cksum_flags & HCK_PARTIALCKSUM) {
218 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
219 			"==> nxge_start: mp $%p len %d "
220 			"cksum_flags 0x%x (partial checksum) ",
221 			mp, MBLKL(mp), cksum_flags));
222 		cksum_on = B_TRUE;
223 	}
224 
225 	pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
226 	pkthdrp->reserved = 0;
227 	tmp_hdrp.value = 0;
228 	nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
229 	    0, 0, pkthdrp,
230 	    start_offset, stuff_offset);
231 
232 	lso_again = B_FALSE;
233 	lso_ngathers = 0;
234 
235 	MUTEX_ENTER(&tx_ring_p->lock);
236 
237 	if (isLDOMservice(nxgep)) {
238 		if (tx_ring_p->tx_ring_offline) {
239 			freemsg(mp);
240 			MUTEX_EXIT(&tx_ring_p->lock);
241 			return (status);
242 		}
243 	}
244 
245 	cur_index_lso = tx_ring_p->wr_index;
246 	lso_tail_wrap = tx_ring_p->wr_index_wrap;
247 start_again:
248 	ngathers = 0;
249 	sop_index = tx_ring_p->wr_index;
250 #ifdef	NXGE_DEBUG
251 	if (tx_ring_p->descs_pending) {
252 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
253 			"desc pending %d ", tx_ring_p->descs_pending));
254 	}
255 
256 	dump_len = (int)(MBLKL(mp));
257 	dump_len = (dump_len > 128) ? 128: dump_len;
258 
259 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
260 		"==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
261 		"(Before header reserve: ORIGINAL LEN %d)",
262 		tx_ring_p->tdc,
263 		mp->b_rptr,
264 		dump_len));
265 
266 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
267 		"(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
268 		nxge_dump_packet((char *)mp->b_rptr, dump_len)));
269 #endif
270 
271 	tdc_stats = tx_ring_p->tdc_stats;
272 	mark_mode = (tx_ring_p->descs_pending &&
273 		((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending)
274 		< nxge_tx_minfree));
275 
276 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
277 		"TX Descriptor ring is channel %d mark mode %d",
278 		tx_ring_p->tdc, mark_mode));
279 
280 	if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) {
281 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
282 			"TX Descriptor ring is full: channel %d",
283 			tx_ring_p->tdc));
284 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
285 			"TX Descriptor ring is full: channel %d",
286 			tx_ring_p->tdc));
287 		if (is_lso) {
288 			/* free the current mp and mp_chain if not FULL */
289 			tdc_stats->tx_no_desc++;
290 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
291 			    "LSO packet: TX Descriptor ring is full: "
292 			    "channel %d",
293 			    tx_ring_p->tdc));
294 			goto nxge_start_fail_lso;
295 		} else {
296 			boolean_t skip_sched = B_FALSE;
297 
298 			cas32((uint32_t *)&tx_ring_p->queueing, 0, 1);
299 			tdc_stats->tx_no_desc++;
300 			if (isLDOMservice(nxgep) &&
301 				tx_ring_p->tx_ring_offline) {
302 				(void) atomic_swap_32(
303 				    &tx_ring_p->tx_ring_offline,
304 				    NXGE_TX_RING_OFFLINED);
305 				skip_sched = B_TRUE;
306 			}
307 
308 			MUTEX_EXIT(&tx_ring_p->lock);
309 			if (nxgep->resched_needed &&
310 			    !nxgep->resched_running && !skip_sched) {
311 				nxgep->resched_running = B_TRUE;
312 				ddi_trigger_softintr(nxgep->resched_id);
313 			}
314 			status = 1;
315 			goto nxge_start_fail1;
316 		}
317 	}
318 
319 	nmp = mp;
320 	i = sop_index = tx_ring_p->wr_index;
321 	nmblks = 0;
322 	ngathers = 0;
323 	pkt_len = 0;
324 	pack_len = 0;
325 	clen = 0;
326 	last_bidx = -1;
327 	good_packet = B_TRUE;
328 
329 	desc_area = tx_ring_p->tdc_desc;
330 	npi_handle = desc_area.npi_handle;
331 	npi_desc_handle.regh = (nxge_os_acc_handle_t)
332 			DMA_COMMON_ACC_HANDLE(desc_area);
333 	tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
334 	tx_desc_dma_handle = (nxge_os_dma_handle_t)
335 			DMA_COMMON_HANDLE(desc_area);
336 	tx_msg_ring = tx_ring_p->tx_msg_ring;
337 
338 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
339 		sop_index, i));
340 
341 #ifdef	NXGE_DEBUG
342 	msgsize = msgdsize(nmp);
343 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
344 		"==> nxge_start(1): wr_index %d i %d msgdsize %d",
345 		sop_index, i, msgsize));
346 #endif
347 	/*
348 	 * The first 16 bytes of the premapped buffer are reserved
349 	 * for header. No padding will be used.
350 	 */
351 	pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
352 	if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
353 		bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
354 	} else {
355 		bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
356 	}
357 	while (nmp) {
358 		good_packet = B_TRUE;
359 		b_rptr = nmp->b_rptr;
360 		len = MBLKL(nmp);
361 		if (len <= 0) {
362 			nmp = nmp->b_cont;
363 			continue;
364 		}
365 		nmblks++;
366 
367 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
368 			"len %d pkt_len %d pack_len %d",
369 			nmblks, len, pkt_len, pack_len));
370 		/*
371 		 * Hardware limits the transfer length to 4K for NIU and
372 		 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
373 		 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
374 		 * If len is longer than the limit, then we break nmp into
375 		 * two chunks: Make the first chunk equal to the limit and
376 		 * the second chunk for the remaining data. If the second
377 		 * chunk is still larger than the limit, then it will be
378 		 * broken into two in the next pass.
379 		 */
380 		if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
381 			if ((t_mp = dupb(nmp)) != NULL) {
382 				nmp->b_wptr = nmp->b_rptr +
383 				    (TX_MAX_TRANSFER_LENGTH
384 				    - TX_PKT_HEADER_SIZE);
385 				t_mp->b_rptr = nmp->b_wptr;
386 				t_mp->b_cont = nmp->b_cont;
387 				nmp->b_cont = t_mp;
388 				len = MBLKL(nmp);
389 			} else {
390 				if (is_lso) {
391 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
392 					    "LSO packet: dupb failed: "
393 					    "channel %d",
394 					    tx_ring_p->tdc));
395 					mp = nmp;
396 					goto nxge_start_fail_lso;
397 				} else {
398 					good_packet = B_FALSE;
399 					goto nxge_start_fail2;
400 				}
401 			}
402 		}
403 		tx_desc.value = 0;
404 		tx_desc_p = &tx_desc_ring_vp[i];
405 #ifdef	NXGE_DEBUG
406 		tx_desc_pp = &tx_desc_ring_pp[i];
407 #endif
408 		tx_msg_p = &tx_msg_ring[i];
409 #if defined(__i386)
410 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
411 #else
412 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
413 #endif
414 		if (!header_set &&
415 			((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
416 				(len >= bcopy_thresh))) {
417 			header_set = B_TRUE;
418 			bcopy_thresh += TX_PKT_HEADER_SIZE;
419 			boff = 0;
420 			pack_len = 0;
421 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
422 			hdrp = (p_tx_pkt_header_t)kaddr;
423 			clen = pkt_len;
424 			dma_handle = tx_msg_p->buf_dma_handle;
425 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
426 			(void) ddi_dma_sync(dma_handle,
427 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
428 				DDI_DMA_SYNC_FORDEV);
429 
430 			tx_msg_p->flags.dma_type = USE_BCOPY;
431 			goto nxge_start_control_header_only;
432 		}
433 
434 		pkt_len += len;
435 		pack_len += len;
436 
437 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
438 			"desc entry %d "
439 			"DESC IOADDR $%p "
440 			"desc_vp $%p tx_desc_p $%p "
441 			"desc_pp $%p tx_desc_pp $%p "
442 			"len %d pkt_len %d pack_len %d",
443 			i,
444 			DMA_COMMON_IOADDR(desc_area),
445 			tx_desc_ring_vp, tx_desc_p,
446 			tx_desc_ring_pp, tx_desc_pp,
447 			len, pkt_len, pack_len));
448 
449 		if (len < bcopy_thresh) {
450 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
451 				"USE BCOPY: "));
452 			if (nxge_tx_tiny_pack) {
453 				uint32_t blst =
454 					TXDMA_DESC_NEXT_INDEX(i, -1,
455 						tx_ring_p->tx_wrap_mask);
456 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
457 					"==> nxge_start(5): pack"));
458 				if ((pack_len <= bcopy_thresh) &&
459 					(last_bidx == blst)) {
460 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
461 						"==> nxge_start: pack(6) "
462 						"(pkt_len %d pack_len %d)",
463 						pkt_len, pack_len));
464 					i = blst;
465 					tx_desc_p = &tx_desc_ring_vp[i];
466 #ifdef	NXGE_DEBUG
467 					tx_desc_pp = &tx_desc_ring_pp[i];
468 #endif
469 					tx_msg_p = &tx_msg_ring[i];
470 					boff = pack_len - len;
471 					ngathers--;
472 				} else if (pack_len > bcopy_thresh &&
473 					header_set) {
474 					pack_len = len;
475 					boff = 0;
476 					bcopy_thresh = nxge_bcopy_thresh;
477 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
478 						"==> nxge_start(7): > max NEW "
479 						"bcopy thresh %d "
480 						"pkt_len %d pack_len %d(next)",
481 						bcopy_thresh,
482 						pkt_len, pack_len));
483 				}
484 				last_bidx = i;
485 			}
486 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
487 			if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
488 				hdrp = (p_tx_pkt_header_t)kaddr;
489 				header_set = B_TRUE;
490 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
491 					"==> nxge_start(7_x2): "
492 					"pkt_len %d pack_len %d (new hdrp $%p)",
493 					pkt_len, pack_len, hdrp));
494 			}
495 			tx_msg_p->flags.dma_type = USE_BCOPY;
496 			kaddr += boff;
497 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
498 				"USE BCOPY: before bcopy "
499 				"DESC IOADDR $%p entry %d "
500 				"bcopy packets %d "
501 				"bcopy kaddr $%p "
502 				"bcopy ioaddr (SAD) $%p "
503 				"bcopy clen %d "
504 				"bcopy boff %d",
505 				DMA_COMMON_IOADDR(desc_area), i,
506 				tdc_stats->tx_hdr_pkts,
507 				kaddr,
508 				dma_ioaddr,
509 				clen,
510 				boff));
511 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
512 				"1USE BCOPY: "));
513 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
514 				"2USE BCOPY: "));
515 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
516 				"last USE BCOPY: copy from b_rptr $%p "
517 				"to KADDR $%p (len %d offset %d",
518 				b_rptr, kaddr, len, boff));
519 
520 			bcopy(b_rptr, kaddr, len);
521 
522 #ifdef	NXGE_DEBUG
523 			dump_len = (len > 128) ? 128: len;
524 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
525 				"==> nxge_start: dump packets "
526 				"(After BCOPY len %d)"
527 				"(b_rptr $%p): %s", len, nmp->b_rptr,
528 				nxge_dump_packet((char *)nmp->b_rptr,
529 				dump_len)));
530 #endif
531 
532 			dma_handle = tx_msg_p->buf_dma_handle;
533 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
534 			(void) ddi_dma_sync(dma_handle,
535 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
536 					DDI_DMA_SYNC_FORDEV);
537 			clen = len + boff;
538 			tdc_stats->tx_hdr_pkts++;
539 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
540 				"USE BCOPY: "
541 				"DESC IOADDR $%p entry %d "
542 				"bcopy packets %d "
543 				"bcopy kaddr $%p "
544 				"bcopy ioaddr (SAD) $%p "
545 				"bcopy clen %d "
546 				"bcopy boff %d",
547 				DMA_COMMON_IOADDR(desc_area),
548 				i,
549 				tdc_stats->tx_hdr_pkts,
550 				kaddr,
551 				dma_ioaddr,
552 				clen,
553 				boff));
554 		} else {
555 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
556 				"USE DVMA: len %d", len));
557 			tx_msg_p->flags.dma_type = USE_DMA;
558 			dma_flags = DDI_DMA_WRITE;
559 			if (len < nxge_dma_stream_thresh) {
560 				dma_flags |= DDI_DMA_CONSISTENT;
561 			} else {
562 				dma_flags |= DDI_DMA_STREAMING;
563 			}
564 
565 			dma_handle = tx_msg_p->dma_handle;
566 			status = ddi_dma_addr_bind_handle(dma_handle, NULL,
567 				(caddr_t)b_rptr, len, dma_flags,
568 				DDI_DMA_DONTWAIT, NULL,
569 				&dma_cookie, &ncookies);
570 			if (status == DDI_DMA_MAPPED) {
571 				dma_ioaddr = dma_cookie.dmac_laddress;
572 				len = (int)dma_cookie.dmac_size;
573 				clen = (uint32_t)dma_cookie.dmac_size;
574 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
575 					"==> nxge_start(12_1): "
576 					"USE DVMA: len %d clen %d "
577 					"ngathers %d",
578 					len, clen,
579 					ngathers));
580 #if defined(__i386)
581 				npi_desc_handle.regp = (uint32_t)tx_desc_p;
582 #else
583 				npi_desc_handle.regp = (uint64_t)tx_desc_p;
584 #endif
585 				while (ncookies > 1) {
586 					ngathers++;
587 					/*
588 					 * this is the fix for multiple
589 					 * cookies, which are basically
590 					 * a descriptor entry, we don't set
591 					 * SOP bit as well as related fields
592 					 */
593 
594 					(void) npi_txdma_desc_gather_set(
595 						npi_desc_handle,
596 						&tx_desc,
597 						(ngathers -1),
598 						mark_mode,
599 						ngathers,
600 						dma_ioaddr,
601 						clen);
602 
603 					tx_msg_p->tx_msg_size = clen;
604 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
605 						"==> nxge_start:  DMA "
606 						"ncookie %d "
607 						"ngathers %d "
608 						"dma_ioaddr $%p len %d"
609 						"desc $%p descp $%p (%d)",
610 						ncookies,
611 						ngathers,
612 						dma_ioaddr, clen,
613 						*tx_desc_p, tx_desc_p, i));
614 
615 					ddi_dma_nextcookie(dma_handle,
616 							&dma_cookie);
617 					dma_ioaddr =
618 						dma_cookie.dmac_laddress;
619 
620 					len = (int)dma_cookie.dmac_size;
621 					clen = (uint32_t)dma_cookie.dmac_size;
622 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
623 						"==> nxge_start(12_2): "
624 						"USE DVMA: len %d clen %d ",
625 						len, clen));
626 
627 					i = TXDMA_DESC_NEXT_INDEX(i, 1,
628 						tx_ring_p->tx_wrap_mask);
629 					tx_desc_p = &tx_desc_ring_vp[i];
630 
631 					npi_desc_handle.regp =
632 #if defined(__i386)
633 						(uint32_t)tx_desc_p;
634 #else
635 						(uint64_t)tx_desc_p;
636 #endif
637 					tx_msg_p = &tx_msg_ring[i];
638 					tx_msg_p->flags.dma_type = USE_NONE;
639 					tx_desc.value = 0;
640 
641 					ncookies--;
642 				}
643 				tdc_stats->tx_ddi_pkts++;
644 				NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
645 					"DMA: ddi packets %d",
646 					tdc_stats->tx_ddi_pkts));
647 			} else {
648 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
649 				    "dma mapping failed for %d "
650 				    "bytes addr $%p flags %x (%d)",
651 				    len, b_rptr, status, status));
652 				good_packet = B_FALSE;
653 				tdc_stats->tx_dma_bind_fail++;
654 				tx_msg_p->flags.dma_type = USE_NONE;
655 				if (is_lso) {
656 					mp = nmp;
657 					goto nxge_start_fail_lso;
658 				} else {
659 					goto nxge_start_fail2;
660 				}
661 			}
662 		} /* ddi dvma */
663 
664 		if (is_lso) {
665 			nmp_lso_save = nmp;
666 		}
667 		nmp = nmp->b_cont;
668 nxge_start_control_header_only:
669 #if defined(__i386)
670 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
671 #else
672 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
673 #endif
674 		ngathers++;
675 
676 		if (ngathers == 1) {
677 #ifdef	NXGE_DEBUG
678 			save_desc_p = &sop_tx_desc;
679 #endif
680 			sop_tx_desc_p = &sop_tx_desc;
681 			sop_tx_desc_p->value = 0;
682 			sop_tx_desc_p->bits.hdw.tr_len = clen;
683 			sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
684 			sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
685 		} else {
686 #ifdef	NXGE_DEBUG
687 			save_desc_p = &tx_desc;
688 #endif
689 			tmp_desc_p = &tx_desc;
690 			tmp_desc_p->value = 0;
691 			tmp_desc_p->bits.hdw.tr_len = clen;
692 			tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
693 			tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
694 
695 			tx_desc_p->value = tmp_desc_p->value;
696 		}
697 
698 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
699 			"Desc_entry %d ngathers %d "
700 			"desc_vp $%p tx_desc_p $%p "
701 			"len %d clen %d pkt_len %d pack_len %d nmblks %d "
702 			"dma_ioaddr (SAD) $%p mark %d",
703 			i, ngathers,
704 			tx_desc_ring_vp, tx_desc_p,
705 			len, clen, pkt_len, pack_len, nmblks,
706 			dma_ioaddr, mark_mode));
707 
708 #ifdef NXGE_DEBUG
709 		npi_desc_handle.nxgep = nxgep;
710 		npi_desc_handle.function.function = nxgep->function_num;
711 		npi_desc_handle.function.instance = nxgep->instance;
712 		sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
713 		xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
714 			TX_PKT_DESC_TR_LEN_SHIFT);
715 
716 
717 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
718 			"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
719 			"mark %d sop %d\n",
720 			save_desc_p->value,
721 			sad,
722 			save_desc_p->bits.hdw.tr_len,
723 			xfer_len,
724 			save_desc_p->bits.hdw.num_ptr,
725 			save_desc_p->bits.hdw.mark,
726 			save_desc_p->bits.hdw.sop));
727 
728 		npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
729 #endif
730 
731 		tx_msg_p->tx_msg_size = clen;
732 		i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
733 		if (ngathers > nxge_tx_max_gathers) {
734 			good_packet = B_FALSE;
735 			hcksum_retrieve(mp, NULL, NULL, &start_offset,
736 				&stuff_offset, &end_offset, &value,
737 				&cksum_flags);
738 
739 			NXGE_DEBUG_MSG((NULL, TX_CTL,
740 				"==> nxge_start(14): pull msg - "
741 				"len %d pkt_len %d ngathers %d",
742 				len, pkt_len, ngathers));
743 			/* Pull all message blocks from b_cont */
744 			if (is_lso) {
745 				mp = nmp_lso_save;
746 				goto nxge_start_fail_lso;
747 			}
748 			if ((msgpullup(mp, -1)) == NULL) {
749 				goto nxge_start_fail2;
750 			}
751 			goto nxge_start_fail2;
752 		}
753 	} /* while (nmp) */
754 
755 	tx_msg_p->tx_message = mp;
756 	tx_desc_p = &tx_desc_ring_vp[sop_index];
757 #if defined(__i386)
758 	npi_desc_handle.regp = (uint32_t)tx_desc_p;
759 #else
760 	npi_desc_handle.regp = (uint64_t)tx_desc_p;
761 #endif
762 
763 	pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
764 	pkthdrp->reserved = 0;
765 	hdrp->value = 0;
766 	bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));
767 
768 	if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
769 		tdc_stats->tx_jumbo_pkts++;
770 	}
771 
772 	min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
773 	if (pkt_len < min_len) {
774 		/* Assume we use bcopy to premapped buffers */
775 		kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
776 		NXGE_DEBUG_MSG((NULL, TX_CTL,
777 			"==> nxge_start(14-1): < (msg_min + 16)"
778 			"len %d pkt_len %d min_len %d bzero %d ngathers %d",
779 			len, pkt_len, min_len, (min_len - pkt_len), ngathers));
780 		bzero((kaddr + pkt_len), (min_len - pkt_len));
781 		pkt_len = tx_msg_p->tx_msg_size = min_len;
782 
783 		sop_tx_desc_p->bits.hdw.tr_len = min_len;
784 
785 		NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
786 		tx_desc_p->value = sop_tx_desc_p->value;
787 
788 		NXGE_DEBUG_MSG((NULL, TX_CTL,
789 			"==> nxge_start(14-2): < msg_min - "
790 			"len %d pkt_len %d min_len %d ngathers %d",
791 			len, pkt_len, min_len, ngathers));
792 	}
793 
794 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
795 		cksum_flags));
796 	{
797 		uint64_t	tmp_len;
798 
799 		/* pkt_len already includes 16 + paddings!! */
800 		/* Update the control header length */
801 		tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
802 		tmp_len = hdrp->value |
803 			(tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
804 
805 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
806 			"==> nxge_start(15_x1): setting SOP "
807 			"tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
808 			"0x%llx hdrp->value 0x%llx",
809 			tot_xfer_len, tot_xfer_len, pkt_len,
810 			tmp_len, hdrp->value));
811 #if defined(_BIG_ENDIAN)
812 		hdrp->value = ddi_swap64(tmp_len);
813 #else
814 		hdrp->value = tmp_len;
815 #endif
816 		NXGE_DEBUG_MSG((nxgep,
817 			TX_CTL, "==> nxge_start(15_x2): setting SOP "
818 			"after SWAP: tot_xfer_len 0x%llx pkt_len %d "
819 			"tmp_len 0x%llx hdrp->value 0x%llx",
820 			tot_xfer_len, pkt_len,
821 			tmp_len, hdrp->value));
822 	}
823 
824 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
825 		"wr_index %d "
826 		"tot_xfer_len (%d) pkt_len %d npads %d",
827 		sop_index,
828 		tot_xfer_len, pkt_len,
829 		npads));
830 
831 	sop_tx_desc_p->bits.hdw.sop = 1;
832 	sop_tx_desc_p->bits.hdw.mark = mark_mode;
833 	sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
834 
835 	NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
836 
837 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
838 
839 #ifdef NXGE_DEBUG
840 	npi_desc_handle.nxgep = nxgep;
841 	npi_desc_handle.function.function = nxgep->function_num;
842 	npi_desc_handle.function.instance = nxgep->instance;
843 
844 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
845 		"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
846 		save_desc_p->value,
847 		sad,
848 		save_desc_p->bits.hdw.tr_len,
849 		xfer_len,
850 		save_desc_p->bits.hdw.num_ptr,
851 		save_desc_p->bits.hdw.mark,
852 		save_desc_p->bits.hdw.sop));
853 	(void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
854 
855 	dump_len = (pkt_len > 128) ? 128: pkt_len;
856 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
857 		"==> nxge_start: dump packets(17) (after sop set, len "
858 		" (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
859 		"ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
860 		(char *)hdrp,
861 		nxge_dump_packet((char *)hdrp, dump_len)));
862 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
863 		"==> nxge_start(18): TX desc sync: sop_index %d",
864 			sop_index));
865 #endif
866 
867 	if ((ngathers == 1) || tx_ring_p->wr_index < i) {
868 		(void) ddi_dma_sync(tx_desc_dma_handle,
869 			sop_index * sizeof (tx_desc_t),
870 			ngathers * sizeof (tx_desc_t),
871 			DDI_DMA_SYNC_FORDEV);
872 
873 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
874 			"cs_off = 0x%02X cs_s_off = 0x%02X "
875 			"pkt_len %d ngathers %d sop_index %d\n",
876 			stuff_offset, start_offset,
877 			pkt_len, ngathers, sop_index));
878 	} else { /* more than one descriptor and wrap around */
879 		uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
880 		(void) ddi_dma_sync(tx_desc_dma_handle,
881 			sop_index * sizeof (tx_desc_t),
882 			nsdescs * sizeof (tx_desc_t),
883 			DDI_DMA_SYNC_FORDEV);
884 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
885 			"cs_off = 0x%02X cs_s_off = 0x%02X "
886 			"pkt_len %d ngathers %d sop_index %d\n",
887 			stuff_offset, start_offset,
888 				pkt_len, ngathers, sop_index));
889 
890 		(void) ddi_dma_sync(tx_desc_dma_handle,
891 			0,
892 			(ngathers - nsdescs) * sizeof (tx_desc_t),
893 			DDI_DMA_SYNC_FORDEV);
894 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
895 			"cs_off = 0x%02X cs_s_off = 0x%02X "
896 			"pkt_len %d ngathers %d sop_index %d\n",
897 			stuff_offset, start_offset,
898 			pkt_len, ngathers, sop_index));
899 	}
900 
901 	tail_index = tx_ring_p->wr_index;
902 	tail_wrap = tx_ring_p->wr_index_wrap;
903 
904 	tx_ring_p->wr_index = i;
905 	if (tx_ring_p->wr_index <= tail_index) {
906 		tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
907 						B_FALSE : B_TRUE);
908 	}
909 
910 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
911 		"channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
912 		tx_ring_p->tdc,
913 		tx_ring_p->wr_index,
914 		tx_ring_p->wr_index_wrap,
915 		ngathers,
916 		tx_ring_p->descs_pending));
917 
918 	if (is_lso) {
919 		lso_ngathers += ngathers;
920 		if (mp_chain != NULL) {
921 			mp = mp_chain;
922 			mp_chain = mp_chain->b_next;
923 			mp->b_next = NULL;
924 			if (nxge_lso_kick_cnt == lso_ngathers) {
925 				tx_ring_p->descs_pending += lso_ngathers;
926 				{
927 					tx_ring_kick_t		kick;
928 
929 					kick.value = 0;
930 					kick.bits.ldw.wrap =
931 					    tx_ring_p->wr_index_wrap;
932 					kick.bits.ldw.tail =
933 					    (uint16_t)tx_ring_p->wr_index;
934 
935 					/* Kick the Transmit kick register */
936 					TXDMA_REG_WRITE64(
937 					    NXGE_DEV_NPI_HANDLE(nxgep),
938 					    TX_RING_KICK_REG,
939 					    (uint8_t)tx_ring_p->tdc,
940 					    kick.value);
941 					tdc_stats->tx_starts++;
942 
943 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
944 					    "==> nxge_start: more LSO: "
945 					    "LSO_CNT %d",
946 					    lso_ngathers));
947 				}
948 				lso_ngathers = 0;
949 				ngathers = 0;
950 				cur_index_lso = sop_index = tx_ring_p->wr_index;
951 				lso_tail_wrap = tx_ring_p->wr_index_wrap;
952 			}
953 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
954 			    "==> nxge_start: lso again: "
955 			    "lso_gathers %d ngathers %d cur_index_lso %d "
956 			    "wr_index %d sop_index %d",
957 			    lso_ngathers, ngathers, cur_index_lso,
958 			    tx_ring_p->wr_index, sop_index));
959 
960 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
961 			    "==> nxge_start: next : count %d",
962 			    lso_ngathers));
963 			lso_again = B_TRUE;
964 			goto start_again;
965 		}
966 		ngathers = lso_ngathers;
967 	}
968 
969 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
970 
971 	{
972 		tx_ring_kick_t		kick;
973 
974 		kick.value = 0;
975 		kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
976 		kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
977 
978 		/* Kick start the Transmit kick register */
979 		TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
980 			TX_RING_KICK_REG,
981 			(uint8_t)tx_ring_p->tdc,
982 			kick.value);
983 	}
984 
985 	tx_ring_p->descs_pending += ngathers;
986 	tdc_stats->tx_starts++;
987 
988 	if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline)
989 		(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
990 		    NXGE_TX_RING_OFFLINED);
991 
992 	MUTEX_EXIT(&tx_ring_p->lock);
993 
994 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
995 
996 	return (status);
997 
998 nxge_start_fail_lso:
999 	status = 0;
1000 	good_packet = B_FALSE;
1001 	if (mp != NULL) {
1002 		freemsg(mp);
1003 	}
1004 	if (mp_chain != NULL) {
1005 		freemsg(mp_chain);
1006 	}
1007 	if (!lso_again && !ngathers) {
1008 		if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline)
1009 			(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1010 			    NXGE_TX_RING_OFFLINED);
1011 		MUTEX_EXIT(&tx_ring_p->lock);
1012 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1013 		    "==> nxge_start: lso exit (nothing changed)"));
1014 		goto nxge_start_fail1;
1015 	}
1016 
1017 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
1018 	    "==> nxge_start (channel %d): before lso "
1019 	    "lso_gathers %d ngathers %d cur_index_lso %d "
1020 	    "wr_index %d sop_index %d lso_again %d",
1021 	    tx_ring_p->tdc,
1022 	    lso_ngathers, ngathers, cur_index_lso,
1023 	    tx_ring_p->wr_index, sop_index, lso_again));
1024 
1025 	if (lso_again) {
1026 		lso_ngathers += ngathers;
1027 		ngathers = lso_ngathers;
1028 		sop_index = cur_index_lso;
1029 		tx_ring_p->wr_index = sop_index;
1030 		tx_ring_p->wr_index_wrap = lso_tail_wrap;
1031 	}
1032 
1033 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
1034 	    "==> nxge_start (channel %d): after lso "
1035 	    "lso_gathers %d ngathers %d cur_index_lso %d "
1036 	    "wr_index %d sop_index %d lso_again %d",
1037 	    tx_ring_p->tdc,
1038 	    lso_ngathers, ngathers, cur_index_lso,
1039 	    tx_ring_p->wr_index, sop_index, lso_again));
1040 
1041 nxge_start_fail2:
1042 	if (good_packet == B_FALSE) {
1043 		cur_index = sop_index;
1044 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
1045 		for (i = 0; i < ngathers; i++) {
1046 			tx_desc_p = &tx_desc_ring_vp[cur_index];
1047 #if defined(__i386)
1048 			npi_handle.regp = (uint32_t)tx_desc_p;
1049 #else
1050 			npi_handle.regp = (uint64_t)tx_desc_p;
1051 #endif
1052 			tx_msg_p = &tx_msg_ring[cur_index];
1053 			(void) npi_txdma_desc_set_zero(npi_handle, 1);
1054 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
1055 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
1056 				    "tx_desc_p = %X index = %d",
1057 				    tx_desc_p, tx_ring_p->rd_index));
1058 				(void) dvma_unload(tx_msg_p->dvma_handle,
1059 				    0, -1);
1060 				tx_msg_p->dvma_handle = NULL;
1061 				if (tx_ring_p->dvma_wr_index ==
1062 				    tx_ring_p->dvma_wrap_mask)
1063 					tx_ring_p->dvma_wr_index = 0;
1064 				else
1065 					tx_ring_p->dvma_wr_index++;
1066 				tx_ring_p->dvma_pending--;
1067 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
1068 				if (ddi_dma_unbind_handle(
1069 				    tx_msg_p->dma_handle)) {
1070 					cmn_err(CE_WARN, "!nxge_start: "
1071 					    "ddi_dma_unbind_handle failed");
1072 				}
1073 			}
1074 			tx_msg_p->flags.dma_type = USE_NONE;
1075 			cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
1076 				tx_ring_p->tx_wrap_mask);
1077 
1078 		}
1079 
1080 		nxgep->resched_needed = B_TRUE;
1081 	}
1082 
1083 
1084 	if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline)
1085 		(void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1086 		    NXGE_TX_RING_OFFLINED);
1087 
1088 	MUTEX_EXIT(&tx_ring_p->lock);
1089 
1090 nxge_start_fail1:
1091 	/* Add FMA to check the access handle nxge_hregh */
1092 
1093 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1094 
1095 	return (status);
1096 }
1097 
1098 int
1099 nxge_serial_tx(mblk_t *mp, void *arg)
1100 {
1101 	p_tx_ring_t		tx_ring_p = (p_tx_ring_t)arg;
1102 	p_nxge_t		nxgep = tx_ring_p->nxgep;
1103 	int			status = 0;
1104 
1105 	if (isLDOMservice(nxgep)) {
1106 		if (tx_ring_p->tx_ring_offline) {
1107 			freemsg(mp);
1108 			return (status);
1109 		}
1110 	}
1111 
1112 	status = nxge_start(nxgep, tx_ring_p, mp);
1113 
1114 	return (status);
1115 }
1116 
1117 boolean_t
1118 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp)
1119 {
1120 	p_tx_ring_t 		*tx_rings;
1121 	uint8_t			ring_index;
1122 	p_tx_ring_t		tx_ring_p;
1123 	nxge_grp_t		*group;
1124 
1125 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send"));
1126 
1127 	ASSERT(mp->b_next == NULL);
1128 
1129 	group = nxgep->tx_set.group[0];	/* The default group */
1130 	ring_index = nxge_tx_lb_ring_1(mp, group->count, hp);
1131 
1132 	tx_rings = nxgep->tx_rings->rings;
1133 	tx_ring_p = tx_rings[group->legend[ring_index]];
1134 
1135 	if (isLDOMservice(nxgep)) {
1136 		if (tx_ring_p->tx_ring_offline) {
1137 			/*
1138 			 * OFFLINE means that it is in the process of being
1139 			 * shared - that is, it has been claimed by the HIO
1140 			 * code, but hasn't been unlinked from <group> yet.
1141 			 * So in this case use the first TDC, which always
1142 			 * belongs to the service domain and can't be shared.
1143 			 */
1144 			ring_index = 0;
1145 			tx_ring_p = tx_rings[group->legend[ring_index]];
1146 		}
1147 	}
1148 
1149 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p",
1150 		(int)group->count, group->legend[ring_index], tx_ring_p));
1151 
1152 	switch (nxge_tx_scheme) {
1153 	case NXGE_USE_START:
1154 		if (nxge_start(nxgep, tx_ring_p, mp)) {
1155 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed "
1156 				"ring index %d", ring_index));
1157 			return (B_FALSE);
1158 		}
1159 		break;
1160 
1161 	case NXGE_USE_SERIAL:
1162 	default:
1163 		nxge_serialize_enter(tx_ring_p->serial, mp);
1164 		break;
1165 	}
1166 
1167 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d",
1168 		ring_index));
1169 
1170 	return (B_TRUE);
1171 }
1172 
1173 /*
1174  * nxge_m_tx() - send a chain of packets
1175  */
1176 mblk_t *
1177 nxge_m_tx(void *arg, mblk_t *mp)
1178 {
1179 	p_nxge_t 		nxgep = (p_nxge_t)arg;
1180 	mblk_t 			*next;
1181 	mac_tx_hint_t		hint;
1182 
1183 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx"));
1184 
1185 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
1186 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
1187 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
1188 		    "==> nxge_m_tx: hardware not initialized"));
1189 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
1190 		    "<== nxge_m_tx"));
1191 		freemsgchain(mp);
1192 		mp = NULL;
1193 		return (mp);
1194 	}
1195 
1196 	hint.hash =  NULL;
1197 	hint.vid =  0;
1198 	hint.sap =  0;
1199 
1200 	while (mp != NULL) {
1201 		next = mp->b_next;
1202 		mp->b_next = NULL;
1203 
1204 		/*
1205 		 * Until Nemo tx resource works, the mac driver
1206 		 * does the load balancing based on TCP port,
1207 		 * or CPU. For debugging, we use a system
1208 		 * configurable parameter.
1209 		 */
1210 		if (!nxge_send(nxgep, mp, &hint)) {
1211 			mp->b_next = next;
1212 			break;
1213 		}
1214 
1215 		mp = next;
1216 
1217 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1218 		    "==> nxge_m_tx: (go back to loop) mp $%p next $%p",
1219 		    mp, next));
1220 	}
1221 
1222 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx"));
1223 	return (mp);
1224 }
1225 
1226 int
1227 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp)
1228 {
1229 	uint8_t 		ring_index = 0;
1230 	uint8_t 		*tcp_port;
1231 	p_mblk_t 		nmp;
1232 	size_t 			mblk_len;
1233 	size_t 			iph_len;
1234 	size_t 			hdrs_size;
1235 	uint8_t			hdrs_buf[sizeof (struct  ether_header) +
1236 					IP_MAX_HDR_LENGTH + sizeof (uint32_t)];
1237 				/*
1238 				 * allocate space big enough to cover
1239 				 * the max ip header length and the first
1240 				 * 4 bytes of the TCP/IP header.
1241 				 */
1242 
1243 	boolean_t		qos = B_FALSE;
1244 
1245 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring"));
1246 
1247 	if (hp->vid) {
1248 		qos = B_TRUE;
1249 	}
1250 	switch (nxge_tx_lb_policy) {
1251 	case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */
1252 	default:
1253 		tcp_port = mp->b_rptr;
1254 		if (!nxge_no_tx_lb && !qos &&
1255 			(ntohs(((p_ether_header_t)tcp_port)->ether_type)
1256 				== ETHERTYPE_IP)) {
1257 			nmp = mp;
1258 			mblk_len = MBLKL(nmp);
1259 			tcp_port = NULL;
1260 			if (mblk_len > sizeof (struct ether_header) +
1261 					sizeof (uint8_t)) {
1262 				tcp_port = nmp->b_rptr +
1263 					sizeof (struct ether_header);
1264 				mblk_len -= sizeof (struct ether_header);
1265 				iph_len = ((*tcp_port) & 0x0f) << 2;
1266 				if (mblk_len > (iph_len + sizeof (uint32_t))) {
1267 					tcp_port = nmp->b_rptr;
1268 				} else {
1269 					tcp_port = NULL;
1270 				}
1271 			}
1272 			if (tcp_port == NULL) {
1273 				hdrs_size = 0;
1274 				((p_ether_header_t)hdrs_buf)->ether_type = 0;
1275 				while ((nmp) && (hdrs_size <
1276 						sizeof (hdrs_buf))) {
1277 					mblk_len = MBLKL(nmp);
1278 					if (mblk_len >=
1279 						(sizeof (hdrs_buf) - hdrs_size))
1280 						mblk_len = sizeof (hdrs_buf) -
1281 							hdrs_size;
1282 					bcopy(nmp->b_rptr,
1283 						&hdrs_buf[hdrs_size], mblk_len);
1284 					hdrs_size += mblk_len;
1285 					nmp = nmp->b_cont;
1286 				}
1287 				tcp_port = hdrs_buf;
1288 			}
1289 			tcp_port += sizeof (ether_header_t);
1290 			if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) {
1291 				switch (tcp_port[9]) {
1292 				case IPPROTO_TCP:
1293 				case IPPROTO_UDP:
1294 				case IPPROTO_ESP:
1295 					tcp_port += ((*tcp_port) & 0x0f) << 2;
1296 					ring_index =
1297 					    ((tcp_port[0] ^
1298 					    tcp_port[1] ^
1299 					    tcp_port[2] ^
1300 					    tcp_port[3]) % maxtdcs);
1301 					break;
1302 
1303 				case IPPROTO_AH:
1304 					/* SPI starts at the 4th byte */
1305 					tcp_port += ((*tcp_port) & 0x0f) << 2;
1306 					ring_index =
1307 					    ((tcp_port[4] ^
1308 					    tcp_port[5] ^
1309 					    tcp_port[6] ^
1310 					    tcp_port[7]) % maxtdcs);
1311 					break;
1312 
1313 				default:
1314 					ring_index = tcp_port[19] % maxtdcs;
1315 					break;
1316 				}
1317 			} else { /* fragmented packet */
1318 				ring_index = tcp_port[19] % maxtdcs;
1319 			}
1320 		} else {
1321 			ring_index = mp->b_band % maxtdcs;
1322 		}
1323 		break;
1324 
1325 	case NXGE_TX_LB_HASH:
1326 		if (hp->hash) {
1327 #if defined(__i386)
1328 			ring_index = ((uint32_t)(hp->hash) % maxtdcs);
1329 #else
1330 			ring_index = ((uint64_t)(hp->hash) % maxtdcs);
1331 #endif
1332 		} else {
1333 			ring_index = mp->b_band % maxtdcs;
1334 		}
1335 		break;
1336 
1337 	case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */
1338 		tcp_port = mp->b_rptr;
1339 		ring_index = tcp_port[5] % maxtdcs;
1340 		break;
1341 	}
1342 
1343 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring"));
1344 
1345 	return (ring_index);
1346 }
1347 
1348 uint_t
1349 nxge_reschedule(caddr_t arg)
1350 {
1351 	p_nxge_t nxgep;
1352 
1353 	nxgep = (p_nxge_t)arg;
1354 
1355 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule"));
1356 
1357 	if (nxgep->nxge_mac_state == NXGE_MAC_STARTED &&
1358 	    nxgep->resched_needed) {
1359 		if (!isLDOMguest(nxgep))
1360 			mac_tx_update(nxgep->mach);
1361 #if defined(sun4v)
1362 		else {		/* isLDOMguest(nxgep) */
1363 			nxge_hio_data_t *nhd = (nxge_hio_data_t *)
1364 			    nxgep->nxge_hw_p->hio;
1365 			nx_vio_fp_t *vio = &nhd->hio.vio;
1366 
1367 			/* Call back vnet. */
1368 			if (vio->cb.vio_net_tx_update) {
1369 				(*vio->cb.vio_net_tx_update)
1370 				    (nxgep->hio_vr->vhp);
1371 			}
1372 		}
1373 #endif
1374 		nxgep->resched_needed = B_FALSE;
1375 		nxgep->resched_running = B_FALSE;
1376 	}
1377 
1378 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule"));
1379 	return (DDI_INTR_CLAIMED);
1380 }
1381 
1382 
1383 /* Software LSO starts here */
1384 static void
1385 nxge_hcksum_retrieve(mblk_t *mp,
1386     uint32_t *start, uint32_t *stuff, uint32_t *end,
1387     uint32_t *value, uint32_t *flags)
1388 {
1389 	if (mp->b_datap->db_type == M_DATA) {
1390 		if (flags != NULL) {
1391 			*flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
1392 			    HCK_PARTIALCKSUM | HCK_FULLCKSUM |
1393 			    HCK_FULLCKSUM_OK);
1394 			if ((*flags & (HCK_PARTIALCKSUM |
1395 			    HCK_FULLCKSUM)) != 0) {
1396 				if (value != NULL)
1397 					*value = (uint32_t)DB_CKSUM16(mp);
1398 				if ((*flags & HCK_PARTIALCKSUM) != 0) {
1399 					if (start != NULL)
1400 						*start =
1401 						    (uint32_t)DB_CKSUMSTART(mp);
1402 					if (stuff != NULL)
1403 						*stuff =
1404 						    (uint32_t)DB_CKSUMSTUFF(mp);
1405 					if (end != NULL)
1406 						*end =
1407 						    (uint32_t)DB_CKSUMEND(mp);
1408 				}
1409 			}
1410 		}
1411 	}
1412 }
1413 
1414 static void
1415 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
1416 {
1417 	ASSERT(DB_TYPE(mp) == M_DATA);
1418 
1419 	*mss = 0;
1420 	if (flags != NULL) {
1421 		*flags = DB_CKSUMFLAGS(mp) & HW_LSO;
1422 		if ((*flags != 0) && (mss != NULL)) {
1423 			*mss = (uint32_t)DB_LSOMSS(mp);
1424 		}
1425 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1426 		    "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
1427 		    *mss, *flags));
1428 	}
1429 
1430 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1431 	    "<== nxge_lso_info_get: mss %d", *mss));
1432 }
1433 
1434 /*
1435  * Do Soft LSO on the oversized packet.
1436  *
1437  * 1. Create a chain of message for headers.
1438  * 2. Fill up header messages with proper information.
1439  * 3. Copy Eithernet, IP, and TCP headers from the original message to
1440  *    each new message with necessary adjustments.
1441  *    * Unchange the ethernet header for DIX frames. (by default)
1442  *    * IP Total Length field is updated to MSS or less(only for the last one).
1443  *    * IP Identification value is incremented by one for each packet.
1444  *    * TCP sequence Number is recalculated according to the payload length.
1445  *    * Set FIN and/or PSH flags for the *last* packet if applied.
1446  *    * TCP partial Checksum
1447  * 4. Update LSO information in the first message header.
1448  * 5. Release the original message header.
1449  */
1450 static mblk_t *
1451 nxge_do_softlso(mblk_t *mp, uint32_t mss)
1452 {
1453 	uint32_t	hckflags;
1454 	int		pktlen;
1455 	int		hdrlen;
1456 	int		segnum;
1457 	int		i;
1458 	struct ether_vlan_header *evh;
1459 	int		ehlen, iphlen, tcphlen;
1460 	struct ip	*oiph, *niph;
1461 	struct tcphdr *otcph, *ntcph;
1462 	int		available, len, left;
1463 	uint16_t	ip_id;
1464 	uint32_t	tcp_seq;
1465 #ifdef __sparc
1466 	uint32_t	tcp_seq_tmp;
1467 #endif
1468 	mblk_t		*datamp;
1469 	uchar_t		*rptr;
1470 	mblk_t		*nmp;
1471 	mblk_t		*cmp;
1472 	mblk_t		*mp_chain;
1473 	boolean_t do_cleanup = B_FALSE;
1474 	t_uscalar_t start_offset = 0;
1475 	t_uscalar_t stuff_offset = 0;
1476 	t_uscalar_t value = 0;
1477 	uint16_t	l4_len;
1478 	ipaddr_t	src, dst;
1479 	uint32_t	cksum, sum, l4cksum;
1480 
1481 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1482 	    "==> nxge_do_softlso"));
1483 	/*
1484 	 * check the length of LSO packet payload and calculate the number of
1485 	 * segments to be generated.
1486 	 */
1487 	pktlen = msgsize(mp);
1488 	evh = (struct ether_vlan_header *)mp->b_rptr;
1489 
1490 	/* VLAN? */
1491 	if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
1492 		ehlen = sizeof (struct ether_vlan_header);
1493 	else
1494 		ehlen = sizeof (struct ether_header);
1495 	oiph = (struct ip *)(mp->b_rptr + ehlen);
1496 	iphlen = oiph->ip_hl * 4;
1497 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1498 	tcphlen = otcph->th_off * 4;
1499 
1500 	l4_len = pktlen - ehlen - iphlen;
1501 
1502 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1503 	    "==> nxge_do_softlso: mss %d oiph $%p "
1504 	    "original ip_sum oiph->ip_sum 0x%x "
1505 	    "original tcp_sum otcph->th_sum 0x%x "
1506 	    "oiph->ip_len %d pktlen %d ehlen %d "
1507 	    "l4_len %d (0x%x) ip_len - iphlen %d ",
1508 	    mss,
1509 	    oiph,
1510 	    oiph->ip_sum,
1511 	    otcph->th_sum,
1512 	    ntohs(oiph->ip_len), pktlen,
1513 	    ehlen,
1514 	    l4_len,
1515 	    l4_len,
1516 	    ntohs(oiph->ip_len) - iphlen));
1517 
1518 	/* IPv4 + TCP */
1519 	if (!(oiph->ip_v == IPV4_VERSION)) {
1520 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1521 		    "<== nxge_do_softlso: not IPV4 "
1522 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1523 		    ntohs(oiph->ip_len), pktlen, ehlen,
1524 		    tcphlen));
1525 		freemsg(mp);
1526 		return (NULL);
1527 	}
1528 
1529 	if (!(oiph->ip_p == IPPROTO_TCP)) {
1530 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1531 		    "<== nxge_do_softlso: not TCP "
1532 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1533 		    ntohs(oiph->ip_len), pktlen, ehlen,
1534 		    tcphlen));
1535 		freemsg(mp);
1536 		return (NULL);
1537 	}
1538 
1539 	if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
1540 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1541 		    "<== nxge_do_softlso: len not matched  "
1542 		    "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1543 		    ntohs(oiph->ip_len), pktlen, ehlen,
1544 		    tcphlen));
1545 		freemsg(mp);
1546 		return (NULL);
1547 	}
1548 
1549 	otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1550 	tcphlen = otcph->th_off * 4;
1551 
1552 	/* TCP flags can not include URG, RST, or SYN */
1553 	VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);
1554 
1555 	hdrlen = ehlen + iphlen + tcphlen;
1556 
1557 	VERIFY(MBLKL(mp) >= hdrlen);
1558 
1559 	if (MBLKL(mp) > hdrlen) {
1560 		datamp = mp;
1561 		rptr = mp->b_rptr + hdrlen;
1562 	} else { /* = */
1563 		datamp = mp->b_cont;
1564 		rptr = datamp->b_rptr;
1565 	}
1566 
1567 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1568 	    "nxge_do_softlso: otcph $%p pktlen: %d, "
1569 	    "hdrlen %d ehlen %d iphlen %d tcphlen %d "
1570 	    "mblkl(mp): %d, mblkl(datamp): %d",
1571 	    otcph,
1572 	    pktlen, hdrlen, ehlen, iphlen, tcphlen,
1573 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1574 
1575 	hckflags = 0;
1576 	nxge_hcksum_retrieve(mp,
1577 	    &start_offset, &stuff_offset, &value, NULL, &hckflags);
1578 
1579 	dst = oiph->ip_dst.s_addr;
1580 	src = oiph->ip_src.s_addr;
1581 
1582 	cksum = (dst >> 16) + (dst & 0xFFFF) +
1583 	    (src >> 16) + (src & 0xFFFF);
1584 	l4cksum = cksum + IP_TCP_CSUM_COMP;
1585 
1586 	sum = l4_len + l4cksum;
1587 	sum = (sum & 0xFFFF) + (sum >> 16);
1588 
1589 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1590 	    "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
1591 	    "hckflags 0x%x start_offset %d stuff_offset %d "
1592 	    "value (original) 0x%x th_sum 0x%x "
1593 	    "pktlen %d l4_len %d (0x%x) "
1594 	    "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
1595 	    dst, src,
1596 	    (sum & 0xffff), (~sum & 0xffff),
1597 	    hckflags, start_offset, stuff_offset,
1598 	    value, otcph->th_sum,
1599 	    pktlen,
1600 	    l4_len,
1601 	    l4_len,
1602 	    ntohs(oiph->ip_len) - (int)MBLKL(mp),
1603 	    (int)MBLKL(datamp),
1604 	    nxge_dump_packet((char *)evh, 12)));
1605 
1606 	/*
1607 	 * Start to process.
1608 	 */
1609 	available = pktlen - hdrlen;
1610 	segnum = (available - 1) / mss + 1;
1611 
1612 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1613 	    "==> nxge_do_softlso: pktlen %d "
1614 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1615 	    "available %d mss %d segnum %d",
1616 	    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
1617 	    available,
1618 	    mss,
1619 	    segnum));
1620 
1621 	VERIFY(segnum >= 2);
1622 
1623 	/*
1624 	 * Try to pre-allocate all header messages
1625 	 */
1626 	mp_chain = NULL;
1627 	for (i = 0; i < segnum; i++) {
1628 		if ((nmp = allocb(hdrlen, 0)) == NULL) {
1629 			/* Clean up the mp_chain */
1630 			while (mp_chain != NULL) {
1631 				nmp = mp_chain;
1632 				mp_chain = mp_chain->b_next;
1633 				freemsg(nmp);
1634 			}
1635 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1636 			    "<== nxge_do_softlso: "
1637 			    "Could not allocate enough messages for headers!"));
1638 			freemsg(mp);
1639 			return (NULL);
1640 		}
1641 		nmp->b_next = mp_chain;
1642 		mp_chain = nmp;
1643 
1644 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1645 		    "==> nxge_do_softlso: "
1646 		    "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
1647 		    mp, nmp, mp_chain, mp_chain->b_next));
1648 	}
1649 
1650 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1651 	    "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
1652 	    mp, nmp, mp_chain));
1653 
1654 	/*
1655 	 * Associate payload with new packets
1656 	 */
1657 	cmp = mp_chain;
1658 	left = available;
1659 	while (cmp != NULL) {
1660 		nmp = dupb(datamp);
1661 		if (nmp == NULL) {
1662 			do_cleanup = B_TRUE;
1663 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1664 			    "==>nxge_do_softlso: "
1665 			    "Can not dupb(datamp), have to do clean up"));
1666 			goto cleanup_allocated_msgs;
1667 		}
1668 
1669 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1670 		    "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
1671 		    "dupb nmp $%p len %d left %d msd %d ",
1672 		    mp, cmp, nmp, len, left, mss));
1673 
1674 		cmp->b_cont = nmp;
1675 		nmp->b_rptr = rptr;
1676 		len = (left < mss) ? left : mss;
1677 		left -= len;
1678 
1679 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1680 		    "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
1681 		    "dupb nmp $%p len %d left %d mss %d ",
1682 		    mp, cmp, nmp, len, left, mss));
1683 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1684 		    "nxge_do_softlso: before available: %d, "
1685 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1686 		    available, left, len, segnum, (int)MBLKL(nmp)));
1687 
1688 		len -= MBLKL(nmp);
1689 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1690 		    "nxge_do_softlso: after available: %d, "
1691 		    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1692 		    available, left, len, segnum, (int)MBLKL(nmp)));
1693 
1694 		while (len > 0) {
1695 			mblk_t *mmp = NULL;
1696 
1697 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1698 			    "nxge_do_softlso: (4) len > 0 available: %d, "
1699 			    "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1700 			    available, left, len, segnum, (int)MBLKL(nmp)));
1701 
1702 			if (datamp->b_cont != NULL) {
1703 				datamp = datamp->b_cont;
1704 				rptr = datamp->b_rptr;
1705 				mmp = dupb(datamp);
1706 				if (mmp == NULL) {
1707 					do_cleanup = B_TRUE;
1708 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1709 					    "==> nxge_do_softlso: "
1710 					    "Can not dupb(datamp) (1), :"
1711 					    "have to do clean up"));
1712 					NXGE_DEBUG_MSG((NULL, TX_CTL,
1713 					    "==> nxge_do_softlso: "
1714 					    "available: %d, left: %d, "
1715 					    "len: %d, MBLKL(nmp): %d",
1716 					    available, left, len,
1717 					    (int)MBLKL(nmp)));
1718 					goto cleanup_allocated_msgs;
1719 				}
1720 			} else {
1721 				NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1722 				    "==> nxge_do_softlso: "
1723 				    "(1)available: %d, left: %d, "
1724 				    "len: %d, MBLKL(nmp): %d",
1725 				    available, left, len,
1726 				    (int)MBLKL(nmp)));
1727 				cmn_err(CE_PANIC,
1728 				    "==> nxge_do_softlso: "
1729 				    "Pointers must have been corrupted!\n"
1730 				    "datamp: $%p, nmp: $%p, rptr: $%p",
1731 				    (void *)datamp,
1732 				    (void *)nmp,
1733 				    (void *)rptr);
1734 			}
1735 			nmp->b_cont = mmp;
1736 			nmp = mmp;
1737 			len -= MBLKL(nmp);
1738 		}
1739 		if (len < 0) {
1740 			nmp->b_wptr += len;
1741 			rptr = nmp->b_wptr;
1742 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1743 			    "(5) len < 0 (less than 0)"
1744 			    "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
1745 			    available, left, len, (int)MBLKL(nmp)));
1746 
1747 		} else if (len == 0) {
1748 			if (datamp->b_cont != NULL) {
1749 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1750 				    "(5) len == 0"
1751 				    "available: %d, left: %d, len: %d, "
1752 				    "MBLKL(nmp): %d",
1753 				    available, left, len, (int)MBLKL(nmp)));
1754 				datamp = datamp->b_cont;
1755 				rptr = datamp->b_rptr;
1756 			} else {
1757 				NXGE_DEBUG_MSG((NULL, TX_CTL,
1758 				    "(6)available b_cont == NULL : %d, "
1759 				    "left: %d, len: %d, MBLKL(nmp): %d",
1760 				    available, left, len, (int)MBLKL(nmp)));
1761 
1762 				VERIFY(cmp->b_next == NULL);
1763 				VERIFY(left == 0);
1764 				break; /* Done! */
1765 			}
1766 		}
1767 		cmp = cmp->b_next;
1768 
1769 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1770 		    "(7) do_softlso: "
1771 		    "next mp in mp_chain available len != 0 : %d, "
1772 		    "left: %d, len: %d, MBLKL(nmp): %d",
1773 		    available, left, len, (int)MBLKL(nmp)));
1774 	}
1775 
1776 	/*
1777 	 * From now, start to fill up all headers for the first message
1778 	 * Hardware checksum flags need to be updated separately for FULLCKSUM
1779 	 * and PARTIALCKSUM cases. For full checksum, copy the original flags
1780 	 * into every new packet is enough. But for HCK_PARTIALCKSUM, all
1781 	 * required fields need to be updated properly.
1782 	 */
1783 	nmp = mp_chain;
1784 	bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
1785 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1786 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1787 	niph->ip_len = htons(mss + iphlen + tcphlen);
1788 	ip_id = ntohs(niph->ip_id);
1789 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1790 #ifdef __sparc
1791 	bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
1792 	tcp_seq = ntohl(tcp_seq_tmp);
1793 #else
1794 	tcp_seq = ntohl(ntcph->th_seq);
1795 #endif
1796 
1797 	ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);
1798 
1799 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1800 	DB_CKSUMSTART(nmp) = start_offset;
1801 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1802 
1803 	/* calculate IP checksum and TCP pseudo header checksum */
1804 	niph->ip_sum = 0;
1805 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1806 
1807 	l4_len = mss + tcphlen;
1808 	sum = htons(l4_len) + l4cksum;
1809 	sum = (sum & 0xFFFF) + (sum >> 16);
1810 	ntcph->th_sum = (sum & 0xffff);
1811 
1812 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1813 	    "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
1814 	    "mss %d pktlen %d l4_len %d (0x%x) "
1815 	    "MBLKL(mp): %d, MBLKL(datamp): %d "
1816 	    "ip_sum 0x%x "
1817 	    "th_sum 0x%x sum 0x%x ) "
1818 	    "dump first ip->tcp %s",
1819 	    nmp, mp_chain,
1820 	    mss,
1821 	    pktlen,
1822 	    l4_len,
1823 	    l4_len,
1824 	    (int)MBLKL(mp), (int)MBLKL(datamp),
1825 	    niph->ip_sum,
1826 	    ntcph->th_sum,
1827 	    sum,
1828 	    nxge_dump_packet((char *)niph, 52)));
1829 
1830 	cmp = nmp;
1831 	while ((nmp = nmp->b_next)->b_next != NULL) {
1832 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1833 		    "==>nxge_do_softlso: middle l4_len %d ", l4_len));
1834 		bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1835 		nmp->b_wptr = nmp->b_rptr + hdrlen;
1836 		niph = (struct ip *)(nmp->b_rptr + ehlen);
1837 		niph->ip_id = htons(++ip_id);
1838 		niph->ip_len = htons(mss + iphlen + tcphlen);
1839 		ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1840 		tcp_seq += mss;
1841 
1842 		ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);
1843 
1844 #ifdef __sparc
1845 		tcp_seq_tmp = htonl(tcp_seq);
1846 		bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1847 #else
1848 		ntcph->th_seq = htonl(tcp_seq);
1849 #endif
1850 		DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1851 		DB_CKSUMSTART(nmp) = start_offset;
1852 		DB_CKSUMSTUFF(nmp) = stuff_offset;
1853 
1854 		/* calculate IP checksum and TCP pseudo header checksum */
1855 		niph->ip_sum = 0;
1856 		niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1857 		ntcph->th_sum = (sum & 0xffff);
1858 
1859 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1860 		    "==> nxge_do_softlso: middle ip_sum 0x%x "
1861 		    "th_sum 0x%x "
1862 		    " mp $%p (mp_chain $%p) pktlen %d "
1863 		    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1864 		    niph->ip_sum,
1865 		    ntcph->th_sum,
1866 		    nmp, mp_chain,
1867 		    pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
1868 	}
1869 
1870 	/* Last segment */
1871 	/*
1872 	 * Set FIN and/or PSH flags if present only in the last packet.
1873 	 * The ip_len could be different from prior packets.
1874 	 */
1875 	bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1876 	nmp->b_wptr = nmp->b_rptr + hdrlen;
1877 	niph = (struct ip *)(nmp->b_rptr + ehlen);
1878 	niph->ip_id = htons(++ip_id);
1879 	niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
1880 	ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1881 	tcp_seq += mss;
1882 #ifdef __sparc
1883 	tcp_seq_tmp = htonl(tcp_seq);
1884 	bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1885 #else
1886 	ntcph->th_seq = htonl(tcp_seq);
1887 #endif
1888 	ntcph->th_flags = (otcph->th_flags & ~TH_URG);
1889 
1890 	DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1891 	DB_CKSUMSTART(nmp) = start_offset;
1892 	DB_CKSUMSTUFF(nmp) = stuff_offset;
1893 
1894 	/* calculate IP checksum and TCP pseudo header checksum */
1895 	niph->ip_sum = 0;
1896 	niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1897 
1898 	l4_len = ntohs(niph->ip_len) - iphlen;
1899 	sum = htons(l4_len) + l4cksum;
1900 	sum = (sum & 0xFFFF) + (sum >> 16);
1901 	ntcph->th_sum = (sum & 0xffff);
1902 
1903 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1904 	    "==> nxge_do_softlso: last next "
1905 	    "niph->ip_sum 0x%x "
1906 	    "ntcph->th_sum 0x%x sum 0x%x "
1907 	    "dump last ip->tcp %s "
1908 	    "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
1909 	    "l4_len %d (0x%x) "
1910 	    "MBLKL(mp): %d, MBLKL(datamp): %d ",
1911 	    niph->ip_sum,
1912 	    ntcph->th_sum, sum,
1913 	    nxge_dump_packet((char *)niph, 52),
1914 	    cmp, nmp, mp_chain,
1915 	    pktlen, pktlen,
1916 	    l4_len,
1917 	    l4_len,
1918 	    (int)MBLKL(mp), (int)MBLKL(datamp)));
1919 
1920 cleanup_allocated_msgs:
1921 	if (do_cleanup) {
1922 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1923 		    "==> nxge_do_softlso: "
1924 		    "Failed allocating messages, "
1925 		    "have to clean up and fail!"));
1926 		while (mp_chain != NULL) {
1927 			nmp = mp_chain;
1928 			mp_chain = mp_chain->b_next;
1929 			freemsg(nmp);
1930 		}
1931 	}
1932 	/*
1933 	 * We're done here, so just free the original message and return the
1934 	 * new message chain, that could be NULL if failed, back to the caller.
1935 	 */
1936 	freemsg(mp);
1937 
1938 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1939 	    "<== nxge_do_softlso:mp_chain $%p", mp_chain));
1940 	return (mp_chain);
1941 }
1942 
1943 /*
1944  * Will be called before NIC driver do further operation on the message.
1945  * The input message may include LSO information, if so, go to softlso logic
1946  * to eliminate the oversized LSO packet for the incapable underlying h/w.
1947  * The return could be the same non-LSO message or a message chain for LSO case.
1948  *
1949  * The driver needs to call this function per packet and process the whole chain
1950  * if applied.
1951  */
1952 static mblk_t *
1953 nxge_lso_eliminate(mblk_t *mp)
1954 {
1955 	uint32_t lsoflags;
1956 	uint32_t mss;
1957 
1958 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1959 	    "==>nxge_lso_eliminate:"));
1960 	nxge_lso_info_get(mp, &mss, &lsoflags);
1961 
1962 	if (lsoflags & HW_LSO) {
1963 		mblk_t *nmp;
1964 
1965 		NXGE_DEBUG_MSG((NULL, TX_CTL,
1966 		    "==>nxge_lso_eliminate:"
1967 		    "HW_LSO:mss %d mp $%p",
1968 		    mss, mp));
1969 		if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
1970 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1971 			    "<== nxge_lso_eliminate: "
1972 			    "LSO: nmp not NULL nmp $%p mss %d mp $%p",
1973 			    nmp, mss, mp));
1974 			return (nmp);
1975 		} else {
1976 			NXGE_DEBUG_MSG((NULL, TX_CTL,
1977 			    "<== nxge_lso_eliminate_ "
1978 			    "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
1979 			    nmp, mss, mp));
1980 			return (NULL);
1981 		}
1982 	}
1983 
1984 	NXGE_DEBUG_MSG((NULL, TX_CTL,
1985 	    "<== nxge_lso_eliminate"));
1986 	return (mp);
1987 }
1988 
1989 static uint32_t
1990 nxge_csgen(uint16_t *adr, int len)
1991 {
1992 	int		i, odd;
1993 	uint32_t	sum = 0;
1994 	uint32_t	c = 0;
1995 
1996 	odd = len % 2;
1997 	for (i = 0; i < (len / 2); i++) {
1998 		sum += (adr[i] & 0xffff);
1999 	}
2000 	if (odd) {
2001 		sum += adr[len / 2] & 0xff00;
2002 	}
2003 	while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
2004 		sum &= 0xffff;
2005 		sum += c;
2006 	}
2007 	return (~sum & 0xffff);
2008 }
2009