xref: /illumos-gate/usr/src/uts/common/io/nxge/nxge_send.c (revision 53f3d8ec)
144961713Sgirish /*
244961713Sgirish  * CDDL HEADER START
344961713Sgirish  *
444961713Sgirish  * The contents of this file are subject to the terms of the
544961713Sgirish  * Common Development and Distribution License (the "License").
644961713Sgirish  * You may not use this file except in compliance with the License.
744961713Sgirish  *
844961713Sgirish  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
944961713Sgirish  * or http://www.opensolaris.org/os/licensing.
1044961713Sgirish  * See the License for the specific language governing permissions
1144961713Sgirish  * and limitations under the License.
1244961713Sgirish  *
1344961713Sgirish  * When distributing Covered Code, include this CDDL HEADER in each
1444961713Sgirish  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1544961713Sgirish  * If applicable, add the following below this CDDL HEADER, with the
1644961713Sgirish  * fields enclosed by brackets "[]" replaced with your own identifying
1744961713Sgirish  * information: Portions Copyright [yyyy] [name of copyright owner]
1844961713Sgirish  *
1944961713Sgirish  * CDDL HEADER END
2044961713Sgirish  */
2144961713Sgirish /*
22a3c5bd6dSspeer  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
2344961713Sgirish  * Use is subject to license terms.
2444961713Sgirish  */
2544961713Sgirish 
2644961713Sgirish #pragma ident	"%Z%%M%	%I%	%E% SMI"
2744961713Sgirish 
28a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h>
2944961713Sgirish 
3044961713Sgirish extern uint32_t		nxge_reclaim_pending;
3144961713Sgirish extern uint32_t 	nxge_bcopy_thresh;
3244961713Sgirish extern uint32_t 	nxge_dvma_thresh;
3344961713Sgirish extern uint32_t 	nxge_dma_stream_thresh;
3444961713Sgirish extern uint32_t		nxge_tx_minfree;
3544961713Sgirish extern uint32_t		nxge_tx_intr_thres;
3644961713Sgirish extern uint32_t		nxge_tx_max_gathers;
3744961713Sgirish extern uint32_t		nxge_tx_tiny_pack;
3844961713Sgirish extern uint32_t		nxge_tx_use_bcopy;
3944961713Sgirish extern uint32_t		nxge_tx_lb_policy;
4044961713Sgirish extern uint32_t		nxge_no_tx_lb;
411f8914d5Sml extern nxge_tx_mode_t	nxge_tx_scheme;
4244961713Sgirish 
4344961713Sgirish typedef struct _mac_tx_hint {
4444961713Sgirish 	uint16_t	sap;
4544961713Sgirish 	uint16_t	vid;
4644961713Sgirish 	void		*hash;
4744961713Sgirish } mac_tx_hint_t, *p_mac_tx_hint_t;
4844961713Sgirish 
4944961713Sgirish int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t);
5044961713Sgirish 
5144961713Sgirish int
5244961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
5344961713Sgirish {
5444961713Sgirish 	int 			status = 0;
5544961713Sgirish 	p_tx_desc_t 		tx_desc_ring_vp;
5644961713Sgirish 	npi_handle_t		npi_desc_handle;
5744961713Sgirish 	nxge_os_dma_handle_t 	tx_desc_dma_handle;
5844961713Sgirish 	p_tx_desc_t 		tx_desc_p;
5944961713Sgirish 	p_tx_msg_t 		tx_msg_ring;
6044961713Sgirish 	p_tx_msg_t 		tx_msg_p;
6144961713Sgirish 	tx_desc_t		tx_desc, *tmp_desc_p;
6244961713Sgirish 	tx_desc_t		sop_tx_desc, *sop_tx_desc_p;
6344961713Sgirish 	p_tx_pkt_header_t	hdrp;
6444961713Sgirish 	p_tx_pkt_hdr_all_t	pkthdrp;
6544961713Sgirish 	uint8_t			npads = 0;
6644961713Sgirish 	uint64_t 		dma_ioaddr;
6744961713Sgirish 	uint32_t		dma_flags;
6844961713Sgirish 	int			last_bidx;
6944961713Sgirish 	uint8_t 		*b_rptr;
7044961713Sgirish 	caddr_t 		kaddr;
7144961713Sgirish 	uint32_t		nmblks;
7244961713Sgirish 	uint32_t		ngathers;
7344961713Sgirish 	uint32_t		clen;
7444961713Sgirish 	int 			len;
7544961713Sgirish 	uint32_t		pkt_len, pack_len, min_len;
7644961713Sgirish 	uint32_t		bcopy_thresh;
7744961713Sgirish 	int 			i, cur_index, sop_index;
7844961713Sgirish 	uint16_t		tail_index;
7944961713Sgirish 	boolean_t		tail_wrap = B_FALSE;
8044961713Sgirish 	nxge_dma_common_t	desc_area;
8144961713Sgirish 	nxge_os_dma_handle_t 	dma_handle;
8244961713Sgirish 	ddi_dma_cookie_t 	dma_cookie;
8344961713Sgirish 	npi_handle_t		npi_handle;
8444961713Sgirish 	p_mblk_t 		nmp;
8544961713Sgirish 	p_mblk_t		t_mp;
8644961713Sgirish 	uint32_t 		ncookies;
8744961713Sgirish 	boolean_t 		good_packet;
8844961713Sgirish 	boolean_t 		mark_mode = B_FALSE;
8944961713Sgirish 	p_nxge_stats_t 		statsp;
9044961713Sgirish 	p_nxge_tx_ring_stats_t tdc_stats;
9144961713Sgirish 	t_uscalar_t 		start_offset = 0;
9244961713Sgirish 	t_uscalar_t 		stuff_offset = 0;
9344961713Sgirish 	t_uscalar_t 		end_offset = 0;
9444961713Sgirish 	t_uscalar_t 		value = 0;
9544961713Sgirish 	t_uscalar_t 		cksum_flags = 0;
9644961713Sgirish 	boolean_t		cksum_on = B_FALSE;
9744961713Sgirish 	uint32_t		boff = 0;
9844961713Sgirish 	uint64_t		tot_xfer_len = 0, tmp_len = 0;
9944961713Sgirish 	boolean_t		header_set = B_FALSE;
10044961713Sgirish #ifdef NXGE_DEBUG
10144961713Sgirish 	p_tx_desc_t 		tx_desc_ring_pp;
10244961713Sgirish 	p_tx_desc_t 		tx_desc_pp;
10344961713Sgirish 	tx_desc_t		*save_desc_p;
10444961713Sgirish 	int			dump_len;
10544961713Sgirish 	int			sad_len;
10644961713Sgirish 	uint64_t		sad;
10744961713Sgirish 	int			xfer_len;
10844961713Sgirish 	uint32_t		msgsize;
10944961713Sgirish #endif
11044961713Sgirish 
11144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
11244961713Sgirish 		"==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
11344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
11444961713Sgirish 		"==> nxge_start: Starting tdc %d desc pending %d",
11544961713Sgirish 		tx_ring_p->tdc, tx_ring_p->descs_pending));
11644961713Sgirish 
11744961713Sgirish 	statsp = nxgep->statsp;
11844961713Sgirish 
11944961713Sgirish 	if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) {
12014ea4bb7Ssd 		if (!statsp->mac_stats.link_up) {
12144961713Sgirish 			freemsg(mp);
12244961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
12344961713Sgirish 				"link not up or LB mode"));
12444961713Sgirish 			goto nxge_start_fail1;
12544961713Sgirish 		}
12644961713Sgirish 	}
12744961713Sgirish 
12844961713Sgirish 	hcksum_retrieve(mp, NULL, NULL, &start_offset,
12944961713Sgirish 		&stuff_offset, &end_offset, &value, &cksum_flags);
13044961713Sgirish 	if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
13144961713Sgirish 		start_offset += sizeof (ether_header_t);
13244961713Sgirish 		stuff_offset += sizeof (ether_header_t);
13344961713Sgirish 	} else {
13444961713Sgirish 		start_offset += sizeof (struct ether_vlan_header);
13544961713Sgirish 		stuff_offset += sizeof (struct ether_vlan_header);
13644961713Sgirish 	}
13744961713Sgirish 
13844961713Sgirish 	if (cksum_flags & HCK_PARTIALCKSUM) {
13944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
14044961713Sgirish 			"==> nxge_start: cksum_flags 0x%x (partial checksum) ",
14144961713Sgirish 			cksum_flags));
14244961713Sgirish 		cksum_on = B_TRUE;
14344961713Sgirish 	}
14444961713Sgirish 
14544961713Sgirish #ifdef	NXGE_DEBUG
14644961713Sgirish 	if (tx_ring_p->descs_pending) {
14744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
14844961713Sgirish 			"desc pending %d ", tx_ring_p->descs_pending));
14944961713Sgirish 	}
15044961713Sgirish 
15144961713Sgirish 	dump_len = (int)(MBLKL(mp));
15244961713Sgirish 	dump_len = (dump_len > 128) ? 128: dump_len;
15344961713Sgirish 
15444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
15544961713Sgirish 		"==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
15644961713Sgirish 		"(Before header reserve: ORIGINAL LEN %d)",
15744961713Sgirish 		tx_ring_p->tdc,
15844961713Sgirish 		mp->b_rptr,
15944961713Sgirish 		dump_len));
16044961713Sgirish 
16144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
16244961713Sgirish 		"(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
16344961713Sgirish 		nxge_dump_packet((char *)mp->b_rptr, dump_len)));
16444961713Sgirish #endif
16544961713Sgirish 
16644961713Sgirish 	MUTEX_ENTER(&tx_ring_p->lock);
16744961713Sgirish 	tdc_stats = tx_ring_p->tdc_stats;
16844961713Sgirish 	mark_mode = (tx_ring_p->descs_pending &&
16944961713Sgirish 		((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending)
17044961713Sgirish 		< nxge_tx_minfree));
17144961713Sgirish 
17244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
17344961713Sgirish 		"TX Descriptor ring is channel %d mark mode %d",
17444961713Sgirish 		tx_ring_p->tdc, mark_mode));
17544961713Sgirish 
17644961713Sgirish 	if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) {
17744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
17844961713Sgirish 			"TX Descriptor ring is full: channel %d",
17944961713Sgirish 			tx_ring_p->tdc));
18044961713Sgirish 		cas32((uint32_t *)&tx_ring_p->queueing, 0, 1);
18144961713Sgirish 		tdc_stats->tx_no_desc++;
18244961713Sgirish 		MUTEX_EXIT(&tx_ring_p->lock);
18344961713Sgirish 		if (nxgep->resched_needed && !nxgep->resched_running) {
18444961713Sgirish 			nxgep->resched_running = B_TRUE;
18544961713Sgirish 			ddi_trigger_softintr(nxgep->resched_id);
18644961713Sgirish 		}
18744961713Sgirish 		status = 1;
18844961713Sgirish 		goto nxge_start_fail1;
18944961713Sgirish 	}
19044961713Sgirish 
19144961713Sgirish 	nmp = mp;
19244961713Sgirish 	i = sop_index = tx_ring_p->wr_index;
19344961713Sgirish 	nmblks = 0;
19444961713Sgirish 	ngathers = 0;
19544961713Sgirish 	pkt_len = 0;
19644961713Sgirish 	pack_len = 0;
19744961713Sgirish 	clen = 0;
19844961713Sgirish 	last_bidx = -1;
19944961713Sgirish 	good_packet = B_TRUE;
20044961713Sgirish 
20144961713Sgirish 	desc_area = tx_ring_p->tdc_desc;
20244961713Sgirish 	npi_handle = desc_area.npi_handle;
20344961713Sgirish 	npi_desc_handle.regh = (nxge_os_acc_handle_t)
20444961713Sgirish 			DMA_COMMON_ACC_HANDLE(desc_area);
20544961713Sgirish 	tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
20644961713Sgirish #ifdef	NXGE_DEBUG
20744961713Sgirish 	tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area);
20844961713Sgirish #endif
20944961713Sgirish 	tx_desc_dma_handle = (nxge_os_dma_handle_t)
21044961713Sgirish 			DMA_COMMON_HANDLE(desc_area);
21144961713Sgirish 	tx_msg_ring = tx_ring_p->tx_msg_ring;
21244961713Sgirish 
21344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
21444961713Sgirish 		sop_index, i));
21544961713Sgirish 
21644961713Sgirish #ifdef	NXGE_DEBUG
21744961713Sgirish 	msgsize = msgdsize(nmp);
21844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
21944961713Sgirish 		"==> nxge_start(1): wr_index %d i %d msgdsize %d",
22044961713Sgirish 		sop_index, i, msgsize));
22144961713Sgirish #endif
22244961713Sgirish 	/*
22344961713Sgirish 	 * The first 16 bytes of the premapped buffer are reserved
22444961713Sgirish 	 * for header. No padding will be used.
22544961713Sgirish 	 */
22644961713Sgirish 	pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
2271f8914d5Sml 	if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
22844961713Sgirish 		bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
22944961713Sgirish 	} else {
23044961713Sgirish 		bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
23144961713Sgirish 	}
23244961713Sgirish 	while (nmp) {
23344961713Sgirish 		good_packet = B_TRUE;
23444961713Sgirish 		b_rptr = nmp->b_rptr;
23544961713Sgirish 		len = MBLKL(nmp);
23644961713Sgirish 		if (len <= 0) {
23744961713Sgirish 			nmp = nmp->b_cont;
23844961713Sgirish 			continue;
23944961713Sgirish 		}
24044961713Sgirish 		nmblks++;
24144961713Sgirish 
24244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
24344961713Sgirish 			"len %d pkt_len %d pack_len %d",
24444961713Sgirish 			nmblks, len, pkt_len, pack_len));
24544961713Sgirish 		/*
24614ea4bb7Ssd 		 * Hardware limits the transfer length to 4K for NIU and
24714ea4bb7Ssd 		 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
24814ea4bb7Ssd 		 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
24914ea4bb7Ssd 		 * If len is longer than the limit, then we break nmp into
25014ea4bb7Ssd 		 * two chunks: Make the first chunk equal to the limit and
25114ea4bb7Ssd 		 * the second chunk for the remaining data. If the second
25214ea4bb7Ssd 		 * chunk is still larger than the limit, then it will be
25314ea4bb7Ssd 		 * broken into two in the next pass.
25444961713Sgirish 		 */
25514ea4bb7Ssd 		if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
256*53f3d8ecSyc 			if ((t_mp = dupb(nmp)) != NULL) {
257*53f3d8ecSyc 				nmp->b_wptr = nmp->b_rptr +
258*53f3d8ecSyc 				    (TX_MAX_TRANSFER_LENGTH
259*53f3d8ecSyc 				    - TX_PKT_HEADER_SIZE);
260*53f3d8ecSyc 				t_mp->b_rptr = nmp->b_wptr;
261*53f3d8ecSyc 				t_mp->b_cont = nmp->b_cont;
262*53f3d8ecSyc 				nmp->b_cont = t_mp;
263*53f3d8ecSyc 				len = MBLKL(nmp);
264*53f3d8ecSyc 			} else {
265*53f3d8ecSyc 				good_packet = B_FALSE;
266*53f3d8ecSyc 				goto nxge_start_fail2;
267*53f3d8ecSyc 			}
26844961713Sgirish 		}
26944961713Sgirish 		tx_desc.value = 0;
27044961713Sgirish 		tx_desc_p = &tx_desc_ring_vp[i];
27144961713Sgirish #ifdef	NXGE_DEBUG
27244961713Sgirish 		tx_desc_pp = &tx_desc_ring_pp[i];
27344961713Sgirish #endif
27444961713Sgirish 		tx_msg_p = &tx_msg_ring[i];
275adfcba55Sjoycey #if defined(__i386)
276adfcba55Sjoycey 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
277adfcba55Sjoycey #else
27844961713Sgirish 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
279adfcba55Sjoycey #endif
28044961713Sgirish 		if (!header_set &&
28144961713Sgirish 			((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
28244961713Sgirish 				(len >= bcopy_thresh))) {
28344961713Sgirish 			header_set = B_TRUE;
28444961713Sgirish 			bcopy_thresh += TX_PKT_HEADER_SIZE;
28544961713Sgirish 			boff = 0;
28644961713Sgirish 			pack_len = 0;
28744961713Sgirish 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
28844961713Sgirish 			hdrp = (p_tx_pkt_header_t)kaddr;
28944961713Sgirish 			clen = pkt_len;
29044961713Sgirish 			dma_handle = tx_msg_p->buf_dma_handle;
29144961713Sgirish 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
29244961713Sgirish 			(void) ddi_dma_sync(dma_handle,
29344961713Sgirish 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
29444961713Sgirish 				DDI_DMA_SYNC_FORDEV);
29544961713Sgirish 
29644961713Sgirish 			tx_msg_p->flags.dma_type = USE_BCOPY;
29744961713Sgirish 			goto nxge_start_control_header_only;
29844961713Sgirish 		}
29944961713Sgirish 
30044961713Sgirish 		pkt_len += len;
30144961713Sgirish 		pack_len += len;
30244961713Sgirish 
30344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
30444961713Sgirish 			"desc entry %d "
30544961713Sgirish 			"DESC IOADDR $%p "
30644961713Sgirish 			"desc_vp $%p tx_desc_p $%p "
30744961713Sgirish 			"desc_pp $%p tx_desc_pp $%p "
30844961713Sgirish 			"len %d pkt_len %d pack_len %d",
30944961713Sgirish 			i,
31044961713Sgirish 			DMA_COMMON_IOADDR(desc_area),
31144961713Sgirish 			tx_desc_ring_vp, tx_desc_p,
31244961713Sgirish 			tx_desc_ring_pp, tx_desc_pp,
31344961713Sgirish 			len, pkt_len, pack_len));
31444961713Sgirish 
31544961713Sgirish 		if (len < bcopy_thresh) {
31644961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
31744961713Sgirish 				"USE BCOPY: "));
31844961713Sgirish 			if (nxge_tx_tiny_pack) {
31944961713Sgirish 				uint32_t blst =
32044961713Sgirish 					TXDMA_DESC_NEXT_INDEX(i, -1,
32144961713Sgirish 						tx_ring_p->tx_wrap_mask);
32244961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
32344961713Sgirish 					"==> nxge_start(5): pack"));
32444961713Sgirish 				if ((pack_len <= bcopy_thresh) &&
32544961713Sgirish 					(last_bidx == blst)) {
32644961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
32744961713Sgirish 						"==> nxge_start: pack(6) "
32844961713Sgirish 						"(pkt_len %d pack_len %d)",
32944961713Sgirish 						pkt_len, pack_len));
33044961713Sgirish 					i = blst;
33144961713Sgirish 					tx_desc_p = &tx_desc_ring_vp[i];
33244961713Sgirish #ifdef	NXGE_DEBUG
33344961713Sgirish 					tx_desc_pp = &tx_desc_ring_pp[i];
33444961713Sgirish #endif
33544961713Sgirish 					tx_msg_p = &tx_msg_ring[i];
33644961713Sgirish 					boff = pack_len - len;
33744961713Sgirish 					ngathers--;
338a3c5bd6dSspeer 				} else if (pack_len > bcopy_thresh &&
339a3c5bd6dSspeer 					header_set) {
34044961713Sgirish 					pack_len = len;
34144961713Sgirish 					boff = 0;
34244961713Sgirish 					bcopy_thresh = nxge_bcopy_thresh;
34344961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
34444961713Sgirish 						"==> nxge_start(7): > max NEW "
34544961713Sgirish 						"bcopy thresh %d "
34644961713Sgirish 						"pkt_len %d pack_len %d(next)",
34744961713Sgirish 						bcopy_thresh,
34844961713Sgirish 						pkt_len, pack_len));
34944961713Sgirish 				}
35044961713Sgirish 				last_bidx = i;
35144961713Sgirish 			}
35244961713Sgirish 			kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
35344961713Sgirish 			if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
35444961713Sgirish 				hdrp = (p_tx_pkt_header_t)kaddr;
35544961713Sgirish 				header_set = B_TRUE;
35644961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
35744961713Sgirish 					"==> nxge_start(7_x2): "
35844961713Sgirish 					"pkt_len %d pack_len %d (new hdrp $%p)",
35944961713Sgirish 					pkt_len, pack_len, hdrp));
36044961713Sgirish 			}
36144961713Sgirish 			tx_msg_p->flags.dma_type = USE_BCOPY;
36244961713Sgirish 			kaddr += boff;
36344961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
36444961713Sgirish 				"USE BCOPY: before bcopy "
36544961713Sgirish 				"DESC IOADDR $%p entry %d "
36644961713Sgirish 				"bcopy packets %d "
36744961713Sgirish 				"bcopy kaddr $%p "
36844961713Sgirish 				"bcopy ioaddr (SAD) $%p "
36944961713Sgirish 				"bcopy clen %d "
37044961713Sgirish 				"bcopy boff %d",
37144961713Sgirish 				DMA_COMMON_IOADDR(desc_area), i,
37244961713Sgirish 				tdc_stats->tx_hdr_pkts,
37344961713Sgirish 				kaddr,
37444961713Sgirish 				dma_ioaddr,
37544961713Sgirish 				clen,
37644961713Sgirish 				boff));
37744961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
37844961713Sgirish 				"1USE BCOPY: "));
37944961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
38044961713Sgirish 				"2USE BCOPY: "));
38144961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
38244961713Sgirish 				"last USE BCOPY: copy from b_rptr $%p "
38344961713Sgirish 				"to KADDR $%p (len %d offset %d",
38444961713Sgirish 				b_rptr, kaddr, len, boff));
38544961713Sgirish 
38644961713Sgirish 			bcopy(b_rptr, kaddr, len);
38744961713Sgirish 
38844961713Sgirish #ifdef	NXGE_DEBUG
38944961713Sgirish 			dump_len = (len > 128) ? 128: len;
39044961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL,
39144961713Sgirish 				"==> nxge_start: dump packets "
39244961713Sgirish 				"(After BCOPY len %d)"
39344961713Sgirish 				"(b_rptr $%p): %s", len, nmp->b_rptr,
39444961713Sgirish 				nxge_dump_packet((char *)nmp->b_rptr,
39544961713Sgirish 				dump_len)));
39644961713Sgirish #endif
39744961713Sgirish 
39844961713Sgirish 			dma_handle = tx_msg_p->buf_dma_handle;
39944961713Sgirish 			dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
40044961713Sgirish 			(void) ddi_dma_sync(dma_handle,
40144961713Sgirish 				i * nxge_bcopy_thresh, nxge_bcopy_thresh,
40244961713Sgirish 					DDI_DMA_SYNC_FORDEV);
40344961713Sgirish 			clen = len + boff;
40444961713Sgirish 			tdc_stats->tx_hdr_pkts++;
40544961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
40644961713Sgirish 				"USE BCOPY: "
40744961713Sgirish 				"DESC IOADDR $%p entry %d "
40844961713Sgirish 				"bcopy packets %d "
40944961713Sgirish 				"bcopy kaddr $%p "
41044961713Sgirish 				"bcopy ioaddr (SAD) $%p "
41144961713Sgirish 				"bcopy clen %d "
41244961713Sgirish 				"bcopy boff %d",
41344961713Sgirish 				DMA_COMMON_IOADDR(desc_area),
41444961713Sgirish 				i,
41544961713Sgirish 				tdc_stats->tx_hdr_pkts,
41644961713Sgirish 				kaddr,
41744961713Sgirish 				dma_ioaddr,
41844961713Sgirish 				clen,
41944961713Sgirish 				boff));
42044961713Sgirish 		} else {
42144961713Sgirish 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
42244961713Sgirish 				"USE DVMA: len %d", len));
42344961713Sgirish 			tx_msg_p->flags.dma_type = USE_DMA;
42444961713Sgirish 			dma_flags = DDI_DMA_WRITE;
42544961713Sgirish 			if (len < nxge_dma_stream_thresh) {
42644961713Sgirish 				dma_flags |= DDI_DMA_CONSISTENT;
42744961713Sgirish 			} else {
42844961713Sgirish 				dma_flags |= DDI_DMA_STREAMING;
42944961713Sgirish 			}
43044961713Sgirish 
43144961713Sgirish 			dma_handle = tx_msg_p->dma_handle;
43244961713Sgirish 			status = ddi_dma_addr_bind_handle(dma_handle, NULL,
43344961713Sgirish 				(caddr_t)b_rptr, len, dma_flags,
43444961713Sgirish 				DDI_DMA_DONTWAIT, NULL,
43544961713Sgirish 				&dma_cookie, &ncookies);
43644961713Sgirish 			if (status == DDI_DMA_MAPPED) {
43744961713Sgirish 				dma_ioaddr = dma_cookie.dmac_laddress;
43844961713Sgirish 				len = (int)dma_cookie.dmac_size;
43944961713Sgirish 				clen = (uint32_t)dma_cookie.dmac_size;
44044961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
44144961713Sgirish 					"==> nxge_start(12_1): "
44244961713Sgirish 					"USE DVMA: len %d clen %d "
44344961713Sgirish 					"ngathers %d",
44444961713Sgirish 					len, clen,
44544961713Sgirish 					ngathers));
446adfcba55Sjoycey #if defined(__i386)
447adfcba55Sjoycey 				npi_desc_handle.regp = (uint32_t)tx_desc_p;
448adfcba55Sjoycey #else
44944961713Sgirish 				npi_desc_handle.regp = (uint64_t)tx_desc_p;
450adfcba55Sjoycey #endif
45144961713Sgirish 				while (ncookies > 1) {
45244961713Sgirish 					ngathers++;
45344961713Sgirish 					/*
45444961713Sgirish 					 * this is the fix for multiple
45544961713Sgirish 					 * cookies, which are basicaly
45644961713Sgirish 					 * a descriptor entry, we don't set
45744961713Sgirish 					 * SOP bit as well as related fields
45844961713Sgirish 					 */
45944961713Sgirish 
46044961713Sgirish 					(void) npi_txdma_desc_gather_set(
46144961713Sgirish 						npi_desc_handle,
46244961713Sgirish 						&tx_desc,
46344961713Sgirish 						(ngathers -1),
46444961713Sgirish 						mark_mode,
46544961713Sgirish 						ngathers,
46644961713Sgirish 						dma_ioaddr,
46744961713Sgirish 						clen);
46844961713Sgirish 
46944961713Sgirish 					tx_msg_p->tx_msg_size = clen;
47044961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
47144961713Sgirish 						"==> nxge_start:  DMA "
47244961713Sgirish 						"ncookie %d "
47344961713Sgirish 						"ngathers %d "
47444961713Sgirish 						"dma_ioaddr $%p len %d"
47544961713Sgirish 						"desc $%p descp $%p (%d)",
47644961713Sgirish 						ncookies,
47744961713Sgirish 						ngathers,
47844961713Sgirish 						dma_ioaddr, clen,
47944961713Sgirish 						*tx_desc_p, tx_desc_p, i));
48044961713Sgirish 
48144961713Sgirish 					ddi_dma_nextcookie(dma_handle,
48244961713Sgirish 							&dma_cookie);
48344961713Sgirish 					dma_ioaddr =
48444961713Sgirish 						dma_cookie.dmac_laddress;
48544961713Sgirish 
48644961713Sgirish 					len = (int)dma_cookie.dmac_size;
48744961713Sgirish 					clen = (uint32_t)dma_cookie.dmac_size;
48844961713Sgirish 					NXGE_DEBUG_MSG((nxgep, TX_CTL,
48944961713Sgirish 						"==> nxge_start(12_2): "
49044961713Sgirish 						"USE DVMA: len %d clen %d ",
49144961713Sgirish 						len, clen));
49244961713Sgirish 
49344961713Sgirish 					i = TXDMA_DESC_NEXT_INDEX(i, 1,
49444961713Sgirish 						tx_ring_p->tx_wrap_mask);
49544961713Sgirish 					tx_desc_p = &tx_desc_ring_vp[i];
49644961713Sgirish 
49744961713Sgirish 					npi_desc_handle.regp =
498adfcba55Sjoycey #if defined(__i386)
499adfcba55Sjoycey 						(uint32_t)tx_desc_p;
500adfcba55Sjoycey #else
50144961713Sgirish 						(uint64_t)tx_desc_p;
502adfcba55Sjoycey #endif
50344961713Sgirish 					tx_msg_p = &tx_msg_ring[i];
50444961713Sgirish 					tx_msg_p->flags.dma_type = USE_NONE;
50544961713Sgirish 					tx_desc.value = 0;
50644961713Sgirish 
50744961713Sgirish 					ncookies--;
50844961713Sgirish 				}
50944961713Sgirish 				tdc_stats->tx_ddi_pkts++;
51044961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
51144961713Sgirish 					"DMA: ddi packets %d",
51244961713Sgirish 					tdc_stats->tx_ddi_pkts));
51344961713Sgirish 			} else {
51444961713Sgirish 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
51544961713Sgirish 				    "dma mapping failed for %d "
51644961713Sgirish 				    "bytes addr $%p flags %x (%d)",
51744961713Sgirish 				    len, b_rptr, status, status));
51844961713Sgirish 				good_packet = B_FALSE;
51944961713Sgirish 				tdc_stats->tx_dma_bind_fail++;
52044961713Sgirish 				tx_msg_p->flags.dma_type = USE_NONE;
52144961713Sgirish 				goto nxge_start_fail2;
52244961713Sgirish 			}
52344961713Sgirish 		} /* ddi dvma */
52444961713Sgirish 
52544961713Sgirish 		nmp = nmp->b_cont;
52644961713Sgirish nxge_start_control_header_only:
527adfcba55Sjoycey #if defined(__i386)
528adfcba55Sjoycey 		npi_desc_handle.regp = (uint32_t)tx_desc_p;
529adfcba55Sjoycey #else
53044961713Sgirish 		npi_desc_handle.regp = (uint64_t)tx_desc_p;
531adfcba55Sjoycey #endif
53244961713Sgirish 		ngathers++;
53344961713Sgirish 
53444961713Sgirish 		if (ngathers == 1) {
53544961713Sgirish #ifdef	NXGE_DEBUG
53644961713Sgirish 			save_desc_p = &sop_tx_desc;
53744961713Sgirish #endif
53844961713Sgirish 			sop_tx_desc_p = &sop_tx_desc;
53944961713Sgirish 			sop_tx_desc_p->value = 0;
54044961713Sgirish 			sop_tx_desc_p->bits.hdw.tr_len = clen;
54144961713Sgirish 			sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
54244961713Sgirish 			sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
54344961713Sgirish 		} else {
54444961713Sgirish #ifdef	NXGE_DEBUG
54544961713Sgirish 			save_desc_p = &tx_desc;
54644961713Sgirish #endif
54744961713Sgirish 			tmp_desc_p = &tx_desc;
54844961713Sgirish 			tmp_desc_p->value = 0;
54944961713Sgirish 			tmp_desc_p->bits.hdw.tr_len = clen;
55044961713Sgirish 			tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
55144961713Sgirish 			tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
55244961713Sgirish 
55344961713Sgirish 			tx_desc_p->value = tmp_desc_p->value;
55444961713Sgirish 		}
55544961713Sgirish 
55644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
55744961713Sgirish 			"Desc_entry %d ngathers %d "
55844961713Sgirish 			"desc_vp $%p tx_desc_p $%p "
55944961713Sgirish 			"len %d clen %d pkt_len %d pack_len %d nmblks %d "
56044961713Sgirish 			"dma_ioaddr (SAD) $%p mark %d",
56144961713Sgirish 			i, ngathers,
56244961713Sgirish 			tx_desc_ring_vp, tx_desc_p,
56344961713Sgirish 			len, clen, pkt_len, pack_len, nmblks,
56444961713Sgirish 			dma_ioaddr, mark_mode));
56544961713Sgirish 
56644961713Sgirish #ifdef NXGE_DEBUG
56744961713Sgirish 		npi_desc_handle.nxgep = nxgep;
56844961713Sgirish 		npi_desc_handle.function.function = nxgep->function_num;
56944961713Sgirish 		npi_desc_handle.function.instance = nxgep->instance;
57044961713Sgirish 		sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
57144961713Sgirish 		xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
57244961713Sgirish 			TX_PKT_DESC_TR_LEN_SHIFT);
57344961713Sgirish 
57444961713Sgirish 
57544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
57644961713Sgirish 			"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
57744961713Sgirish 			"mark %d sop %d\n",
57844961713Sgirish 			save_desc_p->value,
57944961713Sgirish 			sad,
58044961713Sgirish 			save_desc_p->bits.hdw.tr_len,
58144961713Sgirish 			xfer_len,
58244961713Sgirish 			save_desc_p->bits.hdw.num_ptr,
58344961713Sgirish 			save_desc_p->bits.hdw.mark,
58444961713Sgirish 			save_desc_p->bits.hdw.sop));
58544961713Sgirish 
58644961713Sgirish 		npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
58744961713Sgirish #endif
58844961713Sgirish 
58944961713Sgirish 		tx_msg_p->tx_msg_size = clen;
59044961713Sgirish 		i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
59144961713Sgirish 		if (ngathers > nxge_tx_max_gathers) {
59244961713Sgirish 			good_packet = B_FALSE;
59344961713Sgirish 			hcksum_retrieve(mp, NULL, NULL, &start_offset,
59444961713Sgirish 				&stuff_offset, &end_offset, &value,
59544961713Sgirish 				&cksum_flags);
59644961713Sgirish 
59744961713Sgirish 			NXGE_DEBUG_MSG((NULL, TX_CTL,
59844961713Sgirish 				"==> nxge_start(14): pull msg - "
59944961713Sgirish 				"len %d pkt_len %d ngathers %d",
60044961713Sgirish 				len, pkt_len, ngathers));
60144961713Sgirish 			/* Pull all message blocks from b_cont */
60244961713Sgirish 			if ((msgpullup(mp, -1)) == NULL) {
60344961713Sgirish 				goto nxge_start_fail2;
60444961713Sgirish 			}
60544961713Sgirish 			goto nxge_start_fail2;
60644961713Sgirish 		}
60744961713Sgirish 	} /* while (nmp) */
60844961713Sgirish 
60944961713Sgirish 	tx_msg_p->tx_message = mp;
61044961713Sgirish 	tx_desc_p = &tx_desc_ring_vp[sop_index];
611adfcba55Sjoycey #if defined(__i386)
612adfcba55Sjoycey 	npi_desc_handle.regp = (uint32_t)tx_desc_p;
613adfcba55Sjoycey #else
61444961713Sgirish 	npi_desc_handle.regp = (uint64_t)tx_desc_p;
615adfcba55Sjoycey #endif
61644961713Sgirish 
61744961713Sgirish 	pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
61844961713Sgirish 	pkthdrp->reserved = 0;
61944961713Sgirish 	hdrp->value = 0;
62044961713Sgirish 	(void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
62144961713Sgirish 		(pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp);
62244961713Sgirish 
62344961713Sgirish 	if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
62444961713Sgirish 		tdc_stats->tx_jumbo_pkts++;
62544961713Sgirish 	}
62644961713Sgirish 
62744961713Sgirish 	min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2));
62844961713Sgirish 	if (pkt_len < min_len) {
62944961713Sgirish 		/* Assume we use bcopy to premapped buffers */
63044961713Sgirish 		kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
63144961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
63244961713Sgirish 			"==> nxge_start(14-1): < (msg_min + 16)"
63344961713Sgirish 			"len %d pkt_len %d min_len %d bzero %d ngathers %d",
63444961713Sgirish 			len, pkt_len, min_len, (min_len - pkt_len), ngathers));
63544961713Sgirish 		bzero((kaddr + pkt_len), (min_len - pkt_len));
63644961713Sgirish 		pkt_len = tx_msg_p->tx_msg_size = min_len;
63744961713Sgirish 
63844961713Sgirish 		sop_tx_desc_p->bits.hdw.tr_len = min_len;
63944961713Sgirish 
64044961713Sgirish 		NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
64144961713Sgirish 		tx_desc_p->value = sop_tx_desc_p->value;
64244961713Sgirish 
64344961713Sgirish 		NXGE_DEBUG_MSG((NULL, TX_CTL,
64444961713Sgirish 			"==> nxge_start(14-2): < msg_min - "
64544961713Sgirish 			"len %d pkt_len %d min_len %d ngathers %d",
64644961713Sgirish 			len, pkt_len, min_len, ngathers));
64744961713Sgirish 	}
64844961713Sgirish 
64944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
65044961713Sgirish 		cksum_flags));
65144961713Sgirish 	if (cksum_flags & HCK_PARTIALCKSUM) {
65244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
65344961713Sgirish 			"==> nxge_start: cksum_flags 0x%x (partial checksum) ",
65444961713Sgirish 			cksum_flags));
65544961713Sgirish 		cksum_on = B_TRUE;
65644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
65744961713Sgirish 			"==> nxge_start: from IP cksum_flags 0x%x "
65844961713Sgirish 			"(partial checksum) "
65944961713Sgirish 			"start_offset %d stuff_offset %d",
66044961713Sgirish 			cksum_flags, start_offset, stuff_offset));
66144961713Sgirish 		tmp_len = (uint64_t)(start_offset >> 1);
66244961713Sgirish 		hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT);
66344961713Sgirish 		tmp_len = (uint64_t)(stuff_offset >> 1);
66444961713Sgirish 		hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT);
66544961713Sgirish 
66644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
66744961713Sgirish 			"==> nxge_start: from IP cksum_flags 0x%x "
66844961713Sgirish 			"(partial checksum) "
66944961713Sgirish 			"after SHIFT start_offset %d stuff_offset %d",
67044961713Sgirish 			cksum_flags, start_offset, stuff_offset));
67144961713Sgirish 	}
67244961713Sgirish 	{
67344961713Sgirish 		uint64_t	tmp_len;
67444961713Sgirish 
67544961713Sgirish 		/* pkt_len already includes 16 + paddings!! */
67644961713Sgirish 		/* Update the control header length */
67744961713Sgirish 		tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
67844961713Sgirish 		tmp_len = hdrp->value |
67944961713Sgirish 			(tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
68044961713Sgirish 
68144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
68244961713Sgirish 			"==> nxge_start(15_x1): setting SOP "
68344961713Sgirish 			"tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
68444961713Sgirish 			"0x%llx hdrp->value 0x%llx",
68544961713Sgirish 			tot_xfer_len, tot_xfer_len, pkt_len,
68644961713Sgirish 			tmp_len, hdrp->value));
68744961713Sgirish #if defined(_BIG_ENDIAN)
68844961713Sgirish 		hdrp->value = ddi_swap64(tmp_len);
68944961713Sgirish #else
69044961713Sgirish 		hdrp->value = tmp_len;
69144961713Sgirish #endif
69244961713Sgirish 		NXGE_DEBUG_MSG((nxgep,
69344961713Sgirish 			TX_CTL, "==> nxge_start(15_x2): setting SOP "
69444961713Sgirish 			"after SWAP: tot_xfer_len 0x%llx pkt_len %d "
69544961713Sgirish 			"tmp_len 0x%llx hdrp->value 0x%llx",
69644961713Sgirish 			tot_xfer_len, pkt_len,
69744961713Sgirish 			tmp_len, hdrp->value));
69844961713Sgirish 	}
69944961713Sgirish 
70044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
70144961713Sgirish 		"wr_index %d "
70244961713Sgirish 		"tot_xfer_len (%d) pkt_len %d npads %d",
70344961713Sgirish 		sop_index,
70444961713Sgirish 		tot_xfer_len, pkt_len,
70544961713Sgirish 		npads));
70644961713Sgirish 
70744961713Sgirish 	sop_tx_desc_p->bits.hdw.sop = 1;
70844961713Sgirish 	sop_tx_desc_p->bits.hdw.mark = mark_mode;
70944961713Sgirish 	sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
71044961713Sgirish 
71144961713Sgirish 	NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
71244961713Sgirish 
71344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
71444961713Sgirish 
71544961713Sgirish #ifdef NXGE_DEBUG
71644961713Sgirish 	npi_desc_handle.nxgep = nxgep;
71744961713Sgirish 	npi_desc_handle.function.function = nxgep->function_num;
71844961713Sgirish 	npi_desc_handle.function.instance = nxgep->instance;
71944961713Sgirish 
72044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
72144961713Sgirish 		"\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
72244961713Sgirish 		save_desc_p->value,
72344961713Sgirish 		sad,
72444961713Sgirish 		save_desc_p->bits.hdw.tr_len,
72544961713Sgirish 		xfer_len,
72644961713Sgirish 		save_desc_p->bits.hdw.num_ptr,
72744961713Sgirish 		save_desc_p->bits.hdw.mark,
72844961713Sgirish 		save_desc_p->bits.hdw.sop));
72944961713Sgirish 	(void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
73044961713Sgirish 
73144961713Sgirish 	dump_len = (pkt_len > 128) ? 128: pkt_len;
73244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
73344961713Sgirish 		"==> nxge_start: dump packets(17) (after sop set, len "
73444961713Sgirish 		" (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
73544961713Sgirish 		"ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
73644961713Sgirish 		(char *)hdrp,
73744961713Sgirish 		nxge_dump_packet((char *)hdrp, dump_len)));
73844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL,
73944961713Sgirish 		"==> nxge_start(18): TX desc sync: sop_index %d",
74044961713Sgirish 			sop_index));
74144961713Sgirish #endif
74244961713Sgirish 
74344961713Sgirish 	if ((ngathers == 1) || tx_ring_p->wr_index < i) {
74444961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
74544961713Sgirish 			sop_index * sizeof (tx_desc_t),
74644961713Sgirish 			ngathers * sizeof (tx_desc_t),
74744961713Sgirish 			DDI_DMA_SYNC_FORDEV);
74844961713Sgirish 
74944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
75044961713Sgirish 			"cs_off = 0x%02X cs_s_off = 0x%02X "
75144961713Sgirish 			"pkt_len %d ngathers %d sop_index %d\n",
75244961713Sgirish 			stuff_offset, start_offset,
75344961713Sgirish 			pkt_len, ngathers, sop_index));
75444961713Sgirish 	} else { /* more than one descriptor and wrap around */
75544961713Sgirish 		uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
75644961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
75744961713Sgirish 			sop_index * sizeof (tx_desc_t),
75844961713Sgirish 			nsdescs * sizeof (tx_desc_t),
75944961713Sgirish 			DDI_DMA_SYNC_FORDEV);
76044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
76144961713Sgirish 			"cs_off = 0x%02X cs_s_off = 0x%02X "
76244961713Sgirish 			"pkt_len %d ngathers %d sop_index %d\n",
76344961713Sgirish 			stuff_offset, start_offset,
76444961713Sgirish 				pkt_len, ngathers, sop_index));
76544961713Sgirish 
76644961713Sgirish 		(void) ddi_dma_sync(tx_desc_dma_handle,
76744961713Sgirish 			0,
76844961713Sgirish 			(ngathers - nsdescs) * sizeof (tx_desc_t),
76944961713Sgirish 			DDI_DMA_SYNC_FORDEV);
77044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
77144961713Sgirish 			"cs_off = 0x%02X cs_s_off = 0x%02X "
77244961713Sgirish 			"pkt_len %d ngathers %d sop_index %d\n",
77344961713Sgirish 			stuff_offset, start_offset,
77444961713Sgirish 			pkt_len, ngathers, sop_index));
77544961713Sgirish 	}
77644961713Sgirish 
77744961713Sgirish 	tail_index = tx_ring_p->wr_index;
77844961713Sgirish 	tail_wrap = tx_ring_p->wr_index_wrap;
77944961713Sgirish 
78044961713Sgirish 	tx_ring_p->wr_index = i;
78144961713Sgirish 	if (tx_ring_p->wr_index <= tail_index) {
78244961713Sgirish 		tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
78344961713Sgirish 						B_FALSE : B_TRUE);
78444961713Sgirish 	}
78544961713Sgirish 
78644961713Sgirish 	tx_ring_p->descs_pending += ngathers;
78744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
78844961713Sgirish 		"channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
78944961713Sgirish 		tx_ring_p->tdc,
79044961713Sgirish 		tx_ring_p->wr_index,
79144961713Sgirish 		tx_ring_p->wr_index_wrap,
79244961713Sgirish 		ngathers,
79344961713Sgirish 		tx_ring_p->descs_pending));
79444961713Sgirish 
79544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
79644961713Sgirish 
79744961713Sgirish 	{
79844961713Sgirish 		tx_ring_kick_t		kick;
79944961713Sgirish 
80044961713Sgirish 		kick.value = 0;
80144961713Sgirish 		kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
80244961713Sgirish 		kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
80344961713Sgirish 
80444961713Sgirish 		/* Kick start the Transmit kick register */
80544961713Sgirish 		TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
80644961713Sgirish 			TX_RING_KICK_REG,
80744961713Sgirish 			(uint8_t)tx_ring_p->tdc,
80844961713Sgirish 			kick.value);
80944961713Sgirish 	}
81044961713Sgirish 
81144961713Sgirish 	tdc_stats->tx_starts++;
81244961713Sgirish 
81344961713Sgirish 	MUTEX_EXIT(&tx_ring_p->lock);
81444961713Sgirish 
81544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
81644961713Sgirish 
81744961713Sgirish 	return (status);
81844961713Sgirish 
81944961713Sgirish nxge_start_fail2:
82044961713Sgirish 	if (good_packet == B_FALSE) {
82144961713Sgirish 		cur_index = sop_index;
82244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
82344961713Sgirish 		for (i = 0; i < ngathers; i++) {
82444961713Sgirish 			tx_desc_p = &tx_desc_ring_vp[cur_index];
825adfcba55Sjoycey #if defined(__i386)
826adfcba55Sjoycey 			npi_handle.regp = (uint32_t)tx_desc_p;
827adfcba55Sjoycey #else
82844961713Sgirish 			npi_handle.regp = (uint64_t)tx_desc_p;
829adfcba55Sjoycey #endif
83044961713Sgirish 			tx_msg_p = &tx_msg_ring[cur_index];
83144961713Sgirish 			(void) npi_txdma_desc_set_zero(npi_handle, 1);
83244961713Sgirish 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
83344961713Sgirish 				NXGE_DEBUG_MSG((nxgep, TX_CTL,
834*53f3d8ecSyc 				    "tx_desc_p = %X index = %d",
835*53f3d8ecSyc 				    tx_desc_p, tx_ring_p->rd_index));
836*53f3d8ecSyc 				(void) dvma_unload(tx_msg_p->dvma_handle,
837*53f3d8ecSyc 				    0, -1);
83844961713Sgirish 				tx_msg_p->dvma_handle = NULL;
83944961713Sgirish 				if (tx_ring_p->dvma_wr_index ==
840*53f3d8ecSyc 				    tx_ring_p->dvma_wrap_mask)
84144961713Sgirish 					tx_ring_p->dvma_wr_index = 0;
84244961713Sgirish 				else
84344961713Sgirish 					tx_ring_p->dvma_wr_index++;
84444961713Sgirish 				tx_ring_p->dvma_pending--;
845*53f3d8ecSyc 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
84644961713Sgirish 				if (ddi_dma_unbind_handle(
847*53f3d8ecSyc 				    tx_msg_p->dma_handle)) {
84844961713Sgirish 					cmn_err(CE_WARN, "!nxge_start: "
849*53f3d8ecSyc 					    "ddi_dma_unbind_handle failed");
850*53f3d8ecSyc 				}
85144961713Sgirish 			}
85244961713Sgirish 			tx_msg_p->flags.dma_type = USE_NONE;
85344961713Sgirish 			cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
85444961713Sgirish 				tx_ring_p->tx_wrap_mask);
85544961713Sgirish 
85644961713Sgirish 		}
85744961713Sgirish 
85844961713Sgirish 		nxgep->resched_needed = B_TRUE;
85944961713Sgirish 	}
86044961713Sgirish 
86144961713Sgirish 	MUTEX_EXIT(&tx_ring_p->lock);
86244961713Sgirish 
86344961713Sgirish nxge_start_fail1:
86444961713Sgirish 	/* Add FMA to check the access handle nxge_hregh */
86544961713Sgirish 
86644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
86744961713Sgirish 
86844961713Sgirish 	return (status);
86944961713Sgirish }
87044961713Sgirish 
8711f8914d5Sml int
8721f8914d5Sml nxge_serial_tx(mblk_t *mp, void *arg)
8731f8914d5Sml {
8741f8914d5Sml 	p_tx_ring_t		tx_ring_p = (p_tx_ring_t)arg;
8751f8914d5Sml 	p_nxge_t		nxgep = tx_ring_p->nxgep;
8761f8914d5Sml 
8771f8914d5Sml 	return (nxge_start(nxgep, tx_ring_p, mp));
8781f8914d5Sml }
8791f8914d5Sml 
88044961713Sgirish boolean_t
88144961713Sgirish nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp)
88244961713Sgirish {
88344961713Sgirish 	p_tx_ring_t 		*tx_rings;
88444961713Sgirish 	uint8_t			ring_index;
8851f8914d5Sml 	p_tx_ring_t		tx_ring_p;
88644961713Sgirish 
88744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send"));
88844961713Sgirish 
88944961713Sgirish 	ASSERT(mp->b_next == NULL);
89044961713Sgirish 
89144961713Sgirish 	ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp);
89244961713Sgirish 	tx_rings = nxgep->tx_rings->rings;
89344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p",
89444961713Sgirish 		tx_rings));
89544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d "
89644961713Sgirish 		"ring_index %d", nxgep->max_tdcs, ring_index));
89744961713Sgirish 
8981f8914d5Sml 	switch (nxge_tx_scheme) {
8991f8914d5Sml 	case NXGE_USE_START:
9001f8914d5Sml 		if (nxge_start(nxgep, tx_rings[ring_index], mp)) {
9011f8914d5Sml 			NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed "
9021f8914d5Sml 				"ring index %d", ring_index));
9031f8914d5Sml 			return (B_FALSE);
9041f8914d5Sml 		}
9051f8914d5Sml 		break;
9061f8914d5Sml 
9071f8914d5Sml 	case NXGE_USE_SERIAL:
9081f8914d5Sml 	default:
9091f8914d5Sml 		tx_ring_p = tx_rings[ring_index];
9101f8914d5Sml 		nxge_serialize_enter(tx_ring_p->serial, mp);
9111f8914d5Sml 		break;
91244961713Sgirish 	}
91344961713Sgirish 
91444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d",
91544961713Sgirish 		ring_index));
91644961713Sgirish 
91744961713Sgirish 	return (B_TRUE);
91844961713Sgirish }
91944961713Sgirish 
92044961713Sgirish /*
92144961713Sgirish  * nxge_m_tx() - send a chain of packets
92244961713Sgirish  */
92344961713Sgirish mblk_t *
92444961713Sgirish nxge_m_tx(void *arg, mblk_t *mp)
92544961713Sgirish {
92644961713Sgirish 	p_nxge_t 		nxgep = (p_nxge_t)arg;
92744961713Sgirish 	mblk_t 			*next;
92844961713Sgirish 	mac_tx_hint_t		hint;
92944961713Sgirish 
93044961713Sgirish 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
93144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
93244961713Sgirish 			"==> nxge_m_tx: hardware not initialized"));
93344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, DDI_CTL,
93444961713Sgirish 			"<== nxge_m_tx"));
93544961713Sgirish 		return (mp);
93644961713Sgirish 	}
93744961713Sgirish 
93844961713Sgirish 	hint.hash =  NULL;
93944961713Sgirish 	hint.vid =  0;
94044961713Sgirish 	hint.sap =  0;
94144961713Sgirish 
94244961713Sgirish 	while (mp != NULL) {
94344961713Sgirish 		next = mp->b_next;
94444961713Sgirish 		mp->b_next = NULL;
94544961713Sgirish 
94644961713Sgirish 		/*
94744961713Sgirish 		 * Until Nemo tx resource works, the mac driver
94844961713Sgirish 		 * does the load balancing based on TCP port,
94944961713Sgirish 		 * or CPU. For debugging, we use a system
95044961713Sgirish 		 * configurable parameter.
95144961713Sgirish 		 */
95244961713Sgirish 		if (!nxge_send(nxgep, mp, &hint)) {
95344961713Sgirish 			mp->b_next = next;
95444961713Sgirish 			break;
95544961713Sgirish 		}
95644961713Sgirish 
95744961713Sgirish 		mp = next;
95844961713Sgirish 	}
95944961713Sgirish 
96044961713Sgirish 	return (mp);
96144961713Sgirish }
96244961713Sgirish 
96344961713Sgirish int
96444961713Sgirish nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp)
96544961713Sgirish {
96644961713Sgirish 	uint8_t 		ring_index = 0;
96744961713Sgirish 	uint8_t 		*tcp_port;
96844961713Sgirish 	p_mblk_t 		nmp;
96944961713Sgirish 	size_t 			mblk_len;
97044961713Sgirish 	size_t 			iph_len;
97144961713Sgirish 	size_t 			hdrs_size;
97244961713Sgirish 	uint8_t			hdrs_buf[sizeof (struct  ether_header) +
97344961713Sgirish 					IP_MAX_HDR_LENGTH + sizeof (uint32_t)];
97444961713Sgirish 				/*
97544961713Sgirish 				 * allocate space big enough to cover
97644961713Sgirish 				 * the max ip header length and the first
97744961713Sgirish 				 * 4 bytes of the TCP/IP header.
97844961713Sgirish 				 */
97944961713Sgirish 
98044961713Sgirish 	boolean_t		qos = B_FALSE;
98144961713Sgirish 
98244961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring"));
98344961713Sgirish 
98444961713Sgirish 	if (hp->vid) {
98544961713Sgirish 		qos = B_TRUE;
98644961713Sgirish 	}
98744961713Sgirish 	switch (nxge_tx_lb_policy) {
98844961713Sgirish 	case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */
98944961713Sgirish 	default:
99044961713Sgirish 		tcp_port = mp->b_rptr;
99144961713Sgirish 		if (!nxge_no_tx_lb && !qos &&
99244961713Sgirish 			(ntohs(((p_ether_header_t)tcp_port)->ether_type)
99344961713Sgirish 				== ETHERTYPE_IP)) {
99444961713Sgirish 			nmp = mp;
99544961713Sgirish 			mblk_len = MBLKL(nmp);
99644961713Sgirish 			tcp_port = NULL;
99744961713Sgirish 			if (mblk_len > sizeof (struct ether_header) +
99844961713Sgirish 					sizeof (uint8_t)) {
99944961713Sgirish 				tcp_port = nmp->b_rptr +
100044961713Sgirish 					sizeof (struct ether_header);
100144961713Sgirish 				mblk_len -= sizeof (struct ether_header);
100244961713Sgirish 				iph_len = ((*tcp_port) & 0x0f) << 2;
100344961713Sgirish 				if (mblk_len > (iph_len + sizeof (uint32_t))) {
100444961713Sgirish 					tcp_port = nmp->b_rptr;
100544961713Sgirish 				} else {
100644961713Sgirish 					tcp_port = NULL;
100744961713Sgirish 				}
100844961713Sgirish 			}
100944961713Sgirish 			if (tcp_port == NULL) {
101044961713Sgirish 				hdrs_size = 0;
101144961713Sgirish 				((p_ether_header_t)hdrs_buf)->ether_type = 0;
101244961713Sgirish 				while ((nmp) && (hdrs_size <
101344961713Sgirish 						sizeof (hdrs_buf))) {
101444961713Sgirish 					mblk_len = MBLKL(nmp);
101544961713Sgirish 					if (mblk_len >=
101644961713Sgirish 						(sizeof (hdrs_buf) - hdrs_size))
101744961713Sgirish 						mblk_len = sizeof (hdrs_buf) -
101844961713Sgirish 							hdrs_size;
101944961713Sgirish 					bcopy(nmp->b_rptr,
102044961713Sgirish 						&hdrs_buf[hdrs_size], mblk_len);
102144961713Sgirish 					hdrs_size += mblk_len;
102244961713Sgirish 					nmp = nmp->b_cont;
102344961713Sgirish 				}
102444961713Sgirish 				tcp_port = hdrs_buf;
102544961713Sgirish 			}
102644961713Sgirish 			tcp_port += sizeof (ether_header_t);
102744961713Sgirish 			if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) {
1028958cea9eSml 				switch (tcp_port[9]) {
1029958cea9eSml 				case IPPROTO_TCP:
1030958cea9eSml 				case IPPROTO_UDP:
1031958cea9eSml 				case IPPROTO_ESP:
103244961713Sgirish 					tcp_port += ((*tcp_port) & 0x0f) << 2;
103344961713Sgirish 					ring_index =
1034958cea9eSml 					    ((tcp_port[0] ^
1035958cea9eSml 					    tcp_port[1] ^
1036958cea9eSml 					    tcp_port[2] ^
1037958cea9eSml 					    tcp_port[3]) % maxtdcs);
1038958cea9eSml 					break;
1039958cea9eSml 
1040958cea9eSml 				case IPPROTO_AH:
1041958cea9eSml 					/* SPI starts at the 4th byte */
1042958cea9eSml 					tcp_port += ((*tcp_port) & 0x0f) << 2;
1043958cea9eSml 					ring_index =
1044958cea9eSml 					    ((tcp_port[4] ^
1045958cea9eSml 					    tcp_port[5] ^
1046958cea9eSml 					    tcp_port[6] ^
1047958cea9eSml 					    tcp_port[7]) % maxtdcs);
1048958cea9eSml 					break;
1049958cea9eSml 
1050958cea9eSml 				default:
105144961713Sgirish 					ring_index = tcp_port[19] % maxtdcs;
1052958cea9eSml 					break;
105344961713Sgirish 				}
105444961713Sgirish 			} else { /* fragmented packet */
105544961713Sgirish 				ring_index = tcp_port[19] % maxtdcs;
105644961713Sgirish 			}
105744961713Sgirish 		} else {
105844961713Sgirish 			ring_index = mp->b_band % maxtdcs;
105944961713Sgirish 		}
106044961713Sgirish 		break;
106144961713Sgirish 
106244961713Sgirish 	case NXGE_TX_LB_HASH:
106344961713Sgirish 		if (hp->hash) {
1064adfcba55Sjoycey #if defined(__i386)
1065adfcba55Sjoycey 			ring_index = ((uint32_t)(hp->hash) % maxtdcs);
1066adfcba55Sjoycey #else
106744961713Sgirish 			ring_index = ((uint64_t)(hp->hash) % maxtdcs);
1068adfcba55Sjoycey #endif
106944961713Sgirish 		} else {
107044961713Sgirish 			ring_index = mp->b_band % maxtdcs;
107144961713Sgirish 		}
107244961713Sgirish 		break;
107344961713Sgirish 
107444961713Sgirish 	case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */
107544961713Sgirish 		tcp_port = mp->b_rptr;
107644961713Sgirish 		ring_index = tcp_port[5] % maxtdcs;
107744961713Sgirish 		break;
107844961713Sgirish 	}
107944961713Sgirish 
108044961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring"));
108144961713Sgirish 
108244961713Sgirish 	return (ring_index);
108344961713Sgirish }
108444961713Sgirish 
108544961713Sgirish uint_t
108644961713Sgirish nxge_reschedule(caddr_t arg)
108744961713Sgirish {
108844961713Sgirish 	p_nxge_t nxgep;
108944961713Sgirish 
109044961713Sgirish 	nxgep = (p_nxge_t)arg;
109144961713Sgirish 
109244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule"));
109344961713Sgirish 
109444961713Sgirish 	if (nxgep->nxge_mac_state == NXGE_MAC_STARTED &&
109544961713Sgirish 			nxgep->resched_needed) {
109644961713Sgirish 		mac_tx_update(nxgep->mach);
109744961713Sgirish 		nxgep->resched_needed = B_FALSE;
109844961713Sgirish 		nxgep->resched_running = B_FALSE;
109944961713Sgirish 	}
110044961713Sgirish 
110144961713Sgirish 	NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule"));
110244961713Sgirish 	return (DDI_INTR_CLAIMED);
110344961713Sgirish }
1104