144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 2230ac2e7bSml * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 2644961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2744961713Sgirish 28a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 2944961713Sgirish 3030ac2e7bSml /* Software LSO required header files */ 3130ac2e7bSml #include <netinet/tcp.h> 3230ac2e7bSml #include <inet/ip_impl.h> 3330ac2e7bSml #include <inet/tcp.h> 3430ac2e7bSml 3530ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *); 3630ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 3730ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 3830ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *, 3930ac2e7bSml uint32_t *, uint32_t *, uint32_t *, 4030ac2e7bSml uint32_t *, uint32_t *); 4130ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int); 4230ac2e7bSml 4344961713Sgirish extern uint32_t nxge_reclaim_pending; 4444961713Sgirish extern uint32_t nxge_bcopy_thresh; 4544961713Sgirish extern uint32_t nxge_dvma_thresh; 4644961713Sgirish extern uint32_t nxge_dma_stream_thresh; 4744961713Sgirish extern uint32_t nxge_tx_minfree; 4844961713Sgirish extern uint32_t nxge_tx_intr_thres; 4944961713Sgirish extern uint32_t nxge_tx_max_gathers; 5044961713Sgirish extern uint32_t nxge_tx_tiny_pack; 5144961713Sgirish extern uint32_t nxge_tx_use_bcopy; 5244961713Sgirish extern uint32_t nxge_tx_lb_policy; 5344961713Sgirish extern uint32_t nxge_no_tx_lb; 541f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 5530ac2e7bSml uint32_t nxge_lso_kick_cnt = 2; 5644961713Sgirish 5744961713Sgirish typedef struct _mac_tx_hint { 5844961713Sgirish uint16_t sap; 5944961713Sgirish uint16_t vid; 6044961713Sgirish void *hash; 6144961713Sgirish } mac_tx_hint_t, *p_mac_tx_hint_t; 6244961713Sgirish 6344961713Sgirish int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 6444961713Sgirish 6544961713Sgirish int 6644961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 6744961713Sgirish { 6844961713Sgirish int status = 0; 6944961713Sgirish p_tx_desc_t tx_desc_ring_vp; 7044961713Sgirish npi_handle_t npi_desc_handle; 7144961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 7244961713Sgirish p_tx_desc_t tx_desc_p; 7344961713Sgirish p_tx_msg_t tx_msg_ring; 7444961713Sgirish p_tx_msg_t tx_msg_p; 7544961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 7644961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 7744961713Sgirish p_tx_pkt_header_t hdrp; 7844961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 7944961713Sgirish uint8_t npads = 0; 8044961713Sgirish uint64_t dma_ioaddr; 8144961713Sgirish uint32_t dma_flags; 8244961713Sgirish int last_bidx; 8344961713Sgirish uint8_t *b_rptr; 8444961713Sgirish caddr_t kaddr; 8544961713Sgirish uint32_t nmblks; 8644961713Sgirish uint32_t ngathers; 8744961713Sgirish uint32_t clen; 8844961713Sgirish int len; 8944961713Sgirish uint32_t pkt_len, pack_len, min_len; 9044961713Sgirish uint32_t bcopy_thresh; 9144961713Sgirish int i, cur_index, sop_index; 9244961713Sgirish uint16_t tail_index; 9344961713Sgirish boolean_t tail_wrap = B_FALSE; 9444961713Sgirish nxge_dma_common_t desc_area; 9544961713Sgirish nxge_os_dma_handle_t dma_handle; 9644961713Sgirish ddi_dma_cookie_t dma_cookie; 9744961713Sgirish npi_handle_t npi_handle; 9844961713Sgirish p_mblk_t nmp; 9944961713Sgirish p_mblk_t t_mp; 10044961713Sgirish uint32_t ncookies; 10144961713Sgirish boolean_t good_packet; 10244961713Sgirish boolean_t mark_mode = B_FALSE; 10344961713Sgirish p_nxge_stats_t statsp; 10444961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 10544961713Sgirish t_uscalar_t start_offset = 0; 10644961713Sgirish t_uscalar_t stuff_offset = 0; 10744961713Sgirish t_uscalar_t end_offset = 0; 10844961713Sgirish t_uscalar_t value = 0; 10944961713Sgirish t_uscalar_t cksum_flags = 0; 11044961713Sgirish boolean_t cksum_on = B_FALSE; 11144961713Sgirish uint32_t boff = 0; 11244961713Sgirish uint64_t tot_xfer_len = 0, tmp_len = 0; 11344961713Sgirish boolean_t header_set = B_FALSE; 11444961713Sgirish #ifdef NXGE_DEBUG 11544961713Sgirish p_tx_desc_t tx_desc_ring_pp; 11644961713Sgirish p_tx_desc_t tx_desc_pp; 11744961713Sgirish tx_desc_t *save_desc_p; 11844961713Sgirish int dump_len; 11944961713Sgirish int sad_len; 12044961713Sgirish uint64_t sad; 12144961713Sgirish int xfer_len; 12244961713Sgirish uint32_t msgsize; 12344961713Sgirish #endif 12430ac2e7bSml p_mblk_t mp_chain = NULL; 12530ac2e7bSml boolean_t is_lso = B_FALSE; 12630ac2e7bSml boolean_t lso_again; 12730ac2e7bSml int cur_index_lso; 12830ac2e7bSml p_mblk_t nmp_lso_save; 12930ac2e7bSml uint32_t lso_ngathers; 13030ac2e7bSml boolean_t lso_tail_wrap = B_FALSE; 13144961713Sgirish 13244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 13344961713Sgirish "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 13444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 13544961713Sgirish "==> nxge_start: Starting tdc %d desc pending %d", 13644961713Sgirish tx_ring_p->tdc, tx_ring_p->descs_pending)); 13744961713Sgirish 13844961713Sgirish statsp = nxgep->statsp; 13944961713Sgirish 14044961713Sgirish if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 14114ea4bb7Ssd if (!statsp->mac_stats.link_up) { 14244961713Sgirish freemsg(mp); 14344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 14444961713Sgirish "link not up or LB mode")); 14544961713Sgirish goto nxge_start_fail1; 14644961713Sgirish } 14744961713Sgirish } 14844961713Sgirish 149*3d16f8e7Sml if (nxgep->soft_lso_enable) { 15030ac2e7bSml mp_chain = nxge_lso_eliminate(mp); 15130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 15230ac2e7bSml "==> nxge_start(0): LSO mp $%p mp_chain $%p", 15330ac2e7bSml mp, mp_chain)); 15430ac2e7bSml if (mp_chain == NULL) { 15530ac2e7bSml NXGE_ERROR_MSG((nxgep, TX_CTL, 15630ac2e7bSml "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 15730ac2e7bSml mp_chain, mp)); 15830ac2e7bSml goto nxge_start_fail1; 15930ac2e7bSml } 16030ac2e7bSml if (mp_chain != mp) { 16130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 16230ac2e7bSml "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 16330ac2e7bSml mp_chain, mp)); 16430ac2e7bSml is_lso = B_TRUE; 16530ac2e7bSml mp = mp_chain; 16630ac2e7bSml mp_chain = mp_chain->b_next; 16730ac2e7bSml mp->b_next = NULL; 16830ac2e7bSml } 16930ac2e7bSml } 17030ac2e7bSml 17144961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 17244961713Sgirish &stuff_offset, &end_offset, &value, &cksum_flags); 17344961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 17444961713Sgirish start_offset += sizeof (ether_header_t); 17544961713Sgirish stuff_offset += sizeof (ether_header_t); 17644961713Sgirish } else { 17744961713Sgirish start_offset += sizeof (struct ether_vlan_header); 17844961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 17944961713Sgirish } 18044961713Sgirish 18144961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 18244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 18330ac2e7bSml "==> nxge_start: mp $%p len %d " 18430ac2e7bSml "cksum_flags 0x%x (partial checksum) ", 18530ac2e7bSml mp, MBLKL(mp), cksum_flags)); 18644961713Sgirish cksum_on = B_TRUE; 18744961713Sgirish } 18844961713Sgirish 18930ac2e7bSml lso_again = B_FALSE; 19030ac2e7bSml lso_ngathers = 0; 19130ac2e7bSml 19230ac2e7bSml MUTEX_ENTER(&tx_ring_p->lock); 19330ac2e7bSml cur_index_lso = tx_ring_p->wr_index; 19430ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 19530ac2e7bSml start_again: 19630ac2e7bSml ngathers = 0; 19730ac2e7bSml sop_index = tx_ring_p->wr_index; 19844961713Sgirish #ifdef NXGE_DEBUG 19944961713Sgirish if (tx_ring_p->descs_pending) { 20044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 20144961713Sgirish "desc pending %d ", tx_ring_p->descs_pending)); 20244961713Sgirish } 20344961713Sgirish 20444961713Sgirish dump_len = (int)(MBLKL(mp)); 20544961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 20644961713Sgirish 20744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 20844961713Sgirish "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 20944961713Sgirish "(Before header reserve: ORIGINAL LEN %d)", 21044961713Sgirish tx_ring_p->tdc, 21144961713Sgirish mp->b_rptr, 21244961713Sgirish dump_len)); 21344961713Sgirish 21444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 21544961713Sgirish "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 21644961713Sgirish nxge_dump_packet((char *)mp->b_rptr, dump_len))); 21744961713Sgirish #endif 21844961713Sgirish 21944961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 22044961713Sgirish mark_mode = (tx_ring_p->descs_pending && 22144961713Sgirish ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 22244961713Sgirish < nxge_tx_minfree)); 22344961713Sgirish 22444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 22544961713Sgirish "TX Descriptor ring is channel %d mark mode %d", 22644961713Sgirish tx_ring_p->tdc, mark_mode)); 22744961713Sgirish 22844961713Sgirish if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 22944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 23044961713Sgirish "TX Descriptor ring is full: channel %d", 23144961713Sgirish tx_ring_p->tdc)); 23230ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 23330ac2e7bSml "TX Descriptor ring is full: channel %d", 23430ac2e7bSml tx_ring_p->tdc)); 23530ac2e7bSml if (is_lso) { 23630ac2e7bSml /* free the current mp and mp_chain if not FULL */ 23730ac2e7bSml tdc_stats->tx_no_desc++; 23830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 23930ac2e7bSml "LSO packet: TX Descriptor ring is full: " 24030ac2e7bSml "channel %d", 24130ac2e7bSml tx_ring_p->tdc)); 24230ac2e7bSml goto nxge_start_fail_lso; 24330ac2e7bSml } else { 24430ac2e7bSml cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 24530ac2e7bSml tdc_stats->tx_no_desc++; 24630ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 24730ac2e7bSml if (nxgep->resched_needed && !nxgep->resched_running) { 24830ac2e7bSml nxgep->resched_running = B_TRUE; 24930ac2e7bSml ddi_trigger_softintr(nxgep->resched_id); 25030ac2e7bSml } 25130ac2e7bSml status = 1; 25230ac2e7bSml goto nxge_start_fail1; 25344961713Sgirish } 25444961713Sgirish } 25544961713Sgirish 25644961713Sgirish nmp = mp; 25744961713Sgirish i = sop_index = tx_ring_p->wr_index; 25844961713Sgirish nmblks = 0; 25944961713Sgirish ngathers = 0; 26044961713Sgirish pkt_len = 0; 26144961713Sgirish pack_len = 0; 26244961713Sgirish clen = 0; 26344961713Sgirish last_bidx = -1; 26444961713Sgirish good_packet = B_TRUE; 26544961713Sgirish 26644961713Sgirish desc_area = tx_ring_p->tdc_desc; 26744961713Sgirish npi_handle = desc_area.npi_handle; 26844961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 26944961713Sgirish DMA_COMMON_ACC_HANDLE(desc_area); 27044961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 27144961713Sgirish #ifdef NXGE_DEBUG 27244961713Sgirish tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 27344961713Sgirish #endif 27444961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 27544961713Sgirish DMA_COMMON_HANDLE(desc_area); 27644961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 27744961713Sgirish 27844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 27944961713Sgirish sop_index, i)); 28044961713Sgirish 28144961713Sgirish #ifdef NXGE_DEBUG 28244961713Sgirish msgsize = msgdsize(nmp); 28344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 28444961713Sgirish "==> nxge_start(1): wr_index %d i %d msgdsize %d", 28544961713Sgirish sop_index, i, msgsize)); 28644961713Sgirish #endif 28744961713Sgirish /* 28844961713Sgirish * The first 16 bytes of the premapped buffer are reserved 28944961713Sgirish * for header. No padding will be used. 29044961713Sgirish */ 29144961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 2921f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 29344961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 29444961713Sgirish } else { 29544961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 29644961713Sgirish } 29744961713Sgirish while (nmp) { 29844961713Sgirish good_packet = B_TRUE; 29944961713Sgirish b_rptr = nmp->b_rptr; 30044961713Sgirish len = MBLKL(nmp); 30144961713Sgirish if (len <= 0) { 30244961713Sgirish nmp = nmp->b_cont; 30344961713Sgirish continue; 30444961713Sgirish } 30544961713Sgirish nmblks++; 30644961713Sgirish 30744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 30844961713Sgirish "len %d pkt_len %d pack_len %d", 30944961713Sgirish nmblks, len, pkt_len, pack_len)); 31044961713Sgirish /* 31114ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 31214ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 31314ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 31414ea4bb7Ssd * If len is longer than the limit, then we break nmp into 31514ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 31614ea4bb7Ssd * the second chunk for the remaining data. If the second 31714ea4bb7Ssd * chunk is still larger than the limit, then it will be 31814ea4bb7Ssd * broken into two in the next pass. 31944961713Sgirish */ 32014ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 32153f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 32253f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 32353f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 32453f3d8ecSyc - TX_PKT_HEADER_SIZE); 32553f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 32653f3d8ecSyc t_mp->b_cont = nmp->b_cont; 32753f3d8ecSyc nmp->b_cont = t_mp; 32853f3d8ecSyc len = MBLKL(nmp); 32953f3d8ecSyc } else { 33030ac2e7bSml if (is_lso) { 33130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 33230ac2e7bSml "LSO packet: dupb failed: " 33330ac2e7bSml "channel %d", 33430ac2e7bSml tx_ring_p->tdc)); 33530ac2e7bSml mp = nmp; 33630ac2e7bSml goto nxge_start_fail_lso; 33730ac2e7bSml } else { 33830ac2e7bSml good_packet = B_FALSE; 33930ac2e7bSml goto nxge_start_fail2; 34030ac2e7bSml } 34153f3d8ecSyc } 34244961713Sgirish } 34344961713Sgirish tx_desc.value = 0; 34444961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 34544961713Sgirish #ifdef NXGE_DEBUG 34644961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 34744961713Sgirish #endif 34844961713Sgirish tx_msg_p = &tx_msg_ring[i]; 349adfcba55Sjoycey #if defined(__i386) 350adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 351adfcba55Sjoycey #else 35244961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 353adfcba55Sjoycey #endif 35444961713Sgirish if (!header_set && 35544961713Sgirish ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 35644961713Sgirish (len >= bcopy_thresh))) { 35744961713Sgirish header_set = B_TRUE; 35844961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 35944961713Sgirish boff = 0; 36044961713Sgirish pack_len = 0; 36144961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 36244961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 36344961713Sgirish clen = pkt_len; 36444961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 36544961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 36644961713Sgirish (void) ddi_dma_sync(dma_handle, 36744961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 36844961713Sgirish DDI_DMA_SYNC_FORDEV); 36944961713Sgirish 37044961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 37144961713Sgirish goto nxge_start_control_header_only; 37244961713Sgirish } 37344961713Sgirish 37444961713Sgirish pkt_len += len; 37544961713Sgirish pack_len += len; 37644961713Sgirish 37744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 37844961713Sgirish "desc entry %d " 37944961713Sgirish "DESC IOADDR $%p " 38044961713Sgirish "desc_vp $%p tx_desc_p $%p " 38144961713Sgirish "desc_pp $%p tx_desc_pp $%p " 38244961713Sgirish "len %d pkt_len %d pack_len %d", 38344961713Sgirish i, 38444961713Sgirish DMA_COMMON_IOADDR(desc_area), 38544961713Sgirish tx_desc_ring_vp, tx_desc_p, 38644961713Sgirish tx_desc_ring_pp, tx_desc_pp, 38744961713Sgirish len, pkt_len, pack_len)); 38844961713Sgirish 38944961713Sgirish if (len < bcopy_thresh) { 39044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 39144961713Sgirish "USE BCOPY: ")); 39244961713Sgirish if (nxge_tx_tiny_pack) { 39344961713Sgirish uint32_t blst = 39444961713Sgirish TXDMA_DESC_NEXT_INDEX(i, -1, 39544961713Sgirish tx_ring_p->tx_wrap_mask); 39644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 39744961713Sgirish "==> nxge_start(5): pack")); 39844961713Sgirish if ((pack_len <= bcopy_thresh) && 39944961713Sgirish (last_bidx == blst)) { 40044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 40144961713Sgirish "==> nxge_start: pack(6) " 40244961713Sgirish "(pkt_len %d pack_len %d)", 40344961713Sgirish pkt_len, pack_len)); 40444961713Sgirish i = blst; 40544961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 40644961713Sgirish #ifdef NXGE_DEBUG 40744961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 40844961713Sgirish #endif 40944961713Sgirish tx_msg_p = &tx_msg_ring[i]; 41044961713Sgirish boff = pack_len - len; 41144961713Sgirish ngathers--; 412a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 413a3c5bd6dSspeer header_set) { 41444961713Sgirish pack_len = len; 41544961713Sgirish boff = 0; 41644961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 41744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 41844961713Sgirish "==> nxge_start(7): > max NEW " 41944961713Sgirish "bcopy thresh %d " 42044961713Sgirish "pkt_len %d pack_len %d(next)", 42144961713Sgirish bcopy_thresh, 42244961713Sgirish pkt_len, pack_len)); 42344961713Sgirish } 42444961713Sgirish last_bidx = i; 42544961713Sgirish } 42644961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 42744961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 42844961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 42944961713Sgirish header_set = B_TRUE; 43044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 43144961713Sgirish "==> nxge_start(7_x2): " 43244961713Sgirish "pkt_len %d pack_len %d (new hdrp $%p)", 43344961713Sgirish pkt_len, pack_len, hdrp)); 43444961713Sgirish } 43544961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 43644961713Sgirish kaddr += boff; 43744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 43844961713Sgirish "USE BCOPY: before bcopy " 43944961713Sgirish "DESC IOADDR $%p entry %d " 44044961713Sgirish "bcopy packets %d " 44144961713Sgirish "bcopy kaddr $%p " 44244961713Sgirish "bcopy ioaddr (SAD) $%p " 44344961713Sgirish "bcopy clen %d " 44444961713Sgirish "bcopy boff %d", 44544961713Sgirish DMA_COMMON_IOADDR(desc_area), i, 44644961713Sgirish tdc_stats->tx_hdr_pkts, 44744961713Sgirish kaddr, 44844961713Sgirish dma_ioaddr, 44944961713Sgirish clen, 45044961713Sgirish boff)); 45144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 45244961713Sgirish "1USE BCOPY: ")); 45344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 45444961713Sgirish "2USE BCOPY: ")); 45544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 45644961713Sgirish "last USE BCOPY: copy from b_rptr $%p " 45744961713Sgirish "to KADDR $%p (len %d offset %d", 45844961713Sgirish b_rptr, kaddr, len, boff)); 45944961713Sgirish 46044961713Sgirish bcopy(b_rptr, kaddr, len); 46144961713Sgirish 46244961713Sgirish #ifdef NXGE_DEBUG 46344961713Sgirish dump_len = (len > 128) ? 128: len; 46444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 46544961713Sgirish "==> nxge_start: dump packets " 46644961713Sgirish "(After BCOPY len %d)" 46744961713Sgirish "(b_rptr $%p): %s", len, nmp->b_rptr, 46844961713Sgirish nxge_dump_packet((char *)nmp->b_rptr, 46944961713Sgirish dump_len))); 47044961713Sgirish #endif 47144961713Sgirish 47244961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 47344961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 47444961713Sgirish (void) ddi_dma_sync(dma_handle, 47544961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 47644961713Sgirish DDI_DMA_SYNC_FORDEV); 47744961713Sgirish clen = len + boff; 47844961713Sgirish tdc_stats->tx_hdr_pkts++; 47944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 48044961713Sgirish "USE BCOPY: " 48144961713Sgirish "DESC IOADDR $%p entry %d " 48244961713Sgirish "bcopy packets %d " 48344961713Sgirish "bcopy kaddr $%p " 48444961713Sgirish "bcopy ioaddr (SAD) $%p " 48544961713Sgirish "bcopy clen %d " 48644961713Sgirish "bcopy boff %d", 48744961713Sgirish DMA_COMMON_IOADDR(desc_area), 48844961713Sgirish i, 48944961713Sgirish tdc_stats->tx_hdr_pkts, 49044961713Sgirish kaddr, 49144961713Sgirish dma_ioaddr, 49244961713Sgirish clen, 49344961713Sgirish boff)); 49444961713Sgirish } else { 49544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 49644961713Sgirish "USE DVMA: len %d", len)); 49744961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 49844961713Sgirish dma_flags = DDI_DMA_WRITE; 49944961713Sgirish if (len < nxge_dma_stream_thresh) { 50044961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 50144961713Sgirish } else { 50244961713Sgirish dma_flags |= DDI_DMA_STREAMING; 50344961713Sgirish } 50444961713Sgirish 50544961713Sgirish dma_handle = tx_msg_p->dma_handle; 50644961713Sgirish status = ddi_dma_addr_bind_handle(dma_handle, NULL, 50744961713Sgirish (caddr_t)b_rptr, len, dma_flags, 50844961713Sgirish DDI_DMA_DONTWAIT, NULL, 50944961713Sgirish &dma_cookie, &ncookies); 51044961713Sgirish if (status == DDI_DMA_MAPPED) { 51144961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 51244961713Sgirish len = (int)dma_cookie.dmac_size; 51344961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 51444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 51544961713Sgirish "==> nxge_start(12_1): " 51644961713Sgirish "USE DVMA: len %d clen %d " 51744961713Sgirish "ngathers %d", 51844961713Sgirish len, clen, 51944961713Sgirish ngathers)); 520adfcba55Sjoycey #if defined(__i386) 521adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 522adfcba55Sjoycey #else 52344961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 524adfcba55Sjoycey #endif 52544961713Sgirish while (ncookies > 1) { 52644961713Sgirish ngathers++; 52744961713Sgirish /* 52844961713Sgirish * this is the fix for multiple 52930ac2e7bSml * cookies, which are basically 53044961713Sgirish * a descriptor entry, we don't set 53144961713Sgirish * SOP bit as well as related fields 53244961713Sgirish */ 53344961713Sgirish 53444961713Sgirish (void) npi_txdma_desc_gather_set( 53544961713Sgirish npi_desc_handle, 53644961713Sgirish &tx_desc, 53744961713Sgirish (ngathers -1), 53844961713Sgirish mark_mode, 53944961713Sgirish ngathers, 54044961713Sgirish dma_ioaddr, 54144961713Sgirish clen); 54244961713Sgirish 54344961713Sgirish tx_msg_p->tx_msg_size = clen; 54444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 54544961713Sgirish "==> nxge_start: DMA " 54644961713Sgirish "ncookie %d " 54744961713Sgirish "ngathers %d " 54844961713Sgirish "dma_ioaddr $%p len %d" 54944961713Sgirish "desc $%p descp $%p (%d)", 55044961713Sgirish ncookies, 55144961713Sgirish ngathers, 55244961713Sgirish dma_ioaddr, clen, 55344961713Sgirish *tx_desc_p, tx_desc_p, i)); 55444961713Sgirish 55544961713Sgirish ddi_dma_nextcookie(dma_handle, 55644961713Sgirish &dma_cookie); 55744961713Sgirish dma_ioaddr = 55844961713Sgirish dma_cookie.dmac_laddress; 55944961713Sgirish 56044961713Sgirish len = (int)dma_cookie.dmac_size; 56144961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 56244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 56344961713Sgirish "==> nxge_start(12_2): " 56444961713Sgirish "USE DVMA: len %d clen %d ", 56544961713Sgirish len, clen)); 56644961713Sgirish 56744961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 56844961713Sgirish tx_ring_p->tx_wrap_mask); 56944961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 57044961713Sgirish 57144961713Sgirish npi_desc_handle.regp = 572adfcba55Sjoycey #if defined(__i386) 573adfcba55Sjoycey (uint32_t)tx_desc_p; 574adfcba55Sjoycey #else 57544961713Sgirish (uint64_t)tx_desc_p; 576adfcba55Sjoycey #endif 57744961713Sgirish tx_msg_p = &tx_msg_ring[i]; 57844961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 57944961713Sgirish tx_desc.value = 0; 58044961713Sgirish 58144961713Sgirish ncookies--; 58244961713Sgirish } 58344961713Sgirish tdc_stats->tx_ddi_pkts++; 58444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 58544961713Sgirish "DMA: ddi packets %d", 58644961713Sgirish tdc_stats->tx_ddi_pkts)); 58744961713Sgirish } else { 58844961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 58944961713Sgirish "dma mapping failed for %d " 59044961713Sgirish "bytes addr $%p flags %x (%d)", 59144961713Sgirish len, b_rptr, status, status)); 59244961713Sgirish good_packet = B_FALSE; 59344961713Sgirish tdc_stats->tx_dma_bind_fail++; 59444961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 59530ac2e7bSml if (is_lso) { 59630ac2e7bSml mp = nmp; 59730ac2e7bSml goto nxge_start_fail_lso; 59830ac2e7bSml } else { 59930ac2e7bSml goto nxge_start_fail2; 60030ac2e7bSml } 60144961713Sgirish } 60244961713Sgirish } /* ddi dvma */ 60344961713Sgirish 60430ac2e7bSml if (is_lso) { 60530ac2e7bSml nmp_lso_save = nmp; 60630ac2e7bSml } 60744961713Sgirish nmp = nmp->b_cont; 60844961713Sgirish nxge_start_control_header_only: 609adfcba55Sjoycey #if defined(__i386) 610adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 611adfcba55Sjoycey #else 61244961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 613adfcba55Sjoycey #endif 61444961713Sgirish ngathers++; 61544961713Sgirish 61644961713Sgirish if (ngathers == 1) { 61744961713Sgirish #ifdef NXGE_DEBUG 61844961713Sgirish save_desc_p = &sop_tx_desc; 61944961713Sgirish #endif 62044961713Sgirish sop_tx_desc_p = &sop_tx_desc; 62144961713Sgirish sop_tx_desc_p->value = 0; 62244961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 62344961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 62444961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 62544961713Sgirish } else { 62644961713Sgirish #ifdef NXGE_DEBUG 62744961713Sgirish save_desc_p = &tx_desc; 62844961713Sgirish #endif 62944961713Sgirish tmp_desc_p = &tx_desc; 63044961713Sgirish tmp_desc_p->value = 0; 63144961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 63244961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 63344961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 63444961713Sgirish 63544961713Sgirish tx_desc_p->value = tmp_desc_p->value; 63644961713Sgirish } 63744961713Sgirish 63844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 63944961713Sgirish "Desc_entry %d ngathers %d " 64044961713Sgirish "desc_vp $%p tx_desc_p $%p " 64144961713Sgirish "len %d clen %d pkt_len %d pack_len %d nmblks %d " 64244961713Sgirish "dma_ioaddr (SAD) $%p mark %d", 64344961713Sgirish i, ngathers, 64444961713Sgirish tx_desc_ring_vp, tx_desc_p, 64544961713Sgirish len, clen, pkt_len, pack_len, nmblks, 64644961713Sgirish dma_ioaddr, mark_mode)); 64744961713Sgirish 64844961713Sgirish #ifdef NXGE_DEBUG 64944961713Sgirish npi_desc_handle.nxgep = nxgep; 65044961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 65144961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 65244961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 65344961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 65444961713Sgirish TX_PKT_DESC_TR_LEN_SHIFT); 65544961713Sgirish 65644961713Sgirish 65744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 65844961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 65944961713Sgirish "mark %d sop %d\n", 66044961713Sgirish save_desc_p->value, 66144961713Sgirish sad, 66244961713Sgirish save_desc_p->bits.hdw.tr_len, 66344961713Sgirish xfer_len, 66444961713Sgirish save_desc_p->bits.hdw.num_ptr, 66544961713Sgirish save_desc_p->bits.hdw.mark, 66644961713Sgirish save_desc_p->bits.hdw.sop)); 66744961713Sgirish 66844961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 66944961713Sgirish #endif 67044961713Sgirish 67144961713Sgirish tx_msg_p->tx_msg_size = clen; 67244961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 67344961713Sgirish if (ngathers > nxge_tx_max_gathers) { 67444961713Sgirish good_packet = B_FALSE; 67544961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 67644961713Sgirish &stuff_offset, &end_offset, &value, 67744961713Sgirish &cksum_flags); 67844961713Sgirish 67944961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 68044961713Sgirish "==> nxge_start(14): pull msg - " 68144961713Sgirish "len %d pkt_len %d ngathers %d", 68244961713Sgirish len, pkt_len, ngathers)); 68344961713Sgirish /* Pull all message blocks from b_cont */ 68430ac2e7bSml if (is_lso) { 68530ac2e7bSml mp = nmp_lso_save; 68630ac2e7bSml goto nxge_start_fail_lso; 68730ac2e7bSml } 68844961713Sgirish if ((msgpullup(mp, -1)) == NULL) { 68944961713Sgirish goto nxge_start_fail2; 69044961713Sgirish } 69144961713Sgirish goto nxge_start_fail2; 69244961713Sgirish } 69344961713Sgirish } /* while (nmp) */ 69444961713Sgirish 69544961713Sgirish tx_msg_p->tx_message = mp; 69644961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 697adfcba55Sjoycey #if defined(__i386) 698adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 699adfcba55Sjoycey #else 70044961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 701adfcba55Sjoycey #endif 70244961713Sgirish 70344961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 70444961713Sgirish pkthdrp->reserved = 0; 70544961713Sgirish hdrp->value = 0; 70644961713Sgirish (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 70744961713Sgirish (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 70844961713Sgirish 70944961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 71044961713Sgirish tdc_stats->tx_jumbo_pkts++; 71144961713Sgirish } 71244961713Sgirish 71344961713Sgirish min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 71444961713Sgirish if (pkt_len < min_len) { 71544961713Sgirish /* Assume we use bcopy to premapped buffers */ 71644961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 71744961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 71844961713Sgirish "==> nxge_start(14-1): < (msg_min + 16)" 71944961713Sgirish "len %d pkt_len %d min_len %d bzero %d ngathers %d", 72044961713Sgirish len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 72144961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 72244961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 72344961713Sgirish 72444961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 72544961713Sgirish 72644961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 72744961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 72844961713Sgirish 72944961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 73044961713Sgirish "==> nxge_start(14-2): < msg_min - " 73144961713Sgirish "len %d pkt_len %d min_len %d ngathers %d", 73244961713Sgirish len, pkt_len, min_len, ngathers)); 73344961713Sgirish } 73444961713Sgirish 73544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 73644961713Sgirish cksum_flags)); 73744961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 73844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 73944961713Sgirish "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 74044961713Sgirish cksum_flags)); 74144961713Sgirish cksum_on = B_TRUE; 74244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 74344961713Sgirish "==> nxge_start: from IP cksum_flags 0x%x " 74444961713Sgirish "(partial checksum) " 74544961713Sgirish "start_offset %d stuff_offset %d", 74644961713Sgirish cksum_flags, start_offset, stuff_offset)); 74744961713Sgirish tmp_len = (uint64_t)(start_offset >> 1); 74844961713Sgirish hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 74944961713Sgirish tmp_len = (uint64_t)(stuff_offset >> 1); 75044961713Sgirish hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 75144961713Sgirish 75244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 75344961713Sgirish "==> nxge_start: from IP cksum_flags 0x%x " 75444961713Sgirish "(partial checksum) " 75544961713Sgirish "after SHIFT start_offset %d stuff_offset %d", 75644961713Sgirish cksum_flags, start_offset, stuff_offset)); 75744961713Sgirish } 75844961713Sgirish { 75944961713Sgirish uint64_t tmp_len; 76044961713Sgirish 76144961713Sgirish /* pkt_len already includes 16 + paddings!! */ 76244961713Sgirish /* Update the control header length */ 76344961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 76444961713Sgirish tmp_len = hdrp->value | 76544961713Sgirish (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 76644961713Sgirish 76744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 76844961713Sgirish "==> nxge_start(15_x1): setting SOP " 76944961713Sgirish "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 77044961713Sgirish "0x%llx hdrp->value 0x%llx", 77144961713Sgirish tot_xfer_len, tot_xfer_len, pkt_len, 77244961713Sgirish tmp_len, hdrp->value)); 77344961713Sgirish #if defined(_BIG_ENDIAN) 77444961713Sgirish hdrp->value = ddi_swap64(tmp_len); 77544961713Sgirish #else 77644961713Sgirish hdrp->value = tmp_len; 77744961713Sgirish #endif 77844961713Sgirish NXGE_DEBUG_MSG((nxgep, 77944961713Sgirish TX_CTL, "==> nxge_start(15_x2): setting SOP " 78044961713Sgirish "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 78144961713Sgirish "tmp_len 0x%llx hdrp->value 0x%llx", 78244961713Sgirish tot_xfer_len, pkt_len, 78344961713Sgirish tmp_len, hdrp->value)); 78444961713Sgirish } 78544961713Sgirish 78644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 78744961713Sgirish "wr_index %d " 78844961713Sgirish "tot_xfer_len (%d) pkt_len %d npads %d", 78944961713Sgirish sop_index, 79044961713Sgirish tot_xfer_len, pkt_len, 79144961713Sgirish npads)); 79244961713Sgirish 79344961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 79444961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 79544961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 79644961713Sgirish 79744961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 79844961713Sgirish 79944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 80044961713Sgirish 80144961713Sgirish #ifdef NXGE_DEBUG 80244961713Sgirish npi_desc_handle.nxgep = nxgep; 80344961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 80444961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 80544961713Sgirish 80644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 80744961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 80844961713Sgirish save_desc_p->value, 80944961713Sgirish sad, 81044961713Sgirish save_desc_p->bits.hdw.tr_len, 81144961713Sgirish xfer_len, 81244961713Sgirish save_desc_p->bits.hdw.num_ptr, 81344961713Sgirish save_desc_p->bits.hdw.mark, 81444961713Sgirish save_desc_p->bits.hdw.sop)); 81544961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 81644961713Sgirish 81744961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 81844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 81944961713Sgirish "==> nxge_start: dump packets(17) (after sop set, len " 82044961713Sgirish " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 82144961713Sgirish "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 82244961713Sgirish (char *)hdrp, 82344961713Sgirish nxge_dump_packet((char *)hdrp, dump_len))); 82444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 82544961713Sgirish "==> nxge_start(18): TX desc sync: sop_index %d", 82644961713Sgirish sop_index)); 82744961713Sgirish #endif 82844961713Sgirish 82944961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 83044961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 83144961713Sgirish sop_index * sizeof (tx_desc_t), 83244961713Sgirish ngathers * sizeof (tx_desc_t), 83344961713Sgirish DDI_DMA_SYNC_FORDEV); 83444961713Sgirish 83544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 83644961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 83744961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 83844961713Sgirish stuff_offset, start_offset, 83944961713Sgirish pkt_len, ngathers, sop_index)); 84044961713Sgirish } else { /* more than one descriptor and wrap around */ 84144961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 84244961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 84344961713Sgirish sop_index * sizeof (tx_desc_t), 84444961713Sgirish nsdescs * sizeof (tx_desc_t), 84544961713Sgirish DDI_DMA_SYNC_FORDEV); 84644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 84744961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 84844961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 84944961713Sgirish stuff_offset, start_offset, 85044961713Sgirish pkt_len, ngathers, sop_index)); 85144961713Sgirish 85244961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 85344961713Sgirish 0, 85444961713Sgirish (ngathers - nsdescs) * sizeof (tx_desc_t), 85544961713Sgirish DDI_DMA_SYNC_FORDEV); 85644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 85744961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 85844961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 85944961713Sgirish stuff_offset, start_offset, 86044961713Sgirish pkt_len, ngathers, sop_index)); 86144961713Sgirish } 86244961713Sgirish 86344961713Sgirish tail_index = tx_ring_p->wr_index; 86444961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 86544961713Sgirish 86644961713Sgirish tx_ring_p->wr_index = i; 86744961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 86844961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 86944961713Sgirish B_FALSE : B_TRUE); 87044961713Sgirish } 87144961713Sgirish 87244961713Sgirish tx_ring_p->descs_pending += ngathers; 87344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 87444961713Sgirish "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 87544961713Sgirish tx_ring_p->tdc, 87644961713Sgirish tx_ring_p->wr_index, 87744961713Sgirish tx_ring_p->wr_index_wrap, 87844961713Sgirish ngathers, 87944961713Sgirish tx_ring_p->descs_pending)); 88044961713Sgirish 88130ac2e7bSml if (is_lso) { 88230ac2e7bSml lso_ngathers += ngathers; 88330ac2e7bSml if (mp_chain != NULL) { 88430ac2e7bSml mp = mp_chain; 88530ac2e7bSml mp_chain = mp_chain->b_next; 88630ac2e7bSml mp->b_next = NULL; 88730ac2e7bSml if (nxge_lso_kick_cnt == lso_ngathers) { 88830ac2e7bSml { 88930ac2e7bSml tx_ring_kick_t kick; 89030ac2e7bSml 89130ac2e7bSml kick.value = 0; 89230ac2e7bSml kick.bits.ldw.wrap = 89330ac2e7bSml tx_ring_p->wr_index_wrap; 89430ac2e7bSml kick.bits.ldw.tail = 89530ac2e7bSml (uint16_t)tx_ring_p->wr_index; 89630ac2e7bSml 89730ac2e7bSml /* Kick the Transmit kick register */ 89830ac2e7bSml TXDMA_REG_WRITE64( 89930ac2e7bSml NXGE_DEV_NPI_HANDLE(nxgep), 90030ac2e7bSml TX_RING_KICK_REG, 90130ac2e7bSml (uint8_t)tx_ring_p->tdc, 90230ac2e7bSml kick.value); 90330ac2e7bSml tdc_stats->tx_starts++; 90430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 90530ac2e7bSml "==> nxge_start: more LSO: " 90630ac2e7bSml "LSO_CNT %d", 90730ac2e7bSml lso_gathers)); 90830ac2e7bSml } 90930ac2e7bSml lso_ngathers = 0; 91030ac2e7bSml ngathers = 0; 91130ac2e7bSml cur_index_lso = sop_index = tx_ring_p->wr_index; 91230ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 91330ac2e7bSml } 91430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 91530ac2e7bSml "==> nxge_start: lso again: " 91630ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 91730ac2e7bSml "wr_index %d sop_index %d", 91830ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 91930ac2e7bSml tx_ring_p->wr_index, sop_index)); 92030ac2e7bSml 92130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 92230ac2e7bSml "==> nxge_start: next : count %d", 92330ac2e7bSml lso_gathers)); 92430ac2e7bSml lso_again = B_TRUE; 92530ac2e7bSml goto start_again; 92630ac2e7bSml } 92730ac2e7bSml } 92830ac2e7bSml 92944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 93044961713Sgirish 93144961713Sgirish { 93244961713Sgirish tx_ring_kick_t kick; 93344961713Sgirish 93444961713Sgirish kick.value = 0; 93544961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 93644961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 93744961713Sgirish 93844961713Sgirish /* Kick start the Transmit kick register */ 93944961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 94044961713Sgirish TX_RING_KICK_REG, 94144961713Sgirish (uint8_t)tx_ring_p->tdc, 94244961713Sgirish kick.value); 94344961713Sgirish } 94444961713Sgirish 94544961713Sgirish tdc_stats->tx_starts++; 94644961713Sgirish 94744961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 94844961713Sgirish 94944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 95044961713Sgirish 95144961713Sgirish return (status); 95244961713Sgirish 95330ac2e7bSml nxge_start_fail_lso: 95430ac2e7bSml status = 0; 95530ac2e7bSml good_packet = B_FALSE; 95630ac2e7bSml if (mp != NULL) { 95730ac2e7bSml freemsg(mp); 95830ac2e7bSml } 95930ac2e7bSml if (mp_chain != NULL) { 96030ac2e7bSml freemsg(mp_chain); 96130ac2e7bSml } 96230ac2e7bSml if (!lso_again && !ngathers) { 96330ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 96430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 96530ac2e7bSml "==> nxge_start: lso exit (nothing changed)")); 96630ac2e7bSml goto nxge_start_fail1; 96730ac2e7bSml } 96830ac2e7bSml 96930ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 97030ac2e7bSml "==> nxge_start (channel %d): before lso " 97130ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 97230ac2e7bSml "wr_index %d sop_index %d lso_again %d", 97330ac2e7bSml tx_ring_p->tdc, 97430ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 97530ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 97630ac2e7bSml 97730ac2e7bSml if (lso_again) { 97830ac2e7bSml lso_ngathers += ngathers; 97930ac2e7bSml ngathers = lso_ngathers; 98030ac2e7bSml sop_index = cur_index_lso; 98130ac2e7bSml tx_ring_p->wr_index = sop_index; 98230ac2e7bSml tx_ring_p->wr_index_wrap = lso_tail_wrap; 98330ac2e7bSml } 98430ac2e7bSml 98530ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 98630ac2e7bSml "==> nxge_start (channel %d): after lso " 98730ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 98830ac2e7bSml "wr_index %d sop_index %d lso_again %d", 98930ac2e7bSml tx_ring_p->tdc, 99030ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 99130ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 99230ac2e7bSml 99344961713Sgirish nxge_start_fail2: 99444961713Sgirish if (good_packet == B_FALSE) { 99544961713Sgirish cur_index = sop_index; 99644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 99744961713Sgirish for (i = 0; i < ngathers; i++) { 99844961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 999adfcba55Sjoycey #if defined(__i386) 1000adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 1001adfcba55Sjoycey #else 100244961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 1003adfcba55Sjoycey #endif 100444961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 100544961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 100644961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 100744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 100853f3d8ecSyc "tx_desc_p = %X index = %d", 100953f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 101053f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 101153f3d8ecSyc 0, -1); 101244961713Sgirish tx_msg_p->dvma_handle = NULL; 101344961713Sgirish if (tx_ring_p->dvma_wr_index == 101453f3d8ecSyc tx_ring_p->dvma_wrap_mask) 101544961713Sgirish tx_ring_p->dvma_wr_index = 0; 101644961713Sgirish else 101744961713Sgirish tx_ring_p->dvma_wr_index++; 101844961713Sgirish tx_ring_p->dvma_pending--; 101953f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 102044961713Sgirish if (ddi_dma_unbind_handle( 102153f3d8ecSyc tx_msg_p->dma_handle)) { 102244961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 102353f3d8ecSyc "ddi_dma_unbind_handle failed"); 102453f3d8ecSyc } 102544961713Sgirish } 102644961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 102744961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 102844961713Sgirish tx_ring_p->tx_wrap_mask); 102944961713Sgirish 103044961713Sgirish } 103144961713Sgirish 103244961713Sgirish nxgep->resched_needed = B_TRUE; 103344961713Sgirish } 103444961713Sgirish 103544961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 103644961713Sgirish 103744961713Sgirish nxge_start_fail1: 103844961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 103944961713Sgirish 104044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 104144961713Sgirish 104244961713Sgirish return (status); 104344961713Sgirish } 104444961713Sgirish 10451f8914d5Sml int 10461f8914d5Sml nxge_serial_tx(mblk_t *mp, void *arg) 10471f8914d5Sml { 10481f8914d5Sml p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 10491f8914d5Sml p_nxge_t nxgep = tx_ring_p->nxgep; 10501f8914d5Sml 10511f8914d5Sml return (nxge_start(nxgep, tx_ring_p, mp)); 10521f8914d5Sml } 10531f8914d5Sml 105444961713Sgirish boolean_t 105544961713Sgirish nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 105644961713Sgirish { 105744961713Sgirish p_tx_ring_t *tx_rings; 105844961713Sgirish uint8_t ring_index; 10591f8914d5Sml p_tx_ring_t tx_ring_p; 106044961713Sgirish 106144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 106244961713Sgirish 106344961713Sgirish ASSERT(mp->b_next == NULL); 106444961713Sgirish 106544961713Sgirish ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 106644961713Sgirish tx_rings = nxgep->tx_rings->rings; 106744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 106844961713Sgirish tx_rings)); 106944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 107044961713Sgirish "ring_index %d", nxgep->max_tdcs, ring_index)); 107144961713Sgirish 10721f8914d5Sml switch (nxge_tx_scheme) { 10731f8914d5Sml case NXGE_USE_START: 10741f8914d5Sml if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 10751f8914d5Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 10761f8914d5Sml "ring index %d", ring_index)); 10771f8914d5Sml return (B_FALSE); 10781f8914d5Sml } 10791f8914d5Sml break; 10801f8914d5Sml 10811f8914d5Sml case NXGE_USE_SERIAL: 10821f8914d5Sml default: 10831f8914d5Sml tx_ring_p = tx_rings[ring_index]; 10841f8914d5Sml nxge_serialize_enter(tx_ring_p->serial, mp); 10851f8914d5Sml break; 108644961713Sgirish } 108744961713Sgirish 108844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 108944961713Sgirish ring_index)); 109044961713Sgirish 109144961713Sgirish return (B_TRUE); 109244961713Sgirish } 109344961713Sgirish 109444961713Sgirish /* 109544961713Sgirish * nxge_m_tx() - send a chain of packets 109644961713Sgirish */ 109744961713Sgirish mblk_t * 109844961713Sgirish nxge_m_tx(void *arg, mblk_t *mp) 109944961713Sgirish { 110044961713Sgirish p_nxge_t nxgep = (p_nxge_t)arg; 110144961713Sgirish mblk_t *next; 110244961713Sgirish mac_tx_hint_t hint; 110344961713Sgirish 110444961713Sgirish if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 110544961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 110644961713Sgirish "==> nxge_m_tx: hardware not initialized")); 110744961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 110844961713Sgirish "<== nxge_m_tx")); 110944961713Sgirish return (mp); 111044961713Sgirish } 111144961713Sgirish 111244961713Sgirish hint.hash = NULL; 111344961713Sgirish hint.vid = 0; 111444961713Sgirish hint.sap = 0; 111544961713Sgirish 111644961713Sgirish while (mp != NULL) { 111744961713Sgirish next = mp->b_next; 111844961713Sgirish mp->b_next = NULL; 111944961713Sgirish 112044961713Sgirish /* 112144961713Sgirish * Until Nemo tx resource works, the mac driver 112244961713Sgirish * does the load balancing based on TCP port, 112344961713Sgirish * or CPU. For debugging, we use a system 112444961713Sgirish * configurable parameter. 112544961713Sgirish */ 112644961713Sgirish if (!nxge_send(nxgep, mp, &hint)) { 112744961713Sgirish mp->b_next = next; 112844961713Sgirish break; 112944961713Sgirish } 113044961713Sgirish 113144961713Sgirish mp = next; 113230ac2e7bSml 113330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 113430ac2e7bSml "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 113530ac2e7bSml mp, next)); 113644961713Sgirish } 113744961713Sgirish 113844961713Sgirish return (mp); 113944961713Sgirish } 114044961713Sgirish 114144961713Sgirish int 114244961713Sgirish nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 114344961713Sgirish { 114444961713Sgirish uint8_t ring_index = 0; 114544961713Sgirish uint8_t *tcp_port; 114644961713Sgirish p_mblk_t nmp; 114744961713Sgirish size_t mblk_len; 114844961713Sgirish size_t iph_len; 114944961713Sgirish size_t hdrs_size; 115044961713Sgirish uint8_t hdrs_buf[sizeof (struct ether_header) + 115144961713Sgirish IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 115244961713Sgirish /* 115344961713Sgirish * allocate space big enough to cover 115444961713Sgirish * the max ip header length and the first 115544961713Sgirish * 4 bytes of the TCP/IP header. 115644961713Sgirish */ 115744961713Sgirish 115844961713Sgirish boolean_t qos = B_FALSE; 115944961713Sgirish 116044961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 116144961713Sgirish 116244961713Sgirish if (hp->vid) { 116344961713Sgirish qos = B_TRUE; 116444961713Sgirish } 116544961713Sgirish switch (nxge_tx_lb_policy) { 116644961713Sgirish case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 116744961713Sgirish default: 116844961713Sgirish tcp_port = mp->b_rptr; 116944961713Sgirish if (!nxge_no_tx_lb && !qos && 117044961713Sgirish (ntohs(((p_ether_header_t)tcp_port)->ether_type) 117144961713Sgirish == ETHERTYPE_IP)) { 117244961713Sgirish nmp = mp; 117344961713Sgirish mblk_len = MBLKL(nmp); 117444961713Sgirish tcp_port = NULL; 117544961713Sgirish if (mblk_len > sizeof (struct ether_header) + 117644961713Sgirish sizeof (uint8_t)) { 117744961713Sgirish tcp_port = nmp->b_rptr + 117844961713Sgirish sizeof (struct ether_header); 117944961713Sgirish mblk_len -= sizeof (struct ether_header); 118044961713Sgirish iph_len = ((*tcp_port) & 0x0f) << 2; 118144961713Sgirish if (mblk_len > (iph_len + sizeof (uint32_t))) { 118244961713Sgirish tcp_port = nmp->b_rptr; 118344961713Sgirish } else { 118444961713Sgirish tcp_port = NULL; 118544961713Sgirish } 118644961713Sgirish } 118744961713Sgirish if (tcp_port == NULL) { 118844961713Sgirish hdrs_size = 0; 118944961713Sgirish ((p_ether_header_t)hdrs_buf)->ether_type = 0; 119044961713Sgirish while ((nmp) && (hdrs_size < 119144961713Sgirish sizeof (hdrs_buf))) { 119244961713Sgirish mblk_len = MBLKL(nmp); 119344961713Sgirish if (mblk_len >= 119444961713Sgirish (sizeof (hdrs_buf) - hdrs_size)) 119544961713Sgirish mblk_len = sizeof (hdrs_buf) - 119644961713Sgirish hdrs_size; 119744961713Sgirish bcopy(nmp->b_rptr, 119844961713Sgirish &hdrs_buf[hdrs_size], mblk_len); 119944961713Sgirish hdrs_size += mblk_len; 120044961713Sgirish nmp = nmp->b_cont; 120144961713Sgirish } 120244961713Sgirish tcp_port = hdrs_buf; 120344961713Sgirish } 120444961713Sgirish tcp_port += sizeof (ether_header_t); 120544961713Sgirish if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1206958cea9eSml switch (tcp_port[9]) { 1207958cea9eSml case IPPROTO_TCP: 1208958cea9eSml case IPPROTO_UDP: 1209958cea9eSml case IPPROTO_ESP: 121044961713Sgirish tcp_port += ((*tcp_port) & 0x0f) << 2; 121144961713Sgirish ring_index = 1212958cea9eSml ((tcp_port[0] ^ 1213958cea9eSml tcp_port[1] ^ 1214958cea9eSml tcp_port[2] ^ 1215958cea9eSml tcp_port[3]) % maxtdcs); 1216958cea9eSml break; 1217958cea9eSml 1218958cea9eSml case IPPROTO_AH: 1219958cea9eSml /* SPI starts at the 4th byte */ 1220958cea9eSml tcp_port += ((*tcp_port) & 0x0f) << 2; 1221958cea9eSml ring_index = 1222958cea9eSml ((tcp_port[4] ^ 1223958cea9eSml tcp_port[5] ^ 1224958cea9eSml tcp_port[6] ^ 1225958cea9eSml tcp_port[7]) % maxtdcs); 1226958cea9eSml break; 1227958cea9eSml 1228958cea9eSml default: 122944961713Sgirish ring_index = tcp_port[19] % maxtdcs; 1230958cea9eSml break; 123144961713Sgirish } 123244961713Sgirish } else { /* fragmented packet */ 123344961713Sgirish ring_index = tcp_port[19] % maxtdcs; 123444961713Sgirish } 123544961713Sgirish } else { 123644961713Sgirish ring_index = mp->b_band % maxtdcs; 123744961713Sgirish } 123844961713Sgirish break; 123944961713Sgirish 124044961713Sgirish case NXGE_TX_LB_HASH: 124144961713Sgirish if (hp->hash) { 1242adfcba55Sjoycey #if defined(__i386) 1243adfcba55Sjoycey ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1244adfcba55Sjoycey #else 124544961713Sgirish ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1246adfcba55Sjoycey #endif 124744961713Sgirish } else { 124844961713Sgirish ring_index = mp->b_band % maxtdcs; 124944961713Sgirish } 125044961713Sgirish break; 125144961713Sgirish 125244961713Sgirish case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 125344961713Sgirish tcp_port = mp->b_rptr; 125444961713Sgirish ring_index = tcp_port[5] % maxtdcs; 125544961713Sgirish break; 125644961713Sgirish } 125744961713Sgirish 125844961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 125944961713Sgirish 126044961713Sgirish return (ring_index); 126144961713Sgirish } 126244961713Sgirish 126344961713Sgirish uint_t 126444961713Sgirish nxge_reschedule(caddr_t arg) 126544961713Sgirish { 126644961713Sgirish p_nxge_t nxgep; 126744961713Sgirish 126844961713Sgirish nxgep = (p_nxge_t)arg; 126944961713Sgirish 127044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 127144961713Sgirish 127244961713Sgirish if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 127344961713Sgirish nxgep->resched_needed) { 127444961713Sgirish mac_tx_update(nxgep->mach); 127544961713Sgirish nxgep->resched_needed = B_FALSE; 127644961713Sgirish nxgep->resched_running = B_FALSE; 127744961713Sgirish } 127844961713Sgirish 127944961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 128044961713Sgirish return (DDI_INTR_CLAIMED); 128144961713Sgirish } 128230ac2e7bSml 128330ac2e7bSml 128430ac2e7bSml /* Software LSO starts here */ 128530ac2e7bSml static void 128630ac2e7bSml nxge_hcksum_retrieve(mblk_t *mp, 128730ac2e7bSml uint32_t *start, uint32_t *stuff, uint32_t *end, 128830ac2e7bSml uint32_t *value, uint32_t *flags) 128930ac2e7bSml { 129030ac2e7bSml if (mp->b_datap->db_type == M_DATA) { 129130ac2e7bSml if (flags != NULL) { 129230ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 129330ac2e7bSml HCK_PARTIALCKSUM | HCK_FULLCKSUM | 129430ac2e7bSml HCK_FULLCKSUM_OK); 129530ac2e7bSml if ((*flags & (HCK_PARTIALCKSUM | 129630ac2e7bSml HCK_FULLCKSUM)) != 0) { 129730ac2e7bSml if (value != NULL) 129830ac2e7bSml *value = (uint32_t)DB_CKSUM16(mp); 129930ac2e7bSml if ((*flags & HCK_PARTIALCKSUM) != 0) { 130030ac2e7bSml if (start != NULL) 130130ac2e7bSml *start = 130230ac2e7bSml (uint32_t)DB_CKSUMSTART(mp); 130330ac2e7bSml if (stuff != NULL) 130430ac2e7bSml *stuff = 130530ac2e7bSml (uint32_t)DB_CKSUMSTUFF(mp); 130630ac2e7bSml if (end != NULL) 130730ac2e7bSml *end = 130830ac2e7bSml (uint32_t)DB_CKSUMEND(mp); 130930ac2e7bSml } 131030ac2e7bSml } 131130ac2e7bSml } 131230ac2e7bSml } 131330ac2e7bSml } 131430ac2e7bSml 131530ac2e7bSml static void 131630ac2e7bSml nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 131730ac2e7bSml { 131830ac2e7bSml ASSERT(DB_TYPE(mp) == M_DATA); 131930ac2e7bSml 132030ac2e7bSml *mss = 0; 132130ac2e7bSml if (flags != NULL) { 132230ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 132330ac2e7bSml if ((*flags != 0) && (mss != NULL)) { 132430ac2e7bSml *mss = (uint32_t)DB_LSOMSS(mp); 132530ac2e7bSml } 132630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 132730ac2e7bSml "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 132830ac2e7bSml *mss, *flags)); 132930ac2e7bSml } 133030ac2e7bSml 133130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 133230ac2e7bSml "<== nxge_lso_info_get: mss %d", *mss)); 133330ac2e7bSml } 133430ac2e7bSml 133530ac2e7bSml /* 133630ac2e7bSml * Do Soft LSO on the oversized packet. 133730ac2e7bSml * 133830ac2e7bSml * 1. Create a chain of message for headers. 133930ac2e7bSml * 2. Fill up header messages with proper information. 134030ac2e7bSml * 3. Copy Eithernet, IP, and TCP headers from the original message to 134130ac2e7bSml * each new message with necessary adjustments. 134230ac2e7bSml * * Unchange the ethernet header for DIX frames. (by default) 134330ac2e7bSml * * IP Total Length field is updated to MSS or less(only for the last one). 134430ac2e7bSml * * IP Identification value is incremented by one for each packet. 134530ac2e7bSml * * TCP sequence Number is recalculated according to the payload length. 134630ac2e7bSml * * Set FIN and/or PSH flags for the *last* packet if applied. 134730ac2e7bSml * * TCP partial Checksum 134830ac2e7bSml * 4. Update LSO information in the first message header. 134930ac2e7bSml * 5. Release the original message header. 135030ac2e7bSml */ 135130ac2e7bSml static mblk_t * 135230ac2e7bSml nxge_do_softlso(mblk_t *mp, uint32_t mss) 135330ac2e7bSml { 135430ac2e7bSml uint32_t hckflags; 135530ac2e7bSml int pktlen; 135630ac2e7bSml int hdrlen; 135730ac2e7bSml int segnum; 135830ac2e7bSml int i; 135930ac2e7bSml struct ether_vlan_header *evh; 136030ac2e7bSml int ehlen, iphlen, tcphlen; 136130ac2e7bSml struct ip *oiph, *niph; 136230ac2e7bSml struct tcphdr *otcph, *ntcph; 136330ac2e7bSml int available, len, left; 136430ac2e7bSml uint16_t ip_id; 136530ac2e7bSml uint32_t tcp_seq; 136630ac2e7bSml #ifdef __sparc 136730ac2e7bSml uint32_t tcp_seq_tmp; 136830ac2e7bSml #endif 136930ac2e7bSml mblk_t *datamp; 137030ac2e7bSml uchar_t *rptr; 137130ac2e7bSml mblk_t *nmp; 137230ac2e7bSml mblk_t *cmp; 137330ac2e7bSml mblk_t *mp_chain; 137430ac2e7bSml boolean_t do_cleanup = B_FALSE; 137530ac2e7bSml t_uscalar_t start_offset = 0; 137630ac2e7bSml t_uscalar_t stuff_offset = 0; 137730ac2e7bSml t_uscalar_t value = 0; 137830ac2e7bSml uint16_t l4_len; 137930ac2e7bSml ipaddr_t src, dst; 138030ac2e7bSml uint32_t cksum, sum, l4cksum; 138130ac2e7bSml 138230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 138330ac2e7bSml "==> nxge_do_softlso")); 138430ac2e7bSml /* 138530ac2e7bSml * check the length of LSO packet payload and calculate the number of 138630ac2e7bSml * segments to be generated. 138730ac2e7bSml */ 138830ac2e7bSml pktlen = msgsize(mp); 138930ac2e7bSml evh = (struct ether_vlan_header *)mp->b_rptr; 139030ac2e7bSml 139130ac2e7bSml /* VLAN? */ 139230ac2e7bSml if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 139330ac2e7bSml ehlen = sizeof (struct ether_vlan_header); 139430ac2e7bSml else 139530ac2e7bSml ehlen = sizeof (struct ether_header); 139630ac2e7bSml oiph = (struct ip *)(mp->b_rptr + ehlen); 139730ac2e7bSml iphlen = oiph->ip_hl * 4; 139830ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 139930ac2e7bSml tcphlen = otcph->th_off * 4; 140030ac2e7bSml 140130ac2e7bSml l4_len = pktlen - ehlen - iphlen; 140230ac2e7bSml 140330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 140430ac2e7bSml "==> nxge_do_softlso: mss %d oiph $%p " 140530ac2e7bSml "original ip_sum oiph->ip_sum 0x%x " 140630ac2e7bSml "original tcp_sum otcph->th_sum 0x%x " 140730ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d " 140830ac2e7bSml "l4_len %d (0x%x) ip_len - iphlen %d ", 140930ac2e7bSml mss, 141030ac2e7bSml oiph, 141130ac2e7bSml oiph->ip_sum, 141230ac2e7bSml otcph->th_sum, 141330ac2e7bSml ntohs(oiph->ip_len), pktlen, 141430ac2e7bSml ehlen, 141530ac2e7bSml l4_len, 141630ac2e7bSml l4_len, 141730ac2e7bSml ntohs(oiph->ip_len) - iphlen)); 141830ac2e7bSml 141930ac2e7bSml /* IPv4 + TCP */ 142030ac2e7bSml if (!(oiph->ip_v == IPV4_VERSION)) { 142130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 142230ac2e7bSml "<== nxge_do_softlso: not IPV4 " 142330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 142430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 142530ac2e7bSml tcphlen)); 142630ac2e7bSml freemsg(mp); 142730ac2e7bSml return (NULL); 142830ac2e7bSml } 142930ac2e7bSml 143030ac2e7bSml if (!(oiph->ip_p == IPPROTO_TCP)) { 143130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 143230ac2e7bSml "<== nxge_do_softlso: not TCP " 143330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 143430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 143530ac2e7bSml tcphlen)); 143630ac2e7bSml freemsg(mp); 143730ac2e7bSml return (NULL); 143830ac2e7bSml } 143930ac2e7bSml 144030ac2e7bSml if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 144130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 144230ac2e7bSml "<== nxge_do_softlso: len not matched " 144330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 144430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 144530ac2e7bSml tcphlen)); 144630ac2e7bSml freemsg(mp); 144730ac2e7bSml return (NULL); 144830ac2e7bSml } 144930ac2e7bSml 145030ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 145130ac2e7bSml tcphlen = otcph->th_off * 4; 145230ac2e7bSml 145330ac2e7bSml /* TCP flags can not include URG, RST, or SYN */ 145430ac2e7bSml VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 145530ac2e7bSml 145630ac2e7bSml hdrlen = ehlen + iphlen + tcphlen; 145730ac2e7bSml 145830ac2e7bSml VERIFY(MBLKL(mp) >= hdrlen); 145930ac2e7bSml 146030ac2e7bSml if (MBLKL(mp) > hdrlen) { 146130ac2e7bSml datamp = mp; 146230ac2e7bSml rptr = mp->b_rptr + hdrlen; 146330ac2e7bSml } else { /* = */ 146430ac2e7bSml datamp = mp->b_cont; 146530ac2e7bSml rptr = datamp->b_rptr; 146630ac2e7bSml } 146730ac2e7bSml 146830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 146930ac2e7bSml "nxge_do_softlso: otcph $%p pktlen: %d, " 147030ac2e7bSml "hdrlen %d ehlen %d iphlen %d tcphlen %d " 147130ac2e7bSml "mblkl(mp): %d, mblkl(datamp): %d", 147230ac2e7bSml otcph, 147330ac2e7bSml pktlen, hdrlen, ehlen, iphlen, tcphlen, 147430ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 147530ac2e7bSml 147630ac2e7bSml hckflags = 0; 147730ac2e7bSml nxge_hcksum_retrieve(mp, 147830ac2e7bSml &start_offset, &stuff_offset, &value, NULL, &hckflags); 147930ac2e7bSml 148030ac2e7bSml dst = oiph->ip_dst.s_addr; 148130ac2e7bSml src = oiph->ip_src.s_addr; 148230ac2e7bSml 148330ac2e7bSml cksum = (dst >> 16) + (dst & 0xFFFF) + 148430ac2e7bSml (src >> 16) + (src & 0xFFFF); 148530ac2e7bSml l4cksum = cksum + IP_TCP_CSUM_COMP; 148630ac2e7bSml 148730ac2e7bSml sum = l4_len + l4cksum; 148830ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 148930ac2e7bSml 149030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 149130ac2e7bSml "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 149230ac2e7bSml "hckflags 0x%x start_offset %d stuff_offset %d " 149330ac2e7bSml "value (original) 0x%x th_sum 0x%x " 149430ac2e7bSml "pktlen %d l4_len %d (0x%x) " 149530ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 149630ac2e7bSml dst, src, 149730ac2e7bSml (sum & 0xffff), (~sum & 0xffff), 149830ac2e7bSml hckflags, start_offset, stuff_offset, 149930ac2e7bSml value, otcph->th_sum, 150030ac2e7bSml pktlen, 150130ac2e7bSml l4_len, 150230ac2e7bSml l4_len, 150330ac2e7bSml ntohs(oiph->ip_len) - (int)MBLKL(mp), 150430ac2e7bSml (int)MBLKL(datamp), 150530ac2e7bSml nxge_dump_packet((char *)evh, 12))); 150630ac2e7bSml 150730ac2e7bSml /* 150830ac2e7bSml * Start to process. 150930ac2e7bSml */ 151030ac2e7bSml available = pktlen - hdrlen; 151130ac2e7bSml segnum = (available - 1) / mss + 1; 151230ac2e7bSml 151330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 151430ac2e7bSml "==> nxge_do_softlso: pktlen %d " 151530ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 151630ac2e7bSml "available %d mss %d segnum %d", 151730ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 151830ac2e7bSml available, 151930ac2e7bSml mss, 152030ac2e7bSml segnum)); 152130ac2e7bSml 152230ac2e7bSml VERIFY(segnum >= 2); 152330ac2e7bSml 152430ac2e7bSml /* 152530ac2e7bSml * Try to pre-allocate all header messages 152630ac2e7bSml */ 152730ac2e7bSml mp_chain = NULL; 152830ac2e7bSml for (i = 0; i < segnum; i++) { 152930ac2e7bSml if ((nmp = allocb(hdrlen, 0)) == NULL) { 153030ac2e7bSml /* Clean up the mp_chain */ 153130ac2e7bSml while (mp_chain != NULL) { 153230ac2e7bSml nmp = mp_chain; 153330ac2e7bSml mp_chain = mp_chain->b_next; 153430ac2e7bSml freemsg(nmp); 153530ac2e7bSml } 153630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 153730ac2e7bSml "<== nxge_do_softlso: " 153830ac2e7bSml "Could not allocate enough messages for headers!")); 153930ac2e7bSml freemsg(mp); 154030ac2e7bSml return (NULL); 154130ac2e7bSml } 154230ac2e7bSml nmp->b_next = mp_chain; 154330ac2e7bSml mp_chain = nmp; 154430ac2e7bSml 154530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 154630ac2e7bSml "==> nxge_do_softlso: " 154730ac2e7bSml "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 154830ac2e7bSml mp, nmp, mp_chain, mp_chain->b_next)); 154930ac2e7bSml } 155030ac2e7bSml 155130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 155230ac2e7bSml "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 155330ac2e7bSml mp, nmp, mp_chain)); 155430ac2e7bSml 155530ac2e7bSml /* 155630ac2e7bSml * Associate payload with new packets 155730ac2e7bSml */ 155830ac2e7bSml cmp = mp_chain; 155930ac2e7bSml left = available; 156030ac2e7bSml while (cmp != NULL) { 156130ac2e7bSml nmp = dupb(datamp); 156230ac2e7bSml if (nmp == NULL) { 156330ac2e7bSml do_cleanup = B_TRUE; 156430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 156530ac2e7bSml "==>nxge_do_softlso: " 156630ac2e7bSml "Can not dupb(datamp), have to do clean up")); 156730ac2e7bSml goto cleanup_allocated_msgs; 156830ac2e7bSml } 156930ac2e7bSml 157030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 157130ac2e7bSml "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 157230ac2e7bSml "dupb nmp $%p len %d left %d msd %d ", 157330ac2e7bSml mp, cmp, nmp, len, left, mss)); 157430ac2e7bSml 157530ac2e7bSml cmp->b_cont = nmp; 157630ac2e7bSml nmp->b_rptr = rptr; 157730ac2e7bSml len = (left < mss) ? left : mss; 157830ac2e7bSml left -= len; 157930ac2e7bSml 158030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 158130ac2e7bSml "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 158230ac2e7bSml "dupb nmp $%p len %d left %d mss %d ", 158330ac2e7bSml mp, cmp, nmp, len, left, mss)); 158430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 158530ac2e7bSml "nxge_do_softlso: before available: %d, " 158630ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 158730ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 158830ac2e7bSml 158930ac2e7bSml len -= MBLKL(nmp); 159030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 159130ac2e7bSml "nxge_do_softlso: after available: %d, " 159230ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 159330ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 159430ac2e7bSml 159530ac2e7bSml while (len > 0) { 159630ac2e7bSml mblk_t *mmp = NULL; 159730ac2e7bSml 159830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 159930ac2e7bSml "nxge_do_softlso: (4) len > 0 available: %d, " 160030ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 160130ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 160230ac2e7bSml 160330ac2e7bSml if (datamp->b_cont != NULL) { 160430ac2e7bSml datamp = datamp->b_cont; 160530ac2e7bSml rptr = datamp->b_rptr; 160630ac2e7bSml mmp = dupb(datamp); 160730ac2e7bSml if (mmp == NULL) { 160830ac2e7bSml do_cleanup = B_TRUE; 160930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 161030ac2e7bSml "==> nxge_do_softlso: " 161130ac2e7bSml "Can not dupb(datamp) (1), : 161230ac2e7bSml "have to do clean up")); 161330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 161430ac2e7bSml "==> nxge_do_softlso: " 161530ac2e7bSml "available: %d, left: %d, " 161630ac2e7bSml "len: %d, MBLKL(nmp): %d", 161730ac2e7bSml available, left, len, 161830ac2e7bSml (int)MBLKL(nmp))); 161930ac2e7bSml goto cleanup_allocated_msgs; 162030ac2e7bSml } 162130ac2e7bSml } else { 162230ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 162330ac2e7bSml "==> nxge_do_softlso: " 162430ac2e7bSml "(1)available: %d, left: %d, " 162530ac2e7bSml "len: %d, MBLKL(nmp): %d", 162630ac2e7bSml available, left, len, 162730ac2e7bSml (int)MBLKL(nmp))); 162830ac2e7bSml cmn_err(CE_PANIC, 162930ac2e7bSml "==> nxge_do_softlso: " 163030ac2e7bSml "Pointers must have been corrupted!\n" 163130ac2e7bSml "datamp: $%p, nmp: $%p, rptr: $%p", 163230ac2e7bSml (void *)datamp, 163330ac2e7bSml (void *)nmp, 163430ac2e7bSml (void *)rptr); 163530ac2e7bSml } 163630ac2e7bSml nmp->b_cont = mmp; 163730ac2e7bSml nmp = mmp; 163830ac2e7bSml len -= MBLKL(nmp); 163930ac2e7bSml } 164030ac2e7bSml if (len < 0) { 164130ac2e7bSml nmp->b_wptr += len; 164230ac2e7bSml rptr = nmp->b_wptr; 164330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 164430ac2e7bSml "(5) len < 0 (less than 0)" 164530ac2e7bSml "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 164630ac2e7bSml available, left, len, (int)MBLKL(nmp))); 164730ac2e7bSml 164830ac2e7bSml } else if (len == 0) { 164930ac2e7bSml if (datamp->b_cont != NULL) { 165030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 165130ac2e7bSml "(5) len == 0" 165230ac2e7bSml "available: %d, left: %d, len: %d, " 165330ac2e7bSml "MBLKL(nmp): %d", 165430ac2e7bSml available, left, len, (int)MBLKL(nmp))); 165530ac2e7bSml datamp = datamp->b_cont; 165630ac2e7bSml rptr = datamp->b_rptr; 165730ac2e7bSml } else { 165830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 165930ac2e7bSml "(6)available b_cont == NULL : %d, " 166030ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 166130ac2e7bSml available, left, len, (int)MBLKL(nmp))); 166230ac2e7bSml 166330ac2e7bSml VERIFY(cmp->b_next == NULL); 166430ac2e7bSml VERIFY(left == 0); 166530ac2e7bSml break; /* Done! */ 166630ac2e7bSml } 166730ac2e7bSml } 166830ac2e7bSml cmp = cmp->b_next; 166930ac2e7bSml 167030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 167130ac2e7bSml "(7) do_softlso: " 167230ac2e7bSml "next mp in mp_chain available len != 0 : %d, " 167330ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 167430ac2e7bSml available, left, len, (int)MBLKL(nmp))); 167530ac2e7bSml } 167630ac2e7bSml 167730ac2e7bSml /* 167830ac2e7bSml * From now, start to fill up all headers for the first message 167930ac2e7bSml * Hardware checksum flags need to be updated separately for FULLCKSUM 168030ac2e7bSml * and PARTIALCKSUM cases. For full checksum, copy the original flags 168130ac2e7bSml * into every new packet is enough. But for HCK_PARTIALCKSUM, all 168230ac2e7bSml * required fields need to be updated properly. 168330ac2e7bSml */ 168430ac2e7bSml nmp = mp_chain; 168530ac2e7bSml bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 168630ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 168730ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 168830ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 168930ac2e7bSml ip_id = ntohs(niph->ip_id); 169030ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 169130ac2e7bSml #ifdef __sparc 169230ac2e7bSml bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 169330ac2e7bSml tcp_seq = ntohl(tcp_seq_tmp); 169430ac2e7bSml #else 169530ac2e7bSml tcp_seq = ntohl(ntcph->th_seq); 169630ac2e7bSml #endif 169730ac2e7bSml 169830ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 169930ac2e7bSml 170030ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 170130ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 170230ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 170330ac2e7bSml 170430ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 170530ac2e7bSml niph->ip_sum = 0; 170630ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 170730ac2e7bSml 170830ac2e7bSml l4_len = mss + tcphlen; 170930ac2e7bSml sum = htons(l4_len) + l4cksum; 171030ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 171130ac2e7bSml ntcph->th_sum = (sum & 0xffff); 171230ac2e7bSml 171330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 171430ac2e7bSml "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 171530ac2e7bSml "mss %d pktlen %d l4_len %d (0x%x) " 171630ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 171730ac2e7bSml "ip_sum 0x%x " 171830ac2e7bSml "th_sum 0x%x sum 0x%x ) " 171930ac2e7bSml "dump first ip->tcp %s", 172030ac2e7bSml nmp, mp_chain, 172130ac2e7bSml mss, 172230ac2e7bSml pktlen, 172330ac2e7bSml l4_len, 172430ac2e7bSml l4_len, 172530ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp), 172630ac2e7bSml niph->ip_sum, 172730ac2e7bSml ntcph->th_sum, 172830ac2e7bSml sum, 172930ac2e7bSml nxge_dump_packet((char *)niph, 52))); 173030ac2e7bSml 173130ac2e7bSml cmp = nmp; 173230ac2e7bSml while ((nmp = nmp->b_next)->b_next != NULL) { 173330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 173430ac2e7bSml "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 173530ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 173630ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 173730ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 173830ac2e7bSml niph->ip_id = htons(++ip_id); 173930ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 174030ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 174130ac2e7bSml tcp_seq += mss; 174230ac2e7bSml 174330ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 174430ac2e7bSml 174530ac2e7bSml #ifdef __sparc 174630ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 174730ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 174830ac2e7bSml #else 174930ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 175030ac2e7bSml #endif 175130ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 175230ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 175330ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 175430ac2e7bSml 175530ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 175630ac2e7bSml niph->ip_sum = 0; 175730ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 175830ac2e7bSml ntcph->th_sum = (sum & 0xffff); 175930ac2e7bSml 176030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 176130ac2e7bSml "==> nxge_do_softlso: middle ip_sum 0x%x " 176230ac2e7bSml "th_sum 0x%x " 176330ac2e7bSml " mp $%p (mp_chain $%p) pktlen %d " 176430ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 176530ac2e7bSml niph->ip_sum, 176630ac2e7bSml ntcph->th_sum, 176730ac2e7bSml nmp, mp_chain, 176830ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 176930ac2e7bSml } 177030ac2e7bSml 177130ac2e7bSml /* Last segment */ 177230ac2e7bSml /* 177330ac2e7bSml * Set FIN and/or PSH flags if present only in the last packet. 177430ac2e7bSml * The ip_len could be different from prior packets. 177530ac2e7bSml */ 177630ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 177730ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 177830ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 177930ac2e7bSml niph->ip_id = htons(++ip_id); 178030ac2e7bSml niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 178130ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 178230ac2e7bSml tcp_seq += mss; 178330ac2e7bSml #ifdef __sparc 178430ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 178530ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 178630ac2e7bSml #else 178730ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 178830ac2e7bSml #endif 178930ac2e7bSml ntcph->th_flags = (otcph->th_flags & ~TH_URG); 179030ac2e7bSml 179130ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 179230ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 179330ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 179430ac2e7bSml 179530ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 179630ac2e7bSml niph->ip_sum = 0; 179730ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 179830ac2e7bSml 179930ac2e7bSml l4_len = ntohs(niph->ip_len) - iphlen; 180030ac2e7bSml sum = htons(l4_len) + l4cksum; 180130ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 180230ac2e7bSml ntcph->th_sum = (sum & 0xffff); 180330ac2e7bSml 180430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 180530ac2e7bSml "==> nxge_do_softlso: last next " 180630ac2e7bSml "niph->ip_sum 0x%x " 180730ac2e7bSml "ntcph->th_sum 0x%x sum 0x%x " 180830ac2e7bSml "dump last ip->tcp %s " 180930ac2e7bSml "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 181030ac2e7bSml "l4_len %d (0x%x) " 181130ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 181230ac2e7bSml niph->ip_sum, 181330ac2e7bSml ntcph->th_sum, sum, 181430ac2e7bSml nxge_dump_packet((char *)niph, 52), 181530ac2e7bSml cmp, nmp, mp_chain, 181630ac2e7bSml pktlen, pktlen, 181730ac2e7bSml l4_len, 181830ac2e7bSml l4_len, 181930ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 182030ac2e7bSml 182130ac2e7bSml cleanup_allocated_msgs: 182230ac2e7bSml if (do_cleanup) { 182330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 182430ac2e7bSml "==> nxge_do_softlso: " 182530ac2e7bSml "Failed allocating messages, " 182630ac2e7bSml "have to clean up and fail!")); 182730ac2e7bSml while (mp_chain != NULL) { 182830ac2e7bSml nmp = mp_chain; 182930ac2e7bSml mp_chain = mp_chain->b_next; 183030ac2e7bSml freemsg(nmp); 183130ac2e7bSml } 183230ac2e7bSml } 183330ac2e7bSml /* 183430ac2e7bSml * We're done here, so just free the original message and return the 183530ac2e7bSml * new message chain, that could be NULL if failed, back to the caller. 183630ac2e7bSml */ 183730ac2e7bSml freemsg(mp); 183830ac2e7bSml 183930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 184030ac2e7bSml "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 184130ac2e7bSml return (mp_chain); 184230ac2e7bSml } 184330ac2e7bSml 184430ac2e7bSml /* 184530ac2e7bSml * Will be called before NIC driver do further operation on the message. 184630ac2e7bSml * The input message may include LSO information, if so, go to softlso logic 184730ac2e7bSml * to eliminate the oversized LSO packet for the incapable underlying h/w. 184830ac2e7bSml * The return could be the same non-LSO message or a message chain for LSO case. 184930ac2e7bSml * 185030ac2e7bSml * The driver needs to call this function per packet and process the whole chain 185130ac2e7bSml * if applied. 185230ac2e7bSml */ 185330ac2e7bSml static mblk_t * 185430ac2e7bSml nxge_lso_eliminate(mblk_t *mp) 185530ac2e7bSml { 185630ac2e7bSml uint32_t lsoflags; 185730ac2e7bSml uint32_t mss; 185830ac2e7bSml 185930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 186030ac2e7bSml "==>nxge_lso_eliminate:")); 186130ac2e7bSml nxge_lso_info_get(mp, &mss, &lsoflags); 186230ac2e7bSml 186330ac2e7bSml if (lsoflags & HW_LSO) { 186430ac2e7bSml mblk_t *nmp; 186530ac2e7bSml 186630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 186730ac2e7bSml "==>nxge_lso_eliminate:" 186830ac2e7bSml "HW_LSO:mss %d mp $%p", 186930ac2e7bSml mss, mp)); 187030ac2e7bSml if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 187130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 187230ac2e7bSml "<== nxge_lso_eliminate: " 187330ac2e7bSml "LSO: nmp not NULL nmp $%p mss %d mp $%p", 187430ac2e7bSml nmp, mss, mp)); 187530ac2e7bSml return (nmp); 187630ac2e7bSml } else { 187730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 187830ac2e7bSml "<== nxge_lso_eliminate_ " 187930ac2e7bSml "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 188030ac2e7bSml nmp, mss, mp)); 188130ac2e7bSml return (NULL); 188230ac2e7bSml } 188330ac2e7bSml } 188430ac2e7bSml 188530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 188630ac2e7bSml "<== nxge_lso_eliminate")); 188730ac2e7bSml return (mp); 188830ac2e7bSml } 188930ac2e7bSml 189030ac2e7bSml static uint32_t 189130ac2e7bSml nxge_csgen(uint16_t *adr, int len) 189230ac2e7bSml { 189330ac2e7bSml int i, odd; 189430ac2e7bSml uint32_t sum = 0; 189530ac2e7bSml uint32_t c = 0; 189630ac2e7bSml 189730ac2e7bSml odd = len % 2; 189830ac2e7bSml for (i = 0; i < (len / 2); i++) { 189930ac2e7bSml sum += (adr[i] & 0xffff); 190030ac2e7bSml } 190130ac2e7bSml if (odd) { 190230ac2e7bSml sum += adr[len / 2] & 0xff00; 190330ac2e7bSml } 190430ac2e7bSml while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 190530ac2e7bSml sum &= 0xffff; 190630ac2e7bSml sum += c; 190730ac2e7bSml } 190830ac2e7bSml return (~sum & 0xffff); 190930ac2e7bSml } 1910