144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 22ae6aa22aSVenugopal Iyer * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 26ae6aa22aSVenugopal Iyer #include <sys/mac_provider.h> 27a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 28678453a8Sspeer #include <sys/nxge/nxge_hio.h> 29678453a8Sspeer #include <npi_tx_wr64.h> 3044961713Sgirish 3130ac2e7bSml /* Software LSO required header files */ 3230ac2e7bSml #include <netinet/tcp.h> 3330ac2e7bSml #include <inet/ip_impl.h> 3430ac2e7bSml #include <inet/tcp.h> 3530ac2e7bSml 36ae6aa22aSVenugopal Iyer extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37ae6aa22aSVenugopal Iyer boolean_t is_outbound); 38ae6aa22aSVenugopal Iyer 3930ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *); 4030ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 4130ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 4230ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *, 4330ac2e7bSml uint32_t *, uint32_t *, uint32_t *, 4430ac2e7bSml uint32_t *, uint32_t *); 4530ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int); 4630ac2e7bSml 4744961713Sgirish extern uint32_t nxge_reclaim_pending; 4844961713Sgirish extern uint32_t nxge_bcopy_thresh; 4944961713Sgirish extern uint32_t nxge_dvma_thresh; 5044961713Sgirish extern uint32_t nxge_dma_stream_thresh; 5144961713Sgirish extern uint32_t nxge_tx_minfree; 5244961713Sgirish extern uint32_t nxge_tx_intr_thres; 5344961713Sgirish extern uint32_t nxge_tx_max_gathers; 5444961713Sgirish extern uint32_t nxge_tx_tiny_pack; 5544961713Sgirish extern uint32_t nxge_tx_use_bcopy; 561f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 5730ac2e7bSml uint32_t nxge_lso_kick_cnt = 2; 5844961713Sgirish 5944961713Sgirish 60da14cebeSEric Cheng void 61da14cebeSEric Cheng nxge_tx_ring_task(void *arg) 62da14cebeSEric Cheng { 63da14cebeSEric Cheng p_tx_ring_t ring = (p_tx_ring_t)arg; 64da14cebeSEric Cheng 65da14cebeSEric Cheng MUTEX_ENTER(&ring->lock); 66da14cebeSEric Cheng (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 67da14cebeSEric Cheng MUTEX_EXIT(&ring->lock); 68da14cebeSEric Cheng 69da14cebeSEric Cheng if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 70da14cebeSEric Cheng mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 71da14cebeSEric Cheng #if defined(sun4v) 72da14cebeSEric Cheng else { 73da14cebeSEric Cheng nxge_hio_data_t *nhd = 74da14cebeSEric Cheng (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 75da14cebeSEric Cheng nx_vio_fp_t *vio = &nhd->hio.vio; 76da14cebeSEric Cheng 77da14cebeSEric Cheng /* Call back vnet. */ 78da14cebeSEric Cheng if (vio->cb.vio_net_tx_update) { 79da14cebeSEric Cheng (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 80da14cebeSEric Cheng } 81da14cebeSEric Cheng } 82da14cebeSEric Cheng #endif 83da14cebeSEric Cheng } 84da14cebeSEric Cheng 85da14cebeSEric Cheng static void 86da14cebeSEric Cheng nxge_tx_ring_dispatch(p_tx_ring_t ring) 87da14cebeSEric Cheng { 88da14cebeSEric Cheng /* 89da14cebeSEric Cheng * Kick the ring task to reclaim some buffers. 90da14cebeSEric Cheng */ 91da14cebeSEric Cheng (void) ddi_taskq_dispatch(ring->taskq, 92da14cebeSEric Cheng nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 93da14cebeSEric Cheng } 94da14cebeSEric Cheng 95da14cebeSEric Cheng mblk_t * 96da14cebeSEric Cheng nxge_tx_ring_send(void *arg, mblk_t *mp) 97da14cebeSEric Cheng { 98da14cebeSEric Cheng p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 99da14cebeSEric Cheng p_nxge_t nxgep; 100da14cebeSEric Cheng p_tx_ring_t tx_ring_p; 101da14cebeSEric Cheng int status, channel; 102da14cebeSEric Cheng 103da14cebeSEric Cheng ASSERT(nrhp != NULL); 104da14cebeSEric Cheng nxgep = nrhp->nxgep; 105da14cebeSEric Cheng channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 106da14cebeSEric Cheng tx_ring_p = nxgep->tx_rings->rings[channel]; 107da14cebeSEric Cheng 108da14cebeSEric Cheng ASSERT(nxgep == tx_ring_p->nxgep); 109da14cebeSEric Cheng 110da14cebeSEric Cheng #ifdef DEBUG 111da14cebeSEric Cheng if (isLDOMservice(nxgep)) { 112da14cebeSEric Cheng ASSERT(!tx_ring_p->tx_ring_offline); 113da14cebeSEric Cheng } 114da14cebeSEric Cheng #endif 115da14cebeSEric Cheng 116da14cebeSEric Cheng status = nxge_start(nxgep, tx_ring_p, mp); 117da14cebeSEric Cheng if (status) { 118da14cebeSEric Cheng nxge_tx_ring_dispatch(tx_ring_p); 119da14cebeSEric Cheng return (mp); 120da14cebeSEric Cheng } 121da14cebeSEric Cheng 122da14cebeSEric Cheng return ((mblk_t *)NULL); 123da14cebeSEric Cheng } 124da14cebeSEric Cheng 125da14cebeSEric Cheng #if defined(sun4v) 126da14cebeSEric Cheng 127ae6aa22aSVenugopal Iyer /* 128ae6aa22aSVenugopal Iyer * Hashing policy for load balancing over the set of TX rings 129ae6aa22aSVenugopal Iyer * available to the driver. 130ae6aa22aSVenugopal Iyer */ 131ae6aa22aSVenugopal Iyer static uint8_t nxge_tx_hash_policy = MAC_PKT_HASH_L4; 132ae6aa22aSVenugopal Iyer 133da14cebeSEric Cheng /* 134da14cebeSEric Cheng * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 135da14cebeSEric Cheng * the guest domain. See CR 6778758 for long term solution. 136ae6aa22aSVenugopal Iyer * 137ae6aa22aSVenugopal Iyer * The guest domain driver will for now hash the packet 138ae6aa22aSVenugopal Iyer * to pick a DMA channel from the only group it has group 0. 139da14cebeSEric Cheng */ 140da14cebeSEric Cheng 141da14cebeSEric Cheng mblk_t * 142da14cebeSEric Cheng nxge_m_tx(void *arg, mblk_t *mp) 143da14cebeSEric Cheng { 144da14cebeSEric Cheng p_nxge_t nxgep = (p_nxge_t)arg; 145da14cebeSEric Cheng mblk_t *next; 146ae6aa22aSVenugopal Iyer uint64_t rindex; 147da14cebeSEric Cheng p_tx_ring_t tx_ring_p; 148da14cebeSEric Cheng int status; 149da14cebeSEric Cheng 150da14cebeSEric Cheng NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 151da14cebeSEric Cheng 152da14cebeSEric Cheng /* 153ae6aa22aSVenugopal Iyer * Hash to pick a ring from Group 0, the only TX group 154ae6aa22aSVenugopal Iyer * for a guest domain driver. 155ae6aa22aSVenugopal Iyer */ 156ae6aa22aSVenugopal Iyer rindex = mac_pkt_hash(DL_ETHER, mp, nxge_tx_hash_policy, B_TRUE); 157ae6aa22aSVenugopal Iyer rindex = rindex % nxgep->pt_config.tdc_grps[0].max_tdcs; 158ae6aa22aSVenugopal Iyer 159ae6aa22aSVenugopal Iyer /* 160ae6aa22aSVenugopal Iyer * Get the ring handle. 161da14cebeSEric Cheng */ 162ae6aa22aSVenugopal Iyer tx_ring_p = nxgep->tx_rings->rings[rindex]; 163da14cebeSEric Cheng 164da14cebeSEric Cheng while (mp != NULL) { 165da14cebeSEric Cheng next = mp->b_next; 166da14cebeSEric Cheng mp->b_next = NULL; 167da14cebeSEric Cheng 168da14cebeSEric Cheng status = nxge_start(nxgep, tx_ring_p, mp); 169da14cebeSEric Cheng if (status != 0) { 170da14cebeSEric Cheng mp->b_next = next; 171da14cebeSEric Cheng nxge_tx_ring_dispatch(tx_ring_p); 172da14cebeSEric Cheng return (mp); 173da14cebeSEric Cheng } 174da14cebeSEric Cheng 175da14cebeSEric Cheng mp = next; 176da14cebeSEric Cheng } 177da14cebeSEric Cheng 178da14cebeSEric Cheng NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 179da14cebeSEric Cheng return ((mblk_t *)NULL); 180da14cebeSEric Cheng } 181da14cebeSEric Cheng 182da14cebeSEric Cheng #endif 18344961713Sgirish 18444961713Sgirish int 18544961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 18644961713Sgirish { 18744961713Sgirish int status = 0; 18844961713Sgirish p_tx_desc_t tx_desc_ring_vp; 18944961713Sgirish npi_handle_t npi_desc_handle; 19044961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 19144961713Sgirish p_tx_desc_t tx_desc_p; 19244961713Sgirish p_tx_msg_t tx_msg_ring; 19344961713Sgirish p_tx_msg_t tx_msg_p; 19444961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 19544961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 19644961713Sgirish p_tx_pkt_header_t hdrp; 197ef755e7aStc tx_pkt_hdr_all_t tmp_hdrp; 19844961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 19944961713Sgirish uint8_t npads = 0; 20044961713Sgirish uint64_t dma_ioaddr; 20144961713Sgirish uint32_t dma_flags; 20244961713Sgirish int last_bidx; 20344961713Sgirish uint8_t *b_rptr; 20444961713Sgirish caddr_t kaddr; 20544961713Sgirish uint32_t nmblks; 20644961713Sgirish uint32_t ngathers; 20744961713Sgirish uint32_t clen; 20844961713Sgirish int len; 20944961713Sgirish uint32_t pkt_len, pack_len, min_len; 21044961713Sgirish uint32_t bcopy_thresh; 21144961713Sgirish int i, cur_index, sop_index; 21244961713Sgirish uint16_t tail_index; 21344961713Sgirish boolean_t tail_wrap = B_FALSE; 21444961713Sgirish nxge_dma_common_t desc_area; 21544961713Sgirish nxge_os_dma_handle_t dma_handle; 21644961713Sgirish ddi_dma_cookie_t dma_cookie; 21744961713Sgirish npi_handle_t npi_handle; 21844961713Sgirish p_mblk_t nmp; 21944961713Sgirish p_mblk_t t_mp; 22044961713Sgirish uint32_t ncookies; 22144961713Sgirish boolean_t good_packet; 22244961713Sgirish boolean_t mark_mode = B_FALSE; 22344961713Sgirish p_nxge_stats_t statsp; 22444961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 22544961713Sgirish t_uscalar_t start_offset = 0; 22644961713Sgirish t_uscalar_t stuff_offset = 0; 22744961713Sgirish t_uscalar_t end_offset = 0; 22844961713Sgirish t_uscalar_t value = 0; 22944961713Sgirish t_uscalar_t cksum_flags = 0; 23044961713Sgirish boolean_t cksum_on = B_FALSE; 23144961713Sgirish uint32_t boff = 0; 232b4d05839Sml uint64_t tot_xfer_len = 0; 23344961713Sgirish boolean_t header_set = B_FALSE; 23444961713Sgirish #ifdef NXGE_DEBUG 23544961713Sgirish p_tx_desc_t tx_desc_ring_pp; 23644961713Sgirish p_tx_desc_t tx_desc_pp; 23744961713Sgirish tx_desc_t *save_desc_p; 23844961713Sgirish int dump_len; 23944961713Sgirish int sad_len; 24044961713Sgirish uint64_t sad; 24144961713Sgirish int xfer_len; 24244961713Sgirish uint32_t msgsize; 24344961713Sgirish #endif 24430ac2e7bSml p_mblk_t mp_chain = NULL; 24530ac2e7bSml boolean_t is_lso = B_FALSE; 24630ac2e7bSml boolean_t lso_again; 24730ac2e7bSml int cur_index_lso; 24830ac2e7bSml p_mblk_t nmp_lso_save; 24930ac2e7bSml uint32_t lso_ngathers; 25030ac2e7bSml boolean_t lso_tail_wrap = B_FALSE; 25144961713Sgirish 25244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 253678453a8Sspeer "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 25444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 255678453a8Sspeer "==> nxge_start: Starting tdc %d desc pending %d", 256678453a8Sspeer tx_ring_p->tdc, tx_ring_p->descs_pending)); 25744961713Sgirish 25844961713Sgirish statsp = nxgep->statsp; 25944961713Sgirish 260678453a8Sspeer if (!isLDOMguest(nxgep)) { 261678453a8Sspeer switch (nxgep->mac.portmode) { 262678453a8Sspeer default: 263678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode == 264678453a8Sspeer nxge_lb_normal) { 265678453a8Sspeer if (!statsp->mac_stats.link_up) { 266678453a8Sspeer freemsg(mp); 267678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 268678453a8Sspeer "==> nxge_start: " 269678453a8Sspeer "link not up")); 270678453a8Sspeer goto nxge_start_fail1; 271678453a8Sspeer } 272321febdeSsbehera } 273678453a8Sspeer break; 274678453a8Sspeer case PORT_10G_FIBER: 275678453a8Sspeer /* 276678453a8Sspeer * For the following modes, check the link status 277678453a8Sspeer * before sending the packet out: 278eb1db165Stc * nxge_lb_normal, 279eb1db165Stc * nxge_lb_ext10g, 280eb1db165Stc * nxge_lb_ext1000, 281eb1db165Stc * nxge_lb_ext100, 282eb1db165Stc * nxge_lb_ext10. 283678453a8Sspeer */ 284678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode < 285eb1db165Stc nxge_lb_phy10g) { 286678453a8Sspeer if (!statsp->mac_stats.link_up) { 287678453a8Sspeer freemsg(mp); 288678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 289678453a8Sspeer "==> nxge_start: " 290678453a8Sspeer "link not up")); 291678453a8Sspeer goto nxge_start_fail1; 292678453a8Sspeer } 293321febdeSsbehera } 294678453a8Sspeer break; 29544961713Sgirish } 296678453a8Sspeer } 297678453a8Sspeer 298678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 299678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 300678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 301678453a8Sspeer "==> nxge_start: hardware not initialized or stopped")); 302678453a8Sspeer freemsg(mp); 303678453a8Sspeer goto nxge_start_fail1; 30444961713Sgirish } 30544961713Sgirish 3063d16f8e7Sml if (nxgep->soft_lso_enable) { 30730ac2e7bSml mp_chain = nxge_lso_eliminate(mp); 30830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 30930ac2e7bSml "==> nxge_start(0): LSO mp $%p mp_chain $%p", 31030ac2e7bSml mp, mp_chain)); 31130ac2e7bSml if (mp_chain == NULL) { 31230ac2e7bSml NXGE_ERROR_MSG((nxgep, TX_CTL, 31330ac2e7bSml "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 31430ac2e7bSml mp_chain, mp)); 31530ac2e7bSml goto nxge_start_fail1; 31630ac2e7bSml } 31730ac2e7bSml if (mp_chain != mp) { 31830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 31930ac2e7bSml "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 32030ac2e7bSml mp_chain, mp)); 32130ac2e7bSml is_lso = B_TRUE; 32230ac2e7bSml mp = mp_chain; 32330ac2e7bSml mp_chain = mp_chain->b_next; 32430ac2e7bSml mp->b_next = NULL; 32530ac2e7bSml } 32630ac2e7bSml } 32730ac2e7bSml 32844961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 32952ccf843Smisaki &stuff_offset, &end_offset, &value, &cksum_flags); 33044961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 33144961713Sgirish start_offset += sizeof (ether_header_t); 33244961713Sgirish stuff_offset += sizeof (ether_header_t); 33344961713Sgirish } else { 33444961713Sgirish start_offset += sizeof (struct ether_vlan_header); 33544961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 33644961713Sgirish } 33744961713Sgirish 33844961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 33944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 34052ccf843Smisaki "==> nxge_start: mp $%p len %d " 34152ccf843Smisaki "cksum_flags 0x%x (partial checksum) ", 34252ccf843Smisaki mp, MBLKL(mp), cksum_flags)); 34344961713Sgirish cksum_on = B_TRUE; 34444961713Sgirish } 34544961713Sgirish 346b4d05839Sml pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 347b4d05839Sml pkthdrp->reserved = 0; 348ef755e7aStc tmp_hdrp.pkthdr.value = 0; 349b4d05839Sml nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 350b4d05839Sml 0, 0, pkthdrp, 351b4d05839Sml start_offset, stuff_offset); 352b4d05839Sml 35330ac2e7bSml lso_again = B_FALSE; 35430ac2e7bSml lso_ngathers = 0; 35530ac2e7bSml 35630ac2e7bSml MUTEX_ENTER(&tx_ring_p->lock); 35722c0d73aSspeer 35822c0d73aSspeer if (isLDOMservice(nxgep)) { 3596895688eSspeer tx_ring_p->tx_ring_busy = B_TRUE; 36022c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 36122c0d73aSspeer freemsg(mp); 3626895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 3636895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 3646895688eSspeer NXGE_TX_RING_OFFLINED); 36522c0d73aSspeer MUTEX_EXIT(&tx_ring_p->lock); 36622c0d73aSspeer return (status); 36722c0d73aSspeer } 36822c0d73aSspeer } 36922c0d73aSspeer 37030ac2e7bSml cur_index_lso = tx_ring_p->wr_index; 37130ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 37230ac2e7bSml start_again: 37330ac2e7bSml ngathers = 0; 37430ac2e7bSml sop_index = tx_ring_p->wr_index; 37544961713Sgirish #ifdef NXGE_DEBUG 37644961713Sgirish if (tx_ring_p->descs_pending) { 37744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 37852ccf843Smisaki "desc pending %d ", tx_ring_p->descs_pending)); 37944961713Sgirish } 38044961713Sgirish 38144961713Sgirish dump_len = (int)(MBLKL(mp)); 38244961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 38344961713Sgirish 38444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 38552ccf843Smisaki "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 38652ccf843Smisaki "(Before header reserve: ORIGINAL LEN %d)", 38752ccf843Smisaki tx_ring_p->tdc, 38852ccf843Smisaki mp->b_rptr, 38952ccf843Smisaki dump_len)); 39044961713Sgirish 39144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 39252ccf843Smisaki "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 39352ccf843Smisaki nxge_dump_packet((char *)mp->b_rptr, dump_len))); 39444961713Sgirish #endif 39544961713Sgirish 39644961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 39744961713Sgirish mark_mode = (tx_ring_p->descs_pending && 398*257bdc55SMichael Speer (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) < 399*257bdc55SMichael Speer (int)nxge_tx_minfree)); 40044961713Sgirish 40144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 40252ccf843Smisaki "TX Descriptor ring is channel %d mark mode %d", 40352ccf843Smisaki tx_ring_p->tdc, mark_mode)); 40444961713Sgirish 4057127d9f6Sml if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 4067127d9f6Sml if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 4077127d9f6Sml (nxge_tx_minfree + lso_ngathers))) { 4087127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4097127d9f6Sml "TX Descriptor ring is full: channel %d", 4107127d9f6Sml tx_ring_p->tdc)); 41130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4127127d9f6Sml "TX Descriptor ring is full: channel %d", 41330ac2e7bSml tx_ring_p->tdc)); 4147127d9f6Sml if (is_lso) { 4157127d9f6Sml /* 4167127d9f6Sml * free the current mp and mp_chain if not FULL. 4177127d9f6Sml */ 4187127d9f6Sml tdc_stats->tx_no_desc++; 4197127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4207127d9f6Sml "LSO packet: TX Descriptor ring is full: " 4217127d9f6Sml "channel %d", 4227127d9f6Sml tx_ring_p->tdc)); 4237127d9f6Sml goto nxge_start_fail_lso; 4247127d9f6Sml } else { 4257127d9f6Sml cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 4267127d9f6Sml tdc_stats->tx_no_desc++; 4276895688eSspeer 4287127d9f6Sml if (isLDOMservice(nxgep)) { 4297127d9f6Sml tx_ring_p->tx_ring_busy = B_FALSE; 4307127d9f6Sml if (tx_ring_p->tx_ring_offline) { 4317127d9f6Sml (void) atomic_swap_32( 4327127d9f6Sml &tx_ring_p->tx_ring_offline, 4337127d9f6Sml NXGE_TX_RING_OFFLINED); 4347127d9f6Sml } 4356895688eSspeer } 43622c0d73aSspeer 4377127d9f6Sml MUTEX_EXIT(&tx_ring_p->lock); 4387127d9f6Sml status = 1; 4397127d9f6Sml goto nxge_start_fail1; 44030ac2e7bSml } 44144961713Sgirish } 44244961713Sgirish } 44344961713Sgirish 44444961713Sgirish nmp = mp; 44544961713Sgirish i = sop_index = tx_ring_p->wr_index; 44644961713Sgirish nmblks = 0; 44744961713Sgirish ngathers = 0; 44844961713Sgirish pkt_len = 0; 44944961713Sgirish pack_len = 0; 45044961713Sgirish clen = 0; 45144961713Sgirish last_bidx = -1; 45244961713Sgirish good_packet = B_TRUE; 45344961713Sgirish 45444961713Sgirish desc_area = tx_ring_p->tdc_desc; 45544961713Sgirish npi_handle = desc_area.npi_handle; 45644961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 45752ccf843Smisaki DMA_COMMON_ACC_HANDLE(desc_area); 45844961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 45944961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 46052ccf843Smisaki DMA_COMMON_HANDLE(desc_area); 46144961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 46244961713Sgirish 46344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 46452ccf843Smisaki sop_index, i)); 46544961713Sgirish 46644961713Sgirish #ifdef NXGE_DEBUG 46744961713Sgirish msgsize = msgdsize(nmp); 46844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 46952ccf843Smisaki "==> nxge_start(1): wr_index %d i %d msgdsize %d", 47052ccf843Smisaki sop_index, i, msgsize)); 47144961713Sgirish #endif 47244961713Sgirish /* 47344961713Sgirish * The first 16 bytes of the premapped buffer are reserved 47444961713Sgirish * for header. No padding will be used. 47544961713Sgirish */ 47644961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 4771f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 47844961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 47944961713Sgirish } else { 48044961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 48144961713Sgirish } 48244961713Sgirish while (nmp) { 48344961713Sgirish good_packet = B_TRUE; 48444961713Sgirish b_rptr = nmp->b_rptr; 48544961713Sgirish len = MBLKL(nmp); 48644961713Sgirish if (len <= 0) { 48744961713Sgirish nmp = nmp->b_cont; 48844961713Sgirish continue; 48944961713Sgirish } 49044961713Sgirish nmblks++; 49144961713Sgirish 49244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 49352ccf843Smisaki "len %d pkt_len %d pack_len %d", 49452ccf843Smisaki nmblks, len, pkt_len, pack_len)); 49544961713Sgirish /* 49614ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 49714ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 49814ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 49914ea4bb7Ssd * If len is longer than the limit, then we break nmp into 50014ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 50114ea4bb7Ssd * the second chunk for the remaining data. If the second 50214ea4bb7Ssd * chunk is still larger than the limit, then it will be 50314ea4bb7Ssd * broken into two in the next pass. 50444961713Sgirish */ 50514ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 50653f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 50753f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 50853f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 50953f3d8ecSyc - TX_PKT_HEADER_SIZE); 51053f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 51153f3d8ecSyc t_mp->b_cont = nmp->b_cont; 51253f3d8ecSyc nmp->b_cont = t_mp; 51353f3d8ecSyc len = MBLKL(nmp); 51453f3d8ecSyc } else { 51530ac2e7bSml if (is_lso) { 51630ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 51730ac2e7bSml "LSO packet: dupb failed: " 51830ac2e7bSml "channel %d", 51930ac2e7bSml tx_ring_p->tdc)); 52030ac2e7bSml mp = nmp; 52130ac2e7bSml goto nxge_start_fail_lso; 52230ac2e7bSml } else { 52330ac2e7bSml good_packet = B_FALSE; 52430ac2e7bSml goto nxge_start_fail2; 52530ac2e7bSml } 52653f3d8ecSyc } 52744961713Sgirish } 52844961713Sgirish tx_desc.value = 0; 52944961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 53044961713Sgirish #ifdef NXGE_DEBUG 53144961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 53244961713Sgirish #endif 53344961713Sgirish tx_msg_p = &tx_msg_ring[i]; 534adfcba55Sjoycey #if defined(__i386) 535adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 536adfcba55Sjoycey #else 53744961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 538adfcba55Sjoycey #endif 53944961713Sgirish if (!header_set && 54052ccf843Smisaki ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 54152ccf843Smisaki (len >= bcopy_thresh))) { 54244961713Sgirish header_set = B_TRUE; 54344961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 54444961713Sgirish boff = 0; 54544961713Sgirish pack_len = 0; 54644961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 54744961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 54844961713Sgirish clen = pkt_len; 54944961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 55044961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 55144961713Sgirish (void) ddi_dma_sync(dma_handle, 55252ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 55352ccf843Smisaki DDI_DMA_SYNC_FORDEV); 55444961713Sgirish 55544961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 55644961713Sgirish goto nxge_start_control_header_only; 55744961713Sgirish } 55844961713Sgirish 55944961713Sgirish pkt_len += len; 56044961713Sgirish pack_len += len; 56144961713Sgirish 56244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 56352ccf843Smisaki "desc entry %d " 56452ccf843Smisaki "DESC IOADDR $%p " 56552ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 56652ccf843Smisaki "desc_pp $%p tx_desc_pp $%p " 56752ccf843Smisaki "len %d pkt_len %d pack_len %d", 56852ccf843Smisaki i, 56952ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 57052ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 57152ccf843Smisaki tx_desc_ring_pp, tx_desc_pp, 57252ccf843Smisaki len, pkt_len, pack_len)); 57344961713Sgirish 57444961713Sgirish if (len < bcopy_thresh) { 57544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 57652ccf843Smisaki "USE BCOPY: ")); 57744961713Sgirish if (nxge_tx_tiny_pack) { 57844961713Sgirish uint32_t blst = 57952ccf843Smisaki TXDMA_DESC_NEXT_INDEX(i, -1, 58052ccf843Smisaki tx_ring_p->tx_wrap_mask); 58144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 58252ccf843Smisaki "==> nxge_start(5): pack")); 58344961713Sgirish if ((pack_len <= bcopy_thresh) && 58452ccf843Smisaki (last_bidx == blst)) { 58544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 58652ccf843Smisaki "==> nxge_start: pack(6) " 58752ccf843Smisaki "(pkt_len %d pack_len %d)", 58852ccf843Smisaki pkt_len, pack_len)); 58944961713Sgirish i = blst; 59044961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 59144961713Sgirish #ifdef NXGE_DEBUG 59244961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 59344961713Sgirish #endif 59444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 59544961713Sgirish boff = pack_len - len; 59644961713Sgirish ngathers--; 597a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 59852ccf843Smisaki header_set) { 59944961713Sgirish pack_len = len; 60044961713Sgirish boff = 0; 60144961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 60244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 60352ccf843Smisaki "==> nxge_start(7): > max NEW " 60452ccf843Smisaki "bcopy thresh %d " 60552ccf843Smisaki "pkt_len %d pack_len %d(next)", 60652ccf843Smisaki bcopy_thresh, 60752ccf843Smisaki pkt_len, pack_len)); 60844961713Sgirish } 60944961713Sgirish last_bidx = i; 61044961713Sgirish } 61144961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 61244961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 61344961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 61444961713Sgirish header_set = B_TRUE; 61544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 61652ccf843Smisaki "==> nxge_start(7_x2): " 61752ccf843Smisaki "pkt_len %d pack_len %d (new hdrp $%p)", 61852ccf843Smisaki pkt_len, pack_len, hdrp)); 61944961713Sgirish } 62044961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 62144961713Sgirish kaddr += boff; 62244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 62352ccf843Smisaki "USE BCOPY: before bcopy " 62452ccf843Smisaki "DESC IOADDR $%p entry %d " 62552ccf843Smisaki "bcopy packets %d " 62652ccf843Smisaki "bcopy kaddr $%p " 62752ccf843Smisaki "bcopy ioaddr (SAD) $%p " 62852ccf843Smisaki "bcopy clen %d " 62952ccf843Smisaki "bcopy boff %d", 63052ccf843Smisaki DMA_COMMON_IOADDR(desc_area), i, 63152ccf843Smisaki tdc_stats->tx_hdr_pkts, 63252ccf843Smisaki kaddr, 63352ccf843Smisaki dma_ioaddr, 63452ccf843Smisaki clen, 63552ccf843Smisaki boff)); 63644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 63752ccf843Smisaki "1USE BCOPY: ")); 63844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 63952ccf843Smisaki "2USE BCOPY: ")); 64044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 64152ccf843Smisaki "last USE BCOPY: copy from b_rptr $%p " 64252ccf843Smisaki "to KADDR $%p (len %d offset %d", 64352ccf843Smisaki b_rptr, kaddr, len, boff)); 64444961713Sgirish 64544961713Sgirish bcopy(b_rptr, kaddr, len); 64644961713Sgirish 64744961713Sgirish #ifdef NXGE_DEBUG 64844961713Sgirish dump_len = (len > 128) ? 128: len; 64944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 65052ccf843Smisaki "==> nxge_start: dump packets " 65152ccf843Smisaki "(After BCOPY len %d)" 65252ccf843Smisaki "(b_rptr $%p): %s", len, nmp->b_rptr, 65352ccf843Smisaki nxge_dump_packet((char *)nmp->b_rptr, 65452ccf843Smisaki dump_len))); 65544961713Sgirish #endif 65644961713Sgirish 65744961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 65844961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 65944961713Sgirish (void) ddi_dma_sync(dma_handle, 66052ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 66152ccf843Smisaki DDI_DMA_SYNC_FORDEV); 66244961713Sgirish clen = len + boff; 66344961713Sgirish tdc_stats->tx_hdr_pkts++; 66444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 66552ccf843Smisaki "USE BCOPY: " 66652ccf843Smisaki "DESC IOADDR $%p entry %d " 66752ccf843Smisaki "bcopy packets %d " 66852ccf843Smisaki "bcopy kaddr $%p " 66952ccf843Smisaki "bcopy ioaddr (SAD) $%p " 67052ccf843Smisaki "bcopy clen %d " 67152ccf843Smisaki "bcopy boff %d", 67252ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 67352ccf843Smisaki i, 67452ccf843Smisaki tdc_stats->tx_hdr_pkts, 67552ccf843Smisaki kaddr, 67652ccf843Smisaki dma_ioaddr, 67752ccf843Smisaki clen, 67852ccf843Smisaki boff)); 67944961713Sgirish } else { 68044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 68152ccf843Smisaki "USE DVMA: len %d", len)); 68244961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 68344961713Sgirish dma_flags = DDI_DMA_WRITE; 68444961713Sgirish if (len < nxge_dma_stream_thresh) { 68544961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 68644961713Sgirish } else { 68744961713Sgirish dma_flags |= DDI_DMA_STREAMING; 68844961713Sgirish } 68944961713Sgirish 69044961713Sgirish dma_handle = tx_msg_p->dma_handle; 69144961713Sgirish status = ddi_dma_addr_bind_handle(dma_handle, NULL, 69252ccf843Smisaki (caddr_t)b_rptr, len, dma_flags, 69352ccf843Smisaki DDI_DMA_DONTWAIT, NULL, 69452ccf843Smisaki &dma_cookie, &ncookies); 69544961713Sgirish if (status == DDI_DMA_MAPPED) { 69644961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 69744961713Sgirish len = (int)dma_cookie.dmac_size; 69844961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 69944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 70052ccf843Smisaki "==> nxge_start(12_1): " 70152ccf843Smisaki "USE DVMA: len %d clen %d " 70252ccf843Smisaki "ngathers %d", 70352ccf843Smisaki len, clen, 70452ccf843Smisaki ngathers)); 705adfcba55Sjoycey #if defined(__i386) 706adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 707adfcba55Sjoycey #else 70844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 709adfcba55Sjoycey #endif 71044961713Sgirish while (ncookies > 1) { 71144961713Sgirish ngathers++; 71244961713Sgirish /* 71344961713Sgirish * this is the fix for multiple 71430ac2e7bSml * cookies, which are basically 71544961713Sgirish * a descriptor entry, we don't set 71644961713Sgirish * SOP bit as well as related fields 71744961713Sgirish */ 71844961713Sgirish 71944961713Sgirish (void) npi_txdma_desc_gather_set( 72052ccf843Smisaki npi_desc_handle, 72152ccf843Smisaki &tx_desc, 72252ccf843Smisaki (ngathers -1), 72352ccf843Smisaki mark_mode, 72452ccf843Smisaki ngathers, 72552ccf843Smisaki dma_ioaddr, 72652ccf843Smisaki clen); 72744961713Sgirish 72844961713Sgirish tx_msg_p->tx_msg_size = clen; 72944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 73052ccf843Smisaki "==> nxge_start: DMA " 73152ccf843Smisaki "ncookie %d " 73252ccf843Smisaki "ngathers %d " 73352ccf843Smisaki "dma_ioaddr $%p len %d" 73452ccf843Smisaki "desc $%p descp $%p (%d)", 73552ccf843Smisaki ncookies, 73652ccf843Smisaki ngathers, 73752ccf843Smisaki dma_ioaddr, clen, 73852ccf843Smisaki *tx_desc_p, tx_desc_p, i)); 73944961713Sgirish 74044961713Sgirish ddi_dma_nextcookie(dma_handle, 74152ccf843Smisaki &dma_cookie); 74244961713Sgirish dma_ioaddr = 74352ccf843Smisaki dma_cookie.dmac_laddress; 74444961713Sgirish 74544961713Sgirish len = (int)dma_cookie.dmac_size; 74644961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 74744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 74852ccf843Smisaki "==> nxge_start(12_2): " 74952ccf843Smisaki "USE DVMA: len %d clen %d ", 75052ccf843Smisaki len, clen)); 75144961713Sgirish 75244961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 75352ccf843Smisaki tx_ring_p->tx_wrap_mask); 75444961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 75544961713Sgirish 756adfcba55Sjoycey #if defined(__i386) 75752ccf843Smisaki npi_desc_handle.regp = 75852ccf843Smisaki (uint32_t)tx_desc_p; 759adfcba55Sjoycey #else 76052ccf843Smisaki npi_desc_handle.regp = 76152ccf843Smisaki (uint64_t)tx_desc_p; 762adfcba55Sjoycey #endif 76344961713Sgirish tx_msg_p = &tx_msg_ring[i]; 76444961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 76544961713Sgirish tx_desc.value = 0; 76644961713Sgirish 76744961713Sgirish ncookies--; 76844961713Sgirish } 76944961713Sgirish tdc_stats->tx_ddi_pkts++; 77044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 77152ccf843Smisaki "DMA: ddi packets %d", 77252ccf843Smisaki tdc_stats->tx_ddi_pkts)); 77344961713Sgirish } else { 77444961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 77544961713Sgirish "dma mapping failed for %d " 77644961713Sgirish "bytes addr $%p flags %x (%d)", 77744961713Sgirish len, b_rptr, status, status)); 77844961713Sgirish good_packet = B_FALSE; 77944961713Sgirish tdc_stats->tx_dma_bind_fail++; 78044961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 78130ac2e7bSml if (is_lso) { 78230ac2e7bSml mp = nmp; 78330ac2e7bSml goto nxge_start_fail_lso; 78430ac2e7bSml } else { 78530ac2e7bSml goto nxge_start_fail2; 78630ac2e7bSml } 78744961713Sgirish } 78844961713Sgirish } /* ddi dvma */ 78944961713Sgirish 79030ac2e7bSml if (is_lso) { 79130ac2e7bSml nmp_lso_save = nmp; 79230ac2e7bSml } 79344961713Sgirish nmp = nmp->b_cont; 79444961713Sgirish nxge_start_control_header_only: 795adfcba55Sjoycey #if defined(__i386) 796adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 797adfcba55Sjoycey #else 79844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 799adfcba55Sjoycey #endif 80044961713Sgirish ngathers++; 80144961713Sgirish 80244961713Sgirish if (ngathers == 1) { 80344961713Sgirish #ifdef NXGE_DEBUG 80444961713Sgirish save_desc_p = &sop_tx_desc; 80544961713Sgirish #endif 80644961713Sgirish sop_tx_desc_p = &sop_tx_desc; 80744961713Sgirish sop_tx_desc_p->value = 0; 80844961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 80944961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 81044961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 81144961713Sgirish } else { 81244961713Sgirish #ifdef NXGE_DEBUG 81344961713Sgirish save_desc_p = &tx_desc; 81444961713Sgirish #endif 81544961713Sgirish tmp_desc_p = &tx_desc; 81644961713Sgirish tmp_desc_p->value = 0; 81744961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 81844961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 81944961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 82044961713Sgirish 82144961713Sgirish tx_desc_p->value = tmp_desc_p->value; 82244961713Sgirish } 82344961713Sgirish 82444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 82552ccf843Smisaki "Desc_entry %d ngathers %d " 82652ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 82752ccf843Smisaki "len %d clen %d pkt_len %d pack_len %d nmblks %d " 82852ccf843Smisaki "dma_ioaddr (SAD) $%p mark %d", 82952ccf843Smisaki i, ngathers, 83052ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 83152ccf843Smisaki len, clen, pkt_len, pack_len, nmblks, 83252ccf843Smisaki dma_ioaddr, mark_mode)); 83344961713Sgirish 83444961713Sgirish #ifdef NXGE_DEBUG 83544961713Sgirish npi_desc_handle.nxgep = nxgep; 83644961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 83744961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 83844961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 83944961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 84052ccf843Smisaki TX_PKT_DESC_TR_LEN_SHIFT); 84144961713Sgirish 84244961713Sgirish 84344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 84452ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 84552ccf843Smisaki "mark %d sop %d\n", 84652ccf843Smisaki save_desc_p->value, 84752ccf843Smisaki sad, 84852ccf843Smisaki save_desc_p->bits.hdw.tr_len, 84952ccf843Smisaki xfer_len, 85052ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 85152ccf843Smisaki save_desc_p->bits.hdw.mark, 85252ccf843Smisaki save_desc_p->bits.hdw.sop)); 85344961713Sgirish 85444961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 85544961713Sgirish #endif 85644961713Sgirish 85744961713Sgirish tx_msg_p->tx_msg_size = clen; 85844961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 85944961713Sgirish if (ngathers > nxge_tx_max_gathers) { 86044961713Sgirish good_packet = B_FALSE; 86144961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 86252ccf843Smisaki &stuff_offset, &end_offset, &value, 86352ccf843Smisaki &cksum_flags); 86444961713Sgirish 86544961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 86652ccf843Smisaki "==> nxge_start(14): pull msg - " 86752ccf843Smisaki "len %d pkt_len %d ngathers %d", 86852ccf843Smisaki len, pkt_len, ngathers)); 86944961713Sgirish /* Pull all message blocks from b_cont */ 87030ac2e7bSml if (is_lso) { 87130ac2e7bSml mp = nmp_lso_save; 87230ac2e7bSml goto nxge_start_fail_lso; 87330ac2e7bSml } 87444961713Sgirish if ((msgpullup(mp, -1)) == NULL) { 87544961713Sgirish goto nxge_start_fail2; 87644961713Sgirish } 87744961713Sgirish goto nxge_start_fail2; 87844961713Sgirish } 87944961713Sgirish } /* while (nmp) */ 88044961713Sgirish 88144961713Sgirish tx_msg_p->tx_message = mp; 88244961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 883adfcba55Sjoycey #if defined(__i386) 884adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 885adfcba55Sjoycey #else 88644961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 887adfcba55Sjoycey #endif 88844961713Sgirish 88944961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 89044961713Sgirish pkthdrp->reserved = 0; 89144961713Sgirish hdrp->value = 0; 892b4d05839Sml bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 89344961713Sgirish 89444961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 89544961713Sgirish tdc_stats->tx_jumbo_pkts++; 89644961713Sgirish } 89744961713Sgirish 898678453a8Sspeer min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 89944961713Sgirish if (pkt_len < min_len) { 90044961713Sgirish /* Assume we use bcopy to premapped buffers */ 90144961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 90244961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 90352ccf843Smisaki "==> nxge_start(14-1): < (msg_min + 16)" 90452ccf843Smisaki "len %d pkt_len %d min_len %d bzero %d ngathers %d", 90552ccf843Smisaki len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 90644961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 90744961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 90844961713Sgirish 90944961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 91044961713Sgirish 91144961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 91244961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 91344961713Sgirish 91444961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 91552ccf843Smisaki "==> nxge_start(14-2): < msg_min - " 91652ccf843Smisaki "len %d pkt_len %d min_len %d ngathers %d", 91752ccf843Smisaki len, pkt_len, min_len, ngathers)); 91844961713Sgirish } 91944961713Sgirish 92044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 92152ccf843Smisaki cksum_flags)); 92244961713Sgirish { 92344961713Sgirish uint64_t tmp_len; 92444961713Sgirish 92544961713Sgirish /* pkt_len already includes 16 + paddings!! */ 92644961713Sgirish /* Update the control header length */ 92744961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 92844961713Sgirish tmp_len = hdrp->value | 92952ccf843Smisaki (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 93044961713Sgirish 93144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 93252ccf843Smisaki "==> nxge_start(15_x1): setting SOP " 93352ccf843Smisaki "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 93452ccf843Smisaki "0x%llx hdrp->value 0x%llx", 93552ccf843Smisaki tot_xfer_len, tot_xfer_len, pkt_len, 93652ccf843Smisaki tmp_len, hdrp->value)); 93744961713Sgirish #if defined(_BIG_ENDIAN) 93844961713Sgirish hdrp->value = ddi_swap64(tmp_len); 93944961713Sgirish #else 94044961713Sgirish hdrp->value = tmp_len; 94144961713Sgirish #endif 94244961713Sgirish NXGE_DEBUG_MSG((nxgep, 94352ccf843Smisaki TX_CTL, "==> nxge_start(15_x2): setting SOP " 94452ccf843Smisaki "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 94552ccf843Smisaki "tmp_len 0x%llx hdrp->value 0x%llx", 94652ccf843Smisaki tot_xfer_len, pkt_len, 94752ccf843Smisaki tmp_len, hdrp->value)); 94844961713Sgirish } 94944961713Sgirish 95044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 95152ccf843Smisaki "wr_index %d " 95252ccf843Smisaki "tot_xfer_len (%d) pkt_len %d npads %d", 95352ccf843Smisaki sop_index, 95452ccf843Smisaki tot_xfer_len, pkt_len, 95552ccf843Smisaki npads)); 95644961713Sgirish 95744961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 95844961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 95944961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 96044961713Sgirish 96144961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 96244961713Sgirish 96344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 96444961713Sgirish 96544961713Sgirish #ifdef NXGE_DEBUG 96644961713Sgirish npi_desc_handle.nxgep = nxgep; 96744961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 96844961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 96944961713Sgirish 97044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 97152ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 97252ccf843Smisaki save_desc_p->value, 97352ccf843Smisaki sad, 97452ccf843Smisaki save_desc_p->bits.hdw.tr_len, 97552ccf843Smisaki xfer_len, 97652ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 97752ccf843Smisaki save_desc_p->bits.hdw.mark, 97852ccf843Smisaki save_desc_p->bits.hdw.sop)); 97944961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 98044961713Sgirish 98144961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 98244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 98352ccf843Smisaki "==> nxge_start: dump packets(17) (after sop set, len " 98452ccf843Smisaki " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 98552ccf843Smisaki "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 98652ccf843Smisaki (char *)hdrp, 98752ccf843Smisaki nxge_dump_packet((char *)hdrp, dump_len))); 98844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 98952ccf843Smisaki "==> nxge_start(18): TX desc sync: sop_index %d", 99052ccf843Smisaki sop_index)); 99144961713Sgirish #endif 99244961713Sgirish 99344961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 99444961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 99552ccf843Smisaki sop_index * sizeof (tx_desc_t), 99652ccf843Smisaki ngathers * sizeof (tx_desc_t), 99752ccf843Smisaki DDI_DMA_SYNC_FORDEV); 99844961713Sgirish 99944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 100052ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 100152ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 100252ccf843Smisaki stuff_offset, start_offset, 100352ccf843Smisaki pkt_len, ngathers, sop_index)); 100444961713Sgirish } else { /* more than one descriptor and wrap around */ 100544961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 100644961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 100752ccf843Smisaki sop_index * sizeof (tx_desc_t), 100852ccf843Smisaki nsdescs * sizeof (tx_desc_t), 100952ccf843Smisaki DDI_DMA_SYNC_FORDEV); 101044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 101152ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 101252ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 101352ccf843Smisaki stuff_offset, start_offset, 101452ccf843Smisaki pkt_len, ngathers, sop_index)); 101544961713Sgirish 101644961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 101752ccf843Smisaki 0, 101852ccf843Smisaki (ngathers - nsdescs) * sizeof (tx_desc_t), 101952ccf843Smisaki DDI_DMA_SYNC_FORDEV); 102044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 102152ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 102252ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 102352ccf843Smisaki stuff_offset, start_offset, 102452ccf843Smisaki pkt_len, ngathers, sop_index)); 102544961713Sgirish } 102644961713Sgirish 102744961713Sgirish tail_index = tx_ring_p->wr_index; 102844961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 102944961713Sgirish 103044961713Sgirish tx_ring_p->wr_index = i; 103144961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 103244961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 103352ccf843Smisaki B_FALSE : B_TRUE); 103444961713Sgirish } 103544961713Sgirish 103644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 103752ccf843Smisaki "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 103852ccf843Smisaki tx_ring_p->tdc, 103952ccf843Smisaki tx_ring_p->wr_index, 104052ccf843Smisaki tx_ring_p->wr_index_wrap, 104152ccf843Smisaki ngathers, 104252ccf843Smisaki tx_ring_p->descs_pending)); 104344961713Sgirish 104430ac2e7bSml if (is_lso) { 104530ac2e7bSml lso_ngathers += ngathers; 104630ac2e7bSml if (mp_chain != NULL) { 104730ac2e7bSml mp = mp_chain; 104830ac2e7bSml mp_chain = mp_chain->b_next; 104930ac2e7bSml mp->b_next = NULL; 105030ac2e7bSml if (nxge_lso_kick_cnt == lso_ngathers) { 10517a6dff21Sml tx_ring_p->descs_pending += lso_ngathers; 105230ac2e7bSml { 105330ac2e7bSml tx_ring_kick_t kick; 105430ac2e7bSml 105530ac2e7bSml kick.value = 0; 105630ac2e7bSml kick.bits.ldw.wrap = 105730ac2e7bSml tx_ring_p->wr_index_wrap; 105830ac2e7bSml kick.bits.ldw.tail = 105930ac2e7bSml (uint16_t)tx_ring_p->wr_index; 106030ac2e7bSml 106130ac2e7bSml /* Kick the Transmit kick register */ 106230ac2e7bSml TXDMA_REG_WRITE64( 106330ac2e7bSml NXGE_DEV_NPI_HANDLE(nxgep), 106430ac2e7bSml TX_RING_KICK_REG, 106530ac2e7bSml (uint8_t)tx_ring_p->tdc, 106630ac2e7bSml kick.value); 106730ac2e7bSml tdc_stats->tx_starts++; 1068678453a8Sspeer 106930ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 107030ac2e7bSml "==> nxge_start: more LSO: " 107130ac2e7bSml "LSO_CNT %d", 1072678453a8Sspeer lso_ngathers)); 107330ac2e7bSml } 107430ac2e7bSml lso_ngathers = 0; 107530ac2e7bSml ngathers = 0; 107630ac2e7bSml cur_index_lso = sop_index = tx_ring_p->wr_index; 107730ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 107830ac2e7bSml } 107930ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 108030ac2e7bSml "==> nxge_start: lso again: " 108130ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 108230ac2e7bSml "wr_index %d sop_index %d", 108330ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 108430ac2e7bSml tx_ring_p->wr_index, sop_index)); 108530ac2e7bSml 108630ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 108730ac2e7bSml "==> nxge_start: next : count %d", 1088678453a8Sspeer lso_ngathers)); 108930ac2e7bSml lso_again = B_TRUE; 109030ac2e7bSml goto start_again; 109130ac2e7bSml } 10927a6dff21Sml ngathers = lso_ngathers; 109330ac2e7bSml } 109430ac2e7bSml 109544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 109644961713Sgirish 109744961713Sgirish { 109844961713Sgirish tx_ring_kick_t kick; 109944961713Sgirish 110044961713Sgirish kick.value = 0; 110144961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 110244961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 110344961713Sgirish 110444961713Sgirish /* Kick start the Transmit kick register */ 110544961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 110652ccf843Smisaki TX_RING_KICK_REG, 110752ccf843Smisaki (uint8_t)tx_ring_p->tdc, 110852ccf843Smisaki kick.value); 110944961713Sgirish } 111044961713Sgirish 11117a6dff21Sml tx_ring_p->descs_pending += ngathers; 111244961713Sgirish tdc_stats->tx_starts++; 111344961713Sgirish 11146895688eSspeer if (isLDOMservice(nxgep)) { 11156895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 11166895688eSspeer if (tx_ring_p->tx_ring_offline) { 11176895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 11186895688eSspeer NXGE_TX_RING_OFFLINED); 11196895688eSspeer } 11206895688eSspeer } 1121678453a8Sspeer 112244961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 112344961713Sgirish 112444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 112544961713Sgirish return (status); 112644961713Sgirish 112730ac2e7bSml nxge_start_fail_lso: 112830ac2e7bSml status = 0; 112930ac2e7bSml good_packet = B_FALSE; 113030ac2e7bSml if (mp != NULL) { 113130ac2e7bSml freemsg(mp); 113230ac2e7bSml } 113330ac2e7bSml if (mp_chain != NULL) { 113430ac2e7bSml freemsg(mp_chain); 113530ac2e7bSml } 113630ac2e7bSml if (!lso_again && !ngathers) { 11376895688eSspeer if (isLDOMservice(nxgep)) { 11386895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 11396895688eSspeer if (tx_ring_p->tx_ring_offline) { 11406895688eSspeer (void) atomic_swap_32( 11416895688eSspeer &tx_ring_p->tx_ring_offline, 11426895688eSspeer NXGE_TX_RING_OFFLINED); 11436895688eSspeer } 11446895688eSspeer } 11456895688eSspeer 114630ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 114730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 114830ac2e7bSml "==> nxge_start: lso exit (nothing changed)")); 114930ac2e7bSml goto nxge_start_fail1; 115030ac2e7bSml } 115130ac2e7bSml 115230ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 115330ac2e7bSml "==> nxge_start (channel %d): before lso " 115430ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 115530ac2e7bSml "wr_index %d sop_index %d lso_again %d", 115630ac2e7bSml tx_ring_p->tdc, 115730ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 115830ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 115930ac2e7bSml 116030ac2e7bSml if (lso_again) { 116130ac2e7bSml lso_ngathers += ngathers; 116230ac2e7bSml ngathers = lso_ngathers; 116330ac2e7bSml sop_index = cur_index_lso; 116430ac2e7bSml tx_ring_p->wr_index = sop_index; 116530ac2e7bSml tx_ring_p->wr_index_wrap = lso_tail_wrap; 116630ac2e7bSml } 116730ac2e7bSml 116830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 116930ac2e7bSml "==> nxge_start (channel %d): after lso " 117030ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 117130ac2e7bSml "wr_index %d sop_index %d lso_again %d", 117230ac2e7bSml tx_ring_p->tdc, 117330ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 117430ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 117530ac2e7bSml 117644961713Sgirish nxge_start_fail2: 117744961713Sgirish if (good_packet == B_FALSE) { 117844961713Sgirish cur_index = sop_index; 117944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 118044961713Sgirish for (i = 0; i < ngathers; i++) { 118144961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 1182adfcba55Sjoycey #if defined(__i386) 1183adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 1184adfcba55Sjoycey #else 118544961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 1186adfcba55Sjoycey #endif 118744961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 118844961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 118944961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 119044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 119153f3d8ecSyc "tx_desc_p = %X index = %d", 119253f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 119353f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 119453f3d8ecSyc 0, -1); 119544961713Sgirish tx_msg_p->dvma_handle = NULL; 119644961713Sgirish if (tx_ring_p->dvma_wr_index == 119753f3d8ecSyc tx_ring_p->dvma_wrap_mask) 119844961713Sgirish tx_ring_p->dvma_wr_index = 0; 119944961713Sgirish else 120044961713Sgirish tx_ring_p->dvma_wr_index++; 120144961713Sgirish tx_ring_p->dvma_pending--; 120253f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 120344961713Sgirish if (ddi_dma_unbind_handle( 120453f3d8ecSyc tx_msg_p->dma_handle)) { 120544961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 120653f3d8ecSyc "ddi_dma_unbind_handle failed"); 120753f3d8ecSyc } 120844961713Sgirish } 120944961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 121044961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 121152ccf843Smisaki tx_ring_p->tx_wrap_mask); 121244961713Sgirish 121344961713Sgirish } 121444961713Sgirish } 121544961713Sgirish 12166895688eSspeer if (isLDOMservice(nxgep)) { 12176895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 12186895688eSspeer if (tx_ring_p->tx_ring_offline) { 12196895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 12206895688eSspeer NXGE_TX_RING_OFFLINED); 12216895688eSspeer } 12226895688eSspeer } 1223678453a8Sspeer 122444961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 122544961713Sgirish 122644961713Sgirish nxge_start_fail1: 122744961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 122844961713Sgirish 122944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 123022c0d73aSspeer return (status); 12311f8914d5Sml } 12321f8914d5Sml 123330ac2e7bSml /* Software LSO starts here */ 123430ac2e7bSml static void 123530ac2e7bSml nxge_hcksum_retrieve(mblk_t *mp, 123630ac2e7bSml uint32_t *start, uint32_t *stuff, uint32_t *end, 123730ac2e7bSml uint32_t *value, uint32_t *flags) 123830ac2e7bSml { 123930ac2e7bSml if (mp->b_datap->db_type == M_DATA) { 124030ac2e7bSml if (flags != NULL) { 124130ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 124230ac2e7bSml HCK_PARTIALCKSUM | HCK_FULLCKSUM | 124330ac2e7bSml HCK_FULLCKSUM_OK); 124430ac2e7bSml if ((*flags & (HCK_PARTIALCKSUM | 124530ac2e7bSml HCK_FULLCKSUM)) != 0) { 124630ac2e7bSml if (value != NULL) 124730ac2e7bSml *value = (uint32_t)DB_CKSUM16(mp); 124830ac2e7bSml if ((*flags & HCK_PARTIALCKSUM) != 0) { 124930ac2e7bSml if (start != NULL) 125030ac2e7bSml *start = 125130ac2e7bSml (uint32_t)DB_CKSUMSTART(mp); 125230ac2e7bSml if (stuff != NULL) 125330ac2e7bSml *stuff = 125430ac2e7bSml (uint32_t)DB_CKSUMSTUFF(mp); 125530ac2e7bSml if (end != NULL) 125630ac2e7bSml *end = 125730ac2e7bSml (uint32_t)DB_CKSUMEND(mp); 125830ac2e7bSml } 125930ac2e7bSml } 126030ac2e7bSml } 126130ac2e7bSml } 126230ac2e7bSml } 126330ac2e7bSml 126430ac2e7bSml static void 126530ac2e7bSml nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 126630ac2e7bSml { 126730ac2e7bSml ASSERT(DB_TYPE(mp) == M_DATA); 126830ac2e7bSml 126930ac2e7bSml *mss = 0; 127030ac2e7bSml if (flags != NULL) { 127130ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 127230ac2e7bSml if ((*flags != 0) && (mss != NULL)) { 127330ac2e7bSml *mss = (uint32_t)DB_LSOMSS(mp); 127430ac2e7bSml } 127530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 127630ac2e7bSml "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 127730ac2e7bSml *mss, *flags)); 127830ac2e7bSml } 127930ac2e7bSml 128030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 128130ac2e7bSml "<== nxge_lso_info_get: mss %d", *mss)); 128230ac2e7bSml } 128330ac2e7bSml 128430ac2e7bSml /* 128530ac2e7bSml * Do Soft LSO on the oversized packet. 128630ac2e7bSml * 128730ac2e7bSml * 1. Create a chain of message for headers. 128830ac2e7bSml * 2. Fill up header messages with proper information. 128930ac2e7bSml * 3. Copy Eithernet, IP, and TCP headers from the original message to 129030ac2e7bSml * each new message with necessary adjustments. 129130ac2e7bSml * * Unchange the ethernet header for DIX frames. (by default) 129230ac2e7bSml * * IP Total Length field is updated to MSS or less(only for the last one). 129330ac2e7bSml * * IP Identification value is incremented by one for each packet. 129430ac2e7bSml * * TCP sequence Number is recalculated according to the payload length. 129530ac2e7bSml * * Set FIN and/or PSH flags for the *last* packet if applied. 129630ac2e7bSml * * TCP partial Checksum 129730ac2e7bSml * 4. Update LSO information in the first message header. 129830ac2e7bSml * 5. Release the original message header. 129930ac2e7bSml */ 130030ac2e7bSml static mblk_t * 130130ac2e7bSml nxge_do_softlso(mblk_t *mp, uint32_t mss) 130230ac2e7bSml { 130330ac2e7bSml uint32_t hckflags; 130430ac2e7bSml int pktlen; 130530ac2e7bSml int hdrlen; 130630ac2e7bSml int segnum; 130730ac2e7bSml int i; 130830ac2e7bSml struct ether_vlan_header *evh; 130930ac2e7bSml int ehlen, iphlen, tcphlen; 131030ac2e7bSml struct ip *oiph, *niph; 131130ac2e7bSml struct tcphdr *otcph, *ntcph; 131230ac2e7bSml int available, len, left; 131330ac2e7bSml uint16_t ip_id; 131430ac2e7bSml uint32_t tcp_seq; 131530ac2e7bSml #ifdef __sparc 131630ac2e7bSml uint32_t tcp_seq_tmp; 131730ac2e7bSml #endif 131830ac2e7bSml mblk_t *datamp; 131930ac2e7bSml uchar_t *rptr; 132030ac2e7bSml mblk_t *nmp; 132130ac2e7bSml mblk_t *cmp; 132230ac2e7bSml mblk_t *mp_chain; 132330ac2e7bSml boolean_t do_cleanup = B_FALSE; 132430ac2e7bSml t_uscalar_t start_offset = 0; 132530ac2e7bSml t_uscalar_t stuff_offset = 0; 132630ac2e7bSml t_uscalar_t value = 0; 132730ac2e7bSml uint16_t l4_len; 132830ac2e7bSml ipaddr_t src, dst; 132930ac2e7bSml uint32_t cksum, sum, l4cksum; 133030ac2e7bSml 133130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 133230ac2e7bSml "==> nxge_do_softlso")); 133330ac2e7bSml /* 133430ac2e7bSml * check the length of LSO packet payload and calculate the number of 133530ac2e7bSml * segments to be generated. 133630ac2e7bSml */ 133730ac2e7bSml pktlen = msgsize(mp); 133830ac2e7bSml evh = (struct ether_vlan_header *)mp->b_rptr; 133930ac2e7bSml 134030ac2e7bSml /* VLAN? */ 134130ac2e7bSml if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 134230ac2e7bSml ehlen = sizeof (struct ether_vlan_header); 134330ac2e7bSml else 134430ac2e7bSml ehlen = sizeof (struct ether_header); 134530ac2e7bSml oiph = (struct ip *)(mp->b_rptr + ehlen); 134630ac2e7bSml iphlen = oiph->ip_hl * 4; 134730ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 134830ac2e7bSml tcphlen = otcph->th_off * 4; 134930ac2e7bSml 135030ac2e7bSml l4_len = pktlen - ehlen - iphlen; 135130ac2e7bSml 135230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 135330ac2e7bSml "==> nxge_do_softlso: mss %d oiph $%p " 135430ac2e7bSml "original ip_sum oiph->ip_sum 0x%x " 135530ac2e7bSml "original tcp_sum otcph->th_sum 0x%x " 135630ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d " 135730ac2e7bSml "l4_len %d (0x%x) ip_len - iphlen %d ", 135830ac2e7bSml mss, 135930ac2e7bSml oiph, 136030ac2e7bSml oiph->ip_sum, 136130ac2e7bSml otcph->th_sum, 136230ac2e7bSml ntohs(oiph->ip_len), pktlen, 136330ac2e7bSml ehlen, 136430ac2e7bSml l4_len, 136530ac2e7bSml l4_len, 136630ac2e7bSml ntohs(oiph->ip_len) - iphlen)); 136730ac2e7bSml 136830ac2e7bSml /* IPv4 + TCP */ 136930ac2e7bSml if (!(oiph->ip_v == IPV4_VERSION)) { 137030ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 137130ac2e7bSml "<== nxge_do_softlso: not IPV4 " 137230ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 137330ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 137430ac2e7bSml tcphlen)); 137530ac2e7bSml freemsg(mp); 137630ac2e7bSml return (NULL); 137730ac2e7bSml } 137830ac2e7bSml 137930ac2e7bSml if (!(oiph->ip_p == IPPROTO_TCP)) { 138030ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 138130ac2e7bSml "<== nxge_do_softlso: not TCP " 138230ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 138330ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 138430ac2e7bSml tcphlen)); 138530ac2e7bSml freemsg(mp); 138630ac2e7bSml return (NULL); 138730ac2e7bSml } 138830ac2e7bSml 138930ac2e7bSml if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 139030ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 139130ac2e7bSml "<== nxge_do_softlso: len not matched " 139230ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 139330ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 139430ac2e7bSml tcphlen)); 139530ac2e7bSml freemsg(mp); 139630ac2e7bSml return (NULL); 139730ac2e7bSml } 139830ac2e7bSml 139930ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 140030ac2e7bSml tcphlen = otcph->th_off * 4; 140130ac2e7bSml 140230ac2e7bSml /* TCP flags can not include URG, RST, or SYN */ 140330ac2e7bSml VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 140430ac2e7bSml 140530ac2e7bSml hdrlen = ehlen + iphlen + tcphlen; 140630ac2e7bSml 140730ac2e7bSml VERIFY(MBLKL(mp) >= hdrlen); 140830ac2e7bSml 140930ac2e7bSml if (MBLKL(mp) > hdrlen) { 141030ac2e7bSml datamp = mp; 141130ac2e7bSml rptr = mp->b_rptr + hdrlen; 141230ac2e7bSml } else { /* = */ 141330ac2e7bSml datamp = mp->b_cont; 141430ac2e7bSml rptr = datamp->b_rptr; 141530ac2e7bSml } 141630ac2e7bSml 141730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 141830ac2e7bSml "nxge_do_softlso: otcph $%p pktlen: %d, " 141930ac2e7bSml "hdrlen %d ehlen %d iphlen %d tcphlen %d " 142030ac2e7bSml "mblkl(mp): %d, mblkl(datamp): %d", 142130ac2e7bSml otcph, 142230ac2e7bSml pktlen, hdrlen, ehlen, iphlen, tcphlen, 142330ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 142430ac2e7bSml 142530ac2e7bSml hckflags = 0; 142630ac2e7bSml nxge_hcksum_retrieve(mp, 142730ac2e7bSml &start_offset, &stuff_offset, &value, NULL, &hckflags); 142830ac2e7bSml 142930ac2e7bSml dst = oiph->ip_dst.s_addr; 143030ac2e7bSml src = oiph->ip_src.s_addr; 143130ac2e7bSml 143230ac2e7bSml cksum = (dst >> 16) + (dst & 0xFFFF) + 143330ac2e7bSml (src >> 16) + (src & 0xFFFF); 143430ac2e7bSml l4cksum = cksum + IP_TCP_CSUM_COMP; 143530ac2e7bSml 143630ac2e7bSml sum = l4_len + l4cksum; 143730ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 143830ac2e7bSml 143930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 144030ac2e7bSml "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 144130ac2e7bSml "hckflags 0x%x start_offset %d stuff_offset %d " 144230ac2e7bSml "value (original) 0x%x th_sum 0x%x " 144330ac2e7bSml "pktlen %d l4_len %d (0x%x) " 144430ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 144530ac2e7bSml dst, src, 144630ac2e7bSml (sum & 0xffff), (~sum & 0xffff), 144730ac2e7bSml hckflags, start_offset, stuff_offset, 144830ac2e7bSml value, otcph->th_sum, 144930ac2e7bSml pktlen, 145030ac2e7bSml l4_len, 145130ac2e7bSml l4_len, 145230ac2e7bSml ntohs(oiph->ip_len) - (int)MBLKL(mp), 145330ac2e7bSml (int)MBLKL(datamp), 145430ac2e7bSml nxge_dump_packet((char *)evh, 12))); 145530ac2e7bSml 145630ac2e7bSml /* 145730ac2e7bSml * Start to process. 145830ac2e7bSml */ 145930ac2e7bSml available = pktlen - hdrlen; 146030ac2e7bSml segnum = (available - 1) / mss + 1; 146130ac2e7bSml 146230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 146330ac2e7bSml "==> nxge_do_softlso: pktlen %d " 146430ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 146530ac2e7bSml "available %d mss %d segnum %d", 146630ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 146730ac2e7bSml available, 146830ac2e7bSml mss, 146930ac2e7bSml segnum)); 147030ac2e7bSml 147130ac2e7bSml VERIFY(segnum >= 2); 147230ac2e7bSml 147330ac2e7bSml /* 147430ac2e7bSml * Try to pre-allocate all header messages 147530ac2e7bSml */ 147630ac2e7bSml mp_chain = NULL; 147730ac2e7bSml for (i = 0; i < segnum; i++) { 147830ac2e7bSml if ((nmp = allocb(hdrlen, 0)) == NULL) { 147930ac2e7bSml /* Clean up the mp_chain */ 148030ac2e7bSml while (mp_chain != NULL) { 148130ac2e7bSml nmp = mp_chain; 148230ac2e7bSml mp_chain = mp_chain->b_next; 148330ac2e7bSml freemsg(nmp); 148430ac2e7bSml } 148530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 148630ac2e7bSml "<== nxge_do_softlso: " 148730ac2e7bSml "Could not allocate enough messages for headers!")); 148830ac2e7bSml freemsg(mp); 148930ac2e7bSml return (NULL); 149030ac2e7bSml } 149130ac2e7bSml nmp->b_next = mp_chain; 149230ac2e7bSml mp_chain = nmp; 149330ac2e7bSml 149430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 149530ac2e7bSml "==> nxge_do_softlso: " 149630ac2e7bSml "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 149730ac2e7bSml mp, nmp, mp_chain, mp_chain->b_next)); 149830ac2e7bSml } 149930ac2e7bSml 150030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 150130ac2e7bSml "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 150230ac2e7bSml mp, nmp, mp_chain)); 150330ac2e7bSml 150430ac2e7bSml /* 150530ac2e7bSml * Associate payload with new packets 150630ac2e7bSml */ 150730ac2e7bSml cmp = mp_chain; 150830ac2e7bSml left = available; 150930ac2e7bSml while (cmp != NULL) { 151030ac2e7bSml nmp = dupb(datamp); 151130ac2e7bSml if (nmp == NULL) { 151230ac2e7bSml do_cleanup = B_TRUE; 151330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 151430ac2e7bSml "==>nxge_do_softlso: " 151530ac2e7bSml "Can not dupb(datamp), have to do clean up")); 151630ac2e7bSml goto cleanup_allocated_msgs; 151730ac2e7bSml } 151830ac2e7bSml 151930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 152030ac2e7bSml "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 152130ac2e7bSml "dupb nmp $%p len %d left %d msd %d ", 152230ac2e7bSml mp, cmp, nmp, len, left, mss)); 152330ac2e7bSml 152430ac2e7bSml cmp->b_cont = nmp; 152530ac2e7bSml nmp->b_rptr = rptr; 152630ac2e7bSml len = (left < mss) ? left : mss; 152730ac2e7bSml left -= len; 152830ac2e7bSml 152930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 153030ac2e7bSml "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 153130ac2e7bSml "dupb nmp $%p len %d left %d mss %d ", 153230ac2e7bSml mp, cmp, nmp, len, left, mss)); 153330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 153430ac2e7bSml "nxge_do_softlso: before available: %d, " 153530ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 153630ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 153730ac2e7bSml 153830ac2e7bSml len -= MBLKL(nmp); 153930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 154030ac2e7bSml "nxge_do_softlso: after available: %d, " 154130ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 154230ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 154330ac2e7bSml 154430ac2e7bSml while (len > 0) { 154530ac2e7bSml mblk_t *mmp = NULL; 154630ac2e7bSml 154730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 154830ac2e7bSml "nxge_do_softlso: (4) len > 0 available: %d, " 154930ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 155030ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 155130ac2e7bSml 155230ac2e7bSml if (datamp->b_cont != NULL) { 155330ac2e7bSml datamp = datamp->b_cont; 155430ac2e7bSml rptr = datamp->b_rptr; 155530ac2e7bSml mmp = dupb(datamp); 155630ac2e7bSml if (mmp == NULL) { 155730ac2e7bSml do_cleanup = B_TRUE; 155830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 155930ac2e7bSml "==> nxge_do_softlso: " 1560678453a8Sspeer "Can not dupb(datamp) (1), :" 156130ac2e7bSml "have to do clean up")); 156230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 156330ac2e7bSml "==> nxge_do_softlso: " 156430ac2e7bSml "available: %d, left: %d, " 156530ac2e7bSml "len: %d, MBLKL(nmp): %d", 156630ac2e7bSml available, left, len, 156730ac2e7bSml (int)MBLKL(nmp))); 156830ac2e7bSml goto cleanup_allocated_msgs; 156930ac2e7bSml } 157030ac2e7bSml } else { 157130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 157230ac2e7bSml "==> nxge_do_softlso: " 157330ac2e7bSml "(1)available: %d, left: %d, " 157430ac2e7bSml "len: %d, MBLKL(nmp): %d", 157530ac2e7bSml available, left, len, 157630ac2e7bSml (int)MBLKL(nmp))); 157730ac2e7bSml cmn_err(CE_PANIC, 157830ac2e7bSml "==> nxge_do_softlso: " 157930ac2e7bSml "Pointers must have been corrupted!\n" 158030ac2e7bSml "datamp: $%p, nmp: $%p, rptr: $%p", 158130ac2e7bSml (void *)datamp, 158230ac2e7bSml (void *)nmp, 158330ac2e7bSml (void *)rptr); 158430ac2e7bSml } 158530ac2e7bSml nmp->b_cont = mmp; 158630ac2e7bSml nmp = mmp; 158730ac2e7bSml len -= MBLKL(nmp); 158830ac2e7bSml } 158930ac2e7bSml if (len < 0) { 159030ac2e7bSml nmp->b_wptr += len; 159130ac2e7bSml rptr = nmp->b_wptr; 159230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 159330ac2e7bSml "(5) len < 0 (less than 0)" 159430ac2e7bSml "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 159530ac2e7bSml available, left, len, (int)MBLKL(nmp))); 159630ac2e7bSml 159730ac2e7bSml } else if (len == 0) { 159830ac2e7bSml if (datamp->b_cont != NULL) { 159930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 160030ac2e7bSml "(5) len == 0" 160130ac2e7bSml "available: %d, left: %d, len: %d, " 160230ac2e7bSml "MBLKL(nmp): %d", 160330ac2e7bSml available, left, len, (int)MBLKL(nmp))); 160430ac2e7bSml datamp = datamp->b_cont; 160530ac2e7bSml rptr = datamp->b_rptr; 160630ac2e7bSml } else { 160730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 160830ac2e7bSml "(6)available b_cont == NULL : %d, " 160930ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 161030ac2e7bSml available, left, len, (int)MBLKL(nmp))); 161130ac2e7bSml 161230ac2e7bSml VERIFY(cmp->b_next == NULL); 161330ac2e7bSml VERIFY(left == 0); 161430ac2e7bSml break; /* Done! */ 161530ac2e7bSml } 161630ac2e7bSml } 161730ac2e7bSml cmp = cmp->b_next; 161830ac2e7bSml 161930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 162030ac2e7bSml "(7) do_softlso: " 162130ac2e7bSml "next mp in mp_chain available len != 0 : %d, " 162230ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 162330ac2e7bSml available, left, len, (int)MBLKL(nmp))); 162430ac2e7bSml } 162530ac2e7bSml 162630ac2e7bSml /* 162730ac2e7bSml * From now, start to fill up all headers for the first message 162830ac2e7bSml * Hardware checksum flags need to be updated separately for FULLCKSUM 162930ac2e7bSml * and PARTIALCKSUM cases. For full checksum, copy the original flags 163030ac2e7bSml * into every new packet is enough. But for HCK_PARTIALCKSUM, all 163130ac2e7bSml * required fields need to be updated properly. 163230ac2e7bSml */ 163330ac2e7bSml nmp = mp_chain; 163430ac2e7bSml bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 163530ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 163630ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 163730ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 163830ac2e7bSml ip_id = ntohs(niph->ip_id); 163930ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 164030ac2e7bSml #ifdef __sparc 164130ac2e7bSml bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 164230ac2e7bSml tcp_seq = ntohl(tcp_seq_tmp); 164330ac2e7bSml #else 164430ac2e7bSml tcp_seq = ntohl(ntcph->th_seq); 164530ac2e7bSml #endif 164630ac2e7bSml 164730ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 164830ac2e7bSml 164930ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 165030ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 165130ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 165230ac2e7bSml 165330ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 165430ac2e7bSml niph->ip_sum = 0; 165530ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 165630ac2e7bSml 165730ac2e7bSml l4_len = mss + tcphlen; 165830ac2e7bSml sum = htons(l4_len) + l4cksum; 165930ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 166030ac2e7bSml ntcph->th_sum = (sum & 0xffff); 166130ac2e7bSml 166230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 166330ac2e7bSml "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 166430ac2e7bSml "mss %d pktlen %d l4_len %d (0x%x) " 166530ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 166630ac2e7bSml "ip_sum 0x%x " 166730ac2e7bSml "th_sum 0x%x sum 0x%x ) " 166830ac2e7bSml "dump first ip->tcp %s", 166930ac2e7bSml nmp, mp_chain, 167030ac2e7bSml mss, 167130ac2e7bSml pktlen, 167230ac2e7bSml l4_len, 167330ac2e7bSml l4_len, 167430ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp), 167530ac2e7bSml niph->ip_sum, 167630ac2e7bSml ntcph->th_sum, 167730ac2e7bSml sum, 167830ac2e7bSml nxge_dump_packet((char *)niph, 52))); 167930ac2e7bSml 168030ac2e7bSml cmp = nmp; 168130ac2e7bSml while ((nmp = nmp->b_next)->b_next != NULL) { 168230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168330ac2e7bSml "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 168430ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 168530ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 168630ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 168730ac2e7bSml niph->ip_id = htons(++ip_id); 168830ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 168930ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 169030ac2e7bSml tcp_seq += mss; 169130ac2e7bSml 169230ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 169330ac2e7bSml 169430ac2e7bSml #ifdef __sparc 169530ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 169630ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 169730ac2e7bSml #else 169830ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 169930ac2e7bSml #endif 170030ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 170130ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 170230ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 170330ac2e7bSml 170430ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 170530ac2e7bSml niph->ip_sum = 0; 170630ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 170730ac2e7bSml ntcph->th_sum = (sum & 0xffff); 170830ac2e7bSml 170930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 171030ac2e7bSml "==> nxge_do_softlso: middle ip_sum 0x%x " 171130ac2e7bSml "th_sum 0x%x " 171230ac2e7bSml " mp $%p (mp_chain $%p) pktlen %d " 171330ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 171430ac2e7bSml niph->ip_sum, 171530ac2e7bSml ntcph->th_sum, 171630ac2e7bSml nmp, mp_chain, 171730ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 171830ac2e7bSml } 171930ac2e7bSml 172030ac2e7bSml /* Last segment */ 172130ac2e7bSml /* 172230ac2e7bSml * Set FIN and/or PSH flags if present only in the last packet. 172330ac2e7bSml * The ip_len could be different from prior packets. 172430ac2e7bSml */ 172530ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 172630ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 172730ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 172830ac2e7bSml niph->ip_id = htons(++ip_id); 172930ac2e7bSml niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 173030ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 173130ac2e7bSml tcp_seq += mss; 173230ac2e7bSml #ifdef __sparc 173330ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 173430ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 173530ac2e7bSml #else 173630ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 173730ac2e7bSml #endif 173830ac2e7bSml ntcph->th_flags = (otcph->th_flags & ~TH_URG); 173930ac2e7bSml 174030ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 174130ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 174230ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 174330ac2e7bSml 174430ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 174530ac2e7bSml niph->ip_sum = 0; 174630ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 174730ac2e7bSml 174830ac2e7bSml l4_len = ntohs(niph->ip_len) - iphlen; 174930ac2e7bSml sum = htons(l4_len) + l4cksum; 175030ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 175130ac2e7bSml ntcph->th_sum = (sum & 0xffff); 175230ac2e7bSml 175330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 175430ac2e7bSml "==> nxge_do_softlso: last next " 175530ac2e7bSml "niph->ip_sum 0x%x " 175630ac2e7bSml "ntcph->th_sum 0x%x sum 0x%x " 175730ac2e7bSml "dump last ip->tcp %s " 175830ac2e7bSml "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 175930ac2e7bSml "l4_len %d (0x%x) " 176030ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 176130ac2e7bSml niph->ip_sum, 176230ac2e7bSml ntcph->th_sum, sum, 176330ac2e7bSml nxge_dump_packet((char *)niph, 52), 176430ac2e7bSml cmp, nmp, mp_chain, 176530ac2e7bSml pktlen, pktlen, 176630ac2e7bSml l4_len, 176730ac2e7bSml l4_len, 176830ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 176930ac2e7bSml 177030ac2e7bSml cleanup_allocated_msgs: 177130ac2e7bSml if (do_cleanup) { 177230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 177330ac2e7bSml "==> nxge_do_softlso: " 177430ac2e7bSml "Failed allocating messages, " 177530ac2e7bSml "have to clean up and fail!")); 177630ac2e7bSml while (mp_chain != NULL) { 177730ac2e7bSml nmp = mp_chain; 177830ac2e7bSml mp_chain = mp_chain->b_next; 177930ac2e7bSml freemsg(nmp); 178030ac2e7bSml } 178130ac2e7bSml } 178230ac2e7bSml /* 178330ac2e7bSml * We're done here, so just free the original message and return the 178430ac2e7bSml * new message chain, that could be NULL if failed, back to the caller. 178530ac2e7bSml */ 178630ac2e7bSml freemsg(mp); 178730ac2e7bSml 178830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 178930ac2e7bSml "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 179030ac2e7bSml return (mp_chain); 179130ac2e7bSml } 179230ac2e7bSml 179330ac2e7bSml /* 179430ac2e7bSml * Will be called before NIC driver do further operation on the message. 179530ac2e7bSml * The input message may include LSO information, if so, go to softlso logic 179630ac2e7bSml * to eliminate the oversized LSO packet for the incapable underlying h/w. 179730ac2e7bSml * The return could be the same non-LSO message or a message chain for LSO case. 179830ac2e7bSml * 179930ac2e7bSml * The driver needs to call this function per packet and process the whole chain 180030ac2e7bSml * if applied. 180130ac2e7bSml */ 180230ac2e7bSml static mblk_t * 180330ac2e7bSml nxge_lso_eliminate(mblk_t *mp) 180430ac2e7bSml { 180530ac2e7bSml uint32_t lsoflags; 180630ac2e7bSml uint32_t mss; 180730ac2e7bSml 180830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 180930ac2e7bSml "==>nxge_lso_eliminate:")); 181030ac2e7bSml nxge_lso_info_get(mp, &mss, &lsoflags); 181130ac2e7bSml 181230ac2e7bSml if (lsoflags & HW_LSO) { 181330ac2e7bSml mblk_t *nmp; 181430ac2e7bSml 181530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 181630ac2e7bSml "==>nxge_lso_eliminate:" 181730ac2e7bSml "HW_LSO:mss %d mp $%p", 181830ac2e7bSml mss, mp)); 181930ac2e7bSml if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 182030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 182130ac2e7bSml "<== nxge_lso_eliminate: " 182230ac2e7bSml "LSO: nmp not NULL nmp $%p mss %d mp $%p", 182330ac2e7bSml nmp, mss, mp)); 182430ac2e7bSml return (nmp); 182530ac2e7bSml } else { 182630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 182730ac2e7bSml "<== nxge_lso_eliminate_ " 182830ac2e7bSml "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 182930ac2e7bSml nmp, mss, mp)); 183030ac2e7bSml return (NULL); 183130ac2e7bSml } 183230ac2e7bSml } 183330ac2e7bSml 183430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 183530ac2e7bSml "<== nxge_lso_eliminate")); 183630ac2e7bSml return (mp); 183730ac2e7bSml } 183830ac2e7bSml 183930ac2e7bSml static uint32_t 184030ac2e7bSml nxge_csgen(uint16_t *adr, int len) 184130ac2e7bSml { 184230ac2e7bSml int i, odd; 184330ac2e7bSml uint32_t sum = 0; 184430ac2e7bSml uint32_t c = 0; 184530ac2e7bSml 184630ac2e7bSml odd = len % 2; 184730ac2e7bSml for (i = 0; i < (len / 2); i++) { 184830ac2e7bSml sum += (adr[i] & 0xffff); 184930ac2e7bSml } 185030ac2e7bSml if (odd) { 185130ac2e7bSml sum += adr[len / 2] & 0xff00; 185230ac2e7bSml } 185330ac2e7bSml while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 185430ac2e7bSml sum &= 0xffff; 185530ac2e7bSml sum += c; 185630ac2e7bSml } 185730ac2e7bSml return (~sum & 0xffff); 185830ac2e7bSml } 1859