144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 22ae6aa22aSVenugopal Iyer * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 26ae6aa22aSVenugopal Iyer #include <sys/mac_provider.h> 27a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 28678453a8Sspeer #include <sys/nxge/nxge_hio.h> 29678453a8Sspeer #include <npi_tx_wr64.h> 3044961713Sgirish 3130ac2e7bSml /* Software LSO required header files */ 3230ac2e7bSml #include <netinet/tcp.h> 3330ac2e7bSml #include <inet/ip_impl.h> 3430ac2e7bSml #include <inet/tcp.h> 3530ac2e7bSml 36ae6aa22aSVenugopal Iyer extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy, 37ae6aa22aSVenugopal Iyer boolean_t is_outbound); 38ae6aa22aSVenugopal Iyer 3930ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *); 4030ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 4130ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 4230ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *, 4330ac2e7bSml uint32_t *, uint32_t *, uint32_t *, 4430ac2e7bSml uint32_t *, uint32_t *); 4530ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int); 4630ac2e7bSml 4744961713Sgirish extern uint32_t nxge_reclaim_pending; 4844961713Sgirish extern uint32_t nxge_bcopy_thresh; 4944961713Sgirish extern uint32_t nxge_dvma_thresh; 5044961713Sgirish extern uint32_t nxge_dma_stream_thresh; 5144961713Sgirish extern uint32_t nxge_tx_minfree; 5244961713Sgirish extern uint32_t nxge_tx_intr_thres; 5344961713Sgirish extern uint32_t nxge_tx_max_gathers; 5444961713Sgirish extern uint32_t nxge_tx_tiny_pack; 5544961713Sgirish extern uint32_t nxge_tx_use_bcopy; 561f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 5730ac2e7bSml uint32_t nxge_lso_kick_cnt = 2; 5844961713Sgirish 5944961713Sgirish 60da14cebeSEric Cheng void 61da14cebeSEric Cheng nxge_tx_ring_task(void *arg) 62da14cebeSEric Cheng { 63da14cebeSEric Cheng p_tx_ring_t ring = (p_tx_ring_t)arg; 64da14cebeSEric Cheng 65da14cebeSEric Cheng MUTEX_ENTER(&ring->lock); 66da14cebeSEric Cheng (void) nxge_txdma_reclaim(ring->nxgep, ring, 0); 67da14cebeSEric Cheng MUTEX_EXIT(&ring->lock); 68da14cebeSEric Cheng 69da14cebeSEric Cheng if (!isLDOMguest(ring->nxgep) && !ring->tx_ring_offline) 70da14cebeSEric Cheng mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle); 71da14cebeSEric Cheng #if defined(sun4v) 72da14cebeSEric Cheng else { 73da14cebeSEric Cheng nxge_hio_data_t *nhd = 74da14cebeSEric Cheng (nxge_hio_data_t *)ring->nxgep->nxge_hw_p->hio; 75da14cebeSEric Cheng nx_vio_fp_t *vio = &nhd->hio.vio; 76da14cebeSEric Cheng 77da14cebeSEric Cheng /* Call back vnet. */ 78da14cebeSEric Cheng if (vio->cb.vio_net_tx_update) { 79da14cebeSEric Cheng (*vio->cb.vio_net_tx_update)(ring->nxgep->hio_vr->vhp); 80da14cebeSEric Cheng } 81da14cebeSEric Cheng } 82da14cebeSEric Cheng #endif 83da14cebeSEric Cheng } 84da14cebeSEric Cheng 85da14cebeSEric Cheng static void 86da14cebeSEric Cheng nxge_tx_ring_dispatch(p_tx_ring_t ring) 87da14cebeSEric Cheng { 88da14cebeSEric Cheng /* 89da14cebeSEric Cheng * Kick the ring task to reclaim some buffers. 90da14cebeSEric Cheng */ 91da14cebeSEric Cheng (void) ddi_taskq_dispatch(ring->taskq, 92da14cebeSEric Cheng nxge_tx_ring_task, (void *)ring, DDI_SLEEP); 93da14cebeSEric Cheng } 94da14cebeSEric Cheng 95da14cebeSEric Cheng mblk_t * 96da14cebeSEric Cheng nxge_tx_ring_send(void *arg, mblk_t *mp) 97da14cebeSEric Cheng { 98da14cebeSEric Cheng p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg; 99da14cebeSEric Cheng p_nxge_t nxgep; 100da14cebeSEric Cheng p_tx_ring_t tx_ring_p; 101da14cebeSEric Cheng int status, channel; 102da14cebeSEric Cheng 103da14cebeSEric Cheng ASSERT(nrhp != NULL); 104da14cebeSEric Cheng nxgep = nrhp->nxgep; 105da14cebeSEric Cheng channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index; 106da14cebeSEric Cheng tx_ring_p = nxgep->tx_rings->rings[channel]; 107da14cebeSEric Cheng 10848056c53SMichael Speer /* 10948056c53SMichael Speer * We may be in a transition from offlined DMA to onlined 11048056c53SMichael Speer * DMA. 11148056c53SMichael Speer */ 11248056c53SMichael Speer if (tx_ring_p == NULL) { 11348056c53SMichael Speer ASSERT(tx_ring_p != NULL); 11448056c53SMichael Speer freemsg(mp); 11548056c53SMichael Speer return ((mblk_t *)NULL); 11648056c53SMichael Speer } 11748056c53SMichael Speer 11848056c53SMichael Speer /* 11948056c53SMichael Speer * Valid DMA? 12048056c53SMichael Speer */ 121da14cebeSEric Cheng ASSERT(nxgep == tx_ring_p->nxgep); 122da14cebeSEric Cheng 12348056c53SMichael Speer /* 12448056c53SMichael Speer * Make sure DMA is not offlined. 12548056c53SMichael Speer */ 12648056c53SMichael Speer if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) { 127da14cebeSEric Cheng ASSERT(!tx_ring_p->tx_ring_offline); 12848056c53SMichael Speer freemsg(mp); 12948056c53SMichael Speer return ((mblk_t *)NULL); 130da14cebeSEric Cheng } 131da14cebeSEric Cheng 13248056c53SMichael Speer /* 13348056c53SMichael Speer * Transmit the packet. 13448056c53SMichael Speer */ 135da14cebeSEric Cheng status = nxge_start(nxgep, tx_ring_p, mp); 136da14cebeSEric Cheng if (status) { 137da14cebeSEric Cheng nxge_tx_ring_dispatch(tx_ring_p); 138da14cebeSEric Cheng return (mp); 139da14cebeSEric Cheng } 140da14cebeSEric Cheng 141da14cebeSEric Cheng return ((mblk_t *)NULL); 142da14cebeSEric Cheng } 143da14cebeSEric Cheng 144da14cebeSEric Cheng #if defined(sun4v) 145da14cebeSEric Cheng 146ae6aa22aSVenugopal Iyer /* 147ae6aa22aSVenugopal Iyer * Hashing policy for load balancing over the set of TX rings 148ae6aa22aSVenugopal Iyer * available to the driver. 149ae6aa22aSVenugopal Iyer */ 150ae6aa22aSVenugopal Iyer static uint8_t nxge_tx_hash_policy = MAC_PKT_HASH_L4; 151ae6aa22aSVenugopal Iyer 152da14cebeSEric Cheng /* 153da14cebeSEric Cheng * nxge_m_tx() is needed for Hybrid I/O operation of the vnet in 154da14cebeSEric Cheng * the guest domain. See CR 6778758 for long term solution. 155ae6aa22aSVenugopal Iyer * 156ae6aa22aSVenugopal Iyer * The guest domain driver will for now hash the packet 157ae6aa22aSVenugopal Iyer * to pick a DMA channel from the only group it has group 0. 158da14cebeSEric Cheng */ 159da14cebeSEric Cheng 160da14cebeSEric Cheng mblk_t * 161da14cebeSEric Cheng nxge_m_tx(void *arg, mblk_t *mp) 162da14cebeSEric Cheng { 163da14cebeSEric Cheng p_nxge_t nxgep = (p_nxge_t)arg; 164da14cebeSEric Cheng mblk_t *next; 165ae6aa22aSVenugopal Iyer uint64_t rindex; 166da14cebeSEric Cheng p_tx_ring_t tx_ring_p; 167da14cebeSEric Cheng int status; 168da14cebeSEric Cheng 169da14cebeSEric Cheng NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_m_tx")); 170da14cebeSEric Cheng 171da14cebeSEric Cheng /* 172ae6aa22aSVenugopal Iyer * Hash to pick a ring from Group 0, the only TX group 173ae6aa22aSVenugopal Iyer * for a guest domain driver. 174ae6aa22aSVenugopal Iyer */ 175ae6aa22aSVenugopal Iyer rindex = mac_pkt_hash(DL_ETHER, mp, nxge_tx_hash_policy, B_TRUE); 176ae6aa22aSVenugopal Iyer rindex = rindex % nxgep->pt_config.tdc_grps[0].max_tdcs; 177ae6aa22aSVenugopal Iyer 178ae6aa22aSVenugopal Iyer /* 179ae6aa22aSVenugopal Iyer * Get the ring handle. 180da14cebeSEric Cheng */ 181ae6aa22aSVenugopal Iyer tx_ring_p = nxgep->tx_rings->rings[rindex]; 182da14cebeSEric Cheng 183da14cebeSEric Cheng while (mp != NULL) { 184da14cebeSEric Cheng next = mp->b_next; 185da14cebeSEric Cheng mp->b_next = NULL; 186da14cebeSEric Cheng 187da14cebeSEric Cheng status = nxge_start(nxgep, tx_ring_p, mp); 188da14cebeSEric Cheng if (status != 0) { 189da14cebeSEric Cheng mp->b_next = next; 190da14cebeSEric Cheng nxge_tx_ring_dispatch(tx_ring_p); 191da14cebeSEric Cheng return (mp); 192da14cebeSEric Cheng } 193da14cebeSEric Cheng 194da14cebeSEric Cheng mp = next; 195da14cebeSEric Cheng } 196da14cebeSEric Cheng 197da14cebeSEric Cheng NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_m_tx")); 198da14cebeSEric Cheng return ((mblk_t *)NULL); 199da14cebeSEric Cheng } 200da14cebeSEric Cheng 201da14cebeSEric Cheng #endif 20244961713Sgirish 20344961713Sgirish int 20444961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 20544961713Sgirish { 206*31f519f9SMichael Speer int dma_status, status = 0; 20744961713Sgirish p_tx_desc_t tx_desc_ring_vp; 20844961713Sgirish npi_handle_t npi_desc_handle; 20944961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 21044961713Sgirish p_tx_desc_t tx_desc_p; 21144961713Sgirish p_tx_msg_t tx_msg_ring; 21244961713Sgirish p_tx_msg_t tx_msg_p; 21344961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 21444961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 21544961713Sgirish p_tx_pkt_header_t hdrp; 216ef755e7aStc tx_pkt_hdr_all_t tmp_hdrp; 21744961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 21844961713Sgirish uint8_t npads = 0; 21944961713Sgirish uint64_t dma_ioaddr; 22044961713Sgirish uint32_t dma_flags; 22144961713Sgirish int last_bidx; 22244961713Sgirish uint8_t *b_rptr; 22344961713Sgirish caddr_t kaddr; 22444961713Sgirish uint32_t nmblks; 22544961713Sgirish uint32_t ngathers; 22644961713Sgirish uint32_t clen; 22744961713Sgirish int len; 22844961713Sgirish uint32_t pkt_len, pack_len, min_len; 22944961713Sgirish uint32_t bcopy_thresh; 23044961713Sgirish int i, cur_index, sop_index; 23144961713Sgirish uint16_t tail_index; 23244961713Sgirish boolean_t tail_wrap = B_FALSE; 23344961713Sgirish nxge_dma_common_t desc_area; 23444961713Sgirish nxge_os_dma_handle_t dma_handle; 23544961713Sgirish ddi_dma_cookie_t dma_cookie; 23644961713Sgirish npi_handle_t npi_handle; 23744961713Sgirish p_mblk_t nmp; 23844961713Sgirish p_mblk_t t_mp; 23944961713Sgirish uint32_t ncookies; 24044961713Sgirish boolean_t good_packet; 24144961713Sgirish boolean_t mark_mode = B_FALSE; 24244961713Sgirish p_nxge_stats_t statsp; 24344961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 24444961713Sgirish t_uscalar_t start_offset = 0; 24544961713Sgirish t_uscalar_t stuff_offset = 0; 24644961713Sgirish t_uscalar_t end_offset = 0; 24744961713Sgirish t_uscalar_t value = 0; 24844961713Sgirish t_uscalar_t cksum_flags = 0; 24944961713Sgirish boolean_t cksum_on = B_FALSE; 25044961713Sgirish uint32_t boff = 0; 251b4d05839Sml uint64_t tot_xfer_len = 0; 25244961713Sgirish boolean_t header_set = B_FALSE; 25344961713Sgirish #ifdef NXGE_DEBUG 25444961713Sgirish p_tx_desc_t tx_desc_ring_pp; 25544961713Sgirish p_tx_desc_t tx_desc_pp; 25644961713Sgirish tx_desc_t *save_desc_p; 25744961713Sgirish int dump_len; 25844961713Sgirish int sad_len; 25944961713Sgirish uint64_t sad; 26044961713Sgirish int xfer_len; 26144961713Sgirish uint32_t msgsize; 26244961713Sgirish #endif 26330ac2e7bSml p_mblk_t mp_chain = NULL; 26430ac2e7bSml boolean_t is_lso = B_FALSE; 26530ac2e7bSml boolean_t lso_again; 26630ac2e7bSml int cur_index_lso; 26730ac2e7bSml p_mblk_t nmp_lso_save; 26830ac2e7bSml uint32_t lso_ngathers; 26930ac2e7bSml boolean_t lso_tail_wrap = B_FALSE; 27044961713Sgirish 27144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 272678453a8Sspeer "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 27344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 274678453a8Sspeer "==> nxge_start: Starting tdc %d desc pending %d", 275678453a8Sspeer tx_ring_p->tdc, tx_ring_p->descs_pending)); 27644961713Sgirish 27744961713Sgirish statsp = nxgep->statsp; 27844961713Sgirish 279678453a8Sspeer if (!isLDOMguest(nxgep)) { 280678453a8Sspeer switch (nxgep->mac.portmode) { 281678453a8Sspeer default: 282678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode == 283678453a8Sspeer nxge_lb_normal) { 284678453a8Sspeer if (!statsp->mac_stats.link_up) { 285678453a8Sspeer freemsg(mp); 286678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 287678453a8Sspeer "==> nxge_start: " 288678453a8Sspeer "link not up")); 289678453a8Sspeer goto nxge_start_fail1; 290678453a8Sspeer } 291321febdeSsbehera } 292678453a8Sspeer break; 293678453a8Sspeer case PORT_10G_FIBER: 294678453a8Sspeer /* 295678453a8Sspeer * For the following modes, check the link status 296678453a8Sspeer * before sending the packet out: 297eb1db165Stc * nxge_lb_normal, 298eb1db165Stc * nxge_lb_ext10g, 299eb1db165Stc * nxge_lb_ext1000, 300eb1db165Stc * nxge_lb_ext100, 301eb1db165Stc * nxge_lb_ext10. 302678453a8Sspeer */ 303678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode < 304eb1db165Stc nxge_lb_phy10g) { 305678453a8Sspeer if (!statsp->mac_stats.link_up) { 306678453a8Sspeer freemsg(mp); 307678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 308678453a8Sspeer "==> nxge_start: " 309678453a8Sspeer "link not up")); 310678453a8Sspeer goto nxge_start_fail1; 311678453a8Sspeer } 312321febdeSsbehera } 313678453a8Sspeer break; 31444961713Sgirish } 315678453a8Sspeer } 316678453a8Sspeer 317678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 318678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 319678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 320678453a8Sspeer "==> nxge_start: hardware not initialized or stopped")); 321678453a8Sspeer freemsg(mp); 322678453a8Sspeer goto nxge_start_fail1; 32344961713Sgirish } 32444961713Sgirish 3253d16f8e7Sml if (nxgep->soft_lso_enable) { 32630ac2e7bSml mp_chain = nxge_lso_eliminate(mp); 32730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 32830ac2e7bSml "==> nxge_start(0): LSO mp $%p mp_chain $%p", 32930ac2e7bSml mp, mp_chain)); 33030ac2e7bSml if (mp_chain == NULL) { 33130ac2e7bSml NXGE_ERROR_MSG((nxgep, TX_CTL, 33230ac2e7bSml "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 33330ac2e7bSml mp_chain, mp)); 33430ac2e7bSml goto nxge_start_fail1; 33530ac2e7bSml } 33630ac2e7bSml if (mp_chain != mp) { 33730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 33830ac2e7bSml "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 33930ac2e7bSml mp_chain, mp)); 34030ac2e7bSml is_lso = B_TRUE; 34130ac2e7bSml mp = mp_chain; 34230ac2e7bSml mp_chain = mp_chain->b_next; 34330ac2e7bSml mp->b_next = NULL; 34430ac2e7bSml } 34530ac2e7bSml } 34630ac2e7bSml 34744961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 34852ccf843Smisaki &stuff_offset, &end_offset, &value, &cksum_flags); 34944961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 35044961713Sgirish start_offset += sizeof (ether_header_t); 35144961713Sgirish stuff_offset += sizeof (ether_header_t); 35244961713Sgirish } else { 35344961713Sgirish start_offset += sizeof (struct ether_vlan_header); 35444961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 35544961713Sgirish } 35644961713Sgirish 35744961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 35844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 35952ccf843Smisaki "==> nxge_start: mp $%p len %d " 36052ccf843Smisaki "cksum_flags 0x%x (partial checksum) ", 36152ccf843Smisaki mp, MBLKL(mp), cksum_flags)); 36244961713Sgirish cksum_on = B_TRUE; 36344961713Sgirish } 36444961713Sgirish 365b4d05839Sml pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 366b4d05839Sml pkthdrp->reserved = 0; 367ef755e7aStc tmp_hdrp.pkthdr.value = 0; 368b4d05839Sml nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 369b4d05839Sml 0, 0, pkthdrp, 370b4d05839Sml start_offset, stuff_offset); 371b4d05839Sml 37230ac2e7bSml lso_again = B_FALSE; 37330ac2e7bSml lso_ngathers = 0; 37430ac2e7bSml 37530ac2e7bSml MUTEX_ENTER(&tx_ring_p->lock); 37622c0d73aSspeer 37722c0d73aSspeer if (isLDOMservice(nxgep)) { 3786895688eSspeer tx_ring_p->tx_ring_busy = B_TRUE; 37922c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 38022c0d73aSspeer freemsg(mp); 3816895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 3826895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 3836895688eSspeer NXGE_TX_RING_OFFLINED); 38422c0d73aSspeer MUTEX_EXIT(&tx_ring_p->lock); 38522c0d73aSspeer return (status); 38622c0d73aSspeer } 38722c0d73aSspeer } 38822c0d73aSspeer 38930ac2e7bSml cur_index_lso = tx_ring_p->wr_index; 39030ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 39130ac2e7bSml start_again: 39230ac2e7bSml ngathers = 0; 39330ac2e7bSml sop_index = tx_ring_p->wr_index; 39444961713Sgirish #ifdef NXGE_DEBUG 39544961713Sgirish if (tx_ring_p->descs_pending) { 39644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 39752ccf843Smisaki "desc pending %d ", tx_ring_p->descs_pending)); 39844961713Sgirish } 39944961713Sgirish 40044961713Sgirish dump_len = (int)(MBLKL(mp)); 40144961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 40244961713Sgirish 40344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 40452ccf843Smisaki "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 40552ccf843Smisaki "(Before header reserve: ORIGINAL LEN %d)", 40652ccf843Smisaki tx_ring_p->tdc, 40752ccf843Smisaki mp->b_rptr, 40852ccf843Smisaki dump_len)); 40944961713Sgirish 41044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 41152ccf843Smisaki "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 41252ccf843Smisaki nxge_dump_packet((char *)mp->b_rptr, dump_len))); 41344961713Sgirish #endif 41444961713Sgirish 41544961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 41644961713Sgirish mark_mode = (tx_ring_p->descs_pending && 417257bdc55SMichael Speer (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) < 418257bdc55SMichael Speer (int)nxge_tx_minfree)); 41944961713Sgirish 42044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 42152ccf843Smisaki "TX Descriptor ring is channel %d mark mode %d", 42252ccf843Smisaki tx_ring_p->tdc, mark_mode)); 42344961713Sgirish 4247127d9f6Sml if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 4257127d9f6Sml if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 4267127d9f6Sml (nxge_tx_minfree + lso_ngathers))) { 4277127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4287127d9f6Sml "TX Descriptor ring is full: channel %d", 4297127d9f6Sml tx_ring_p->tdc)); 43030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4317127d9f6Sml "TX Descriptor ring is full: channel %d", 43230ac2e7bSml tx_ring_p->tdc)); 4337127d9f6Sml if (is_lso) { 4347127d9f6Sml /* 4357127d9f6Sml * free the current mp and mp_chain if not FULL. 4367127d9f6Sml */ 4377127d9f6Sml tdc_stats->tx_no_desc++; 4387127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 4397127d9f6Sml "LSO packet: TX Descriptor ring is full: " 4407127d9f6Sml "channel %d", 4417127d9f6Sml tx_ring_p->tdc)); 4427127d9f6Sml goto nxge_start_fail_lso; 4437127d9f6Sml } else { 444952a2464SMichael Speer (void) cas32((uint32_t *)&tx_ring_p->queueing, 445952a2464SMichael Speer 0, 1); 4467127d9f6Sml tdc_stats->tx_no_desc++; 4476895688eSspeer 4487127d9f6Sml if (isLDOMservice(nxgep)) { 4497127d9f6Sml tx_ring_p->tx_ring_busy = B_FALSE; 4507127d9f6Sml if (tx_ring_p->tx_ring_offline) { 4517127d9f6Sml (void) atomic_swap_32( 4527127d9f6Sml &tx_ring_p->tx_ring_offline, 4537127d9f6Sml NXGE_TX_RING_OFFLINED); 4547127d9f6Sml } 4556895688eSspeer } 45622c0d73aSspeer 4577127d9f6Sml MUTEX_EXIT(&tx_ring_p->lock); 4587127d9f6Sml status = 1; 4597127d9f6Sml goto nxge_start_fail1; 46030ac2e7bSml } 46144961713Sgirish } 46244961713Sgirish } 46344961713Sgirish 46444961713Sgirish nmp = mp; 46544961713Sgirish i = sop_index = tx_ring_p->wr_index; 46644961713Sgirish nmblks = 0; 46744961713Sgirish ngathers = 0; 46844961713Sgirish pkt_len = 0; 46944961713Sgirish pack_len = 0; 47044961713Sgirish clen = 0; 47144961713Sgirish last_bidx = -1; 47244961713Sgirish good_packet = B_TRUE; 47344961713Sgirish 47444961713Sgirish desc_area = tx_ring_p->tdc_desc; 47544961713Sgirish npi_handle = desc_area.npi_handle; 47644961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 47752ccf843Smisaki DMA_COMMON_ACC_HANDLE(desc_area); 47844961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 47944961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 48052ccf843Smisaki DMA_COMMON_HANDLE(desc_area); 48144961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 48244961713Sgirish 48344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 48452ccf843Smisaki sop_index, i)); 48544961713Sgirish 48644961713Sgirish #ifdef NXGE_DEBUG 48744961713Sgirish msgsize = msgdsize(nmp); 48844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 48952ccf843Smisaki "==> nxge_start(1): wr_index %d i %d msgdsize %d", 49052ccf843Smisaki sop_index, i, msgsize)); 49144961713Sgirish #endif 49244961713Sgirish /* 49344961713Sgirish * The first 16 bytes of the premapped buffer are reserved 49444961713Sgirish * for header. No padding will be used. 49544961713Sgirish */ 49644961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 4971f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 49844961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 49944961713Sgirish } else { 50044961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 50144961713Sgirish } 50244961713Sgirish while (nmp) { 50344961713Sgirish good_packet = B_TRUE; 50444961713Sgirish b_rptr = nmp->b_rptr; 50544961713Sgirish len = MBLKL(nmp); 50644961713Sgirish if (len <= 0) { 50744961713Sgirish nmp = nmp->b_cont; 50844961713Sgirish continue; 50944961713Sgirish } 51044961713Sgirish nmblks++; 51144961713Sgirish 51244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 51352ccf843Smisaki "len %d pkt_len %d pack_len %d", 51452ccf843Smisaki nmblks, len, pkt_len, pack_len)); 51544961713Sgirish /* 51614ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 51714ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 51814ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 51914ea4bb7Ssd * If len is longer than the limit, then we break nmp into 52014ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 52114ea4bb7Ssd * the second chunk for the remaining data. If the second 52214ea4bb7Ssd * chunk is still larger than the limit, then it will be 52314ea4bb7Ssd * broken into two in the next pass. 52444961713Sgirish */ 52514ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 52653f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 52753f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 52853f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 52953f3d8ecSyc - TX_PKT_HEADER_SIZE); 53053f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 53153f3d8ecSyc t_mp->b_cont = nmp->b_cont; 53253f3d8ecSyc nmp->b_cont = t_mp; 53353f3d8ecSyc len = MBLKL(nmp); 53453f3d8ecSyc } else { 53530ac2e7bSml if (is_lso) { 53630ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 53730ac2e7bSml "LSO packet: dupb failed: " 53830ac2e7bSml "channel %d", 53930ac2e7bSml tx_ring_p->tdc)); 54030ac2e7bSml mp = nmp; 54130ac2e7bSml goto nxge_start_fail_lso; 54230ac2e7bSml } else { 54330ac2e7bSml good_packet = B_FALSE; 54430ac2e7bSml goto nxge_start_fail2; 54530ac2e7bSml } 54653f3d8ecSyc } 54744961713Sgirish } 54844961713Sgirish tx_desc.value = 0; 54944961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 55044961713Sgirish #ifdef NXGE_DEBUG 55144961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 55244961713Sgirish #endif 55344961713Sgirish tx_msg_p = &tx_msg_ring[i]; 554adfcba55Sjoycey #if defined(__i386) 555adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 556adfcba55Sjoycey #else 55744961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 558adfcba55Sjoycey #endif 55944961713Sgirish if (!header_set && 56052ccf843Smisaki ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 56152ccf843Smisaki (len >= bcopy_thresh))) { 56244961713Sgirish header_set = B_TRUE; 56344961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 56444961713Sgirish boff = 0; 56544961713Sgirish pack_len = 0; 56644961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 56744961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 56844961713Sgirish clen = pkt_len; 56944961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 57044961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 57144961713Sgirish (void) ddi_dma_sync(dma_handle, 57252ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 57352ccf843Smisaki DDI_DMA_SYNC_FORDEV); 57444961713Sgirish 57544961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 57644961713Sgirish goto nxge_start_control_header_only; 57744961713Sgirish } 57844961713Sgirish 57944961713Sgirish pkt_len += len; 58044961713Sgirish pack_len += len; 58144961713Sgirish 58244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 58352ccf843Smisaki "desc entry %d " 58452ccf843Smisaki "DESC IOADDR $%p " 58552ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 58652ccf843Smisaki "desc_pp $%p tx_desc_pp $%p " 58752ccf843Smisaki "len %d pkt_len %d pack_len %d", 58852ccf843Smisaki i, 58952ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 59052ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 59152ccf843Smisaki tx_desc_ring_pp, tx_desc_pp, 59252ccf843Smisaki len, pkt_len, pack_len)); 59344961713Sgirish 59444961713Sgirish if (len < bcopy_thresh) { 59544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 59652ccf843Smisaki "USE BCOPY: ")); 59744961713Sgirish if (nxge_tx_tiny_pack) { 59844961713Sgirish uint32_t blst = 59952ccf843Smisaki TXDMA_DESC_NEXT_INDEX(i, -1, 60052ccf843Smisaki tx_ring_p->tx_wrap_mask); 60144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 60252ccf843Smisaki "==> nxge_start(5): pack")); 60344961713Sgirish if ((pack_len <= bcopy_thresh) && 60452ccf843Smisaki (last_bidx == blst)) { 60544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 60652ccf843Smisaki "==> nxge_start: pack(6) " 60752ccf843Smisaki "(pkt_len %d pack_len %d)", 60852ccf843Smisaki pkt_len, pack_len)); 60944961713Sgirish i = blst; 61044961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 61144961713Sgirish #ifdef NXGE_DEBUG 61244961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 61344961713Sgirish #endif 61444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 61544961713Sgirish boff = pack_len - len; 61644961713Sgirish ngathers--; 617a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 61852ccf843Smisaki header_set) { 61944961713Sgirish pack_len = len; 62044961713Sgirish boff = 0; 62144961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 62244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 62352ccf843Smisaki "==> nxge_start(7): > max NEW " 62452ccf843Smisaki "bcopy thresh %d " 62552ccf843Smisaki "pkt_len %d pack_len %d(next)", 62652ccf843Smisaki bcopy_thresh, 62752ccf843Smisaki pkt_len, pack_len)); 62844961713Sgirish } 62944961713Sgirish last_bidx = i; 63044961713Sgirish } 63144961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 63244961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 63344961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 63444961713Sgirish header_set = B_TRUE; 63544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 63652ccf843Smisaki "==> nxge_start(7_x2): " 63752ccf843Smisaki "pkt_len %d pack_len %d (new hdrp $%p)", 63852ccf843Smisaki pkt_len, pack_len, hdrp)); 63944961713Sgirish } 64044961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 64144961713Sgirish kaddr += boff; 64244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 64352ccf843Smisaki "USE BCOPY: before bcopy " 64452ccf843Smisaki "DESC IOADDR $%p entry %d " 64552ccf843Smisaki "bcopy packets %d " 64652ccf843Smisaki "bcopy kaddr $%p " 64752ccf843Smisaki "bcopy ioaddr (SAD) $%p " 64852ccf843Smisaki "bcopy clen %d " 64952ccf843Smisaki "bcopy boff %d", 65052ccf843Smisaki DMA_COMMON_IOADDR(desc_area), i, 65152ccf843Smisaki tdc_stats->tx_hdr_pkts, 65252ccf843Smisaki kaddr, 65352ccf843Smisaki dma_ioaddr, 65452ccf843Smisaki clen, 65552ccf843Smisaki boff)); 65644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 65752ccf843Smisaki "1USE BCOPY: ")); 65844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 65952ccf843Smisaki "2USE BCOPY: ")); 66044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 66152ccf843Smisaki "last USE BCOPY: copy from b_rptr $%p " 66252ccf843Smisaki "to KADDR $%p (len %d offset %d", 66352ccf843Smisaki b_rptr, kaddr, len, boff)); 66444961713Sgirish 66544961713Sgirish bcopy(b_rptr, kaddr, len); 66644961713Sgirish 66744961713Sgirish #ifdef NXGE_DEBUG 66844961713Sgirish dump_len = (len > 128) ? 128: len; 66944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 67052ccf843Smisaki "==> nxge_start: dump packets " 67152ccf843Smisaki "(After BCOPY len %d)" 67252ccf843Smisaki "(b_rptr $%p): %s", len, nmp->b_rptr, 67352ccf843Smisaki nxge_dump_packet((char *)nmp->b_rptr, 67452ccf843Smisaki dump_len))); 67544961713Sgirish #endif 67644961713Sgirish 67744961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 67844961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 67944961713Sgirish (void) ddi_dma_sync(dma_handle, 68052ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 68152ccf843Smisaki DDI_DMA_SYNC_FORDEV); 68244961713Sgirish clen = len + boff; 68344961713Sgirish tdc_stats->tx_hdr_pkts++; 68444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 68552ccf843Smisaki "USE BCOPY: " 68652ccf843Smisaki "DESC IOADDR $%p entry %d " 68752ccf843Smisaki "bcopy packets %d " 68852ccf843Smisaki "bcopy kaddr $%p " 68952ccf843Smisaki "bcopy ioaddr (SAD) $%p " 69052ccf843Smisaki "bcopy clen %d " 69152ccf843Smisaki "bcopy boff %d", 69252ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 69352ccf843Smisaki i, 69452ccf843Smisaki tdc_stats->tx_hdr_pkts, 69552ccf843Smisaki kaddr, 69652ccf843Smisaki dma_ioaddr, 69752ccf843Smisaki clen, 69852ccf843Smisaki boff)); 69944961713Sgirish } else { 70044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 70152ccf843Smisaki "USE DVMA: len %d", len)); 70244961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 70344961713Sgirish dma_flags = DDI_DMA_WRITE; 70444961713Sgirish if (len < nxge_dma_stream_thresh) { 70544961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 70644961713Sgirish } else { 70744961713Sgirish dma_flags |= DDI_DMA_STREAMING; 70844961713Sgirish } 70944961713Sgirish 71044961713Sgirish dma_handle = tx_msg_p->dma_handle; 711*31f519f9SMichael Speer dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL, 71252ccf843Smisaki (caddr_t)b_rptr, len, dma_flags, 71352ccf843Smisaki DDI_DMA_DONTWAIT, NULL, 71452ccf843Smisaki &dma_cookie, &ncookies); 715*31f519f9SMichael Speer if (dma_status == DDI_DMA_MAPPED) { 71644961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 71744961713Sgirish len = (int)dma_cookie.dmac_size; 71844961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 71944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 72052ccf843Smisaki "==> nxge_start(12_1): " 72152ccf843Smisaki "USE DVMA: len %d clen %d " 72252ccf843Smisaki "ngathers %d", 72352ccf843Smisaki len, clen, 72452ccf843Smisaki ngathers)); 725adfcba55Sjoycey #if defined(__i386) 726adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 727adfcba55Sjoycey #else 72844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 729adfcba55Sjoycey #endif 73044961713Sgirish while (ncookies > 1) { 73144961713Sgirish ngathers++; 73244961713Sgirish /* 73344961713Sgirish * this is the fix for multiple 73430ac2e7bSml * cookies, which are basically 73544961713Sgirish * a descriptor entry, we don't set 73644961713Sgirish * SOP bit as well as related fields 73744961713Sgirish */ 73844961713Sgirish 73944961713Sgirish (void) npi_txdma_desc_gather_set( 74052ccf843Smisaki npi_desc_handle, 74152ccf843Smisaki &tx_desc, 74252ccf843Smisaki (ngathers -1), 74352ccf843Smisaki mark_mode, 74452ccf843Smisaki ngathers, 74552ccf843Smisaki dma_ioaddr, 74652ccf843Smisaki clen); 74744961713Sgirish 74844961713Sgirish tx_msg_p->tx_msg_size = clen; 74944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 75052ccf843Smisaki "==> nxge_start: DMA " 75152ccf843Smisaki "ncookie %d " 75252ccf843Smisaki "ngathers %d " 75352ccf843Smisaki "dma_ioaddr $%p len %d" 75452ccf843Smisaki "desc $%p descp $%p (%d)", 75552ccf843Smisaki ncookies, 75652ccf843Smisaki ngathers, 75752ccf843Smisaki dma_ioaddr, clen, 75852ccf843Smisaki *tx_desc_p, tx_desc_p, i)); 75944961713Sgirish 76044961713Sgirish ddi_dma_nextcookie(dma_handle, 76152ccf843Smisaki &dma_cookie); 76244961713Sgirish dma_ioaddr = 76352ccf843Smisaki dma_cookie.dmac_laddress; 76444961713Sgirish 76544961713Sgirish len = (int)dma_cookie.dmac_size; 76644961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 76744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 76852ccf843Smisaki "==> nxge_start(12_2): " 76952ccf843Smisaki "USE DVMA: len %d clen %d ", 77052ccf843Smisaki len, clen)); 77144961713Sgirish 77244961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 77352ccf843Smisaki tx_ring_p->tx_wrap_mask); 77444961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 77544961713Sgirish 776adfcba55Sjoycey #if defined(__i386) 77752ccf843Smisaki npi_desc_handle.regp = 77852ccf843Smisaki (uint32_t)tx_desc_p; 779adfcba55Sjoycey #else 78052ccf843Smisaki npi_desc_handle.regp = 78152ccf843Smisaki (uint64_t)tx_desc_p; 782adfcba55Sjoycey #endif 78344961713Sgirish tx_msg_p = &tx_msg_ring[i]; 78444961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 78544961713Sgirish tx_desc.value = 0; 78644961713Sgirish 78744961713Sgirish ncookies--; 78844961713Sgirish } 78944961713Sgirish tdc_stats->tx_ddi_pkts++; 79044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 79152ccf843Smisaki "DMA: ddi packets %d", 79252ccf843Smisaki tdc_stats->tx_ddi_pkts)); 79344961713Sgirish } else { 79444961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 79544961713Sgirish "dma mapping failed for %d " 79644961713Sgirish "bytes addr $%p flags %x (%d)", 79744961713Sgirish len, b_rptr, status, status)); 79844961713Sgirish good_packet = B_FALSE; 79944961713Sgirish tdc_stats->tx_dma_bind_fail++; 80044961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 80130ac2e7bSml if (is_lso) { 80230ac2e7bSml mp = nmp; 80330ac2e7bSml goto nxge_start_fail_lso; 80430ac2e7bSml } else { 805*31f519f9SMichael Speer status = 1; 80630ac2e7bSml goto nxge_start_fail2; 80730ac2e7bSml } 80844961713Sgirish } 80944961713Sgirish } /* ddi dvma */ 81044961713Sgirish 81130ac2e7bSml if (is_lso) { 81230ac2e7bSml nmp_lso_save = nmp; 81330ac2e7bSml } 81444961713Sgirish nmp = nmp->b_cont; 81544961713Sgirish nxge_start_control_header_only: 816adfcba55Sjoycey #if defined(__i386) 817adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 818adfcba55Sjoycey #else 81944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 820adfcba55Sjoycey #endif 82144961713Sgirish ngathers++; 82244961713Sgirish 82344961713Sgirish if (ngathers == 1) { 82444961713Sgirish #ifdef NXGE_DEBUG 82544961713Sgirish save_desc_p = &sop_tx_desc; 82644961713Sgirish #endif 82744961713Sgirish sop_tx_desc_p = &sop_tx_desc; 82844961713Sgirish sop_tx_desc_p->value = 0; 82944961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 83044961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 83144961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 83244961713Sgirish } else { 83344961713Sgirish #ifdef NXGE_DEBUG 83444961713Sgirish save_desc_p = &tx_desc; 83544961713Sgirish #endif 83644961713Sgirish tmp_desc_p = &tx_desc; 83744961713Sgirish tmp_desc_p->value = 0; 83844961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 83944961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 84044961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 84144961713Sgirish 84244961713Sgirish tx_desc_p->value = tmp_desc_p->value; 84344961713Sgirish } 84444961713Sgirish 84544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 84652ccf843Smisaki "Desc_entry %d ngathers %d " 84752ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 84852ccf843Smisaki "len %d clen %d pkt_len %d pack_len %d nmblks %d " 84952ccf843Smisaki "dma_ioaddr (SAD) $%p mark %d", 85052ccf843Smisaki i, ngathers, 85152ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 85252ccf843Smisaki len, clen, pkt_len, pack_len, nmblks, 85352ccf843Smisaki dma_ioaddr, mark_mode)); 85444961713Sgirish 85544961713Sgirish #ifdef NXGE_DEBUG 85644961713Sgirish npi_desc_handle.nxgep = nxgep; 85744961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 85844961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 85944961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 86044961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 86152ccf843Smisaki TX_PKT_DESC_TR_LEN_SHIFT); 86244961713Sgirish 86344961713Sgirish 86444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 86552ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 86652ccf843Smisaki "mark %d sop %d\n", 86752ccf843Smisaki save_desc_p->value, 86852ccf843Smisaki sad, 86952ccf843Smisaki save_desc_p->bits.hdw.tr_len, 87052ccf843Smisaki xfer_len, 87152ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 87252ccf843Smisaki save_desc_p->bits.hdw.mark, 87352ccf843Smisaki save_desc_p->bits.hdw.sop)); 87444961713Sgirish 87544961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 87644961713Sgirish #endif 87744961713Sgirish 87844961713Sgirish tx_msg_p->tx_msg_size = clen; 87944961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 88044961713Sgirish if (ngathers > nxge_tx_max_gathers) { 88144961713Sgirish good_packet = B_FALSE; 88244961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 88352ccf843Smisaki &stuff_offset, &end_offset, &value, 88452ccf843Smisaki &cksum_flags); 88544961713Sgirish 88644961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 88752ccf843Smisaki "==> nxge_start(14): pull msg - " 88852ccf843Smisaki "len %d pkt_len %d ngathers %d", 88952ccf843Smisaki len, pkt_len, ngathers)); 890*31f519f9SMichael Speer 891*31f519f9SMichael Speer /* 892*31f519f9SMichael Speer * Just give up on this packet. 893*31f519f9SMichael Speer */ 89430ac2e7bSml if (is_lso) { 89530ac2e7bSml mp = nmp_lso_save; 89630ac2e7bSml goto nxge_start_fail_lso; 89730ac2e7bSml } 898*31f519f9SMichael Speer status = 0; 89944961713Sgirish goto nxge_start_fail2; 90044961713Sgirish } 90144961713Sgirish } /* while (nmp) */ 90244961713Sgirish 90344961713Sgirish tx_msg_p->tx_message = mp; 90444961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 905adfcba55Sjoycey #if defined(__i386) 906adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 907adfcba55Sjoycey #else 90844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 909adfcba55Sjoycey #endif 91044961713Sgirish 91144961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 91244961713Sgirish pkthdrp->reserved = 0; 91344961713Sgirish hdrp->value = 0; 914b4d05839Sml bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 91544961713Sgirish 91644961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 91744961713Sgirish tdc_stats->tx_jumbo_pkts++; 91844961713Sgirish } 91944961713Sgirish 920678453a8Sspeer min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 92144961713Sgirish if (pkt_len < min_len) { 92244961713Sgirish /* Assume we use bcopy to premapped buffers */ 92344961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 92444961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 92552ccf843Smisaki "==> nxge_start(14-1): < (msg_min + 16)" 92652ccf843Smisaki "len %d pkt_len %d min_len %d bzero %d ngathers %d", 92752ccf843Smisaki len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 92844961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 92944961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 93044961713Sgirish 93144961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 93244961713Sgirish 93344961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 93444961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 93544961713Sgirish 93644961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 93752ccf843Smisaki "==> nxge_start(14-2): < msg_min - " 93852ccf843Smisaki "len %d pkt_len %d min_len %d ngathers %d", 93952ccf843Smisaki len, pkt_len, min_len, ngathers)); 94044961713Sgirish } 94144961713Sgirish 94244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 94352ccf843Smisaki cksum_flags)); 94444961713Sgirish { 94544961713Sgirish uint64_t tmp_len; 94644961713Sgirish 94744961713Sgirish /* pkt_len already includes 16 + paddings!! */ 94844961713Sgirish /* Update the control header length */ 94944961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 95044961713Sgirish tmp_len = hdrp->value | 95152ccf843Smisaki (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 95244961713Sgirish 95344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 95452ccf843Smisaki "==> nxge_start(15_x1): setting SOP " 95552ccf843Smisaki "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 95652ccf843Smisaki "0x%llx hdrp->value 0x%llx", 95752ccf843Smisaki tot_xfer_len, tot_xfer_len, pkt_len, 95852ccf843Smisaki tmp_len, hdrp->value)); 95944961713Sgirish #if defined(_BIG_ENDIAN) 96044961713Sgirish hdrp->value = ddi_swap64(tmp_len); 96144961713Sgirish #else 96244961713Sgirish hdrp->value = tmp_len; 96344961713Sgirish #endif 96444961713Sgirish NXGE_DEBUG_MSG((nxgep, 96552ccf843Smisaki TX_CTL, "==> nxge_start(15_x2): setting SOP " 96652ccf843Smisaki "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 96752ccf843Smisaki "tmp_len 0x%llx hdrp->value 0x%llx", 96852ccf843Smisaki tot_xfer_len, pkt_len, 96952ccf843Smisaki tmp_len, hdrp->value)); 97044961713Sgirish } 97144961713Sgirish 97244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 97352ccf843Smisaki "wr_index %d " 97452ccf843Smisaki "tot_xfer_len (%d) pkt_len %d npads %d", 97552ccf843Smisaki sop_index, 97652ccf843Smisaki tot_xfer_len, pkt_len, 97752ccf843Smisaki npads)); 97844961713Sgirish 97944961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 98044961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 98144961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 98244961713Sgirish 98344961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 98444961713Sgirish 98544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 98644961713Sgirish 98744961713Sgirish #ifdef NXGE_DEBUG 98844961713Sgirish npi_desc_handle.nxgep = nxgep; 98944961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 99044961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 99144961713Sgirish 99244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 99352ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 99452ccf843Smisaki save_desc_p->value, 99552ccf843Smisaki sad, 99652ccf843Smisaki save_desc_p->bits.hdw.tr_len, 99752ccf843Smisaki xfer_len, 99852ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 99952ccf843Smisaki save_desc_p->bits.hdw.mark, 100052ccf843Smisaki save_desc_p->bits.hdw.sop)); 100144961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 100244961713Sgirish 100344961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 100444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 100552ccf843Smisaki "==> nxge_start: dump packets(17) (after sop set, len " 100652ccf843Smisaki " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 100752ccf843Smisaki "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 100852ccf843Smisaki (char *)hdrp, 100952ccf843Smisaki nxge_dump_packet((char *)hdrp, dump_len))); 101044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 101152ccf843Smisaki "==> nxge_start(18): TX desc sync: sop_index %d", 101252ccf843Smisaki sop_index)); 101344961713Sgirish #endif 101444961713Sgirish 101544961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 101644961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 101752ccf843Smisaki sop_index * sizeof (tx_desc_t), 101852ccf843Smisaki ngathers * sizeof (tx_desc_t), 101952ccf843Smisaki DDI_DMA_SYNC_FORDEV); 102044961713Sgirish 102144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 102252ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 102352ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 102452ccf843Smisaki stuff_offset, start_offset, 102552ccf843Smisaki pkt_len, ngathers, sop_index)); 102644961713Sgirish } else { /* more than one descriptor and wrap around */ 102744961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 102844961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 102952ccf843Smisaki sop_index * sizeof (tx_desc_t), 103052ccf843Smisaki nsdescs * sizeof (tx_desc_t), 103152ccf843Smisaki DDI_DMA_SYNC_FORDEV); 103244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 103352ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 103452ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 103552ccf843Smisaki stuff_offset, start_offset, 103652ccf843Smisaki pkt_len, ngathers, sop_index)); 103744961713Sgirish 103844961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 103952ccf843Smisaki 0, 104052ccf843Smisaki (ngathers - nsdescs) * sizeof (tx_desc_t), 104152ccf843Smisaki DDI_DMA_SYNC_FORDEV); 104244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 104352ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 104452ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 104552ccf843Smisaki stuff_offset, start_offset, 104652ccf843Smisaki pkt_len, ngathers, sop_index)); 104744961713Sgirish } 104844961713Sgirish 104944961713Sgirish tail_index = tx_ring_p->wr_index; 105044961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 105144961713Sgirish 105244961713Sgirish tx_ring_p->wr_index = i; 105344961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 105444961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 105552ccf843Smisaki B_FALSE : B_TRUE); 105644961713Sgirish } 105744961713Sgirish 105844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 105952ccf843Smisaki "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 106052ccf843Smisaki tx_ring_p->tdc, 106152ccf843Smisaki tx_ring_p->wr_index, 106252ccf843Smisaki tx_ring_p->wr_index_wrap, 106352ccf843Smisaki ngathers, 106452ccf843Smisaki tx_ring_p->descs_pending)); 106544961713Sgirish 106630ac2e7bSml if (is_lso) { 106730ac2e7bSml lso_ngathers += ngathers; 106830ac2e7bSml if (mp_chain != NULL) { 106930ac2e7bSml mp = mp_chain; 107030ac2e7bSml mp_chain = mp_chain->b_next; 107130ac2e7bSml mp->b_next = NULL; 107230ac2e7bSml if (nxge_lso_kick_cnt == lso_ngathers) { 10737a6dff21Sml tx_ring_p->descs_pending += lso_ngathers; 107430ac2e7bSml { 107530ac2e7bSml tx_ring_kick_t kick; 107630ac2e7bSml 107730ac2e7bSml kick.value = 0; 107830ac2e7bSml kick.bits.ldw.wrap = 107930ac2e7bSml tx_ring_p->wr_index_wrap; 108030ac2e7bSml kick.bits.ldw.tail = 108130ac2e7bSml (uint16_t)tx_ring_p->wr_index; 108230ac2e7bSml 108330ac2e7bSml /* Kick the Transmit kick register */ 108430ac2e7bSml TXDMA_REG_WRITE64( 108530ac2e7bSml NXGE_DEV_NPI_HANDLE(nxgep), 108630ac2e7bSml TX_RING_KICK_REG, 108730ac2e7bSml (uint8_t)tx_ring_p->tdc, 108830ac2e7bSml kick.value); 108930ac2e7bSml tdc_stats->tx_starts++; 1090678453a8Sspeer 109130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 109230ac2e7bSml "==> nxge_start: more LSO: " 109330ac2e7bSml "LSO_CNT %d", 1094678453a8Sspeer lso_ngathers)); 109530ac2e7bSml } 109630ac2e7bSml lso_ngathers = 0; 109730ac2e7bSml ngathers = 0; 109830ac2e7bSml cur_index_lso = sop_index = tx_ring_p->wr_index; 109930ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 110030ac2e7bSml } 110130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 110230ac2e7bSml "==> nxge_start: lso again: " 110330ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 110430ac2e7bSml "wr_index %d sop_index %d", 110530ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 110630ac2e7bSml tx_ring_p->wr_index, sop_index)); 110730ac2e7bSml 110830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 110930ac2e7bSml "==> nxge_start: next : count %d", 1110678453a8Sspeer lso_ngathers)); 111130ac2e7bSml lso_again = B_TRUE; 111230ac2e7bSml goto start_again; 111330ac2e7bSml } 11147a6dff21Sml ngathers = lso_ngathers; 111530ac2e7bSml } 111630ac2e7bSml 111744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 111844961713Sgirish 111944961713Sgirish { 112044961713Sgirish tx_ring_kick_t kick; 112144961713Sgirish 112244961713Sgirish kick.value = 0; 112344961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 112444961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 112544961713Sgirish 112644961713Sgirish /* Kick start the Transmit kick register */ 112744961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 112852ccf843Smisaki TX_RING_KICK_REG, 112952ccf843Smisaki (uint8_t)tx_ring_p->tdc, 113052ccf843Smisaki kick.value); 113144961713Sgirish } 113244961713Sgirish 11337a6dff21Sml tx_ring_p->descs_pending += ngathers; 113444961713Sgirish tdc_stats->tx_starts++; 113544961713Sgirish 11366895688eSspeer if (isLDOMservice(nxgep)) { 11376895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 11386895688eSspeer if (tx_ring_p->tx_ring_offline) { 11396895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 11406895688eSspeer NXGE_TX_RING_OFFLINED); 11416895688eSspeer } 11426895688eSspeer } 1143678453a8Sspeer 114444961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 114544961713Sgirish 114644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 114744961713Sgirish return (status); 114844961713Sgirish 114930ac2e7bSml nxge_start_fail_lso: 115030ac2e7bSml status = 0; 115130ac2e7bSml good_packet = B_FALSE; 1152*31f519f9SMichael Speer if (mp != NULL) 115330ac2e7bSml freemsg(mp); 1154*31f519f9SMichael Speer if (mp_chain != NULL) 1155*31f519f9SMichael Speer freemsgchain(mp_chain); 1156*31f519f9SMichael Speer 115730ac2e7bSml if (!lso_again && !ngathers) { 11586895688eSspeer if (isLDOMservice(nxgep)) { 11596895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 11606895688eSspeer if (tx_ring_p->tx_ring_offline) { 11616895688eSspeer (void) atomic_swap_32( 11626895688eSspeer &tx_ring_p->tx_ring_offline, 11636895688eSspeer NXGE_TX_RING_OFFLINED); 11646895688eSspeer } 11656895688eSspeer } 11666895688eSspeer 116730ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 116830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 116930ac2e7bSml "==> nxge_start: lso exit (nothing changed)")); 117030ac2e7bSml goto nxge_start_fail1; 117130ac2e7bSml } 117230ac2e7bSml 117330ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 117430ac2e7bSml "==> nxge_start (channel %d): before lso " 117530ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 117630ac2e7bSml "wr_index %d sop_index %d lso_again %d", 117730ac2e7bSml tx_ring_p->tdc, 117830ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 117930ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 118030ac2e7bSml 118130ac2e7bSml if (lso_again) { 118230ac2e7bSml lso_ngathers += ngathers; 118330ac2e7bSml ngathers = lso_ngathers; 118430ac2e7bSml sop_index = cur_index_lso; 118530ac2e7bSml tx_ring_p->wr_index = sop_index; 118630ac2e7bSml tx_ring_p->wr_index_wrap = lso_tail_wrap; 118730ac2e7bSml } 118830ac2e7bSml 118930ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 119030ac2e7bSml "==> nxge_start (channel %d): after lso " 119130ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 119230ac2e7bSml "wr_index %d sop_index %d lso_again %d", 119330ac2e7bSml tx_ring_p->tdc, 119430ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 119530ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 119630ac2e7bSml 119744961713Sgirish nxge_start_fail2: 119844961713Sgirish if (good_packet == B_FALSE) { 119944961713Sgirish cur_index = sop_index; 120044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 120144961713Sgirish for (i = 0; i < ngathers; i++) { 120244961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 1203adfcba55Sjoycey #if defined(__i386) 1204adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 1205adfcba55Sjoycey #else 120644961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 1207adfcba55Sjoycey #endif 120844961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 120944961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 121044961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 121144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 121253f3d8ecSyc "tx_desc_p = %X index = %d", 121353f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 121453f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 121553f3d8ecSyc 0, -1); 121644961713Sgirish tx_msg_p->dvma_handle = NULL; 121744961713Sgirish if (tx_ring_p->dvma_wr_index == 121853f3d8ecSyc tx_ring_p->dvma_wrap_mask) 121944961713Sgirish tx_ring_p->dvma_wr_index = 0; 122044961713Sgirish else 122144961713Sgirish tx_ring_p->dvma_wr_index++; 122244961713Sgirish tx_ring_p->dvma_pending--; 122353f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 122444961713Sgirish if (ddi_dma_unbind_handle( 122553f3d8ecSyc tx_msg_p->dma_handle)) { 122644961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 122753f3d8ecSyc "ddi_dma_unbind_handle failed"); 122853f3d8ecSyc } 122944961713Sgirish } 123044961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 123144961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 123252ccf843Smisaki tx_ring_p->tx_wrap_mask); 123344961713Sgirish 123444961713Sgirish } 123544961713Sgirish } 123644961713Sgirish 12376895688eSspeer if (isLDOMservice(nxgep)) { 12386895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 12396895688eSspeer if (tx_ring_p->tx_ring_offline) { 12406895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 12416895688eSspeer NXGE_TX_RING_OFFLINED); 12426895688eSspeer } 12436895688eSspeer } 1244678453a8Sspeer 124544961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 124644961713Sgirish 124744961713Sgirish nxge_start_fail1: 124844961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 124944961713Sgirish 125044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 125122c0d73aSspeer return (status); 12521f8914d5Sml } 12531f8914d5Sml 125430ac2e7bSml /* Software LSO starts here */ 125530ac2e7bSml static void 125630ac2e7bSml nxge_hcksum_retrieve(mblk_t *mp, 125730ac2e7bSml uint32_t *start, uint32_t *stuff, uint32_t *end, 125830ac2e7bSml uint32_t *value, uint32_t *flags) 125930ac2e7bSml { 126030ac2e7bSml if (mp->b_datap->db_type == M_DATA) { 126130ac2e7bSml if (flags != NULL) { 126230ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 126330ac2e7bSml HCK_PARTIALCKSUM | HCK_FULLCKSUM | 126430ac2e7bSml HCK_FULLCKSUM_OK); 126530ac2e7bSml if ((*flags & (HCK_PARTIALCKSUM | 126630ac2e7bSml HCK_FULLCKSUM)) != 0) { 126730ac2e7bSml if (value != NULL) 126830ac2e7bSml *value = (uint32_t)DB_CKSUM16(mp); 126930ac2e7bSml if ((*flags & HCK_PARTIALCKSUM) != 0) { 127030ac2e7bSml if (start != NULL) 127130ac2e7bSml *start = 127230ac2e7bSml (uint32_t)DB_CKSUMSTART(mp); 127330ac2e7bSml if (stuff != NULL) 127430ac2e7bSml *stuff = 127530ac2e7bSml (uint32_t)DB_CKSUMSTUFF(mp); 127630ac2e7bSml if (end != NULL) 127730ac2e7bSml *end = 127830ac2e7bSml (uint32_t)DB_CKSUMEND(mp); 127930ac2e7bSml } 128030ac2e7bSml } 128130ac2e7bSml } 128230ac2e7bSml } 128330ac2e7bSml } 128430ac2e7bSml 128530ac2e7bSml static void 128630ac2e7bSml nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 128730ac2e7bSml { 128830ac2e7bSml ASSERT(DB_TYPE(mp) == M_DATA); 128930ac2e7bSml 129030ac2e7bSml *mss = 0; 129130ac2e7bSml if (flags != NULL) { 129230ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 129330ac2e7bSml if ((*flags != 0) && (mss != NULL)) { 129430ac2e7bSml *mss = (uint32_t)DB_LSOMSS(mp); 129530ac2e7bSml } 129630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 129730ac2e7bSml "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 129830ac2e7bSml *mss, *flags)); 129930ac2e7bSml } 130030ac2e7bSml 130130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 130230ac2e7bSml "<== nxge_lso_info_get: mss %d", *mss)); 130330ac2e7bSml } 130430ac2e7bSml 130530ac2e7bSml /* 130630ac2e7bSml * Do Soft LSO on the oversized packet. 130730ac2e7bSml * 130830ac2e7bSml * 1. Create a chain of message for headers. 130930ac2e7bSml * 2. Fill up header messages with proper information. 131030ac2e7bSml * 3. Copy Eithernet, IP, and TCP headers from the original message to 131130ac2e7bSml * each new message with necessary adjustments. 131230ac2e7bSml * * Unchange the ethernet header for DIX frames. (by default) 131330ac2e7bSml * * IP Total Length field is updated to MSS or less(only for the last one). 131430ac2e7bSml * * IP Identification value is incremented by one for each packet. 131530ac2e7bSml * * TCP sequence Number is recalculated according to the payload length. 131630ac2e7bSml * * Set FIN and/or PSH flags for the *last* packet if applied. 131730ac2e7bSml * * TCP partial Checksum 131830ac2e7bSml * 4. Update LSO information in the first message header. 131930ac2e7bSml * 5. Release the original message header. 132030ac2e7bSml */ 132130ac2e7bSml static mblk_t * 132230ac2e7bSml nxge_do_softlso(mblk_t *mp, uint32_t mss) 132330ac2e7bSml { 132430ac2e7bSml uint32_t hckflags; 132530ac2e7bSml int pktlen; 132630ac2e7bSml int hdrlen; 132730ac2e7bSml int segnum; 132830ac2e7bSml int i; 132930ac2e7bSml struct ether_vlan_header *evh; 133030ac2e7bSml int ehlen, iphlen, tcphlen; 133130ac2e7bSml struct ip *oiph, *niph; 133230ac2e7bSml struct tcphdr *otcph, *ntcph; 133330ac2e7bSml int available, len, left; 133430ac2e7bSml uint16_t ip_id; 133530ac2e7bSml uint32_t tcp_seq; 133630ac2e7bSml #ifdef __sparc 133730ac2e7bSml uint32_t tcp_seq_tmp; 133830ac2e7bSml #endif 133930ac2e7bSml mblk_t *datamp; 134030ac2e7bSml uchar_t *rptr; 134130ac2e7bSml mblk_t *nmp; 134230ac2e7bSml mblk_t *cmp; 134330ac2e7bSml mblk_t *mp_chain; 134430ac2e7bSml boolean_t do_cleanup = B_FALSE; 134530ac2e7bSml t_uscalar_t start_offset = 0; 134630ac2e7bSml t_uscalar_t stuff_offset = 0; 134730ac2e7bSml t_uscalar_t value = 0; 134830ac2e7bSml uint16_t l4_len; 134930ac2e7bSml ipaddr_t src, dst; 135030ac2e7bSml uint32_t cksum, sum, l4cksum; 135130ac2e7bSml 135230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 135330ac2e7bSml "==> nxge_do_softlso")); 135430ac2e7bSml /* 135530ac2e7bSml * check the length of LSO packet payload and calculate the number of 135630ac2e7bSml * segments to be generated. 135730ac2e7bSml */ 135830ac2e7bSml pktlen = msgsize(mp); 135930ac2e7bSml evh = (struct ether_vlan_header *)mp->b_rptr; 136030ac2e7bSml 136130ac2e7bSml /* VLAN? */ 136230ac2e7bSml if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 136330ac2e7bSml ehlen = sizeof (struct ether_vlan_header); 136430ac2e7bSml else 136530ac2e7bSml ehlen = sizeof (struct ether_header); 136630ac2e7bSml oiph = (struct ip *)(mp->b_rptr + ehlen); 136730ac2e7bSml iphlen = oiph->ip_hl * 4; 136830ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 136930ac2e7bSml tcphlen = otcph->th_off * 4; 137030ac2e7bSml 137130ac2e7bSml l4_len = pktlen - ehlen - iphlen; 137230ac2e7bSml 137330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 137430ac2e7bSml "==> nxge_do_softlso: mss %d oiph $%p " 137530ac2e7bSml "original ip_sum oiph->ip_sum 0x%x " 137630ac2e7bSml "original tcp_sum otcph->th_sum 0x%x " 137730ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d " 137830ac2e7bSml "l4_len %d (0x%x) ip_len - iphlen %d ", 137930ac2e7bSml mss, 138030ac2e7bSml oiph, 138130ac2e7bSml oiph->ip_sum, 138230ac2e7bSml otcph->th_sum, 138330ac2e7bSml ntohs(oiph->ip_len), pktlen, 138430ac2e7bSml ehlen, 138530ac2e7bSml l4_len, 138630ac2e7bSml l4_len, 138730ac2e7bSml ntohs(oiph->ip_len) - iphlen)); 138830ac2e7bSml 138930ac2e7bSml /* IPv4 + TCP */ 139030ac2e7bSml if (!(oiph->ip_v == IPV4_VERSION)) { 139130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 139230ac2e7bSml "<== nxge_do_softlso: not IPV4 " 139330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 139430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 139530ac2e7bSml tcphlen)); 139630ac2e7bSml freemsg(mp); 139730ac2e7bSml return (NULL); 139830ac2e7bSml } 139930ac2e7bSml 140030ac2e7bSml if (!(oiph->ip_p == IPPROTO_TCP)) { 140130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 140230ac2e7bSml "<== nxge_do_softlso: not TCP " 140330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 140430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 140530ac2e7bSml tcphlen)); 140630ac2e7bSml freemsg(mp); 140730ac2e7bSml return (NULL); 140830ac2e7bSml } 140930ac2e7bSml 141030ac2e7bSml if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 141130ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 141230ac2e7bSml "<== nxge_do_softlso: len not matched " 141330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 141430ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 141530ac2e7bSml tcphlen)); 141630ac2e7bSml freemsg(mp); 141730ac2e7bSml return (NULL); 141830ac2e7bSml } 141930ac2e7bSml 142030ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 142130ac2e7bSml tcphlen = otcph->th_off * 4; 142230ac2e7bSml 142330ac2e7bSml /* TCP flags can not include URG, RST, or SYN */ 142430ac2e7bSml VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 142530ac2e7bSml 142630ac2e7bSml hdrlen = ehlen + iphlen + tcphlen; 142730ac2e7bSml 142830ac2e7bSml VERIFY(MBLKL(mp) >= hdrlen); 142930ac2e7bSml 143030ac2e7bSml if (MBLKL(mp) > hdrlen) { 143130ac2e7bSml datamp = mp; 143230ac2e7bSml rptr = mp->b_rptr + hdrlen; 143330ac2e7bSml } else { /* = */ 143430ac2e7bSml datamp = mp->b_cont; 143530ac2e7bSml rptr = datamp->b_rptr; 143630ac2e7bSml } 143730ac2e7bSml 143830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 143930ac2e7bSml "nxge_do_softlso: otcph $%p pktlen: %d, " 144030ac2e7bSml "hdrlen %d ehlen %d iphlen %d tcphlen %d " 144130ac2e7bSml "mblkl(mp): %d, mblkl(datamp): %d", 144230ac2e7bSml otcph, 144330ac2e7bSml pktlen, hdrlen, ehlen, iphlen, tcphlen, 144430ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 144530ac2e7bSml 144630ac2e7bSml hckflags = 0; 144730ac2e7bSml nxge_hcksum_retrieve(mp, 144830ac2e7bSml &start_offset, &stuff_offset, &value, NULL, &hckflags); 144930ac2e7bSml 145030ac2e7bSml dst = oiph->ip_dst.s_addr; 145130ac2e7bSml src = oiph->ip_src.s_addr; 145230ac2e7bSml 145330ac2e7bSml cksum = (dst >> 16) + (dst & 0xFFFF) + 145430ac2e7bSml (src >> 16) + (src & 0xFFFF); 145530ac2e7bSml l4cksum = cksum + IP_TCP_CSUM_COMP; 145630ac2e7bSml 145730ac2e7bSml sum = l4_len + l4cksum; 145830ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 145930ac2e7bSml 146030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 146130ac2e7bSml "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 146230ac2e7bSml "hckflags 0x%x start_offset %d stuff_offset %d " 146330ac2e7bSml "value (original) 0x%x th_sum 0x%x " 146430ac2e7bSml "pktlen %d l4_len %d (0x%x) " 146530ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 146630ac2e7bSml dst, src, 146730ac2e7bSml (sum & 0xffff), (~sum & 0xffff), 146830ac2e7bSml hckflags, start_offset, stuff_offset, 146930ac2e7bSml value, otcph->th_sum, 147030ac2e7bSml pktlen, 147130ac2e7bSml l4_len, 147230ac2e7bSml l4_len, 147330ac2e7bSml ntohs(oiph->ip_len) - (int)MBLKL(mp), 147430ac2e7bSml (int)MBLKL(datamp), 147530ac2e7bSml nxge_dump_packet((char *)evh, 12))); 147630ac2e7bSml 147730ac2e7bSml /* 147830ac2e7bSml * Start to process. 147930ac2e7bSml */ 148030ac2e7bSml available = pktlen - hdrlen; 148130ac2e7bSml segnum = (available - 1) / mss + 1; 148230ac2e7bSml 148330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 148430ac2e7bSml "==> nxge_do_softlso: pktlen %d " 148530ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 148630ac2e7bSml "available %d mss %d segnum %d", 148730ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 148830ac2e7bSml available, 148930ac2e7bSml mss, 149030ac2e7bSml segnum)); 149130ac2e7bSml 149230ac2e7bSml VERIFY(segnum >= 2); 149330ac2e7bSml 149430ac2e7bSml /* 149530ac2e7bSml * Try to pre-allocate all header messages 149630ac2e7bSml */ 149730ac2e7bSml mp_chain = NULL; 149830ac2e7bSml for (i = 0; i < segnum; i++) { 149930ac2e7bSml if ((nmp = allocb(hdrlen, 0)) == NULL) { 150030ac2e7bSml /* Clean up the mp_chain */ 150130ac2e7bSml while (mp_chain != NULL) { 150230ac2e7bSml nmp = mp_chain; 150330ac2e7bSml mp_chain = mp_chain->b_next; 150430ac2e7bSml freemsg(nmp); 150530ac2e7bSml } 150630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 150730ac2e7bSml "<== nxge_do_softlso: " 150830ac2e7bSml "Could not allocate enough messages for headers!")); 150930ac2e7bSml freemsg(mp); 151030ac2e7bSml return (NULL); 151130ac2e7bSml } 151230ac2e7bSml nmp->b_next = mp_chain; 151330ac2e7bSml mp_chain = nmp; 151430ac2e7bSml 151530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 151630ac2e7bSml "==> nxge_do_softlso: " 151730ac2e7bSml "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 151830ac2e7bSml mp, nmp, mp_chain, mp_chain->b_next)); 151930ac2e7bSml } 152030ac2e7bSml 152130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 152230ac2e7bSml "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 152330ac2e7bSml mp, nmp, mp_chain)); 152430ac2e7bSml 152530ac2e7bSml /* 152630ac2e7bSml * Associate payload with new packets 152730ac2e7bSml */ 152830ac2e7bSml cmp = mp_chain; 152930ac2e7bSml left = available; 153030ac2e7bSml while (cmp != NULL) { 153130ac2e7bSml nmp = dupb(datamp); 153230ac2e7bSml if (nmp == NULL) { 153330ac2e7bSml do_cleanup = B_TRUE; 153430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 153530ac2e7bSml "==>nxge_do_softlso: " 153630ac2e7bSml "Can not dupb(datamp), have to do clean up")); 153730ac2e7bSml goto cleanup_allocated_msgs; 153830ac2e7bSml } 153930ac2e7bSml 154030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 154130ac2e7bSml "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 154230ac2e7bSml "dupb nmp $%p len %d left %d msd %d ", 154330ac2e7bSml mp, cmp, nmp, len, left, mss)); 154430ac2e7bSml 154530ac2e7bSml cmp->b_cont = nmp; 154630ac2e7bSml nmp->b_rptr = rptr; 154730ac2e7bSml len = (left < mss) ? left : mss; 154830ac2e7bSml left -= len; 154930ac2e7bSml 155030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 155130ac2e7bSml "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 155230ac2e7bSml "dupb nmp $%p len %d left %d mss %d ", 155330ac2e7bSml mp, cmp, nmp, len, left, mss)); 155430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 155530ac2e7bSml "nxge_do_softlso: before available: %d, " 155630ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 155730ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 155830ac2e7bSml 155930ac2e7bSml len -= MBLKL(nmp); 156030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 156130ac2e7bSml "nxge_do_softlso: after available: %d, " 156230ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 156330ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 156430ac2e7bSml 156530ac2e7bSml while (len > 0) { 156630ac2e7bSml mblk_t *mmp = NULL; 156730ac2e7bSml 156830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 156930ac2e7bSml "nxge_do_softlso: (4) len > 0 available: %d, " 157030ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 157130ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 157230ac2e7bSml 157330ac2e7bSml if (datamp->b_cont != NULL) { 157430ac2e7bSml datamp = datamp->b_cont; 157530ac2e7bSml rptr = datamp->b_rptr; 157630ac2e7bSml mmp = dupb(datamp); 157730ac2e7bSml if (mmp == NULL) { 157830ac2e7bSml do_cleanup = B_TRUE; 157930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 158030ac2e7bSml "==> nxge_do_softlso: " 1581678453a8Sspeer "Can not dupb(datamp) (1), :" 158230ac2e7bSml "have to do clean up")); 158330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 158430ac2e7bSml "==> nxge_do_softlso: " 158530ac2e7bSml "available: %d, left: %d, " 158630ac2e7bSml "len: %d, MBLKL(nmp): %d", 158730ac2e7bSml available, left, len, 158830ac2e7bSml (int)MBLKL(nmp))); 158930ac2e7bSml goto cleanup_allocated_msgs; 159030ac2e7bSml } 159130ac2e7bSml } else { 159230ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 159330ac2e7bSml "==> nxge_do_softlso: " 159430ac2e7bSml "(1)available: %d, left: %d, " 159530ac2e7bSml "len: %d, MBLKL(nmp): %d", 159630ac2e7bSml available, left, len, 159730ac2e7bSml (int)MBLKL(nmp))); 159830ac2e7bSml cmn_err(CE_PANIC, 159930ac2e7bSml "==> nxge_do_softlso: " 160030ac2e7bSml "Pointers must have been corrupted!\n" 160130ac2e7bSml "datamp: $%p, nmp: $%p, rptr: $%p", 160230ac2e7bSml (void *)datamp, 160330ac2e7bSml (void *)nmp, 160430ac2e7bSml (void *)rptr); 160530ac2e7bSml } 160630ac2e7bSml nmp->b_cont = mmp; 160730ac2e7bSml nmp = mmp; 160830ac2e7bSml len -= MBLKL(nmp); 160930ac2e7bSml } 161030ac2e7bSml if (len < 0) { 161130ac2e7bSml nmp->b_wptr += len; 161230ac2e7bSml rptr = nmp->b_wptr; 161330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 161430ac2e7bSml "(5) len < 0 (less than 0)" 161530ac2e7bSml "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 161630ac2e7bSml available, left, len, (int)MBLKL(nmp))); 161730ac2e7bSml 161830ac2e7bSml } else if (len == 0) { 161930ac2e7bSml if (datamp->b_cont != NULL) { 162030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 162130ac2e7bSml "(5) len == 0" 162230ac2e7bSml "available: %d, left: %d, len: %d, " 162330ac2e7bSml "MBLKL(nmp): %d", 162430ac2e7bSml available, left, len, (int)MBLKL(nmp))); 162530ac2e7bSml datamp = datamp->b_cont; 162630ac2e7bSml rptr = datamp->b_rptr; 162730ac2e7bSml } else { 162830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 162930ac2e7bSml "(6)available b_cont == NULL : %d, " 163030ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 163130ac2e7bSml available, left, len, (int)MBLKL(nmp))); 163230ac2e7bSml 163330ac2e7bSml VERIFY(cmp->b_next == NULL); 163430ac2e7bSml VERIFY(left == 0); 163530ac2e7bSml break; /* Done! */ 163630ac2e7bSml } 163730ac2e7bSml } 163830ac2e7bSml cmp = cmp->b_next; 163930ac2e7bSml 164030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 164130ac2e7bSml "(7) do_softlso: " 164230ac2e7bSml "next mp in mp_chain available len != 0 : %d, " 164330ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 164430ac2e7bSml available, left, len, (int)MBLKL(nmp))); 164530ac2e7bSml } 164630ac2e7bSml 164730ac2e7bSml /* 164830ac2e7bSml * From now, start to fill up all headers for the first message 164930ac2e7bSml * Hardware checksum flags need to be updated separately for FULLCKSUM 165030ac2e7bSml * and PARTIALCKSUM cases. For full checksum, copy the original flags 165130ac2e7bSml * into every new packet is enough. But for HCK_PARTIALCKSUM, all 165230ac2e7bSml * required fields need to be updated properly. 165330ac2e7bSml */ 165430ac2e7bSml nmp = mp_chain; 165530ac2e7bSml bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 165630ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 165730ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 165830ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 165930ac2e7bSml ip_id = ntohs(niph->ip_id); 166030ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 166130ac2e7bSml #ifdef __sparc 166230ac2e7bSml bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 166330ac2e7bSml tcp_seq = ntohl(tcp_seq_tmp); 166430ac2e7bSml #else 166530ac2e7bSml tcp_seq = ntohl(ntcph->th_seq); 166630ac2e7bSml #endif 166730ac2e7bSml 166830ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 166930ac2e7bSml 167030ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 167130ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 167230ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 167330ac2e7bSml 167430ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 167530ac2e7bSml niph->ip_sum = 0; 167630ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 167730ac2e7bSml 167830ac2e7bSml l4_len = mss + tcphlen; 167930ac2e7bSml sum = htons(l4_len) + l4cksum; 168030ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 168130ac2e7bSml ntcph->th_sum = (sum & 0xffff); 168230ac2e7bSml 168330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168430ac2e7bSml "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 168530ac2e7bSml "mss %d pktlen %d l4_len %d (0x%x) " 168630ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 168730ac2e7bSml "ip_sum 0x%x " 168830ac2e7bSml "th_sum 0x%x sum 0x%x ) " 168930ac2e7bSml "dump first ip->tcp %s", 169030ac2e7bSml nmp, mp_chain, 169130ac2e7bSml mss, 169230ac2e7bSml pktlen, 169330ac2e7bSml l4_len, 169430ac2e7bSml l4_len, 169530ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp), 169630ac2e7bSml niph->ip_sum, 169730ac2e7bSml ntcph->th_sum, 169830ac2e7bSml sum, 169930ac2e7bSml nxge_dump_packet((char *)niph, 52))); 170030ac2e7bSml 170130ac2e7bSml cmp = nmp; 170230ac2e7bSml while ((nmp = nmp->b_next)->b_next != NULL) { 170330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 170430ac2e7bSml "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 170530ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 170630ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 170730ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 170830ac2e7bSml niph->ip_id = htons(++ip_id); 170930ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 171030ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 171130ac2e7bSml tcp_seq += mss; 171230ac2e7bSml 171330ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 171430ac2e7bSml 171530ac2e7bSml #ifdef __sparc 171630ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 171730ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 171830ac2e7bSml #else 171930ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 172030ac2e7bSml #endif 172130ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 172230ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 172330ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 172430ac2e7bSml 172530ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 172630ac2e7bSml niph->ip_sum = 0; 172730ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 172830ac2e7bSml ntcph->th_sum = (sum & 0xffff); 172930ac2e7bSml 173030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 173130ac2e7bSml "==> nxge_do_softlso: middle ip_sum 0x%x " 173230ac2e7bSml "th_sum 0x%x " 173330ac2e7bSml " mp $%p (mp_chain $%p) pktlen %d " 173430ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 173530ac2e7bSml niph->ip_sum, 173630ac2e7bSml ntcph->th_sum, 173730ac2e7bSml nmp, mp_chain, 173830ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 173930ac2e7bSml } 174030ac2e7bSml 174130ac2e7bSml /* Last segment */ 174230ac2e7bSml /* 174330ac2e7bSml * Set FIN and/or PSH flags if present only in the last packet. 174430ac2e7bSml * The ip_len could be different from prior packets. 174530ac2e7bSml */ 174630ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 174730ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 174830ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 174930ac2e7bSml niph->ip_id = htons(++ip_id); 175030ac2e7bSml niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 175130ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 175230ac2e7bSml tcp_seq += mss; 175330ac2e7bSml #ifdef __sparc 175430ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 175530ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 175630ac2e7bSml #else 175730ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 175830ac2e7bSml #endif 175930ac2e7bSml ntcph->th_flags = (otcph->th_flags & ~TH_URG); 176030ac2e7bSml 176130ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 176230ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 176330ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 176430ac2e7bSml 176530ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 176630ac2e7bSml niph->ip_sum = 0; 176730ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 176830ac2e7bSml 176930ac2e7bSml l4_len = ntohs(niph->ip_len) - iphlen; 177030ac2e7bSml sum = htons(l4_len) + l4cksum; 177130ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 177230ac2e7bSml ntcph->th_sum = (sum & 0xffff); 177330ac2e7bSml 177430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 177530ac2e7bSml "==> nxge_do_softlso: last next " 177630ac2e7bSml "niph->ip_sum 0x%x " 177730ac2e7bSml "ntcph->th_sum 0x%x sum 0x%x " 177830ac2e7bSml "dump last ip->tcp %s " 177930ac2e7bSml "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 178030ac2e7bSml "l4_len %d (0x%x) " 178130ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 178230ac2e7bSml niph->ip_sum, 178330ac2e7bSml ntcph->th_sum, sum, 178430ac2e7bSml nxge_dump_packet((char *)niph, 52), 178530ac2e7bSml cmp, nmp, mp_chain, 178630ac2e7bSml pktlen, pktlen, 178730ac2e7bSml l4_len, 178830ac2e7bSml l4_len, 178930ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 179030ac2e7bSml 179130ac2e7bSml cleanup_allocated_msgs: 179230ac2e7bSml if (do_cleanup) { 179330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 179430ac2e7bSml "==> nxge_do_softlso: " 179530ac2e7bSml "Failed allocating messages, " 179630ac2e7bSml "have to clean up and fail!")); 179730ac2e7bSml while (mp_chain != NULL) { 179830ac2e7bSml nmp = mp_chain; 179930ac2e7bSml mp_chain = mp_chain->b_next; 180030ac2e7bSml freemsg(nmp); 180130ac2e7bSml } 180230ac2e7bSml } 180330ac2e7bSml /* 180430ac2e7bSml * We're done here, so just free the original message and return the 180530ac2e7bSml * new message chain, that could be NULL if failed, back to the caller. 180630ac2e7bSml */ 180730ac2e7bSml freemsg(mp); 180830ac2e7bSml 180930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 181030ac2e7bSml "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 181130ac2e7bSml return (mp_chain); 181230ac2e7bSml } 181330ac2e7bSml 181430ac2e7bSml /* 181530ac2e7bSml * Will be called before NIC driver do further operation on the message. 181630ac2e7bSml * The input message may include LSO information, if so, go to softlso logic 181730ac2e7bSml * to eliminate the oversized LSO packet for the incapable underlying h/w. 181830ac2e7bSml * The return could be the same non-LSO message or a message chain for LSO case. 181930ac2e7bSml * 182030ac2e7bSml * The driver needs to call this function per packet and process the whole chain 182130ac2e7bSml * if applied. 182230ac2e7bSml */ 182330ac2e7bSml static mblk_t * 182430ac2e7bSml nxge_lso_eliminate(mblk_t *mp) 182530ac2e7bSml { 182630ac2e7bSml uint32_t lsoflags; 182730ac2e7bSml uint32_t mss; 182830ac2e7bSml 182930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 183030ac2e7bSml "==>nxge_lso_eliminate:")); 183130ac2e7bSml nxge_lso_info_get(mp, &mss, &lsoflags); 183230ac2e7bSml 183330ac2e7bSml if (lsoflags & HW_LSO) { 183430ac2e7bSml mblk_t *nmp; 183530ac2e7bSml 183630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 183730ac2e7bSml "==>nxge_lso_eliminate:" 183830ac2e7bSml "HW_LSO:mss %d mp $%p", 183930ac2e7bSml mss, mp)); 184030ac2e7bSml if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 184130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 184230ac2e7bSml "<== nxge_lso_eliminate: " 184330ac2e7bSml "LSO: nmp not NULL nmp $%p mss %d mp $%p", 184430ac2e7bSml nmp, mss, mp)); 184530ac2e7bSml return (nmp); 184630ac2e7bSml } else { 184730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 184830ac2e7bSml "<== nxge_lso_eliminate_ " 184930ac2e7bSml "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 185030ac2e7bSml nmp, mss, mp)); 185130ac2e7bSml return (NULL); 185230ac2e7bSml } 185330ac2e7bSml } 185430ac2e7bSml 185530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 185630ac2e7bSml "<== nxge_lso_eliminate")); 185730ac2e7bSml return (mp); 185830ac2e7bSml } 185930ac2e7bSml 186030ac2e7bSml static uint32_t 186130ac2e7bSml nxge_csgen(uint16_t *adr, int len) 186230ac2e7bSml { 186330ac2e7bSml int i, odd; 186430ac2e7bSml uint32_t sum = 0; 186530ac2e7bSml uint32_t c = 0; 186630ac2e7bSml 186730ac2e7bSml odd = len % 2; 186830ac2e7bSml for (i = 0; i < (len / 2); i++) { 186930ac2e7bSml sum += (adr[i] & 0xffff); 187030ac2e7bSml } 187130ac2e7bSml if (odd) { 187230ac2e7bSml sum += adr[len / 2] & 0xff00; 187330ac2e7bSml } 187430ac2e7bSml while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 187530ac2e7bSml sum &= 0xffff; 187630ac2e7bSml sum += c; 187730ac2e7bSml } 187830ac2e7bSml return (~sum & 0xffff); 187930ac2e7bSml } 1880