144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 2230ac2e7bSml * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 2644961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2744961713Sgirish 28a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 29678453a8Sspeer #include <sys/nxge/nxge_hio.h> 30678453a8Sspeer #include <npi_tx_wr64.h> 3144961713Sgirish 3230ac2e7bSml /* Software LSO required header files */ 3330ac2e7bSml #include <netinet/tcp.h> 3430ac2e7bSml #include <inet/ip_impl.h> 3530ac2e7bSml #include <inet/tcp.h> 3630ac2e7bSml 3730ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *); 3830ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 3930ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 4030ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *, 4130ac2e7bSml uint32_t *, uint32_t *, uint32_t *, 4230ac2e7bSml uint32_t *, uint32_t *); 4330ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int); 4430ac2e7bSml 4544961713Sgirish extern uint32_t nxge_reclaim_pending; 4644961713Sgirish extern uint32_t nxge_bcopy_thresh; 4744961713Sgirish extern uint32_t nxge_dvma_thresh; 4844961713Sgirish extern uint32_t nxge_dma_stream_thresh; 4944961713Sgirish extern uint32_t nxge_tx_minfree; 5044961713Sgirish extern uint32_t nxge_tx_intr_thres; 5144961713Sgirish extern uint32_t nxge_tx_max_gathers; 5244961713Sgirish extern uint32_t nxge_tx_tiny_pack; 5344961713Sgirish extern uint32_t nxge_tx_use_bcopy; 5444961713Sgirish extern uint32_t nxge_tx_lb_policy; 5544961713Sgirish extern uint32_t nxge_no_tx_lb; 561f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 5730ac2e7bSml uint32_t nxge_lso_kick_cnt = 2; 5844961713Sgirish 5944961713Sgirish typedef struct _mac_tx_hint { 6044961713Sgirish uint16_t sap; 6144961713Sgirish uint16_t vid; 6244961713Sgirish void *hash; 6344961713Sgirish } mac_tx_hint_t, *p_mac_tx_hint_t; 6444961713Sgirish 6544961713Sgirish int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 6644961713Sgirish 6744961713Sgirish int 6844961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 6944961713Sgirish { 7044961713Sgirish int status = 0; 7144961713Sgirish p_tx_desc_t tx_desc_ring_vp; 7244961713Sgirish npi_handle_t npi_desc_handle; 7344961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 7444961713Sgirish p_tx_desc_t tx_desc_p; 7544961713Sgirish p_tx_msg_t tx_msg_ring; 7644961713Sgirish p_tx_msg_t tx_msg_p; 7744961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 7844961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 7944961713Sgirish p_tx_pkt_header_t hdrp; 80b4d05839Sml tx_pkt_header_t tmp_hdrp; 8144961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 8244961713Sgirish uint8_t npads = 0; 8344961713Sgirish uint64_t dma_ioaddr; 8444961713Sgirish uint32_t dma_flags; 8544961713Sgirish int last_bidx; 8644961713Sgirish uint8_t *b_rptr; 8744961713Sgirish caddr_t kaddr; 8844961713Sgirish uint32_t nmblks; 8944961713Sgirish uint32_t ngathers; 9044961713Sgirish uint32_t clen; 9144961713Sgirish int len; 9244961713Sgirish uint32_t pkt_len, pack_len, min_len; 9344961713Sgirish uint32_t bcopy_thresh; 9444961713Sgirish int i, cur_index, sop_index; 9544961713Sgirish uint16_t tail_index; 9644961713Sgirish boolean_t tail_wrap = B_FALSE; 9744961713Sgirish nxge_dma_common_t desc_area; 9844961713Sgirish nxge_os_dma_handle_t dma_handle; 9944961713Sgirish ddi_dma_cookie_t dma_cookie; 10044961713Sgirish npi_handle_t npi_handle; 10144961713Sgirish p_mblk_t nmp; 10244961713Sgirish p_mblk_t t_mp; 10344961713Sgirish uint32_t ncookies; 10444961713Sgirish boolean_t good_packet; 10544961713Sgirish boolean_t mark_mode = B_FALSE; 10644961713Sgirish p_nxge_stats_t statsp; 10744961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 10844961713Sgirish t_uscalar_t start_offset = 0; 10944961713Sgirish t_uscalar_t stuff_offset = 0; 11044961713Sgirish t_uscalar_t end_offset = 0; 11144961713Sgirish t_uscalar_t value = 0; 11244961713Sgirish t_uscalar_t cksum_flags = 0; 11344961713Sgirish boolean_t cksum_on = B_FALSE; 11444961713Sgirish uint32_t boff = 0; 115b4d05839Sml uint64_t tot_xfer_len = 0; 11644961713Sgirish boolean_t header_set = B_FALSE; 11744961713Sgirish #ifdef NXGE_DEBUG 11844961713Sgirish p_tx_desc_t tx_desc_ring_pp; 11944961713Sgirish p_tx_desc_t tx_desc_pp; 12044961713Sgirish tx_desc_t *save_desc_p; 12144961713Sgirish int dump_len; 12244961713Sgirish int sad_len; 12344961713Sgirish uint64_t sad; 12444961713Sgirish int xfer_len; 12544961713Sgirish uint32_t msgsize; 12644961713Sgirish #endif 12730ac2e7bSml p_mblk_t mp_chain = NULL; 12830ac2e7bSml boolean_t is_lso = B_FALSE; 12930ac2e7bSml boolean_t lso_again; 13030ac2e7bSml int cur_index_lso; 13130ac2e7bSml p_mblk_t nmp_lso_save; 13230ac2e7bSml uint32_t lso_ngathers; 13330ac2e7bSml boolean_t lso_tail_wrap = B_FALSE; 13444961713Sgirish 13544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 136678453a8Sspeer "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 13744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 138678453a8Sspeer "==> nxge_start: Starting tdc %d desc pending %d", 139678453a8Sspeer tx_ring_p->tdc, tx_ring_p->descs_pending)); 14044961713Sgirish 14144961713Sgirish statsp = nxgep->statsp; 14244961713Sgirish 143678453a8Sspeer if (!isLDOMguest(nxgep)) { 144678453a8Sspeer switch (nxgep->mac.portmode) { 145678453a8Sspeer default: 146678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode == 147678453a8Sspeer nxge_lb_normal) { 148678453a8Sspeer if (!statsp->mac_stats.link_up) { 149678453a8Sspeer freemsg(mp); 150678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 151678453a8Sspeer "==> nxge_start: " 152678453a8Sspeer "link not up")); 153678453a8Sspeer goto nxge_start_fail1; 154678453a8Sspeer } 155321febdeSsbehera } 156678453a8Sspeer break; 157678453a8Sspeer case PORT_10G_FIBER: 158678453a8Sspeer /* 159678453a8Sspeer * For the following modes, check the link status 160678453a8Sspeer * before sending the packet out: 161678453a8Sspeer * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 162678453a8Sspeer */ 163678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode < 164678453a8Sspeer nxge_lb_serdes10g) { 165678453a8Sspeer if (!statsp->mac_stats.link_up) { 166678453a8Sspeer freemsg(mp); 167678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 168678453a8Sspeer "==> nxge_start: " 169678453a8Sspeer "link not up")); 170678453a8Sspeer goto nxge_start_fail1; 171678453a8Sspeer } 172321febdeSsbehera } 173678453a8Sspeer break; 17444961713Sgirish } 175678453a8Sspeer } 176678453a8Sspeer 177678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 178678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 179678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 180678453a8Sspeer "==> nxge_start: hardware not initialized or stopped")); 181678453a8Sspeer freemsg(mp); 182678453a8Sspeer goto nxge_start_fail1; 18344961713Sgirish } 18444961713Sgirish 1853d16f8e7Sml if (nxgep->soft_lso_enable) { 18630ac2e7bSml mp_chain = nxge_lso_eliminate(mp); 18730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 18830ac2e7bSml "==> nxge_start(0): LSO mp $%p mp_chain $%p", 18930ac2e7bSml mp, mp_chain)); 19030ac2e7bSml if (mp_chain == NULL) { 19130ac2e7bSml NXGE_ERROR_MSG((nxgep, TX_CTL, 19230ac2e7bSml "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 19330ac2e7bSml mp_chain, mp)); 19430ac2e7bSml goto nxge_start_fail1; 19530ac2e7bSml } 19630ac2e7bSml if (mp_chain != mp) { 19730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 19830ac2e7bSml "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 19930ac2e7bSml mp_chain, mp)); 20030ac2e7bSml is_lso = B_TRUE; 20130ac2e7bSml mp = mp_chain; 20230ac2e7bSml mp_chain = mp_chain->b_next; 20330ac2e7bSml mp->b_next = NULL; 20430ac2e7bSml } 20530ac2e7bSml } 20630ac2e7bSml 20744961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 20844961713Sgirish &stuff_offset, &end_offset, &value, &cksum_flags); 20944961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 21044961713Sgirish start_offset += sizeof (ether_header_t); 21144961713Sgirish stuff_offset += sizeof (ether_header_t); 21244961713Sgirish } else { 21344961713Sgirish start_offset += sizeof (struct ether_vlan_header); 21444961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 21544961713Sgirish } 21644961713Sgirish 21744961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 21844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 21930ac2e7bSml "==> nxge_start: mp $%p len %d " 22030ac2e7bSml "cksum_flags 0x%x (partial checksum) ", 22130ac2e7bSml mp, MBLKL(mp), cksum_flags)); 22244961713Sgirish cksum_on = B_TRUE; 22344961713Sgirish } 22444961713Sgirish 225b4d05839Sml pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 226b4d05839Sml pkthdrp->reserved = 0; 227b4d05839Sml tmp_hdrp.value = 0; 228b4d05839Sml nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 229b4d05839Sml 0, 0, pkthdrp, 230b4d05839Sml start_offset, stuff_offset); 231b4d05839Sml 23230ac2e7bSml lso_again = B_FALSE; 23330ac2e7bSml lso_ngathers = 0; 23430ac2e7bSml 23530ac2e7bSml MUTEX_ENTER(&tx_ring_p->lock); 23622c0d73aSspeer 23722c0d73aSspeer if (isLDOMservice(nxgep)) { 238*6895688eSspeer tx_ring_p->tx_ring_busy = B_TRUE; 23922c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 24022c0d73aSspeer freemsg(mp); 241*6895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 242*6895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 243*6895688eSspeer NXGE_TX_RING_OFFLINED); 24422c0d73aSspeer MUTEX_EXIT(&tx_ring_p->lock); 24522c0d73aSspeer return (status); 24622c0d73aSspeer } 24722c0d73aSspeer } 24822c0d73aSspeer 24930ac2e7bSml cur_index_lso = tx_ring_p->wr_index; 25030ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 25130ac2e7bSml start_again: 25230ac2e7bSml ngathers = 0; 25330ac2e7bSml sop_index = tx_ring_p->wr_index; 25444961713Sgirish #ifdef NXGE_DEBUG 25544961713Sgirish if (tx_ring_p->descs_pending) { 25644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 25744961713Sgirish "desc pending %d ", tx_ring_p->descs_pending)); 25844961713Sgirish } 25944961713Sgirish 26044961713Sgirish dump_len = (int)(MBLKL(mp)); 26144961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 26244961713Sgirish 26344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 26444961713Sgirish "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 26544961713Sgirish "(Before header reserve: ORIGINAL LEN %d)", 26644961713Sgirish tx_ring_p->tdc, 26744961713Sgirish mp->b_rptr, 26844961713Sgirish dump_len)); 26944961713Sgirish 27044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 27144961713Sgirish "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 27244961713Sgirish nxge_dump_packet((char *)mp->b_rptr, dump_len))); 27344961713Sgirish #endif 27444961713Sgirish 27544961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 27644961713Sgirish mark_mode = (tx_ring_p->descs_pending && 27744961713Sgirish ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 27844961713Sgirish < nxge_tx_minfree)); 27944961713Sgirish 28044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 28144961713Sgirish "TX Descriptor ring is channel %d mark mode %d", 28244961713Sgirish tx_ring_p->tdc, mark_mode)); 28344961713Sgirish 28444961713Sgirish if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 28544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 28644961713Sgirish "TX Descriptor ring is full: channel %d", 28744961713Sgirish tx_ring_p->tdc)); 28830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 28930ac2e7bSml "TX Descriptor ring is full: channel %d", 29030ac2e7bSml tx_ring_p->tdc)); 29130ac2e7bSml if (is_lso) { 29230ac2e7bSml /* free the current mp and mp_chain if not FULL */ 29330ac2e7bSml tdc_stats->tx_no_desc++; 29430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 29530ac2e7bSml "LSO packet: TX Descriptor ring is full: " 29630ac2e7bSml "channel %d", 29730ac2e7bSml tx_ring_p->tdc)); 29830ac2e7bSml goto nxge_start_fail_lso; 29930ac2e7bSml } else { 30022c0d73aSspeer boolean_t skip_sched = B_FALSE; 30122c0d73aSspeer 30230ac2e7bSml cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 30330ac2e7bSml tdc_stats->tx_no_desc++; 304*6895688eSspeer 305*6895688eSspeer if (isLDOMservice(nxgep)) { 306*6895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 307*6895688eSspeer if (tx_ring_p->tx_ring_offline) { 308*6895688eSspeer (void) atomic_swap_32( 309*6895688eSspeer &tx_ring_p->tx_ring_offline, 310*6895688eSspeer NXGE_TX_RING_OFFLINED); 311*6895688eSspeer skip_sched = B_TRUE; 312*6895688eSspeer } 31322c0d73aSspeer } 31422c0d73aSspeer 31530ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 31622c0d73aSspeer if (nxgep->resched_needed && 31722c0d73aSspeer !nxgep->resched_running && !skip_sched) { 31830ac2e7bSml nxgep->resched_running = B_TRUE; 31930ac2e7bSml ddi_trigger_softintr(nxgep->resched_id); 32030ac2e7bSml } 32130ac2e7bSml status = 1; 32230ac2e7bSml goto nxge_start_fail1; 32344961713Sgirish } 32444961713Sgirish } 32544961713Sgirish 32644961713Sgirish nmp = mp; 32744961713Sgirish i = sop_index = tx_ring_p->wr_index; 32844961713Sgirish nmblks = 0; 32944961713Sgirish ngathers = 0; 33044961713Sgirish pkt_len = 0; 33144961713Sgirish pack_len = 0; 33244961713Sgirish clen = 0; 33344961713Sgirish last_bidx = -1; 33444961713Sgirish good_packet = B_TRUE; 33544961713Sgirish 33644961713Sgirish desc_area = tx_ring_p->tdc_desc; 33744961713Sgirish npi_handle = desc_area.npi_handle; 33844961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 33944961713Sgirish DMA_COMMON_ACC_HANDLE(desc_area); 34044961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 34144961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 34244961713Sgirish DMA_COMMON_HANDLE(desc_area); 34344961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 34444961713Sgirish 34544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 34644961713Sgirish sop_index, i)); 34744961713Sgirish 34844961713Sgirish #ifdef NXGE_DEBUG 34944961713Sgirish msgsize = msgdsize(nmp); 35044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 35144961713Sgirish "==> nxge_start(1): wr_index %d i %d msgdsize %d", 35244961713Sgirish sop_index, i, msgsize)); 35344961713Sgirish #endif 35444961713Sgirish /* 35544961713Sgirish * The first 16 bytes of the premapped buffer are reserved 35644961713Sgirish * for header. No padding will be used. 35744961713Sgirish */ 35844961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 3591f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 36044961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 36144961713Sgirish } else { 36244961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 36344961713Sgirish } 36444961713Sgirish while (nmp) { 36544961713Sgirish good_packet = B_TRUE; 36644961713Sgirish b_rptr = nmp->b_rptr; 36744961713Sgirish len = MBLKL(nmp); 36844961713Sgirish if (len <= 0) { 36944961713Sgirish nmp = nmp->b_cont; 37044961713Sgirish continue; 37144961713Sgirish } 37244961713Sgirish nmblks++; 37344961713Sgirish 37444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 37544961713Sgirish "len %d pkt_len %d pack_len %d", 37644961713Sgirish nmblks, len, pkt_len, pack_len)); 37744961713Sgirish /* 37814ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 37914ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 38014ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 38114ea4bb7Ssd * If len is longer than the limit, then we break nmp into 38214ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 38314ea4bb7Ssd * the second chunk for the remaining data. If the second 38414ea4bb7Ssd * chunk is still larger than the limit, then it will be 38514ea4bb7Ssd * broken into two in the next pass. 38644961713Sgirish */ 38714ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 38853f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 38953f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 39053f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 39153f3d8ecSyc - TX_PKT_HEADER_SIZE); 39253f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 39353f3d8ecSyc t_mp->b_cont = nmp->b_cont; 39453f3d8ecSyc nmp->b_cont = t_mp; 39553f3d8ecSyc len = MBLKL(nmp); 39653f3d8ecSyc } else { 39730ac2e7bSml if (is_lso) { 39830ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 39930ac2e7bSml "LSO packet: dupb failed: " 40030ac2e7bSml "channel %d", 40130ac2e7bSml tx_ring_p->tdc)); 40230ac2e7bSml mp = nmp; 40330ac2e7bSml goto nxge_start_fail_lso; 40430ac2e7bSml } else { 40530ac2e7bSml good_packet = B_FALSE; 40630ac2e7bSml goto nxge_start_fail2; 40730ac2e7bSml } 40853f3d8ecSyc } 40944961713Sgirish } 41044961713Sgirish tx_desc.value = 0; 41144961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 41244961713Sgirish #ifdef NXGE_DEBUG 41344961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 41444961713Sgirish #endif 41544961713Sgirish tx_msg_p = &tx_msg_ring[i]; 416adfcba55Sjoycey #if defined(__i386) 417adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 418adfcba55Sjoycey #else 41944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 420adfcba55Sjoycey #endif 42144961713Sgirish if (!header_set && 42244961713Sgirish ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 42344961713Sgirish (len >= bcopy_thresh))) { 42444961713Sgirish header_set = B_TRUE; 42544961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 42644961713Sgirish boff = 0; 42744961713Sgirish pack_len = 0; 42844961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 42944961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 43044961713Sgirish clen = pkt_len; 43144961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 43244961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 43344961713Sgirish (void) ddi_dma_sync(dma_handle, 43444961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 43544961713Sgirish DDI_DMA_SYNC_FORDEV); 43644961713Sgirish 43744961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 43844961713Sgirish goto nxge_start_control_header_only; 43944961713Sgirish } 44044961713Sgirish 44144961713Sgirish pkt_len += len; 44244961713Sgirish pack_len += len; 44344961713Sgirish 44444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 44544961713Sgirish "desc entry %d " 44644961713Sgirish "DESC IOADDR $%p " 44744961713Sgirish "desc_vp $%p tx_desc_p $%p " 44844961713Sgirish "desc_pp $%p tx_desc_pp $%p " 44944961713Sgirish "len %d pkt_len %d pack_len %d", 45044961713Sgirish i, 45144961713Sgirish DMA_COMMON_IOADDR(desc_area), 45244961713Sgirish tx_desc_ring_vp, tx_desc_p, 45344961713Sgirish tx_desc_ring_pp, tx_desc_pp, 45444961713Sgirish len, pkt_len, pack_len)); 45544961713Sgirish 45644961713Sgirish if (len < bcopy_thresh) { 45744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 45844961713Sgirish "USE BCOPY: ")); 45944961713Sgirish if (nxge_tx_tiny_pack) { 46044961713Sgirish uint32_t blst = 46144961713Sgirish TXDMA_DESC_NEXT_INDEX(i, -1, 46244961713Sgirish tx_ring_p->tx_wrap_mask); 46344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 46444961713Sgirish "==> nxge_start(5): pack")); 46544961713Sgirish if ((pack_len <= bcopy_thresh) && 46644961713Sgirish (last_bidx == blst)) { 46744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 46844961713Sgirish "==> nxge_start: pack(6) " 46944961713Sgirish "(pkt_len %d pack_len %d)", 47044961713Sgirish pkt_len, pack_len)); 47144961713Sgirish i = blst; 47244961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 47344961713Sgirish #ifdef NXGE_DEBUG 47444961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 47544961713Sgirish #endif 47644961713Sgirish tx_msg_p = &tx_msg_ring[i]; 47744961713Sgirish boff = pack_len - len; 47844961713Sgirish ngathers--; 479a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 480a3c5bd6dSspeer header_set) { 48144961713Sgirish pack_len = len; 48244961713Sgirish boff = 0; 48344961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 48444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 48544961713Sgirish "==> nxge_start(7): > max NEW " 48644961713Sgirish "bcopy thresh %d " 48744961713Sgirish "pkt_len %d pack_len %d(next)", 48844961713Sgirish bcopy_thresh, 48944961713Sgirish pkt_len, pack_len)); 49044961713Sgirish } 49144961713Sgirish last_bidx = i; 49244961713Sgirish } 49344961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 49444961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 49544961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 49644961713Sgirish header_set = B_TRUE; 49744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 49844961713Sgirish "==> nxge_start(7_x2): " 49944961713Sgirish "pkt_len %d pack_len %d (new hdrp $%p)", 50044961713Sgirish pkt_len, pack_len, hdrp)); 50144961713Sgirish } 50244961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 50344961713Sgirish kaddr += boff; 50444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 50544961713Sgirish "USE BCOPY: before bcopy " 50644961713Sgirish "DESC IOADDR $%p entry %d " 50744961713Sgirish "bcopy packets %d " 50844961713Sgirish "bcopy kaddr $%p " 50944961713Sgirish "bcopy ioaddr (SAD) $%p " 51044961713Sgirish "bcopy clen %d " 51144961713Sgirish "bcopy boff %d", 51244961713Sgirish DMA_COMMON_IOADDR(desc_area), i, 51344961713Sgirish tdc_stats->tx_hdr_pkts, 51444961713Sgirish kaddr, 51544961713Sgirish dma_ioaddr, 51644961713Sgirish clen, 51744961713Sgirish boff)); 51844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 51944961713Sgirish "1USE BCOPY: ")); 52044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 52144961713Sgirish "2USE BCOPY: ")); 52244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 52344961713Sgirish "last USE BCOPY: copy from b_rptr $%p " 52444961713Sgirish "to KADDR $%p (len %d offset %d", 52544961713Sgirish b_rptr, kaddr, len, boff)); 52644961713Sgirish 52744961713Sgirish bcopy(b_rptr, kaddr, len); 52844961713Sgirish 52944961713Sgirish #ifdef NXGE_DEBUG 53044961713Sgirish dump_len = (len > 128) ? 128: len; 53144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 53244961713Sgirish "==> nxge_start: dump packets " 53344961713Sgirish "(After BCOPY len %d)" 53444961713Sgirish "(b_rptr $%p): %s", len, nmp->b_rptr, 53544961713Sgirish nxge_dump_packet((char *)nmp->b_rptr, 53644961713Sgirish dump_len))); 53744961713Sgirish #endif 53844961713Sgirish 53944961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 54044961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 54144961713Sgirish (void) ddi_dma_sync(dma_handle, 54244961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 54344961713Sgirish DDI_DMA_SYNC_FORDEV); 54444961713Sgirish clen = len + boff; 54544961713Sgirish tdc_stats->tx_hdr_pkts++; 54644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 54744961713Sgirish "USE BCOPY: " 54844961713Sgirish "DESC IOADDR $%p entry %d " 54944961713Sgirish "bcopy packets %d " 55044961713Sgirish "bcopy kaddr $%p " 55144961713Sgirish "bcopy ioaddr (SAD) $%p " 55244961713Sgirish "bcopy clen %d " 55344961713Sgirish "bcopy boff %d", 55444961713Sgirish DMA_COMMON_IOADDR(desc_area), 55544961713Sgirish i, 55644961713Sgirish tdc_stats->tx_hdr_pkts, 55744961713Sgirish kaddr, 55844961713Sgirish dma_ioaddr, 55944961713Sgirish clen, 56044961713Sgirish boff)); 56144961713Sgirish } else { 56244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 56344961713Sgirish "USE DVMA: len %d", len)); 56444961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 56544961713Sgirish dma_flags = DDI_DMA_WRITE; 56644961713Sgirish if (len < nxge_dma_stream_thresh) { 56744961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 56844961713Sgirish } else { 56944961713Sgirish dma_flags |= DDI_DMA_STREAMING; 57044961713Sgirish } 57144961713Sgirish 57244961713Sgirish dma_handle = tx_msg_p->dma_handle; 57344961713Sgirish status = ddi_dma_addr_bind_handle(dma_handle, NULL, 57444961713Sgirish (caddr_t)b_rptr, len, dma_flags, 57544961713Sgirish DDI_DMA_DONTWAIT, NULL, 57644961713Sgirish &dma_cookie, &ncookies); 57744961713Sgirish if (status == DDI_DMA_MAPPED) { 57844961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 57944961713Sgirish len = (int)dma_cookie.dmac_size; 58044961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 58144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 58244961713Sgirish "==> nxge_start(12_1): " 58344961713Sgirish "USE DVMA: len %d clen %d " 58444961713Sgirish "ngathers %d", 58544961713Sgirish len, clen, 58644961713Sgirish ngathers)); 587adfcba55Sjoycey #if defined(__i386) 588adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 589adfcba55Sjoycey #else 59044961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 591adfcba55Sjoycey #endif 59244961713Sgirish while (ncookies > 1) { 59344961713Sgirish ngathers++; 59444961713Sgirish /* 59544961713Sgirish * this is the fix for multiple 59630ac2e7bSml * cookies, which are basically 59744961713Sgirish * a descriptor entry, we don't set 59844961713Sgirish * SOP bit as well as related fields 59944961713Sgirish */ 60044961713Sgirish 60144961713Sgirish (void) npi_txdma_desc_gather_set( 60244961713Sgirish npi_desc_handle, 60344961713Sgirish &tx_desc, 60444961713Sgirish (ngathers -1), 60544961713Sgirish mark_mode, 60644961713Sgirish ngathers, 60744961713Sgirish dma_ioaddr, 60844961713Sgirish clen); 60944961713Sgirish 61044961713Sgirish tx_msg_p->tx_msg_size = clen; 61144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 61244961713Sgirish "==> nxge_start: DMA " 61344961713Sgirish "ncookie %d " 61444961713Sgirish "ngathers %d " 61544961713Sgirish "dma_ioaddr $%p len %d" 61644961713Sgirish "desc $%p descp $%p (%d)", 61744961713Sgirish ncookies, 61844961713Sgirish ngathers, 61944961713Sgirish dma_ioaddr, clen, 62044961713Sgirish *tx_desc_p, tx_desc_p, i)); 62144961713Sgirish 62244961713Sgirish ddi_dma_nextcookie(dma_handle, 62344961713Sgirish &dma_cookie); 62444961713Sgirish dma_ioaddr = 62544961713Sgirish dma_cookie.dmac_laddress; 62644961713Sgirish 62744961713Sgirish len = (int)dma_cookie.dmac_size; 62844961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 62944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 63044961713Sgirish "==> nxge_start(12_2): " 63144961713Sgirish "USE DVMA: len %d clen %d ", 63244961713Sgirish len, clen)); 63344961713Sgirish 63444961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 63544961713Sgirish tx_ring_p->tx_wrap_mask); 63644961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 63744961713Sgirish 63844961713Sgirish npi_desc_handle.regp = 639adfcba55Sjoycey #if defined(__i386) 640adfcba55Sjoycey (uint32_t)tx_desc_p; 641adfcba55Sjoycey #else 64244961713Sgirish (uint64_t)tx_desc_p; 643adfcba55Sjoycey #endif 64444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 64544961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 64644961713Sgirish tx_desc.value = 0; 64744961713Sgirish 64844961713Sgirish ncookies--; 64944961713Sgirish } 65044961713Sgirish tdc_stats->tx_ddi_pkts++; 65144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 65244961713Sgirish "DMA: ddi packets %d", 65344961713Sgirish tdc_stats->tx_ddi_pkts)); 65444961713Sgirish } else { 65544961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 65644961713Sgirish "dma mapping failed for %d " 65744961713Sgirish "bytes addr $%p flags %x (%d)", 65844961713Sgirish len, b_rptr, status, status)); 65944961713Sgirish good_packet = B_FALSE; 66044961713Sgirish tdc_stats->tx_dma_bind_fail++; 66144961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 66230ac2e7bSml if (is_lso) { 66330ac2e7bSml mp = nmp; 66430ac2e7bSml goto nxge_start_fail_lso; 66530ac2e7bSml } else { 66630ac2e7bSml goto nxge_start_fail2; 66730ac2e7bSml } 66844961713Sgirish } 66944961713Sgirish } /* ddi dvma */ 67044961713Sgirish 67130ac2e7bSml if (is_lso) { 67230ac2e7bSml nmp_lso_save = nmp; 67330ac2e7bSml } 67444961713Sgirish nmp = nmp->b_cont; 67544961713Sgirish nxge_start_control_header_only: 676adfcba55Sjoycey #if defined(__i386) 677adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 678adfcba55Sjoycey #else 67944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 680adfcba55Sjoycey #endif 68144961713Sgirish ngathers++; 68244961713Sgirish 68344961713Sgirish if (ngathers == 1) { 68444961713Sgirish #ifdef NXGE_DEBUG 68544961713Sgirish save_desc_p = &sop_tx_desc; 68644961713Sgirish #endif 68744961713Sgirish sop_tx_desc_p = &sop_tx_desc; 68844961713Sgirish sop_tx_desc_p->value = 0; 68944961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 69044961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 69144961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 69244961713Sgirish } else { 69344961713Sgirish #ifdef NXGE_DEBUG 69444961713Sgirish save_desc_p = &tx_desc; 69544961713Sgirish #endif 69644961713Sgirish tmp_desc_p = &tx_desc; 69744961713Sgirish tmp_desc_p->value = 0; 69844961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 69944961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 70044961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 70144961713Sgirish 70244961713Sgirish tx_desc_p->value = tmp_desc_p->value; 70344961713Sgirish } 70444961713Sgirish 70544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 70644961713Sgirish "Desc_entry %d ngathers %d " 70744961713Sgirish "desc_vp $%p tx_desc_p $%p " 70844961713Sgirish "len %d clen %d pkt_len %d pack_len %d nmblks %d " 70944961713Sgirish "dma_ioaddr (SAD) $%p mark %d", 71044961713Sgirish i, ngathers, 71144961713Sgirish tx_desc_ring_vp, tx_desc_p, 71244961713Sgirish len, clen, pkt_len, pack_len, nmblks, 71344961713Sgirish dma_ioaddr, mark_mode)); 71444961713Sgirish 71544961713Sgirish #ifdef NXGE_DEBUG 71644961713Sgirish npi_desc_handle.nxgep = nxgep; 71744961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 71844961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 71944961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 72044961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 72144961713Sgirish TX_PKT_DESC_TR_LEN_SHIFT); 72244961713Sgirish 72344961713Sgirish 72444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 72544961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 72644961713Sgirish "mark %d sop %d\n", 72744961713Sgirish save_desc_p->value, 72844961713Sgirish sad, 72944961713Sgirish save_desc_p->bits.hdw.tr_len, 73044961713Sgirish xfer_len, 73144961713Sgirish save_desc_p->bits.hdw.num_ptr, 73244961713Sgirish save_desc_p->bits.hdw.mark, 73344961713Sgirish save_desc_p->bits.hdw.sop)); 73444961713Sgirish 73544961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 73644961713Sgirish #endif 73744961713Sgirish 73844961713Sgirish tx_msg_p->tx_msg_size = clen; 73944961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 74044961713Sgirish if (ngathers > nxge_tx_max_gathers) { 74144961713Sgirish good_packet = B_FALSE; 74244961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 74344961713Sgirish &stuff_offset, &end_offset, &value, 74444961713Sgirish &cksum_flags); 74544961713Sgirish 74644961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 74744961713Sgirish "==> nxge_start(14): pull msg - " 74844961713Sgirish "len %d pkt_len %d ngathers %d", 74944961713Sgirish len, pkt_len, ngathers)); 75044961713Sgirish /* Pull all message blocks from b_cont */ 75130ac2e7bSml if (is_lso) { 75230ac2e7bSml mp = nmp_lso_save; 75330ac2e7bSml goto nxge_start_fail_lso; 75430ac2e7bSml } 75544961713Sgirish if ((msgpullup(mp, -1)) == NULL) { 75644961713Sgirish goto nxge_start_fail2; 75744961713Sgirish } 75844961713Sgirish goto nxge_start_fail2; 75944961713Sgirish } 76044961713Sgirish } /* while (nmp) */ 76144961713Sgirish 76244961713Sgirish tx_msg_p->tx_message = mp; 76344961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 764adfcba55Sjoycey #if defined(__i386) 765adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 766adfcba55Sjoycey #else 76744961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 768adfcba55Sjoycey #endif 76944961713Sgirish 77044961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 77144961713Sgirish pkthdrp->reserved = 0; 77244961713Sgirish hdrp->value = 0; 773b4d05839Sml bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 77444961713Sgirish 77544961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 77644961713Sgirish tdc_stats->tx_jumbo_pkts++; 77744961713Sgirish } 77844961713Sgirish 779678453a8Sspeer min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 78044961713Sgirish if (pkt_len < min_len) { 78144961713Sgirish /* Assume we use bcopy to premapped buffers */ 78244961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 78344961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 78444961713Sgirish "==> nxge_start(14-1): < (msg_min + 16)" 78544961713Sgirish "len %d pkt_len %d min_len %d bzero %d ngathers %d", 78644961713Sgirish len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 78744961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 78844961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 78944961713Sgirish 79044961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 79144961713Sgirish 79244961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 79344961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 79444961713Sgirish 79544961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 79644961713Sgirish "==> nxge_start(14-2): < msg_min - " 79744961713Sgirish "len %d pkt_len %d min_len %d ngathers %d", 79844961713Sgirish len, pkt_len, min_len, ngathers)); 79944961713Sgirish } 80044961713Sgirish 80144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 80244961713Sgirish cksum_flags)); 80344961713Sgirish { 80444961713Sgirish uint64_t tmp_len; 80544961713Sgirish 80644961713Sgirish /* pkt_len already includes 16 + paddings!! */ 80744961713Sgirish /* Update the control header length */ 80844961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 80944961713Sgirish tmp_len = hdrp->value | 81044961713Sgirish (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 81144961713Sgirish 81244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 81344961713Sgirish "==> nxge_start(15_x1): setting SOP " 81444961713Sgirish "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 81544961713Sgirish "0x%llx hdrp->value 0x%llx", 81644961713Sgirish tot_xfer_len, tot_xfer_len, pkt_len, 81744961713Sgirish tmp_len, hdrp->value)); 81844961713Sgirish #if defined(_BIG_ENDIAN) 81944961713Sgirish hdrp->value = ddi_swap64(tmp_len); 82044961713Sgirish #else 82144961713Sgirish hdrp->value = tmp_len; 82244961713Sgirish #endif 82344961713Sgirish NXGE_DEBUG_MSG((nxgep, 82444961713Sgirish TX_CTL, "==> nxge_start(15_x2): setting SOP " 82544961713Sgirish "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 82644961713Sgirish "tmp_len 0x%llx hdrp->value 0x%llx", 82744961713Sgirish tot_xfer_len, pkt_len, 82844961713Sgirish tmp_len, hdrp->value)); 82944961713Sgirish } 83044961713Sgirish 83144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 83244961713Sgirish "wr_index %d " 83344961713Sgirish "tot_xfer_len (%d) pkt_len %d npads %d", 83444961713Sgirish sop_index, 83544961713Sgirish tot_xfer_len, pkt_len, 83644961713Sgirish npads)); 83744961713Sgirish 83844961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 83944961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 84044961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 84144961713Sgirish 84244961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 84344961713Sgirish 84444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 84544961713Sgirish 84644961713Sgirish #ifdef NXGE_DEBUG 84744961713Sgirish npi_desc_handle.nxgep = nxgep; 84844961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 84944961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 85044961713Sgirish 85144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 85244961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 85344961713Sgirish save_desc_p->value, 85444961713Sgirish sad, 85544961713Sgirish save_desc_p->bits.hdw.tr_len, 85644961713Sgirish xfer_len, 85744961713Sgirish save_desc_p->bits.hdw.num_ptr, 85844961713Sgirish save_desc_p->bits.hdw.mark, 85944961713Sgirish save_desc_p->bits.hdw.sop)); 86044961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 86144961713Sgirish 86244961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 86344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 86444961713Sgirish "==> nxge_start: dump packets(17) (after sop set, len " 86544961713Sgirish " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 86644961713Sgirish "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 86744961713Sgirish (char *)hdrp, 86844961713Sgirish nxge_dump_packet((char *)hdrp, dump_len))); 86944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 87044961713Sgirish "==> nxge_start(18): TX desc sync: sop_index %d", 87144961713Sgirish sop_index)); 87244961713Sgirish #endif 87344961713Sgirish 87444961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 87544961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 87644961713Sgirish sop_index * sizeof (tx_desc_t), 87744961713Sgirish ngathers * sizeof (tx_desc_t), 87844961713Sgirish DDI_DMA_SYNC_FORDEV); 87944961713Sgirish 88044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 88144961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 88244961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 88344961713Sgirish stuff_offset, start_offset, 88444961713Sgirish pkt_len, ngathers, sop_index)); 88544961713Sgirish } else { /* more than one descriptor and wrap around */ 88644961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 88744961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 88844961713Sgirish sop_index * sizeof (tx_desc_t), 88944961713Sgirish nsdescs * sizeof (tx_desc_t), 89044961713Sgirish DDI_DMA_SYNC_FORDEV); 89144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 89244961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 89344961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 89444961713Sgirish stuff_offset, start_offset, 89544961713Sgirish pkt_len, ngathers, sop_index)); 89644961713Sgirish 89744961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 89844961713Sgirish 0, 89944961713Sgirish (ngathers - nsdescs) * sizeof (tx_desc_t), 90044961713Sgirish DDI_DMA_SYNC_FORDEV); 90144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 90244961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 90344961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 90444961713Sgirish stuff_offset, start_offset, 90544961713Sgirish pkt_len, ngathers, sop_index)); 90644961713Sgirish } 90744961713Sgirish 90844961713Sgirish tail_index = tx_ring_p->wr_index; 90944961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 91044961713Sgirish 91144961713Sgirish tx_ring_p->wr_index = i; 91244961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 91344961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 91444961713Sgirish B_FALSE : B_TRUE); 91544961713Sgirish } 91644961713Sgirish 91744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 91844961713Sgirish "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 91944961713Sgirish tx_ring_p->tdc, 92044961713Sgirish tx_ring_p->wr_index, 92144961713Sgirish tx_ring_p->wr_index_wrap, 92244961713Sgirish ngathers, 92344961713Sgirish tx_ring_p->descs_pending)); 92444961713Sgirish 92530ac2e7bSml if (is_lso) { 92630ac2e7bSml lso_ngathers += ngathers; 92730ac2e7bSml if (mp_chain != NULL) { 92830ac2e7bSml mp = mp_chain; 92930ac2e7bSml mp_chain = mp_chain->b_next; 93030ac2e7bSml mp->b_next = NULL; 93130ac2e7bSml if (nxge_lso_kick_cnt == lso_ngathers) { 9327a6dff21Sml tx_ring_p->descs_pending += lso_ngathers; 93330ac2e7bSml { 93430ac2e7bSml tx_ring_kick_t kick; 93530ac2e7bSml 93630ac2e7bSml kick.value = 0; 93730ac2e7bSml kick.bits.ldw.wrap = 93830ac2e7bSml tx_ring_p->wr_index_wrap; 93930ac2e7bSml kick.bits.ldw.tail = 94030ac2e7bSml (uint16_t)tx_ring_p->wr_index; 94130ac2e7bSml 94230ac2e7bSml /* Kick the Transmit kick register */ 94330ac2e7bSml TXDMA_REG_WRITE64( 94430ac2e7bSml NXGE_DEV_NPI_HANDLE(nxgep), 94530ac2e7bSml TX_RING_KICK_REG, 94630ac2e7bSml (uint8_t)tx_ring_p->tdc, 94730ac2e7bSml kick.value); 94830ac2e7bSml tdc_stats->tx_starts++; 949678453a8Sspeer 95030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 95130ac2e7bSml "==> nxge_start: more LSO: " 95230ac2e7bSml "LSO_CNT %d", 953678453a8Sspeer lso_ngathers)); 95430ac2e7bSml } 95530ac2e7bSml lso_ngathers = 0; 95630ac2e7bSml ngathers = 0; 95730ac2e7bSml cur_index_lso = sop_index = tx_ring_p->wr_index; 95830ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 95930ac2e7bSml } 96030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 96130ac2e7bSml "==> nxge_start: lso again: " 96230ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 96330ac2e7bSml "wr_index %d sop_index %d", 96430ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 96530ac2e7bSml tx_ring_p->wr_index, sop_index)); 96630ac2e7bSml 96730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 96830ac2e7bSml "==> nxge_start: next : count %d", 969678453a8Sspeer lso_ngathers)); 97030ac2e7bSml lso_again = B_TRUE; 97130ac2e7bSml goto start_again; 97230ac2e7bSml } 9737a6dff21Sml ngathers = lso_ngathers; 97430ac2e7bSml } 97530ac2e7bSml 97644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 97744961713Sgirish 97844961713Sgirish { 97944961713Sgirish tx_ring_kick_t kick; 98044961713Sgirish 98144961713Sgirish kick.value = 0; 98244961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 98344961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 98444961713Sgirish 98544961713Sgirish /* Kick start the Transmit kick register */ 98644961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 98744961713Sgirish TX_RING_KICK_REG, 98844961713Sgirish (uint8_t)tx_ring_p->tdc, 98944961713Sgirish kick.value); 99044961713Sgirish } 99144961713Sgirish 9927a6dff21Sml tx_ring_p->descs_pending += ngathers; 99344961713Sgirish tdc_stats->tx_starts++; 99444961713Sgirish 995*6895688eSspeer if (isLDOMservice(nxgep)) { 996*6895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 997*6895688eSspeer if (tx_ring_p->tx_ring_offline) { 998*6895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 999*6895688eSspeer NXGE_TX_RING_OFFLINED); 1000*6895688eSspeer } 1001*6895688eSspeer } 1002678453a8Sspeer 100344961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 100444961713Sgirish 100544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 100644961713Sgirish 100744961713Sgirish return (status); 100844961713Sgirish 100930ac2e7bSml nxge_start_fail_lso: 101030ac2e7bSml status = 0; 101130ac2e7bSml good_packet = B_FALSE; 101230ac2e7bSml if (mp != NULL) { 101330ac2e7bSml freemsg(mp); 101430ac2e7bSml } 101530ac2e7bSml if (mp_chain != NULL) { 101630ac2e7bSml freemsg(mp_chain); 101730ac2e7bSml } 101830ac2e7bSml if (!lso_again && !ngathers) { 1019*6895688eSspeer if (isLDOMservice(nxgep)) { 1020*6895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 1021*6895688eSspeer if (tx_ring_p->tx_ring_offline) { 1022*6895688eSspeer (void) atomic_swap_32( 1023*6895688eSspeer &tx_ring_p->tx_ring_offline, 1024*6895688eSspeer NXGE_TX_RING_OFFLINED); 1025*6895688eSspeer } 1026*6895688eSspeer } 1027*6895688eSspeer 102830ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 102930ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 103030ac2e7bSml "==> nxge_start: lso exit (nothing changed)")); 103130ac2e7bSml goto nxge_start_fail1; 103230ac2e7bSml } 103330ac2e7bSml 103430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 103530ac2e7bSml "==> nxge_start (channel %d): before lso " 103630ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 103730ac2e7bSml "wr_index %d sop_index %d lso_again %d", 103830ac2e7bSml tx_ring_p->tdc, 103930ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 104030ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 104130ac2e7bSml 104230ac2e7bSml if (lso_again) { 104330ac2e7bSml lso_ngathers += ngathers; 104430ac2e7bSml ngathers = lso_ngathers; 104530ac2e7bSml sop_index = cur_index_lso; 104630ac2e7bSml tx_ring_p->wr_index = sop_index; 104730ac2e7bSml tx_ring_p->wr_index_wrap = lso_tail_wrap; 104830ac2e7bSml } 104930ac2e7bSml 105030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 105130ac2e7bSml "==> nxge_start (channel %d): after lso " 105230ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 105330ac2e7bSml "wr_index %d sop_index %d lso_again %d", 105430ac2e7bSml tx_ring_p->tdc, 105530ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 105630ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 105730ac2e7bSml 105844961713Sgirish nxge_start_fail2: 105944961713Sgirish if (good_packet == B_FALSE) { 106044961713Sgirish cur_index = sop_index; 106144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 106244961713Sgirish for (i = 0; i < ngathers; i++) { 106344961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 1064adfcba55Sjoycey #if defined(__i386) 1065adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 1066adfcba55Sjoycey #else 106744961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 1068adfcba55Sjoycey #endif 106944961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 107044961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 107144961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 107244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 107353f3d8ecSyc "tx_desc_p = %X index = %d", 107453f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 107553f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 107653f3d8ecSyc 0, -1); 107744961713Sgirish tx_msg_p->dvma_handle = NULL; 107844961713Sgirish if (tx_ring_p->dvma_wr_index == 107953f3d8ecSyc tx_ring_p->dvma_wrap_mask) 108044961713Sgirish tx_ring_p->dvma_wr_index = 0; 108144961713Sgirish else 108244961713Sgirish tx_ring_p->dvma_wr_index++; 108344961713Sgirish tx_ring_p->dvma_pending--; 108453f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 108544961713Sgirish if (ddi_dma_unbind_handle( 108653f3d8ecSyc tx_msg_p->dma_handle)) { 108744961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 108853f3d8ecSyc "ddi_dma_unbind_handle failed"); 108953f3d8ecSyc } 109044961713Sgirish } 109144961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 109244961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 109344961713Sgirish tx_ring_p->tx_wrap_mask); 109444961713Sgirish 109544961713Sgirish } 109644961713Sgirish 109744961713Sgirish nxgep->resched_needed = B_TRUE; 109844961713Sgirish } 109944961713Sgirish 1100*6895688eSspeer if (isLDOMservice(nxgep)) { 1101*6895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 1102*6895688eSspeer if (tx_ring_p->tx_ring_offline) { 1103*6895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1104*6895688eSspeer NXGE_TX_RING_OFFLINED); 1105*6895688eSspeer } 1106*6895688eSspeer } 1107678453a8Sspeer 110844961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 110944961713Sgirish 111044961713Sgirish nxge_start_fail1: 111144961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 111244961713Sgirish 111344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 111444961713Sgirish 111544961713Sgirish return (status); 111644961713Sgirish } 111744961713Sgirish 11181f8914d5Sml int 11191f8914d5Sml nxge_serial_tx(mblk_t *mp, void *arg) 11201f8914d5Sml { 11211f8914d5Sml p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 11221f8914d5Sml p_nxge_t nxgep = tx_ring_p->nxgep; 112322c0d73aSspeer int status = 0; 112422c0d73aSspeer 112522c0d73aSspeer if (isLDOMservice(nxgep)) { 112622c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 112722c0d73aSspeer freemsg(mp); 112822c0d73aSspeer return (status); 112922c0d73aSspeer } 113022c0d73aSspeer } 113122c0d73aSspeer 113222c0d73aSspeer status = nxge_start(nxgep, tx_ring_p, mp); 113322c0d73aSspeer return (status); 11341f8914d5Sml } 11351f8914d5Sml 113644961713Sgirish boolean_t 113744961713Sgirish nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 113844961713Sgirish { 113944961713Sgirish p_tx_ring_t *tx_rings; 114044961713Sgirish uint8_t ring_index; 11411f8914d5Sml p_tx_ring_t tx_ring_p; 1142678453a8Sspeer nxge_grp_t *group; 114344961713Sgirish 114444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 114544961713Sgirish 114644961713Sgirish ASSERT(mp->b_next == NULL); 114744961713Sgirish 1148678453a8Sspeer group = nxgep->tx_set.group[0]; /* The default group */ 1149678453a8Sspeer ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1150678453a8Sspeer 115144961713Sgirish tx_rings = nxgep->tx_rings->rings; 1152678453a8Sspeer tx_ring_p = tx_rings[group->legend[ring_index]]; 1153678453a8Sspeer 115422c0d73aSspeer if (isLDOMservice(nxgep)) { 115522c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 115622c0d73aSspeer /* 115722c0d73aSspeer * OFFLINE means that it is in the process of being 115822c0d73aSspeer * shared - that is, it has been claimed by the HIO 115922c0d73aSspeer * code, but hasn't been unlinked from <group> yet. 116022c0d73aSspeer * So in this case use the first TDC, which always 116122c0d73aSspeer * belongs to the service domain and can't be shared. 116222c0d73aSspeer */ 116322c0d73aSspeer ring_index = 0; 116422c0d73aSspeer tx_ring_p = tx_rings[group->legend[ring_index]]; 116522c0d73aSspeer } 1166678453a8Sspeer } 1167678453a8Sspeer 1168678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1169678453a8Sspeer (int)group->count, group->legend[ring_index], tx_ring_p)); 117044961713Sgirish 11711f8914d5Sml switch (nxge_tx_scheme) { 11721f8914d5Sml case NXGE_USE_START: 1173678453a8Sspeer if (nxge_start(nxgep, tx_ring_p, mp)) { 11741f8914d5Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 11751f8914d5Sml "ring index %d", ring_index)); 11761f8914d5Sml return (B_FALSE); 11771f8914d5Sml } 11781f8914d5Sml break; 11791f8914d5Sml 11801f8914d5Sml case NXGE_USE_SERIAL: 11811f8914d5Sml default: 11821f8914d5Sml nxge_serialize_enter(tx_ring_p->serial, mp); 11831f8914d5Sml break; 118444961713Sgirish } 118544961713Sgirish 118644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 118744961713Sgirish ring_index)); 118844961713Sgirish 118944961713Sgirish return (B_TRUE); 119044961713Sgirish } 119144961713Sgirish 119244961713Sgirish /* 119344961713Sgirish * nxge_m_tx() - send a chain of packets 119444961713Sgirish */ 119544961713Sgirish mblk_t * 119644961713Sgirish nxge_m_tx(void *arg, mblk_t *mp) 119744961713Sgirish { 119844961713Sgirish p_nxge_t nxgep = (p_nxge_t)arg; 119944961713Sgirish mblk_t *next; 120044961713Sgirish mac_tx_hint_t hint; 120144961713Sgirish 1202678453a8Sspeer NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1203678453a8Sspeer 1204678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1205678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 120644961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1207678453a8Sspeer "==> nxge_m_tx: hardware not initialized")); 120844961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1209678453a8Sspeer "<== nxge_m_tx")); 1210678453a8Sspeer freemsgchain(mp); 1211678453a8Sspeer mp = NULL; 121244961713Sgirish return (mp); 121344961713Sgirish } 121444961713Sgirish 121544961713Sgirish hint.hash = NULL; 121644961713Sgirish hint.vid = 0; 121744961713Sgirish hint.sap = 0; 121844961713Sgirish 121944961713Sgirish while (mp != NULL) { 122044961713Sgirish next = mp->b_next; 122144961713Sgirish mp->b_next = NULL; 122244961713Sgirish 122344961713Sgirish /* 122444961713Sgirish * Until Nemo tx resource works, the mac driver 122544961713Sgirish * does the load balancing based on TCP port, 122644961713Sgirish * or CPU. For debugging, we use a system 122744961713Sgirish * configurable parameter. 122844961713Sgirish */ 122944961713Sgirish if (!nxge_send(nxgep, mp, &hint)) { 123044961713Sgirish mp->b_next = next; 123144961713Sgirish break; 123244961713Sgirish } 123344961713Sgirish 123444961713Sgirish mp = next; 123530ac2e7bSml 123630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 123730ac2e7bSml "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 123830ac2e7bSml mp, next)); 123944961713Sgirish } 124044961713Sgirish 1241678453a8Sspeer NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 124244961713Sgirish return (mp); 124344961713Sgirish } 124444961713Sgirish 124544961713Sgirish int 124644961713Sgirish nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 124744961713Sgirish { 124844961713Sgirish uint8_t ring_index = 0; 124944961713Sgirish uint8_t *tcp_port; 125044961713Sgirish p_mblk_t nmp; 125144961713Sgirish size_t mblk_len; 125244961713Sgirish size_t iph_len; 125344961713Sgirish size_t hdrs_size; 125444961713Sgirish uint8_t hdrs_buf[sizeof (struct ether_header) + 125544961713Sgirish IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 125644961713Sgirish /* 125744961713Sgirish * allocate space big enough to cover 125844961713Sgirish * the max ip header length and the first 125944961713Sgirish * 4 bytes of the TCP/IP header. 126044961713Sgirish */ 126144961713Sgirish 126244961713Sgirish boolean_t qos = B_FALSE; 126344961713Sgirish 126444961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 126544961713Sgirish 126644961713Sgirish if (hp->vid) { 126744961713Sgirish qos = B_TRUE; 126844961713Sgirish } 126944961713Sgirish switch (nxge_tx_lb_policy) { 127044961713Sgirish case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 127144961713Sgirish default: 127244961713Sgirish tcp_port = mp->b_rptr; 127344961713Sgirish if (!nxge_no_tx_lb && !qos && 127444961713Sgirish (ntohs(((p_ether_header_t)tcp_port)->ether_type) 127544961713Sgirish == ETHERTYPE_IP)) { 127644961713Sgirish nmp = mp; 127744961713Sgirish mblk_len = MBLKL(nmp); 127844961713Sgirish tcp_port = NULL; 127944961713Sgirish if (mblk_len > sizeof (struct ether_header) + 128044961713Sgirish sizeof (uint8_t)) { 128144961713Sgirish tcp_port = nmp->b_rptr + 128244961713Sgirish sizeof (struct ether_header); 128344961713Sgirish mblk_len -= sizeof (struct ether_header); 128444961713Sgirish iph_len = ((*tcp_port) & 0x0f) << 2; 128544961713Sgirish if (mblk_len > (iph_len + sizeof (uint32_t))) { 128644961713Sgirish tcp_port = nmp->b_rptr; 128744961713Sgirish } else { 128844961713Sgirish tcp_port = NULL; 128944961713Sgirish } 129044961713Sgirish } 129144961713Sgirish if (tcp_port == NULL) { 129244961713Sgirish hdrs_size = 0; 129344961713Sgirish ((p_ether_header_t)hdrs_buf)->ether_type = 0; 129444961713Sgirish while ((nmp) && (hdrs_size < 129544961713Sgirish sizeof (hdrs_buf))) { 129644961713Sgirish mblk_len = MBLKL(nmp); 129744961713Sgirish if (mblk_len >= 129844961713Sgirish (sizeof (hdrs_buf) - hdrs_size)) 129944961713Sgirish mblk_len = sizeof (hdrs_buf) - 130044961713Sgirish hdrs_size; 130144961713Sgirish bcopy(nmp->b_rptr, 130244961713Sgirish &hdrs_buf[hdrs_size], mblk_len); 130344961713Sgirish hdrs_size += mblk_len; 130444961713Sgirish nmp = nmp->b_cont; 130544961713Sgirish } 130644961713Sgirish tcp_port = hdrs_buf; 130744961713Sgirish } 130844961713Sgirish tcp_port += sizeof (ether_header_t); 130944961713Sgirish if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1310958cea9eSml switch (tcp_port[9]) { 1311958cea9eSml case IPPROTO_TCP: 1312958cea9eSml case IPPROTO_UDP: 1313958cea9eSml case IPPROTO_ESP: 131444961713Sgirish tcp_port += ((*tcp_port) & 0x0f) << 2; 131544961713Sgirish ring_index = 1316958cea9eSml ((tcp_port[0] ^ 1317958cea9eSml tcp_port[1] ^ 1318958cea9eSml tcp_port[2] ^ 1319958cea9eSml tcp_port[3]) % maxtdcs); 1320958cea9eSml break; 1321958cea9eSml 1322958cea9eSml case IPPROTO_AH: 1323958cea9eSml /* SPI starts at the 4th byte */ 1324958cea9eSml tcp_port += ((*tcp_port) & 0x0f) << 2; 1325958cea9eSml ring_index = 1326958cea9eSml ((tcp_port[4] ^ 1327958cea9eSml tcp_port[5] ^ 1328958cea9eSml tcp_port[6] ^ 1329958cea9eSml tcp_port[7]) % maxtdcs); 1330958cea9eSml break; 1331958cea9eSml 1332958cea9eSml default: 133344961713Sgirish ring_index = tcp_port[19] % maxtdcs; 1334958cea9eSml break; 133544961713Sgirish } 133644961713Sgirish } else { /* fragmented packet */ 133744961713Sgirish ring_index = tcp_port[19] % maxtdcs; 133844961713Sgirish } 133944961713Sgirish } else { 134044961713Sgirish ring_index = mp->b_band % maxtdcs; 134144961713Sgirish } 134244961713Sgirish break; 134344961713Sgirish 134444961713Sgirish case NXGE_TX_LB_HASH: 134544961713Sgirish if (hp->hash) { 1346adfcba55Sjoycey #if defined(__i386) 1347adfcba55Sjoycey ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1348adfcba55Sjoycey #else 134944961713Sgirish ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1350adfcba55Sjoycey #endif 135144961713Sgirish } else { 135244961713Sgirish ring_index = mp->b_band % maxtdcs; 135344961713Sgirish } 135444961713Sgirish break; 135544961713Sgirish 135644961713Sgirish case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 135744961713Sgirish tcp_port = mp->b_rptr; 135844961713Sgirish ring_index = tcp_port[5] % maxtdcs; 135944961713Sgirish break; 136044961713Sgirish } 136144961713Sgirish 136244961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 136344961713Sgirish 136444961713Sgirish return (ring_index); 136544961713Sgirish } 136644961713Sgirish 136744961713Sgirish uint_t 136844961713Sgirish nxge_reschedule(caddr_t arg) 136944961713Sgirish { 137044961713Sgirish p_nxge_t nxgep; 137144961713Sgirish 137244961713Sgirish nxgep = (p_nxge_t)arg; 137344961713Sgirish 137444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 137544961713Sgirish 137644961713Sgirish if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1377678453a8Sspeer nxgep->resched_needed) { 1378678453a8Sspeer if (!isLDOMguest(nxgep)) 1379678453a8Sspeer mac_tx_update(nxgep->mach); 1380678453a8Sspeer #if defined(sun4v) 1381678453a8Sspeer else { /* isLDOMguest(nxgep) */ 1382678453a8Sspeer nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1383678453a8Sspeer nxgep->nxge_hw_p->hio; 1384678453a8Sspeer nx_vio_fp_t *vio = &nhd->hio.vio; 1385678453a8Sspeer 1386678453a8Sspeer /* Call back vnet. */ 1387678453a8Sspeer if (vio->cb.vio_net_tx_update) { 1388678453a8Sspeer (*vio->cb.vio_net_tx_update) 1389678453a8Sspeer (nxgep->hio_vr->vhp); 1390678453a8Sspeer } 1391678453a8Sspeer } 1392678453a8Sspeer #endif 139344961713Sgirish nxgep->resched_needed = B_FALSE; 139444961713Sgirish nxgep->resched_running = B_FALSE; 139544961713Sgirish } 139644961713Sgirish 139744961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 139844961713Sgirish return (DDI_INTR_CLAIMED); 139944961713Sgirish } 140030ac2e7bSml 140130ac2e7bSml 140230ac2e7bSml /* Software LSO starts here */ 140330ac2e7bSml static void 140430ac2e7bSml nxge_hcksum_retrieve(mblk_t *mp, 140530ac2e7bSml uint32_t *start, uint32_t *stuff, uint32_t *end, 140630ac2e7bSml uint32_t *value, uint32_t *flags) 140730ac2e7bSml { 140830ac2e7bSml if (mp->b_datap->db_type == M_DATA) { 140930ac2e7bSml if (flags != NULL) { 141030ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 141130ac2e7bSml HCK_PARTIALCKSUM | HCK_FULLCKSUM | 141230ac2e7bSml HCK_FULLCKSUM_OK); 141330ac2e7bSml if ((*flags & (HCK_PARTIALCKSUM | 141430ac2e7bSml HCK_FULLCKSUM)) != 0) { 141530ac2e7bSml if (value != NULL) 141630ac2e7bSml *value = (uint32_t)DB_CKSUM16(mp); 141730ac2e7bSml if ((*flags & HCK_PARTIALCKSUM) != 0) { 141830ac2e7bSml if (start != NULL) 141930ac2e7bSml *start = 142030ac2e7bSml (uint32_t)DB_CKSUMSTART(mp); 142130ac2e7bSml if (stuff != NULL) 142230ac2e7bSml *stuff = 142330ac2e7bSml (uint32_t)DB_CKSUMSTUFF(mp); 142430ac2e7bSml if (end != NULL) 142530ac2e7bSml *end = 142630ac2e7bSml (uint32_t)DB_CKSUMEND(mp); 142730ac2e7bSml } 142830ac2e7bSml } 142930ac2e7bSml } 143030ac2e7bSml } 143130ac2e7bSml } 143230ac2e7bSml 143330ac2e7bSml static void 143430ac2e7bSml nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 143530ac2e7bSml { 143630ac2e7bSml ASSERT(DB_TYPE(mp) == M_DATA); 143730ac2e7bSml 143830ac2e7bSml *mss = 0; 143930ac2e7bSml if (flags != NULL) { 144030ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 144130ac2e7bSml if ((*flags != 0) && (mss != NULL)) { 144230ac2e7bSml *mss = (uint32_t)DB_LSOMSS(mp); 144330ac2e7bSml } 144430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 144530ac2e7bSml "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 144630ac2e7bSml *mss, *flags)); 144730ac2e7bSml } 144830ac2e7bSml 144930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 145030ac2e7bSml "<== nxge_lso_info_get: mss %d", *mss)); 145130ac2e7bSml } 145230ac2e7bSml 145330ac2e7bSml /* 145430ac2e7bSml * Do Soft LSO on the oversized packet. 145530ac2e7bSml * 145630ac2e7bSml * 1. Create a chain of message for headers. 145730ac2e7bSml * 2. Fill up header messages with proper information. 145830ac2e7bSml * 3. Copy Eithernet, IP, and TCP headers from the original message to 145930ac2e7bSml * each new message with necessary adjustments. 146030ac2e7bSml * * Unchange the ethernet header for DIX frames. (by default) 146130ac2e7bSml * * IP Total Length field is updated to MSS or less(only for the last one). 146230ac2e7bSml * * IP Identification value is incremented by one for each packet. 146330ac2e7bSml * * TCP sequence Number is recalculated according to the payload length. 146430ac2e7bSml * * Set FIN and/or PSH flags for the *last* packet if applied. 146530ac2e7bSml * * TCP partial Checksum 146630ac2e7bSml * 4. Update LSO information in the first message header. 146730ac2e7bSml * 5. Release the original message header. 146830ac2e7bSml */ 146930ac2e7bSml static mblk_t * 147030ac2e7bSml nxge_do_softlso(mblk_t *mp, uint32_t mss) 147130ac2e7bSml { 147230ac2e7bSml uint32_t hckflags; 147330ac2e7bSml int pktlen; 147430ac2e7bSml int hdrlen; 147530ac2e7bSml int segnum; 147630ac2e7bSml int i; 147730ac2e7bSml struct ether_vlan_header *evh; 147830ac2e7bSml int ehlen, iphlen, tcphlen; 147930ac2e7bSml struct ip *oiph, *niph; 148030ac2e7bSml struct tcphdr *otcph, *ntcph; 148130ac2e7bSml int available, len, left; 148230ac2e7bSml uint16_t ip_id; 148330ac2e7bSml uint32_t tcp_seq; 148430ac2e7bSml #ifdef __sparc 148530ac2e7bSml uint32_t tcp_seq_tmp; 148630ac2e7bSml #endif 148730ac2e7bSml mblk_t *datamp; 148830ac2e7bSml uchar_t *rptr; 148930ac2e7bSml mblk_t *nmp; 149030ac2e7bSml mblk_t *cmp; 149130ac2e7bSml mblk_t *mp_chain; 149230ac2e7bSml boolean_t do_cleanup = B_FALSE; 149330ac2e7bSml t_uscalar_t start_offset = 0; 149430ac2e7bSml t_uscalar_t stuff_offset = 0; 149530ac2e7bSml t_uscalar_t value = 0; 149630ac2e7bSml uint16_t l4_len; 149730ac2e7bSml ipaddr_t src, dst; 149830ac2e7bSml uint32_t cksum, sum, l4cksum; 149930ac2e7bSml 150030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 150130ac2e7bSml "==> nxge_do_softlso")); 150230ac2e7bSml /* 150330ac2e7bSml * check the length of LSO packet payload and calculate the number of 150430ac2e7bSml * segments to be generated. 150530ac2e7bSml */ 150630ac2e7bSml pktlen = msgsize(mp); 150730ac2e7bSml evh = (struct ether_vlan_header *)mp->b_rptr; 150830ac2e7bSml 150930ac2e7bSml /* VLAN? */ 151030ac2e7bSml if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 151130ac2e7bSml ehlen = sizeof (struct ether_vlan_header); 151230ac2e7bSml else 151330ac2e7bSml ehlen = sizeof (struct ether_header); 151430ac2e7bSml oiph = (struct ip *)(mp->b_rptr + ehlen); 151530ac2e7bSml iphlen = oiph->ip_hl * 4; 151630ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 151730ac2e7bSml tcphlen = otcph->th_off * 4; 151830ac2e7bSml 151930ac2e7bSml l4_len = pktlen - ehlen - iphlen; 152030ac2e7bSml 152130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 152230ac2e7bSml "==> nxge_do_softlso: mss %d oiph $%p " 152330ac2e7bSml "original ip_sum oiph->ip_sum 0x%x " 152430ac2e7bSml "original tcp_sum otcph->th_sum 0x%x " 152530ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d " 152630ac2e7bSml "l4_len %d (0x%x) ip_len - iphlen %d ", 152730ac2e7bSml mss, 152830ac2e7bSml oiph, 152930ac2e7bSml oiph->ip_sum, 153030ac2e7bSml otcph->th_sum, 153130ac2e7bSml ntohs(oiph->ip_len), pktlen, 153230ac2e7bSml ehlen, 153330ac2e7bSml l4_len, 153430ac2e7bSml l4_len, 153530ac2e7bSml ntohs(oiph->ip_len) - iphlen)); 153630ac2e7bSml 153730ac2e7bSml /* IPv4 + TCP */ 153830ac2e7bSml if (!(oiph->ip_v == IPV4_VERSION)) { 153930ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 154030ac2e7bSml "<== nxge_do_softlso: not IPV4 " 154130ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 154230ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 154330ac2e7bSml tcphlen)); 154430ac2e7bSml freemsg(mp); 154530ac2e7bSml return (NULL); 154630ac2e7bSml } 154730ac2e7bSml 154830ac2e7bSml if (!(oiph->ip_p == IPPROTO_TCP)) { 154930ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 155030ac2e7bSml "<== nxge_do_softlso: not TCP " 155130ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 155230ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 155330ac2e7bSml tcphlen)); 155430ac2e7bSml freemsg(mp); 155530ac2e7bSml return (NULL); 155630ac2e7bSml } 155730ac2e7bSml 155830ac2e7bSml if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 155930ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 156030ac2e7bSml "<== nxge_do_softlso: len not matched " 156130ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 156230ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 156330ac2e7bSml tcphlen)); 156430ac2e7bSml freemsg(mp); 156530ac2e7bSml return (NULL); 156630ac2e7bSml } 156730ac2e7bSml 156830ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 156930ac2e7bSml tcphlen = otcph->th_off * 4; 157030ac2e7bSml 157130ac2e7bSml /* TCP flags can not include URG, RST, or SYN */ 157230ac2e7bSml VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 157330ac2e7bSml 157430ac2e7bSml hdrlen = ehlen + iphlen + tcphlen; 157530ac2e7bSml 157630ac2e7bSml VERIFY(MBLKL(mp) >= hdrlen); 157730ac2e7bSml 157830ac2e7bSml if (MBLKL(mp) > hdrlen) { 157930ac2e7bSml datamp = mp; 158030ac2e7bSml rptr = mp->b_rptr + hdrlen; 158130ac2e7bSml } else { /* = */ 158230ac2e7bSml datamp = mp->b_cont; 158330ac2e7bSml rptr = datamp->b_rptr; 158430ac2e7bSml } 158530ac2e7bSml 158630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 158730ac2e7bSml "nxge_do_softlso: otcph $%p pktlen: %d, " 158830ac2e7bSml "hdrlen %d ehlen %d iphlen %d tcphlen %d " 158930ac2e7bSml "mblkl(mp): %d, mblkl(datamp): %d", 159030ac2e7bSml otcph, 159130ac2e7bSml pktlen, hdrlen, ehlen, iphlen, tcphlen, 159230ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 159330ac2e7bSml 159430ac2e7bSml hckflags = 0; 159530ac2e7bSml nxge_hcksum_retrieve(mp, 159630ac2e7bSml &start_offset, &stuff_offset, &value, NULL, &hckflags); 159730ac2e7bSml 159830ac2e7bSml dst = oiph->ip_dst.s_addr; 159930ac2e7bSml src = oiph->ip_src.s_addr; 160030ac2e7bSml 160130ac2e7bSml cksum = (dst >> 16) + (dst & 0xFFFF) + 160230ac2e7bSml (src >> 16) + (src & 0xFFFF); 160330ac2e7bSml l4cksum = cksum + IP_TCP_CSUM_COMP; 160430ac2e7bSml 160530ac2e7bSml sum = l4_len + l4cksum; 160630ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 160730ac2e7bSml 160830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 160930ac2e7bSml "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 161030ac2e7bSml "hckflags 0x%x start_offset %d stuff_offset %d " 161130ac2e7bSml "value (original) 0x%x th_sum 0x%x " 161230ac2e7bSml "pktlen %d l4_len %d (0x%x) " 161330ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 161430ac2e7bSml dst, src, 161530ac2e7bSml (sum & 0xffff), (~sum & 0xffff), 161630ac2e7bSml hckflags, start_offset, stuff_offset, 161730ac2e7bSml value, otcph->th_sum, 161830ac2e7bSml pktlen, 161930ac2e7bSml l4_len, 162030ac2e7bSml l4_len, 162130ac2e7bSml ntohs(oiph->ip_len) - (int)MBLKL(mp), 162230ac2e7bSml (int)MBLKL(datamp), 162330ac2e7bSml nxge_dump_packet((char *)evh, 12))); 162430ac2e7bSml 162530ac2e7bSml /* 162630ac2e7bSml * Start to process. 162730ac2e7bSml */ 162830ac2e7bSml available = pktlen - hdrlen; 162930ac2e7bSml segnum = (available - 1) / mss + 1; 163030ac2e7bSml 163130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 163230ac2e7bSml "==> nxge_do_softlso: pktlen %d " 163330ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 163430ac2e7bSml "available %d mss %d segnum %d", 163530ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 163630ac2e7bSml available, 163730ac2e7bSml mss, 163830ac2e7bSml segnum)); 163930ac2e7bSml 164030ac2e7bSml VERIFY(segnum >= 2); 164130ac2e7bSml 164230ac2e7bSml /* 164330ac2e7bSml * Try to pre-allocate all header messages 164430ac2e7bSml */ 164530ac2e7bSml mp_chain = NULL; 164630ac2e7bSml for (i = 0; i < segnum; i++) { 164730ac2e7bSml if ((nmp = allocb(hdrlen, 0)) == NULL) { 164830ac2e7bSml /* Clean up the mp_chain */ 164930ac2e7bSml while (mp_chain != NULL) { 165030ac2e7bSml nmp = mp_chain; 165130ac2e7bSml mp_chain = mp_chain->b_next; 165230ac2e7bSml freemsg(nmp); 165330ac2e7bSml } 165430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 165530ac2e7bSml "<== nxge_do_softlso: " 165630ac2e7bSml "Could not allocate enough messages for headers!")); 165730ac2e7bSml freemsg(mp); 165830ac2e7bSml return (NULL); 165930ac2e7bSml } 166030ac2e7bSml nmp->b_next = mp_chain; 166130ac2e7bSml mp_chain = nmp; 166230ac2e7bSml 166330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 166430ac2e7bSml "==> nxge_do_softlso: " 166530ac2e7bSml "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 166630ac2e7bSml mp, nmp, mp_chain, mp_chain->b_next)); 166730ac2e7bSml } 166830ac2e7bSml 166930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 167030ac2e7bSml "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 167130ac2e7bSml mp, nmp, mp_chain)); 167230ac2e7bSml 167330ac2e7bSml /* 167430ac2e7bSml * Associate payload with new packets 167530ac2e7bSml */ 167630ac2e7bSml cmp = mp_chain; 167730ac2e7bSml left = available; 167830ac2e7bSml while (cmp != NULL) { 167930ac2e7bSml nmp = dupb(datamp); 168030ac2e7bSml if (nmp == NULL) { 168130ac2e7bSml do_cleanup = B_TRUE; 168230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168330ac2e7bSml "==>nxge_do_softlso: " 168430ac2e7bSml "Can not dupb(datamp), have to do clean up")); 168530ac2e7bSml goto cleanup_allocated_msgs; 168630ac2e7bSml } 168730ac2e7bSml 168830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168930ac2e7bSml "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 169030ac2e7bSml "dupb nmp $%p len %d left %d msd %d ", 169130ac2e7bSml mp, cmp, nmp, len, left, mss)); 169230ac2e7bSml 169330ac2e7bSml cmp->b_cont = nmp; 169430ac2e7bSml nmp->b_rptr = rptr; 169530ac2e7bSml len = (left < mss) ? left : mss; 169630ac2e7bSml left -= len; 169730ac2e7bSml 169830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 169930ac2e7bSml "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 170030ac2e7bSml "dupb nmp $%p len %d left %d mss %d ", 170130ac2e7bSml mp, cmp, nmp, len, left, mss)); 170230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 170330ac2e7bSml "nxge_do_softlso: before available: %d, " 170430ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 170530ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 170630ac2e7bSml 170730ac2e7bSml len -= MBLKL(nmp); 170830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 170930ac2e7bSml "nxge_do_softlso: after available: %d, " 171030ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 171130ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 171230ac2e7bSml 171330ac2e7bSml while (len > 0) { 171430ac2e7bSml mblk_t *mmp = NULL; 171530ac2e7bSml 171630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 171730ac2e7bSml "nxge_do_softlso: (4) len > 0 available: %d, " 171830ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 171930ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 172030ac2e7bSml 172130ac2e7bSml if (datamp->b_cont != NULL) { 172230ac2e7bSml datamp = datamp->b_cont; 172330ac2e7bSml rptr = datamp->b_rptr; 172430ac2e7bSml mmp = dupb(datamp); 172530ac2e7bSml if (mmp == NULL) { 172630ac2e7bSml do_cleanup = B_TRUE; 172730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 172830ac2e7bSml "==> nxge_do_softlso: " 1729678453a8Sspeer "Can not dupb(datamp) (1), :" 173030ac2e7bSml "have to do clean up")); 173130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 173230ac2e7bSml "==> nxge_do_softlso: " 173330ac2e7bSml "available: %d, left: %d, " 173430ac2e7bSml "len: %d, MBLKL(nmp): %d", 173530ac2e7bSml available, left, len, 173630ac2e7bSml (int)MBLKL(nmp))); 173730ac2e7bSml goto cleanup_allocated_msgs; 173830ac2e7bSml } 173930ac2e7bSml } else { 174030ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 174130ac2e7bSml "==> nxge_do_softlso: " 174230ac2e7bSml "(1)available: %d, left: %d, " 174330ac2e7bSml "len: %d, MBLKL(nmp): %d", 174430ac2e7bSml available, left, len, 174530ac2e7bSml (int)MBLKL(nmp))); 174630ac2e7bSml cmn_err(CE_PANIC, 174730ac2e7bSml "==> nxge_do_softlso: " 174830ac2e7bSml "Pointers must have been corrupted!\n" 174930ac2e7bSml "datamp: $%p, nmp: $%p, rptr: $%p", 175030ac2e7bSml (void *)datamp, 175130ac2e7bSml (void *)nmp, 175230ac2e7bSml (void *)rptr); 175330ac2e7bSml } 175430ac2e7bSml nmp->b_cont = mmp; 175530ac2e7bSml nmp = mmp; 175630ac2e7bSml len -= MBLKL(nmp); 175730ac2e7bSml } 175830ac2e7bSml if (len < 0) { 175930ac2e7bSml nmp->b_wptr += len; 176030ac2e7bSml rptr = nmp->b_wptr; 176130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 176230ac2e7bSml "(5) len < 0 (less than 0)" 176330ac2e7bSml "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 176430ac2e7bSml available, left, len, (int)MBLKL(nmp))); 176530ac2e7bSml 176630ac2e7bSml } else if (len == 0) { 176730ac2e7bSml if (datamp->b_cont != NULL) { 176830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 176930ac2e7bSml "(5) len == 0" 177030ac2e7bSml "available: %d, left: %d, len: %d, " 177130ac2e7bSml "MBLKL(nmp): %d", 177230ac2e7bSml available, left, len, (int)MBLKL(nmp))); 177330ac2e7bSml datamp = datamp->b_cont; 177430ac2e7bSml rptr = datamp->b_rptr; 177530ac2e7bSml } else { 177630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 177730ac2e7bSml "(6)available b_cont == NULL : %d, " 177830ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 177930ac2e7bSml available, left, len, (int)MBLKL(nmp))); 178030ac2e7bSml 178130ac2e7bSml VERIFY(cmp->b_next == NULL); 178230ac2e7bSml VERIFY(left == 0); 178330ac2e7bSml break; /* Done! */ 178430ac2e7bSml } 178530ac2e7bSml } 178630ac2e7bSml cmp = cmp->b_next; 178730ac2e7bSml 178830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 178930ac2e7bSml "(7) do_softlso: " 179030ac2e7bSml "next mp in mp_chain available len != 0 : %d, " 179130ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 179230ac2e7bSml available, left, len, (int)MBLKL(nmp))); 179330ac2e7bSml } 179430ac2e7bSml 179530ac2e7bSml /* 179630ac2e7bSml * From now, start to fill up all headers for the first message 179730ac2e7bSml * Hardware checksum flags need to be updated separately for FULLCKSUM 179830ac2e7bSml * and PARTIALCKSUM cases. For full checksum, copy the original flags 179930ac2e7bSml * into every new packet is enough. But for HCK_PARTIALCKSUM, all 180030ac2e7bSml * required fields need to be updated properly. 180130ac2e7bSml */ 180230ac2e7bSml nmp = mp_chain; 180330ac2e7bSml bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 180430ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 180530ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 180630ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 180730ac2e7bSml ip_id = ntohs(niph->ip_id); 180830ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 180930ac2e7bSml #ifdef __sparc 181030ac2e7bSml bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 181130ac2e7bSml tcp_seq = ntohl(tcp_seq_tmp); 181230ac2e7bSml #else 181330ac2e7bSml tcp_seq = ntohl(ntcph->th_seq); 181430ac2e7bSml #endif 181530ac2e7bSml 181630ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 181730ac2e7bSml 181830ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 181930ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 182030ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 182130ac2e7bSml 182230ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 182330ac2e7bSml niph->ip_sum = 0; 182430ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 182530ac2e7bSml 182630ac2e7bSml l4_len = mss + tcphlen; 182730ac2e7bSml sum = htons(l4_len) + l4cksum; 182830ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 182930ac2e7bSml ntcph->th_sum = (sum & 0xffff); 183030ac2e7bSml 183130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 183230ac2e7bSml "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 183330ac2e7bSml "mss %d pktlen %d l4_len %d (0x%x) " 183430ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 183530ac2e7bSml "ip_sum 0x%x " 183630ac2e7bSml "th_sum 0x%x sum 0x%x ) " 183730ac2e7bSml "dump first ip->tcp %s", 183830ac2e7bSml nmp, mp_chain, 183930ac2e7bSml mss, 184030ac2e7bSml pktlen, 184130ac2e7bSml l4_len, 184230ac2e7bSml l4_len, 184330ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp), 184430ac2e7bSml niph->ip_sum, 184530ac2e7bSml ntcph->th_sum, 184630ac2e7bSml sum, 184730ac2e7bSml nxge_dump_packet((char *)niph, 52))); 184830ac2e7bSml 184930ac2e7bSml cmp = nmp; 185030ac2e7bSml while ((nmp = nmp->b_next)->b_next != NULL) { 185130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 185230ac2e7bSml "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 185330ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 185430ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 185530ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 185630ac2e7bSml niph->ip_id = htons(++ip_id); 185730ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 185830ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 185930ac2e7bSml tcp_seq += mss; 186030ac2e7bSml 186130ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 186230ac2e7bSml 186330ac2e7bSml #ifdef __sparc 186430ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 186530ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 186630ac2e7bSml #else 186730ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 186830ac2e7bSml #endif 186930ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 187030ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 187130ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 187230ac2e7bSml 187330ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 187430ac2e7bSml niph->ip_sum = 0; 187530ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 187630ac2e7bSml ntcph->th_sum = (sum & 0xffff); 187730ac2e7bSml 187830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 187930ac2e7bSml "==> nxge_do_softlso: middle ip_sum 0x%x " 188030ac2e7bSml "th_sum 0x%x " 188130ac2e7bSml " mp $%p (mp_chain $%p) pktlen %d " 188230ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 188330ac2e7bSml niph->ip_sum, 188430ac2e7bSml ntcph->th_sum, 188530ac2e7bSml nmp, mp_chain, 188630ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 188730ac2e7bSml } 188830ac2e7bSml 188930ac2e7bSml /* Last segment */ 189030ac2e7bSml /* 189130ac2e7bSml * Set FIN and/or PSH flags if present only in the last packet. 189230ac2e7bSml * The ip_len could be different from prior packets. 189330ac2e7bSml */ 189430ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 189530ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 189630ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 189730ac2e7bSml niph->ip_id = htons(++ip_id); 189830ac2e7bSml niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 189930ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 190030ac2e7bSml tcp_seq += mss; 190130ac2e7bSml #ifdef __sparc 190230ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 190330ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 190430ac2e7bSml #else 190530ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 190630ac2e7bSml #endif 190730ac2e7bSml ntcph->th_flags = (otcph->th_flags & ~TH_URG); 190830ac2e7bSml 190930ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 191030ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 191130ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 191230ac2e7bSml 191330ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 191430ac2e7bSml niph->ip_sum = 0; 191530ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 191630ac2e7bSml 191730ac2e7bSml l4_len = ntohs(niph->ip_len) - iphlen; 191830ac2e7bSml sum = htons(l4_len) + l4cksum; 191930ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 192030ac2e7bSml ntcph->th_sum = (sum & 0xffff); 192130ac2e7bSml 192230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 192330ac2e7bSml "==> nxge_do_softlso: last next " 192430ac2e7bSml "niph->ip_sum 0x%x " 192530ac2e7bSml "ntcph->th_sum 0x%x sum 0x%x " 192630ac2e7bSml "dump last ip->tcp %s " 192730ac2e7bSml "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 192830ac2e7bSml "l4_len %d (0x%x) " 192930ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 193030ac2e7bSml niph->ip_sum, 193130ac2e7bSml ntcph->th_sum, sum, 193230ac2e7bSml nxge_dump_packet((char *)niph, 52), 193330ac2e7bSml cmp, nmp, mp_chain, 193430ac2e7bSml pktlen, pktlen, 193530ac2e7bSml l4_len, 193630ac2e7bSml l4_len, 193730ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 193830ac2e7bSml 193930ac2e7bSml cleanup_allocated_msgs: 194030ac2e7bSml if (do_cleanup) { 194130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 194230ac2e7bSml "==> nxge_do_softlso: " 194330ac2e7bSml "Failed allocating messages, " 194430ac2e7bSml "have to clean up and fail!")); 194530ac2e7bSml while (mp_chain != NULL) { 194630ac2e7bSml nmp = mp_chain; 194730ac2e7bSml mp_chain = mp_chain->b_next; 194830ac2e7bSml freemsg(nmp); 194930ac2e7bSml } 195030ac2e7bSml } 195130ac2e7bSml /* 195230ac2e7bSml * We're done here, so just free the original message and return the 195330ac2e7bSml * new message chain, that could be NULL if failed, back to the caller. 195430ac2e7bSml */ 195530ac2e7bSml freemsg(mp); 195630ac2e7bSml 195730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 195830ac2e7bSml "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 195930ac2e7bSml return (mp_chain); 196030ac2e7bSml } 196130ac2e7bSml 196230ac2e7bSml /* 196330ac2e7bSml * Will be called before NIC driver do further operation on the message. 196430ac2e7bSml * The input message may include LSO information, if so, go to softlso logic 196530ac2e7bSml * to eliminate the oversized LSO packet for the incapable underlying h/w. 196630ac2e7bSml * The return could be the same non-LSO message or a message chain for LSO case. 196730ac2e7bSml * 196830ac2e7bSml * The driver needs to call this function per packet and process the whole chain 196930ac2e7bSml * if applied. 197030ac2e7bSml */ 197130ac2e7bSml static mblk_t * 197230ac2e7bSml nxge_lso_eliminate(mblk_t *mp) 197330ac2e7bSml { 197430ac2e7bSml uint32_t lsoflags; 197530ac2e7bSml uint32_t mss; 197630ac2e7bSml 197730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 197830ac2e7bSml "==>nxge_lso_eliminate:")); 197930ac2e7bSml nxge_lso_info_get(mp, &mss, &lsoflags); 198030ac2e7bSml 198130ac2e7bSml if (lsoflags & HW_LSO) { 198230ac2e7bSml mblk_t *nmp; 198330ac2e7bSml 198430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 198530ac2e7bSml "==>nxge_lso_eliminate:" 198630ac2e7bSml "HW_LSO:mss %d mp $%p", 198730ac2e7bSml mss, mp)); 198830ac2e7bSml if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 198930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 199030ac2e7bSml "<== nxge_lso_eliminate: " 199130ac2e7bSml "LSO: nmp not NULL nmp $%p mss %d mp $%p", 199230ac2e7bSml nmp, mss, mp)); 199330ac2e7bSml return (nmp); 199430ac2e7bSml } else { 199530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 199630ac2e7bSml "<== nxge_lso_eliminate_ " 199730ac2e7bSml "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 199830ac2e7bSml nmp, mss, mp)); 199930ac2e7bSml return (NULL); 200030ac2e7bSml } 200130ac2e7bSml } 200230ac2e7bSml 200330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 200430ac2e7bSml "<== nxge_lso_eliminate")); 200530ac2e7bSml return (mp); 200630ac2e7bSml } 200730ac2e7bSml 200830ac2e7bSml static uint32_t 200930ac2e7bSml nxge_csgen(uint16_t *adr, int len) 201030ac2e7bSml { 201130ac2e7bSml int i, odd; 201230ac2e7bSml uint32_t sum = 0; 201330ac2e7bSml uint32_t c = 0; 201430ac2e7bSml 201530ac2e7bSml odd = len % 2; 201630ac2e7bSml for (i = 0; i < (len / 2); i++) { 201730ac2e7bSml sum += (adr[i] & 0xffff); 201830ac2e7bSml } 201930ac2e7bSml if (odd) { 202030ac2e7bSml sum += adr[len / 2] & 0xff00; 202130ac2e7bSml } 202230ac2e7bSml while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 202330ac2e7bSml sum &= 0xffff; 202430ac2e7bSml sum += c; 202530ac2e7bSml } 202630ac2e7bSml return (~sum & 0xffff); 202730ac2e7bSml } 2028