144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 2230ac2e7bSml * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 26a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 27678453a8Sspeer #include <sys/nxge/nxge_hio.h> 28678453a8Sspeer #include <npi_tx_wr64.h> 2944961713Sgirish 3030ac2e7bSml /* Software LSO required header files */ 3130ac2e7bSml #include <netinet/tcp.h> 3230ac2e7bSml #include <inet/ip_impl.h> 3330ac2e7bSml #include <inet/tcp.h> 3430ac2e7bSml 3530ac2e7bSml static mblk_t *nxge_lso_eliminate(mblk_t *); 3630ac2e7bSml static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 3730ac2e7bSml static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 3830ac2e7bSml static void nxge_hcksum_retrieve(mblk_t *, 3930ac2e7bSml uint32_t *, uint32_t *, uint32_t *, 4030ac2e7bSml uint32_t *, uint32_t *); 4130ac2e7bSml static uint32_t nxge_csgen(uint16_t *, int); 4230ac2e7bSml 432d99c5d4SMichael Speer extern void nxge_txdma_freemsg_task(p_tx_ring_t ringp); 442d99c5d4SMichael Speer 4544961713Sgirish extern uint32_t nxge_reclaim_pending; 4644961713Sgirish extern uint32_t nxge_bcopy_thresh; 4744961713Sgirish extern uint32_t nxge_dvma_thresh; 4844961713Sgirish extern uint32_t nxge_dma_stream_thresh; 4944961713Sgirish extern uint32_t nxge_tx_minfree; 5044961713Sgirish extern uint32_t nxge_tx_intr_thres; 5144961713Sgirish extern uint32_t nxge_tx_max_gathers; 5244961713Sgirish extern uint32_t nxge_tx_tiny_pack; 5344961713Sgirish extern uint32_t nxge_tx_use_bcopy; 5444961713Sgirish extern uint32_t nxge_tx_lb_policy; 5544961713Sgirish extern uint32_t nxge_no_tx_lb; 561f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 5730ac2e7bSml uint32_t nxge_lso_kick_cnt = 2; 5844961713Sgirish 5944961713Sgirish typedef struct _mac_tx_hint { 6044961713Sgirish uint16_t sap; 6144961713Sgirish uint16_t vid; 6244961713Sgirish void *hash; 6344961713Sgirish } mac_tx_hint_t, *p_mac_tx_hint_t; 6444961713Sgirish 6544961713Sgirish int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 6644961713Sgirish 6744961713Sgirish int 6844961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 6944961713Sgirish { 7044961713Sgirish int status = 0; 7144961713Sgirish p_tx_desc_t tx_desc_ring_vp; 7244961713Sgirish npi_handle_t npi_desc_handle; 7344961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 7444961713Sgirish p_tx_desc_t tx_desc_p; 7544961713Sgirish p_tx_msg_t tx_msg_ring; 7644961713Sgirish p_tx_msg_t tx_msg_p; 7744961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 7844961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 7944961713Sgirish p_tx_pkt_header_t hdrp; 80b4d05839Sml tx_pkt_header_t tmp_hdrp; 8144961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 8244961713Sgirish uint8_t npads = 0; 8344961713Sgirish uint64_t dma_ioaddr; 8444961713Sgirish uint32_t dma_flags; 8544961713Sgirish int last_bidx; 8644961713Sgirish uint8_t *b_rptr; 8744961713Sgirish caddr_t kaddr; 8844961713Sgirish uint32_t nmblks; 8944961713Sgirish uint32_t ngathers; 9044961713Sgirish uint32_t clen; 9144961713Sgirish int len; 9244961713Sgirish uint32_t pkt_len, pack_len, min_len; 9344961713Sgirish uint32_t bcopy_thresh; 9444961713Sgirish int i, cur_index, sop_index; 9544961713Sgirish uint16_t tail_index; 9644961713Sgirish boolean_t tail_wrap = B_FALSE; 9744961713Sgirish nxge_dma_common_t desc_area; 9844961713Sgirish nxge_os_dma_handle_t dma_handle; 9944961713Sgirish ddi_dma_cookie_t dma_cookie; 10044961713Sgirish npi_handle_t npi_handle; 10144961713Sgirish p_mblk_t nmp; 10244961713Sgirish p_mblk_t t_mp; 10344961713Sgirish uint32_t ncookies; 10444961713Sgirish boolean_t good_packet; 10544961713Sgirish boolean_t mark_mode = B_FALSE; 10644961713Sgirish p_nxge_stats_t statsp; 10744961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 10844961713Sgirish t_uscalar_t start_offset = 0; 10944961713Sgirish t_uscalar_t stuff_offset = 0; 11044961713Sgirish t_uscalar_t end_offset = 0; 11144961713Sgirish t_uscalar_t value = 0; 11244961713Sgirish t_uscalar_t cksum_flags = 0; 11344961713Sgirish boolean_t cksum_on = B_FALSE; 11444961713Sgirish uint32_t boff = 0; 115b4d05839Sml uint64_t tot_xfer_len = 0; 11644961713Sgirish boolean_t header_set = B_FALSE; 11744961713Sgirish #ifdef NXGE_DEBUG 11844961713Sgirish p_tx_desc_t tx_desc_ring_pp; 11944961713Sgirish p_tx_desc_t tx_desc_pp; 12044961713Sgirish tx_desc_t *save_desc_p; 12144961713Sgirish int dump_len; 12244961713Sgirish int sad_len; 12344961713Sgirish uint64_t sad; 12444961713Sgirish int xfer_len; 12544961713Sgirish uint32_t msgsize; 12644961713Sgirish #endif 12730ac2e7bSml p_mblk_t mp_chain = NULL; 12830ac2e7bSml boolean_t is_lso = B_FALSE; 12930ac2e7bSml boolean_t lso_again; 13030ac2e7bSml int cur_index_lso; 13130ac2e7bSml p_mblk_t nmp_lso_save; 13230ac2e7bSml uint32_t lso_ngathers; 13330ac2e7bSml boolean_t lso_tail_wrap = B_FALSE; 13444961713Sgirish 13544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 136678453a8Sspeer "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 13744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 138678453a8Sspeer "==> nxge_start: Starting tdc %d desc pending %d", 139678453a8Sspeer tx_ring_p->tdc, tx_ring_p->descs_pending)); 14044961713Sgirish 14144961713Sgirish statsp = nxgep->statsp; 14244961713Sgirish 143678453a8Sspeer if (!isLDOMguest(nxgep)) { 144678453a8Sspeer switch (nxgep->mac.portmode) { 145678453a8Sspeer default: 146678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode == 147678453a8Sspeer nxge_lb_normal) { 148678453a8Sspeer if (!statsp->mac_stats.link_up) { 149678453a8Sspeer freemsg(mp); 150678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 151678453a8Sspeer "==> nxge_start: " 152678453a8Sspeer "link not up")); 153678453a8Sspeer goto nxge_start_fail1; 154678453a8Sspeer } 155321febdeSsbehera } 156678453a8Sspeer break; 157678453a8Sspeer case PORT_10G_FIBER: 158678453a8Sspeer /* 159678453a8Sspeer * For the following modes, check the link status 160678453a8Sspeer * before sending the packet out: 161*eb1db165Stc * nxge_lb_normal, 162*eb1db165Stc * nxge_lb_ext10g, 163*eb1db165Stc * nxge_lb_ext1000, 164*eb1db165Stc * nxge_lb_ext100, 165*eb1db165Stc * nxge_lb_ext10. 166678453a8Sspeer */ 167678453a8Sspeer if (nxgep->statsp->port_stats.lb_mode < 168*eb1db165Stc nxge_lb_phy10g) { 169678453a8Sspeer if (!statsp->mac_stats.link_up) { 170678453a8Sspeer freemsg(mp); 171678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 172678453a8Sspeer "==> nxge_start: " 173678453a8Sspeer "link not up")); 174678453a8Sspeer goto nxge_start_fail1; 175678453a8Sspeer } 176321febdeSsbehera } 177678453a8Sspeer break; 17844961713Sgirish } 179678453a8Sspeer } 180678453a8Sspeer 181678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 182678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 183678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 184678453a8Sspeer "==> nxge_start: hardware not initialized or stopped")); 185678453a8Sspeer freemsg(mp); 186678453a8Sspeer goto nxge_start_fail1; 18744961713Sgirish } 18844961713Sgirish 1893d16f8e7Sml if (nxgep->soft_lso_enable) { 19030ac2e7bSml mp_chain = nxge_lso_eliminate(mp); 19130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 19230ac2e7bSml "==> nxge_start(0): LSO mp $%p mp_chain $%p", 19330ac2e7bSml mp, mp_chain)); 19430ac2e7bSml if (mp_chain == NULL) { 19530ac2e7bSml NXGE_ERROR_MSG((nxgep, TX_CTL, 19630ac2e7bSml "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 19730ac2e7bSml mp_chain, mp)); 19830ac2e7bSml goto nxge_start_fail1; 19930ac2e7bSml } 20030ac2e7bSml if (mp_chain != mp) { 20130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 20230ac2e7bSml "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 20330ac2e7bSml mp_chain, mp)); 20430ac2e7bSml is_lso = B_TRUE; 20530ac2e7bSml mp = mp_chain; 20630ac2e7bSml mp_chain = mp_chain->b_next; 20730ac2e7bSml mp->b_next = NULL; 20830ac2e7bSml } 20930ac2e7bSml } 21030ac2e7bSml 21144961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 21252ccf843Smisaki &stuff_offset, &end_offset, &value, &cksum_flags); 21344961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 21444961713Sgirish start_offset += sizeof (ether_header_t); 21544961713Sgirish stuff_offset += sizeof (ether_header_t); 21644961713Sgirish } else { 21744961713Sgirish start_offset += sizeof (struct ether_vlan_header); 21844961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 21944961713Sgirish } 22044961713Sgirish 22144961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 22244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 22352ccf843Smisaki "==> nxge_start: mp $%p len %d " 22452ccf843Smisaki "cksum_flags 0x%x (partial checksum) ", 22552ccf843Smisaki mp, MBLKL(mp), cksum_flags)); 22644961713Sgirish cksum_on = B_TRUE; 22744961713Sgirish } 22844961713Sgirish 229b4d05839Sml pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 230b4d05839Sml pkthdrp->reserved = 0; 231b4d05839Sml tmp_hdrp.value = 0; 232b4d05839Sml nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 233b4d05839Sml 0, 0, pkthdrp, 234b4d05839Sml start_offset, stuff_offset); 235b4d05839Sml 23630ac2e7bSml lso_again = B_FALSE; 23730ac2e7bSml lso_ngathers = 0; 23830ac2e7bSml 23930ac2e7bSml MUTEX_ENTER(&tx_ring_p->lock); 24022c0d73aSspeer 24122c0d73aSspeer if (isLDOMservice(nxgep)) { 2426895688eSspeer tx_ring_p->tx_ring_busy = B_TRUE; 24322c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 24422c0d73aSspeer freemsg(mp); 2456895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 2466895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 2476895688eSspeer NXGE_TX_RING_OFFLINED); 24822c0d73aSspeer MUTEX_EXIT(&tx_ring_p->lock); 24922c0d73aSspeer return (status); 25022c0d73aSspeer } 25122c0d73aSspeer } 25222c0d73aSspeer 25330ac2e7bSml cur_index_lso = tx_ring_p->wr_index; 25430ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 25530ac2e7bSml start_again: 25630ac2e7bSml ngathers = 0; 25730ac2e7bSml sop_index = tx_ring_p->wr_index; 25844961713Sgirish #ifdef NXGE_DEBUG 25944961713Sgirish if (tx_ring_p->descs_pending) { 26044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 26152ccf843Smisaki "desc pending %d ", tx_ring_p->descs_pending)); 26244961713Sgirish } 26344961713Sgirish 26444961713Sgirish dump_len = (int)(MBLKL(mp)); 26544961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 26644961713Sgirish 26744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 26852ccf843Smisaki "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 26952ccf843Smisaki "(Before header reserve: ORIGINAL LEN %d)", 27052ccf843Smisaki tx_ring_p->tdc, 27152ccf843Smisaki mp->b_rptr, 27252ccf843Smisaki dump_len)); 27344961713Sgirish 27444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 27552ccf843Smisaki "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 27652ccf843Smisaki nxge_dump_packet((char *)mp->b_rptr, dump_len))); 27744961713Sgirish #endif 27844961713Sgirish 27944961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 28044961713Sgirish mark_mode = (tx_ring_p->descs_pending && 28152ccf843Smisaki ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 28252ccf843Smisaki < nxge_tx_minfree)); 28344961713Sgirish 28444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 28552ccf843Smisaki "TX Descriptor ring is channel %d mark mode %d", 28652ccf843Smisaki tx_ring_p->tdc, mark_mode)); 28744961713Sgirish 2887127d9f6Sml if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 2897127d9f6Sml if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 2907127d9f6Sml (nxge_tx_minfree + lso_ngathers))) { 2917127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 2927127d9f6Sml "TX Descriptor ring is full: channel %d", 2937127d9f6Sml tx_ring_p->tdc)); 29430ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 2957127d9f6Sml "TX Descriptor ring is full: channel %d", 29630ac2e7bSml tx_ring_p->tdc)); 2977127d9f6Sml if (is_lso) { 2987127d9f6Sml /* 2997127d9f6Sml * free the current mp and mp_chain if not FULL. 3007127d9f6Sml */ 3017127d9f6Sml tdc_stats->tx_no_desc++; 3027127d9f6Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, 3037127d9f6Sml "LSO packet: TX Descriptor ring is full: " 3047127d9f6Sml "channel %d", 3057127d9f6Sml tx_ring_p->tdc)); 3067127d9f6Sml goto nxge_start_fail_lso; 3077127d9f6Sml } else { 3087127d9f6Sml boolean_t skip_sched = B_FALSE; 30922c0d73aSspeer 3107127d9f6Sml cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 3117127d9f6Sml tdc_stats->tx_no_desc++; 3126895688eSspeer 3137127d9f6Sml if (isLDOMservice(nxgep)) { 3147127d9f6Sml tx_ring_p->tx_ring_busy = B_FALSE; 3157127d9f6Sml if (tx_ring_p->tx_ring_offline) { 3167127d9f6Sml (void) atomic_swap_32( 3177127d9f6Sml &tx_ring_p->tx_ring_offline, 3187127d9f6Sml NXGE_TX_RING_OFFLINED); 3197127d9f6Sml skip_sched = B_TRUE; 3207127d9f6Sml } 3216895688eSspeer } 32222c0d73aSspeer 3237127d9f6Sml MUTEX_EXIT(&tx_ring_p->lock); 3247127d9f6Sml if (nxgep->resched_needed && 3257127d9f6Sml !nxgep->resched_running && !skip_sched) { 3267127d9f6Sml nxgep->resched_running = B_TRUE; 3277127d9f6Sml ddi_trigger_softintr(nxgep->resched_id); 3287127d9f6Sml } 3297127d9f6Sml status = 1; 3307127d9f6Sml goto nxge_start_fail1; 33130ac2e7bSml } 33244961713Sgirish } 33344961713Sgirish } 33444961713Sgirish 33544961713Sgirish nmp = mp; 33644961713Sgirish i = sop_index = tx_ring_p->wr_index; 33744961713Sgirish nmblks = 0; 33844961713Sgirish ngathers = 0; 33944961713Sgirish pkt_len = 0; 34044961713Sgirish pack_len = 0; 34144961713Sgirish clen = 0; 34244961713Sgirish last_bidx = -1; 34344961713Sgirish good_packet = B_TRUE; 34444961713Sgirish 34544961713Sgirish desc_area = tx_ring_p->tdc_desc; 34644961713Sgirish npi_handle = desc_area.npi_handle; 34744961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 34852ccf843Smisaki DMA_COMMON_ACC_HANDLE(desc_area); 34944961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 35044961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 35152ccf843Smisaki DMA_COMMON_HANDLE(desc_area); 35244961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 35344961713Sgirish 35444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 35552ccf843Smisaki sop_index, i)); 35644961713Sgirish 35744961713Sgirish #ifdef NXGE_DEBUG 35844961713Sgirish msgsize = msgdsize(nmp); 35944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 36052ccf843Smisaki "==> nxge_start(1): wr_index %d i %d msgdsize %d", 36152ccf843Smisaki sop_index, i, msgsize)); 36244961713Sgirish #endif 36344961713Sgirish /* 36444961713Sgirish * The first 16 bytes of the premapped buffer are reserved 36544961713Sgirish * for header. No padding will be used. 36644961713Sgirish */ 36744961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 3681f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 36944961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 37044961713Sgirish } else { 37144961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 37244961713Sgirish } 37344961713Sgirish while (nmp) { 37444961713Sgirish good_packet = B_TRUE; 37544961713Sgirish b_rptr = nmp->b_rptr; 37644961713Sgirish len = MBLKL(nmp); 37744961713Sgirish if (len <= 0) { 37844961713Sgirish nmp = nmp->b_cont; 37944961713Sgirish continue; 38044961713Sgirish } 38144961713Sgirish nmblks++; 38244961713Sgirish 38344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 38452ccf843Smisaki "len %d pkt_len %d pack_len %d", 38552ccf843Smisaki nmblks, len, pkt_len, pack_len)); 38644961713Sgirish /* 38714ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 38814ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 38914ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 39014ea4bb7Ssd * If len is longer than the limit, then we break nmp into 39114ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 39214ea4bb7Ssd * the second chunk for the remaining data. If the second 39314ea4bb7Ssd * chunk is still larger than the limit, then it will be 39414ea4bb7Ssd * broken into two in the next pass. 39544961713Sgirish */ 39614ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 39753f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 39853f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 39953f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 40053f3d8ecSyc - TX_PKT_HEADER_SIZE); 40153f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 40253f3d8ecSyc t_mp->b_cont = nmp->b_cont; 40353f3d8ecSyc nmp->b_cont = t_mp; 40453f3d8ecSyc len = MBLKL(nmp); 40553f3d8ecSyc } else { 40630ac2e7bSml if (is_lso) { 40730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 40830ac2e7bSml "LSO packet: dupb failed: " 40930ac2e7bSml "channel %d", 41030ac2e7bSml tx_ring_p->tdc)); 41130ac2e7bSml mp = nmp; 41230ac2e7bSml goto nxge_start_fail_lso; 41330ac2e7bSml } else { 41430ac2e7bSml good_packet = B_FALSE; 41530ac2e7bSml goto nxge_start_fail2; 41630ac2e7bSml } 41753f3d8ecSyc } 41844961713Sgirish } 41944961713Sgirish tx_desc.value = 0; 42044961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 42144961713Sgirish #ifdef NXGE_DEBUG 42244961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 42344961713Sgirish #endif 42444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 425adfcba55Sjoycey #if defined(__i386) 426adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 427adfcba55Sjoycey #else 42844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 429adfcba55Sjoycey #endif 43044961713Sgirish if (!header_set && 43152ccf843Smisaki ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 43252ccf843Smisaki (len >= bcopy_thresh))) { 43344961713Sgirish header_set = B_TRUE; 43444961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 43544961713Sgirish boff = 0; 43644961713Sgirish pack_len = 0; 43744961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 43844961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 43944961713Sgirish clen = pkt_len; 44044961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 44144961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 44244961713Sgirish (void) ddi_dma_sync(dma_handle, 44352ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 44452ccf843Smisaki DDI_DMA_SYNC_FORDEV); 44544961713Sgirish 44644961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 44744961713Sgirish goto nxge_start_control_header_only; 44844961713Sgirish } 44944961713Sgirish 45044961713Sgirish pkt_len += len; 45144961713Sgirish pack_len += len; 45244961713Sgirish 45344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 45452ccf843Smisaki "desc entry %d " 45552ccf843Smisaki "DESC IOADDR $%p " 45652ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 45752ccf843Smisaki "desc_pp $%p tx_desc_pp $%p " 45852ccf843Smisaki "len %d pkt_len %d pack_len %d", 45952ccf843Smisaki i, 46052ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 46152ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 46252ccf843Smisaki tx_desc_ring_pp, tx_desc_pp, 46352ccf843Smisaki len, pkt_len, pack_len)); 46444961713Sgirish 46544961713Sgirish if (len < bcopy_thresh) { 46644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 46752ccf843Smisaki "USE BCOPY: ")); 46844961713Sgirish if (nxge_tx_tiny_pack) { 46944961713Sgirish uint32_t blst = 47052ccf843Smisaki TXDMA_DESC_NEXT_INDEX(i, -1, 47152ccf843Smisaki tx_ring_p->tx_wrap_mask); 47244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 47352ccf843Smisaki "==> nxge_start(5): pack")); 47444961713Sgirish if ((pack_len <= bcopy_thresh) && 47552ccf843Smisaki (last_bidx == blst)) { 47644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 47752ccf843Smisaki "==> nxge_start: pack(6) " 47852ccf843Smisaki "(pkt_len %d pack_len %d)", 47952ccf843Smisaki pkt_len, pack_len)); 48044961713Sgirish i = blst; 48144961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 48244961713Sgirish #ifdef NXGE_DEBUG 48344961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 48444961713Sgirish #endif 48544961713Sgirish tx_msg_p = &tx_msg_ring[i]; 48644961713Sgirish boff = pack_len - len; 48744961713Sgirish ngathers--; 488a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 48952ccf843Smisaki header_set) { 49044961713Sgirish pack_len = len; 49144961713Sgirish boff = 0; 49244961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 49344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 49452ccf843Smisaki "==> nxge_start(7): > max NEW " 49552ccf843Smisaki "bcopy thresh %d " 49652ccf843Smisaki "pkt_len %d pack_len %d(next)", 49752ccf843Smisaki bcopy_thresh, 49852ccf843Smisaki pkt_len, pack_len)); 49944961713Sgirish } 50044961713Sgirish last_bidx = i; 50144961713Sgirish } 50244961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 50344961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 50444961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 50544961713Sgirish header_set = B_TRUE; 50644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 50752ccf843Smisaki "==> nxge_start(7_x2): " 50852ccf843Smisaki "pkt_len %d pack_len %d (new hdrp $%p)", 50952ccf843Smisaki pkt_len, pack_len, hdrp)); 51044961713Sgirish } 51144961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 51244961713Sgirish kaddr += boff; 51344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 51452ccf843Smisaki "USE BCOPY: before bcopy " 51552ccf843Smisaki "DESC IOADDR $%p entry %d " 51652ccf843Smisaki "bcopy packets %d " 51752ccf843Smisaki "bcopy kaddr $%p " 51852ccf843Smisaki "bcopy ioaddr (SAD) $%p " 51952ccf843Smisaki "bcopy clen %d " 52052ccf843Smisaki "bcopy boff %d", 52152ccf843Smisaki DMA_COMMON_IOADDR(desc_area), i, 52252ccf843Smisaki tdc_stats->tx_hdr_pkts, 52352ccf843Smisaki kaddr, 52452ccf843Smisaki dma_ioaddr, 52552ccf843Smisaki clen, 52652ccf843Smisaki boff)); 52744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 52852ccf843Smisaki "1USE BCOPY: ")); 52944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 53052ccf843Smisaki "2USE BCOPY: ")); 53144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 53252ccf843Smisaki "last USE BCOPY: copy from b_rptr $%p " 53352ccf843Smisaki "to KADDR $%p (len %d offset %d", 53452ccf843Smisaki b_rptr, kaddr, len, boff)); 53544961713Sgirish 53644961713Sgirish bcopy(b_rptr, kaddr, len); 53744961713Sgirish 53844961713Sgirish #ifdef NXGE_DEBUG 53944961713Sgirish dump_len = (len > 128) ? 128: len; 54044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 54152ccf843Smisaki "==> nxge_start: dump packets " 54252ccf843Smisaki "(After BCOPY len %d)" 54352ccf843Smisaki "(b_rptr $%p): %s", len, nmp->b_rptr, 54452ccf843Smisaki nxge_dump_packet((char *)nmp->b_rptr, 54552ccf843Smisaki dump_len))); 54644961713Sgirish #endif 54744961713Sgirish 54844961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 54944961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 55044961713Sgirish (void) ddi_dma_sync(dma_handle, 55152ccf843Smisaki i * nxge_bcopy_thresh, nxge_bcopy_thresh, 55252ccf843Smisaki DDI_DMA_SYNC_FORDEV); 55344961713Sgirish clen = len + boff; 55444961713Sgirish tdc_stats->tx_hdr_pkts++; 55544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 55652ccf843Smisaki "USE BCOPY: " 55752ccf843Smisaki "DESC IOADDR $%p entry %d " 55852ccf843Smisaki "bcopy packets %d " 55952ccf843Smisaki "bcopy kaddr $%p " 56052ccf843Smisaki "bcopy ioaddr (SAD) $%p " 56152ccf843Smisaki "bcopy clen %d " 56252ccf843Smisaki "bcopy boff %d", 56352ccf843Smisaki DMA_COMMON_IOADDR(desc_area), 56452ccf843Smisaki i, 56552ccf843Smisaki tdc_stats->tx_hdr_pkts, 56652ccf843Smisaki kaddr, 56752ccf843Smisaki dma_ioaddr, 56852ccf843Smisaki clen, 56952ccf843Smisaki boff)); 57044961713Sgirish } else { 57144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 57252ccf843Smisaki "USE DVMA: len %d", len)); 57344961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 57444961713Sgirish dma_flags = DDI_DMA_WRITE; 57544961713Sgirish if (len < nxge_dma_stream_thresh) { 57644961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 57744961713Sgirish } else { 57844961713Sgirish dma_flags |= DDI_DMA_STREAMING; 57944961713Sgirish } 58044961713Sgirish 58144961713Sgirish dma_handle = tx_msg_p->dma_handle; 58244961713Sgirish status = ddi_dma_addr_bind_handle(dma_handle, NULL, 58352ccf843Smisaki (caddr_t)b_rptr, len, dma_flags, 58452ccf843Smisaki DDI_DMA_DONTWAIT, NULL, 58552ccf843Smisaki &dma_cookie, &ncookies); 58644961713Sgirish if (status == DDI_DMA_MAPPED) { 58744961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 58844961713Sgirish len = (int)dma_cookie.dmac_size; 58944961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 59044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 59152ccf843Smisaki "==> nxge_start(12_1): " 59252ccf843Smisaki "USE DVMA: len %d clen %d " 59352ccf843Smisaki "ngathers %d", 59452ccf843Smisaki len, clen, 59552ccf843Smisaki ngathers)); 596adfcba55Sjoycey #if defined(__i386) 597adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 598adfcba55Sjoycey #else 59944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 600adfcba55Sjoycey #endif 60144961713Sgirish while (ncookies > 1) { 60244961713Sgirish ngathers++; 60344961713Sgirish /* 60444961713Sgirish * this is the fix for multiple 60530ac2e7bSml * cookies, which are basically 60644961713Sgirish * a descriptor entry, we don't set 60744961713Sgirish * SOP bit as well as related fields 60844961713Sgirish */ 60944961713Sgirish 61044961713Sgirish (void) npi_txdma_desc_gather_set( 61152ccf843Smisaki npi_desc_handle, 61252ccf843Smisaki &tx_desc, 61352ccf843Smisaki (ngathers -1), 61452ccf843Smisaki mark_mode, 61552ccf843Smisaki ngathers, 61652ccf843Smisaki dma_ioaddr, 61752ccf843Smisaki clen); 61844961713Sgirish 61944961713Sgirish tx_msg_p->tx_msg_size = clen; 62044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 62152ccf843Smisaki "==> nxge_start: DMA " 62252ccf843Smisaki "ncookie %d " 62352ccf843Smisaki "ngathers %d " 62452ccf843Smisaki "dma_ioaddr $%p len %d" 62552ccf843Smisaki "desc $%p descp $%p (%d)", 62652ccf843Smisaki ncookies, 62752ccf843Smisaki ngathers, 62852ccf843Smisaki dma_ioaddr, clen, 62952ccf843Smisaki *tx_desc_p, tx_desc_p, i)); 63044961713Sgirish 63144961713Sgirish ddi_dma_nextcookie(dma_handle, 63252ccf843Smisaki &dma_cookie); 63344961713Sgirish dma_ioaddr = 63452ccf843Smisaki dma_cookie.dmac_laddress; 63544961713Sgirish 63644961713Sgirish len = (int)dma_cookie.dmac_size; 63744961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 63844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 63952ccf843Smisaki "==> nxge_start(12_2): " 64052ccf843Smisaki "USE DVMA: len %d clen %d ", 64152ccf843Smisaki len, clen)); 64244961713Sgirish 64344961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 64452ccf843Smisaki tx_ring_p->tx_wrap_mask); 64544961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 64644961713Sgirish 647adfcba55Sjoycey #if defined(__i386) 64852ccf843Smisaki npi_desc_handle.regp = 64952ccf843Smisaki (uint32_t)tx_desc_p; 650adfcba55Sjoycey #else 65152ccf843Smisaki npi_desc_handle.regp = 65252ccf843Smisaki (uint64_t)tx_desc_p; 653adfcba55Sjoycey #endif 65444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 65544961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 65644961713Sgirish tx_desc.value = 0; 65744961713Sgirish 65844961713Sgirish ncookies--; 65944961713Sgirish } 66044961713Sgirish tdc_stats->tx_ddi_pkts++; 66144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 66252ccf843Smisaki "DMA: ddi packets %d", 66352ccf843Smisaki tdc_stats->tx_ddi_pkts)); 66444961713Sgirish } else { 66544961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 66644961713Sgirish "dma mapping failed for %d " 66744961713Sgirish "bytes addr $%p flags %x (%d)", 66844961713Sgirish len, b_rptr, status, status)); 66944961713Sgirish good_packet = B_FALSE; 67044961713Sgirish tdc_stats->tx_dma_bind_fail++; 67144961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 67230ac2e7bSml if (is_lso) { 67330ac2e7bSml mp = nmp; 67430ac2e7bSml goto nxge_start_fail_lso; 67530ac2e7bSml } else { 67630ac2e7bSml goto nxge_start_fail2; 67730ac2e7bSml } 67844961713Sgirish } 67944961713Sgirish } /* ddi dvma */ 68044961713Sgirish 68130ac2e7bSml if (is_lso) { 68230ac2e7bSml nmp_lso_save = nmp; 68330ac2e7bSml } 68444961713Sgirish nmp = nmp->b_cont; 68544961713Sgirish nxge_start_control_header_only: 686adfcba55Sjoycey #if defined(__i386) 687adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 688adfcba55Sjoycey #else 68944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 690adfcba55Sjoycey #endif 69144961713Sgirish ngathers++; 69244961713Sgirish 69344961713Sgirish if (ngathers == 1) { 69444961713Sgirish #ifdef NXGE_DEBUG 69544961713Sgirish save_desc_p = &sop_tx_desc; 69644961713Sgirish #endif 69744961713Sgirish sop_tx_desc_p = &sop_tx_desc; 69844961713Sgirish sop_tx_desc_p->value = 0; 69944961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 70044961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 70144961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 70244961713Sgirish } else { 70344961713Sgirish #ifdef NXGE_DEBUG 70444961713Sgirish save_desc_p = &tx_desc; 70544961713Sgirish #endif 70644961713Sgirish tmp_desc_p = &tx_desc; 70744961713Sgirish tmp_desc_p->value = 0; 70844961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 70944961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 71044961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 71144961713Sgirish 71244961713Sgirish tx_desc_p->value = tmp_desc_p->value; 71344961713Sgirish } 71444961713Sgirish 71544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 71652ccf843Smisaki "Desc_entry %d ngathers %d " 71752ccf843Smisaki "desc_vp $%p tx_desc_p $%p " 71852ccf843Smisaki "len %d clen %d pkt_len %d pack_len %d nmblks %d " 71952ccf843Smisaki "dma_ioaddr (SAD) $%p mark %d", 72052ccf843Smisaki i, ngathers, 72152ccf843Smisaki tx_desc_ring_vp, tx_desc_p, 72252ccf843Smisaki len, clen, pkt_len, pack_len, nmblks, 72352ccf843Smisaki dma_ioaddr, mark_mode)); 72444961713Sgirish 72544961713Sgirish #ifdef NXGE_DEBUG 72644961713Sgirish npi_desc_handle.nxgep = nxgep; 72744961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 72844961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 72944961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 73044961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 73152ccf843Smisaki TX_PKT_DESC_TR_LEN_SHIFT); 73244961713Sgirish 73344961713Sgirish 73444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 73552ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 73652ccf843Smisaki "mark %d sop %d\n", 73752ccf843Smisaki save_desc_p->value, 73852ccf843Smisaki sad, 73952ccf843Smisaki save_desc_p->bits.hdw.tr_len, 74052ccf843Smisaki xfer_len, 74152ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 74252ccf843Smisaki save_desc_p->bits.hdw.mark, 74352ccf843Smisaki save_desc_p->bits.hdw.sop)); 74444961713Sgirish 74544961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 74644961713Sgirish #endif 74744961713Sgirish 74844961713Sgirish tx_msg_p->tx_msg_size = clen; 74944961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 75044961713Sgirish if (ngathers > nxge_tx_max_gathers) { 75144961713Sgirish good_packet = B_FALSE; 75244961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 75352ccf843Smisaki &stuff_offset, &end_offset, &value, 75452ccf843Smisaki &cksum_flags); 75544961713Sgirish 75644961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 75752ccf843Smisaki "==> nxge_start(14): pull msg - " 75852ccf843Smisaki "len %d pkt_len %d ngathers %d", 75952ccf843Smisaki len, pkt_len, ngathers)); 76044961713Sgirish /* Pull all message blocks from b_cont */ 76130ac2e7bSml if (is_lso) { 76230ac2e7bSml mp = nmp_lso_save; 76330ac2e7bSml goto nxge_start_fail_lso; 76430ac2e7bSml } 76544961713Sgirish if ((msgpullup(mp, -1)) == NULL) { 76644961713Sgirish goto nxge_start_fail2; 76744961713Sgirish } 76844961713Sgirish goto nxge_start_fail2; 76944961713Sgirish } 77044961713Sgirish } /* while (nmp) */ 77144961713Sgirish 77244961713Sgirish tx_msg_p->tx_message = mp; 77344961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 774adfcba55Sjoycey #if defined(__i386) 775adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 776adfcba55Sjoycey #else 77744961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 778adfcba55Sjoycey #endif 77944961713Sgirish 78044961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 78144961713Sgirish pkthdrp->reserved = 0; 78244961713Sgirish hdrp->value = 0; 783b4d05839Sml bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 78444961713Sgirish 78544961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 78644961713Sgirish tdc_stats->tx_jumbo_pkts++; 78744961713Sgirish } 78844961713Sgirish 789678453a8Sspeer min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 79044961713Sgirish if (pkt_len < min_len) { 79144961713Sgirish /* Assume we use bcopy to premapped buffers */ 79244961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 79344961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 79452ccf843Smisaki "==> nxge_start(14-1): < (msg_min + 16)" 79552ccf843Smisaki "len %d pkt_len %d min_len %d bzero %d ngathers %d", 79652ccf843Smisaki len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 79744961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 79844961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 79944961713Sgirish 80044961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 80144961713Sgirish 80244961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 80344961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 80444961713Sgirish 80544961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 80652ccf843Smisaki "==> nxge_start(14-2): < msg_min - " 80752ccf843Smisaki "len %d pkt_len %d min_len %d ngathers %d", 80852ccf843Smisaki len, pkt_len, min_len, ngathers)); 80944961713Sgirish } 81044961713Sgirish 81144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 81252ccf843Smisaki cksum_flags)); 81344961713Sgirish { 81444961713Sgirish uint64_t tmp_len; 81544961713Sgirish 81644961713Sgirish /* pkt_len already includes 16 + paddings!! */ 81744961713Sgirish /* Update the control header length */ 81844961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 81944961713Sgirish tmp_len = hdrp->value | 82052ccf843Smisaki (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 82144961713Sgirish 82244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 82352ccf843Smisaki "==> nxge_start(15_x1): setting SOP " 82452ccf843Smisaki "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 82552ccf843Smisaki "0x%llx hdrp->value 0x%llx", 82652ccf843Smisaki tot_xfer_len, tot_xfer_len, pkt_len, 82752ccf843Smisaki tmp_len, hdrp->value)); 82844961713Sgirish #if defined(_BIG_ENDIAN) 82944961713Sgirish hdrp->value = ddi_swap64(tmp_len); 83044961713Sgirish #else 83144961713Sgirish hdrp->value = tmp_len; 83244961713Sgirish #endif 83344961713Sgirish NXGE_DEBUG_MSG((nxgep, 83452ccf843Smisaki TX_CTL, "==> nxge_start(15_x2): setting SOP " 83552ccf843Smisaki "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 83652ccf843Smisaki "tmp_len 0x%llx hdrp->value 0x%llx", 83752ccf843Smisaki tot_xfer_len, pkt_len, 83852ccf843Smisaki tmp_len, hdrp->value)); 83944961713Sgirish } 84044961713Sgirish 84144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 84252ccf843Smisaki "wr_index %d " 84352ccf843Smisaki "tot_xfer_len (%d) pkt_len %d npads %d", 84452ccf843Smisaki sop_index, 84552ccf843Smisaki tot_xfer_len, pkt_len, 84652ccf843Smisaki npads)); 84744961713Sgirish 84844961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 84944961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 85044961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 85144961713Sgirish 85244961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 85344961713Sgirish 85444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 85544961713Sgirish 85644961713Sgirish #ifdef NXGE_DEBUG 85744961713Sgirish npi_desc_handle.nxgep = nxgep; 85844961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 85944961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 86044961713Sgirish 86144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 86252ccf843Smisaki "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 86352ccf843Smisaki save_desc_p->value, 86452ccf843Smisaki sad, 86552ccf843Smisaki save_desc_p->bits.hdw.tr_len, 86652ccf843Smisaki xfer_len, 86752ccf843Smisaki save_desc_p->bits.hdw.num_ptr, 86852ccf843Smisaki save_desc_p->bits.hdw.mark, 86952ccf843Smisaki save_desc_p->bits.hdw.sop)); 87044961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 87144961713Sgirish 87244961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 87344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 87452ccf843Smisaki "==> nxge_start: dump packets(17) (after sop set, len " 87552ccf843Smisaki " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 87652ccf843Smisaki "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 87752ccf843Smisaki (char *)hdrp, 87852ccf843Smisaki nxge_dump_packet((char *)hdrp, dump_len))); 87944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 88052ccf843Smisaki "==> nxge_start(18): TX desc sync: sop_index %d", 88152ccf843Smisaki sop_index)); 88244961713Sgirish #endif 88344961713Sgirish 88444961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 88544961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 88652ccf843Smisaki sop_index * sizeof (tx_desc_t), 88752ccf843Smisaki ngathers * sizeof (tx_desc_t), 88852ccf843Smisaki DDI_DMA_SYNC_FORDEV); 88944961713Sgirish 89044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 89152ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 89252ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 89352ccf843Smisaki stuff_offset, start_offset, 89452ccf843Smisaki pkt_len, ngathers, sop_index)); 89544961713Sgirish } else { /* more than one descriptor and wrap around */ 89644961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 89744961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 89852ccf843Smisaki sop_index * sizeof (tx_desc_t), 89952ccf843Smisaki nsdescs * sizeof (tx_desc_t), 90052ccf843Smisaki DDI_DMA_SYNC_FORDEV); 90144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 90252ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 90352ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 90452ccf843Smisaki stuff_offset, start_offset, 90552ccf843Smisaki pkt_len, ngathers, sop_index)); 90644961713Sgirish 90744961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 90852ccf843Smisaki 0, 90952ccf843Smisaki (ngathers - nsdescs) * sizeof (tx_desc_t), 91052ccf843Smisaki DDI_DMA_SYNC_FORDEV); 91144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 91252ccf843Smisaki "cs_off = 0x%02X cs_s_off = 0x%02X " 91352ccf843Smisaki "pkt_len %d ngathers %d sop_index %d\n", 91452ccf843Smisaki stuff_offset, start_offset, 91552ccf843Smisaki pkt_len, ngathers, sop_index)); 91644961713Sgirish } 91744961713Sgirish 91844961713Sgirish tail_index = tx_ring_p->wr_index; 91944961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 92044961713Sgirish 92144961713Sgirish tx_ring_p->wr_index = i; 92244961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 92344961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 92452ccf843Smisaki B_FALSE : B_TRUE); 92544961713Sgirish } 92644961713Sgirish 92744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 92852ccf843Smisaki "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 92952ccf843Smisaki tx_ring_p->tdc, 93052ccf843Smisaki tx_ring_p->wr_index, 93152ccf843Smisaki tx_ring_p->wr_index_wrap, 93252ccf843Smisaki ngathers, 93352ccf843Smisaki tx_ring_p->descs_pending)); 93444961713Sgirish 93530ac2e7bSml if (is_lso) { 93630ac2e7bSml lso_ngathers += ngathers; 93730ac2e7bSml if (mp_chain != NULL) { 93830ac2e7bSml mp = mp_chain; 93930ac2e7bSml mp_chain = mp_chain->b_next; 94030ac2e7bSml mp->b_next = NULL; 94130ac2e7bSml if (nxge_lso_kick_cnt == lso_ngathers) { 9427a6dff21Sml tx_ring_p->descs_pending += lso_ngathers; 94330ac2e7bSml { 94430ac2e7bSml tx_ring_kick_t kick; 94530ac2e7bSml 94630ac2e7bSml kick.value = 0; 94730ac2e7bSml kick.bits.ldw.wrap = 94830ac2e7bSml tx_ring_p->wr_index_wrap; 94930ac2e7bSml kick.bits.ldw.tail = 95030ac2e7bSml (uint16_t)tx_ring_p->wr_index; 95130ac2e7bSml 95230ac2e7bSml /* Kick the Transmit kick register */ 95330ac2e7bSml TXDMA_REG_WRITE64( 95430ac2e7bSml NXGE_DEV_NPI_HANDLE(nxgep), 95530ac2e7bSml TX_RING_KICK_REG, 95630ac2e7bSml (uint8_t)tx_ring_p->tdc, 95730ac2e7bSml kick.value); 95830ac2e7bSml tdc_stats->tx_starts++; 959678453a8Sspeer 96030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 96130ac2e7bSml "==> nxge_start: more LSO: " 96230ac2e7bSml "LSO_CNT %d", 963678453a8Sspeer lso_ngathers)); 96430ac2e7bSml } 96530ac2e7bSml lso_ngathers = 0; 96630ac2e7bSml ngathers = 0; 96730ac2e7bSml cur_index_lso = sop_index = tx_ring_p->wr_index; 96830ac2e7bSml lso_tail_wrap = tx_ring_p->wr_index_wrap; 96930ac2e7bSml } 97030ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 97130ac2e7bSml "==> nxge_start: lso again: " 97230ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 97330ac2e7bSml "wr_index %d sop_index %d", 97430ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 97530ac2e7bSml tx_ring_p->wr_index, sop_index)); 97630ac2e7bSml 97730ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 97830ac2e7bSml "==> nxge_start: next : count %d", 979678453a8Sspeer lso_ngathers)); 98030ac2e7bSml lso_again = B_TRUE; 98130ac2e7bSml goto start_again; 98230ac2e7bSml } 9837a6dff21Sml ngathers = lso_ngathers; 98430ac2e7bSml } 98530ac2e7bSml 98644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 98744961713Sgirish 98844961713Sgirish { 98944961713Sgirish tx_ring_kick_t kick; 99044961713Sgirish 99144961713Sgirish kick.value = 0; 99244961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 99344961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 99444961713Sgirish 99544961713Sgirish /* Kick start the Transmit kick register */ 99644961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 99752ccf843Smisaki TX_RING_KICK_REG, 99852ccf843Smisaki (uint8_t)tx_ring_p->tdc, 99952ccf843Smisaki kick.value); 100044961713Sgirish } 100144961713Sgirish 10027a6dff21Sml tx_ring_p->descs_pending += ngathers; 100344961713Sgirish tdc_stats->tx_starts++; 100444961713Sgirish 10056895688eSspeer if (isLDOMservice(nxgep)) { 10066895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 10076895688eSspeer if (tx_ring_p->tx_ring_offline) { 10086895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 10096895688eSspeer NXGE_TX_RING_OFFLINED); 10106895688eSspeer } 10116895688eSspeer } 1012678453a8Sspeer 101344961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 101444961713Sgirish 10152d99c5d4SMichael Speer nxge_txdma_freemsg_task(tx_ring_p); 10162d99c5d4SMichael Speer 101744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 101844961713Sgirish 101944961713Sgirish return (status); 102044961713Sgirish 102130ac2e7bSml nxge_start_fail_lso: 102230ac2e7bSml status = 0; 102330ac2e7bSml good_packet = B_FALSE; 102430ac2e7bSml if (mp != NULL) { 102530ac2e7bSml freemsg(mp); 102630ac2e7bSml } 102730ac2e7bSml if (mp_chain != NULL) { 102830ac2e7bSml freemsg(mp_chain); 102930ac2e7bSml } 103030ac2e7bSml if (!lso_again && !ngathers) { 10316895688eSspeer if (isLDOMservice(nxgep)) { 10326895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 10336895688eSspeer if (tx_ring_p->tx_ring_offline) { 10346895688eSspeer (void) atomic_swap_32( 10356895688eSspeer &tx_ring_p->tx_ring_offline, 10366895688eSspeer NXGE_TX_RING_OFFLINED); 10376895688eSspeer } 10386895688eSspeer } 10396895688eSspeer 104030ac2e7bSml MUTEX_EXIT(&tx_ring_p->lock); 104130ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 104230ac2e7bSml "==> nxge_start: lso exit (nothing changed)")); 104330ac2e7bSml goto nxge_start_fail1; 104430ac2e7bSml } 104530ac2e7bSml 104630ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 104730ac2e7bSml "==> nxge_start (channel %d): before lso " 104830ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 104930ac2e7bSml "wr_index %d sop_index %d lso_again %d", 105030ac2e7bSml tx_ring_p->tdc, 105130ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 105230ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 105330ac2e7bSml 105430ac2e7bSml if (lso_again) { 105530ac2e7bSml lso_ngathers += ngathers; 105630ac2e7bSml ngathers = lso_ngathers; 105730ac2e7bSml sop_index = cur_index_lso; 105830ac2e7bSml tx_ring_p->wr_index = sop_index; 105930ac2e7bSml tx_ring_p->wr_index_wrap = lso_tail_wrap; 106030ac2e7bSml } 106130ac2e7bSml 106230ac2e7bSml NXGE_DEBUG_MSG((nxgep, TX_CTL, 106330ac2e7bSml "==> nxge_start (channel %d): after lso " 106430ac2e7bSml "lso_gathers %d ngathers %d cur_index_lso %d " 106530ac2e7bSml "wr_index %d sop_index %d lso_again %d", 106630ac2e7bSml tx_ring_p->tdc, 106730ac2e7bSml lso_ngathers, ngathers, cur_index_lso, 106830ac2e7bSml tx_ring_p->wr_index, sop_index, lso_again)); 106930ac2e7bSml 107044961713Sgirish nxge_start_fail2: 107144961713Sgirish if (good_packet == B_FALSE) { 107244961713Sgirish cur_index = sop_index; 107344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 107444961713Sgirish for (i = 0; i < ngathers; i++) { 107544961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 1076adfcba55Sjoycey #if defined(__i386) 1077adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 1078adfcba55Sjoycey #else 107944961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 1080adfcba55Sjoycey #endif 108144961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 108244961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 108344961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 108444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 108553f3d8ecSyc "tx_desc_p = %X index = %d", 108653f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 108753f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 108853f3d8ecSyc 0, -1); 108944961713Sgirish tx_msg_p->dvma_handle = NULL; 109044961713Sgirish if (tx_ring_p->dvma_wr_index == 109153f3d8ecSyc tx_ring_p->dvma_wrap_mask) 109244961713Sgirish tx_ring_p->dvma_wr_index = 0; 109344961713Sgirish else 109444961713Sgirish tx_ring_p->dvma_wr_index++; 109544961713Sgirish tx_ring_p->dvma_pending--; 109653f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 109744961713Sgirish if (ddi_dma_unbind_handle( 109853f3d8ecSyc tx_msg_p->dma_handle)) { 109944961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 110053f3d8ecSyc "ddi_dma_unbind_handle failed"); 110153f3d8ecSyc } 110244961713Sgirish } 110344961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 110444961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 110552ccf843Smisaki tx_ring_p->tx_wrap_mask); 110644961713Sgirish 110744961713Sgirish } 110844961713Sgirish 110944961713Sgirish nxgep->resched_needed = B_TRUE; 111044961713Sgirish } 111144961713Sgirish 11126895688eSspeer if (isLDOMservice(nxgep)) { 11136895688eSspeer tx_ring_p->tx_ring_busy = B_FALSE; 11146895688eSspeer if (tx_ring_p->tx_ring_offline) { 11156895688eSspeer (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 11166895688eSspeer NXGE_TX_RING_OFFLINED); 11176895688eSspeer } 11186895688eSspeer } 1119678453a8Sspeer 112044961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 112144961713Sgirish 112244961713Sgirish nxge_start_fail1: 112344961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 112444961713Sgirish 112544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 112644961713Sgirish 112744961713Sgirish return (status); 112844961713Sgirish } 112944961713Sgirish 11301f8914d5Sml int 11311f8914d5Sml nxge_serial_tx(mblk_t *mp, void *arg) 11321f8914d5Sml { 11331f8914d5Sml p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 11341f8914d5Sml p_nxge_t nxgep = tx_ring_p->nxgep; 113522c0d73aSspeer int status = 0; 113622c0d73aSspeer 113722c0d73aSspeer if (isLDOMservice(nxgep)) { 113822c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 113922c0d73aSspeer freemsg(mp); 114022c0d73aSspeer return (status); 114122c0d73aSspeer } 114222c0d73aSspeer } 114322c0d73aSspeer 114422c0d73aSspeer status = nxge_start(nxgep, tx_ring_p, mp); 114522c0d73aSspeer return (status); 11461f8914d5Sml } 11471f8914d5Sml 114844961713Sgirish boolean_t 114944961713Sgirish nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 115044961713Sgirish { 115144961713Sgirish p_tx_ring_t *tx_rings; 115244961713Sgirish uint8_t ring_index; 11531f8914d5Sml p_tx_ring_t tx_ring_p; 1154678453a8Sspeer nxge_grp_t *group; 115544961713Sgirish 115644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 115744961713Sgirish 115844961713Sgirish ASSERT(mp->b_next == NULL); 115944961713Sgirish 1160678453a8Sspeer group = nxgep->tx_set.group[0]; /* The default group */ 1161678453a8Sspeer ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1162678453a8Sspeer 116344961713Sgirish tx_rings = nxgep->tx_rings->rings; 1164678453a8Sspeer tx_ring_p = tx_rings[group->legend[ring_index]]; 1165678453a8Sspeer 116622c0d73aSspeer if (isLDOMservice(nxgep)) { 116722c0d73aSspeer if (tx_ring_p->tx_ring_offline) { 116822c0d73aSspeer /* 116922c0d73aSspeer * OFFLINE means that it is in the process of being 117022c0d73aSspeer * shared - that is, it has been claimed by the HIO 117122c0d73aSspeer * code, but hasn't been unlinked from <group> yet. 117222c0d73aSspeer * So in this case use the first TDC, which always 117322c0d73aSspeer * belongs to the service domain and can't be shared. 117422c0d73aSspeer */ 117522c0d73aSspeer ring_index = 0; 117622c0d73aSspeer tx_ring_p = tx_rings[group->legend[ring_index]]; 117722c0d73aSspeer } 1178678453a8Sspeer } 1179678453a8Sspeer 1180678453a8Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 118152ccf843Smisaki (int)group->count, group->legend[ring_index], tx_ring_p)); 118244961713Sgirish 11831f8914d5Sml switch (nxge_tx_scheme) { 11841f8914d5Sml case NXGE_USE_START: 1185678453a8Sspeer if (nxge_start(nxgep, tx_ring_p, mp)) { 11861f8914d5Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 118752ccf843Smisaki "ring index %d", ring_index)); 11881f8914d5Sml return (B_FALSE); 11891f8914d5Sml } 11901f8914d5Sml break; 11911f8914d5Sml 11921f8914d5Sml case NXGE_USE_SERIAL: 11931f8914d5Sml default: 11941f8914d5Sml nxge_serialize_enter(tx_ring_p->serial, mp); 11951f8914d5Sml break; 119644961713Sgirish } 119744961713Sgirish 119844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 119952ccf843Smisaki ring_index)); 120044961713Sgirish 120144961713Sgirish return (B_TRUE); 120244961713Sgirish } 120344961713Sgirish 120444961713Sgirish /* 120544961713Sgirish * nxge_m_tx() - send a chain of packets 120644961713Sgirish */ 120744961713Sgirish mblk_t * 120844961713Sgirish nxge_m_tx(void *arg, mblk_t *mp) 120944961713Sgirish { 121044961713Sgirish p_nxge_t nxgep = (p_nxge_t)arg; 121144961713Sgirish mblk_t *next; 121244961713Sgirish mac_tx_hint_t hint; 121344961713Sgirish 1214678453a8Sspeer NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1215678453a8Sspeer 1216678453a8Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1217678453a8Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 121844961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1219678453a8Sspeer "==> nxge_m_tx: hardware not initialized")); 122044961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1221678453a8Sspeer "<== nxge_m_tx")); 1222678453a8Sspeer freemsgchain(mp); 1223678453a8Sspeer mp = NULL; 122444961713Sgirish return (mp); 122544961713Sgirish } 122644961713Sgirish 122744961713Sgirish hint.hash = NULL; 122844961713Sgirish hint.vid = 0; 122944961713Sgirish hint.sap = 0; 123044961713Sgirish 123144961713Sgirish while (mp != NULL) { 123244961713Sgirish next = mp->b_next; 123344961713Sgirish mp->b_next = NULL; 123444961713Sgirish 123544961713Sgirish /* 123644961713Sgirish * Until Nemo tx resource works, the mac driver 123744961713Sgirish * does the load balancing based on TCP port, 123844961713Sgirish * or CPU. For debugging, we use a system 123944961713Sgirish * configurable parameter. 124044961713Sgirish */ 124144961713Sgirish if (!nxge_send(nxgep, mp, &hint)) { 124244961713Sgirish mp->b_next = next; 124344961713Sgirish break; 124444961713Sgirish } 124544961713Sgirish 124644961713Sgirish mp = next; 124730ac2e7bSml 124830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 124930ac2e7bSml "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 125030ac2e7bSml mp, next)); 125144961713Sgirish } 125244961713Sgirish 1253678453a8Sspeer NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 125444961713Sgirish return (mp); 125544961713Sgirish } 125644961713Sgirish 125744961713Sgirish int 125844961713Sgirish nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 125944961713Sgirish { 126044961713Sgirish uint8_t ring_index = 0; 126144961713Sgirish uint8_t *tcp_port; 126244961713Sgirish p_mblk_t nmp; 126344961713Sgirish size_t mblk_len; 126444961713Sgirish size_t iph_len; 126544961713Sgirish size_t hdrs_size; 1266dc8400b9SSantwona Behera uint8_t hdrs_buf[sizeof (struct ether_vlan_header) + 126752ccf843Smisaki IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 126844961713Sgirish /* 126944961713Sgirish * allocate space big enough to cover 127044961713Sgirish * the max ip header length and the first 127144961713Sgirish * 4 bytes of the TCP/IP header. 127244961713Sgirish */ 127344961713Sgirish 127444961713Sgirish boolean_t qos = B_FALSE; 1275dc8400b9SSantwona Behera ushort_t eth_type; 1276dc8400b9SSantwona Behera size_t eth_hdr_size; 127744961713Sgirish 127844961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 127944961713Sgirish 128044961713Sgirish if (hp->vid) { 128144961713Sgirish qos = B_TRUE; 128244961713Sgirish } 128344961713Sgirish switch (nxge_tx_lb_policy) { 128444961713Sgirish case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 128544961713Sgirish default: 128644961713Sgirish tcp_port = mp->b_rptr; 1287dc8400b9SSantwona Behera eth_type = ntohs(((struct ether_header *)tcp_port)->ether_type); 1288dc8400b9SSantwona Behera if (eth_type == VLAN_ETHERTYPE) { 1289dc8400b9SSantwona Behera eth_type = ntohs(((struct ether_vlan_header *) 1290dc8400b9SSantwona Behera tcp_port)->ether_type); 1291dc8400b9SSantwona Behera eth_hdr_size = sizeof (struct ether_vlan_header); 1292dc8400b9SSantwona Behera } else { 1293dc8400b9SSantwona Behera eth_hdr_size = sizeof (struct ether_header); 1294dc8400b9SSantwona Behera } 1295dc8400b9SSantwona Behera 1296dc8400b9SSantwona Behera if (!nxge_no_tx_lb && !qos && eth_type == ETHERTYPE_IP) { 129744961713Sgirish nmp = mp; 129844961713Sgirish mblk_len = MBLKL(nmp); 129944961713Sgirish tcp_port = NULL; 1300dc8400b9SSantwona Behera if (mblk_len > eth_hdr_size + sizeof (uint8_t)) { 1301dc8400b9SSantwona Behera tcp_port = nmp->b_rptr + eth_hdr_size; 1302dc8400b9SSantwona Behera mblk_len -= eth_hdr_size; 130344961713Sgirish iph_len = ((*tcp_port) & 0x0f) << 2; 130444961713Sgirish if (mblk_len > (iph_len + sizeof (uint32_t))) { 130544961713Sgirish tcp_port = nmp->b_rptr; 130644961713Sgirish } else { 130744961713Sgirish tcp_port = NULL; 130844961713Sgirish } 130944961713Sgirish } 131044961713Sgirish if (tcp_port == NULL) { 131144961713Sgirish hdrs_size = 0; 131244961713Sgirish while ((nmp) && (hdrs_size < 131352ccf843Smisaki sizeof (hdrs_buf))) { 131444961713Sgirish mblk_len = MBLKL(nmp); 131544961713Sgirish if (mblk_len >= 131652ccf843Smisaki (sizeof (hdrs_buf) - hdrs_size)) 131744961713Sgirish mblk_len = sizeof (hdrs_buf) - 131852ccf843Smisaki hdrs_size; 131944961713Sgirish bcopy(nmp->b_rptr, 132052ccf843Smisaki &hdrs_buf[hdrs_size], mblk_len); 132144961713Sgirish hdrs_size += mblk_len; 132244961713Sgirish nmp = nmp->b_cont; 132344961713Sgirish } 132444961713Sgirish tcp_port = hdrs_buf; 132544961713Sgirish } 1326dc8400b9SSantwona Behera tcp_port += eth_hdr_size; 132744961713Sgirish if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1328958cea9eSml switch (tcp_port[9]) { 1329958cea9eSml case IPPROTO_TCP: 1330958cea9eSml case IPPROTO_UDP: 1331958cea9eSml case IPPROTO_ESP: 133244961713Sgirish tcp_port += ((*tcp_port) & 0x0f) << 2; 133344961713Sgirish ring_index = 1334958cea9eSml ((tcp_port[0] ^ 1335958cea9eSml tcp_port[1] ^ 1336958cea9eSml tcp_port[2] ^ 1337958cea9eSml tcp_port[3]) % maxtdcs); 1338958cea9eSml break; 1339958cea9eSml 1340958cea9eSml case IPPROTO_AH: 1341958cea9eSml /* SPI starts at the 4th byte */ 1342958cea9eSml tcp_port += ((*tcp_port) & 0x0f) << 2; 1343958cea9eSml ring_index = 1344958cea9eSml ((tcp_port[4] ^ 1345958cea9eSml tcp_port[5] ^ 1346958cea9eSml tcp_port[6] ^ 1347958cea9eSml tcp_port[7]) % maxtdcs); 1348958cea9eSml break; 1349958cea9eSml 1350958cea9eSml default: 135144961713Sgirish ring_index = tcp_port[19] % maxtdcs; 1352958cea9eSml break; 135344961713Sgirish } 135444961713Sgirish } else { /* fragmented packet */ 135544961713Sgirish ring_index = tcp_port[19] % maxtdcs; 135644961713Sgirish } 135744961713Sgirish } else { 135844961713Sgirish ring_index = mp->b_band % maxtdcs; 135944961713Sgirish } 136044961713Sgirish break; 136144961713Sgirish 136244961713Sgirish case NXGE_TX_LB_HASH: 136344961713Sgirish if (hp->hash) { 1364adfcba55Sjoycey #if defined(__i386) 1365adfcba55Sjoycey ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1366adfcba55Sjoycey #else 136744961713Sgirish ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1368adfcba55Sjoycey #endif 136944961713Sgirish } else { 137044961713Sgirish ring_index = mp->b_band % maxtdcs; 137144961713Sgirish } 137244961713Sgirish break; 137344961713Sgirish 137444961713Sgirish case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 137544961713Sgirish tcp_port = mp->b_rptr; 137644961713Sgirish ring_index = tcp_port[5] % maxtdcs; 137744961713Sgirish break; 137844961713Sgirish } 137944961713Sgirish 138044961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 138144961713Sgirish 138244961713Sgirish return (ring_index); 138344961713Sgirish } 138444961713Sgirish 138544961713Sgirish uint_t 138644961713Sgirish nxge_reschedule(caddr_t arg) 138744961713Sgirish { 138844961713Sgirish p_nxge_t nxgep; 138944961713Sgirish 139044961713Sgirish nxgep = (p_nxge_t)arg; 139144961713Sgirish 139244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 139344961713Sgirish 139444961713Sgirish if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1395678453a8Sspeer nxgep->resched_needed) { 1396678453a8Sspeer if (!isLDOMguest(nxgep)) 1397678453a8Sspeer mac_tx_update(nxgep->mach); 1398678453a8Sspeer #if defined(sun4v) 1399678453a8Sspeer else { /* isLDOMguest(nxgep) */ 1400678453a8Sspeer nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1401678453a8Sspeer nxgep->nxge_hw_p->hio; 1402678453a8Sspeer nx_vio_fp_t *vio = &nhd->hio.vio; 1403678453a8Sspeer 1404678453a8Sspeer /* Call back vnet. */ 1405678453a8Sspeer if (vio->cb.vio_net_tx_update) { 1406678453a8Sspeer (*vio->cb.vio_net_tx_update) 1407678453a8Sspeer (nxgep->hio_vr->vhp); 1408678453a8Sspeer } 1409678453a8Sspeer } 1410678453a8Sspeer #endif 141144961713Sgirish nxgep->resched_needed = B_FALSE; 141244961713Sgirish nxgep->resched_running = B_FALSE; 141344961713Sgirish } 141444961713Sgirish 141544961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 141644961713Sgirish return (DDI_INTR_CLAIMED); 141744961713Sgirish } 141830ac2e7bSml 141930ac2e7bSml 142030ac2e7bSml /* Software LSO starts here */ 142130ac2e7bSml static void 142230ac2e7bSml nxge_hcksum_retrieve(mblk_t *mp, 142330ac2e7bSml uint32_t *start, uint32_t *stuff, uint32_t *end, 142430ac2e7bSml uint32_t *value, uint32_t *flags) 142530ac2e7bSml { 142630ac2e7bSml if (mp->b_datap->db_type == M_DATA) { 142730ac2e7bSml if (flags != NULL) { 142830ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 142930ac2e7bSml HCK_PARTIALCKSUM | HCK_FULLCKSUM | 143030ac2e7bSml HCK_FULLCKSUM_OK); 143130ac2e7bSml if ((*flags & (HCK_PARTIALCKSUM | 143230ac2e7bSml HCK_FULLCKSUM)) != 0) { 143330ac2e7bSml if (value != NULL) 143430ac2e7bSml *value = (uint32_t)DB_CKSUM16(mp); 143530ac2e7bSml if ((*flags & HCK_PARTIALCKSUM) != 0) { 143630ac2e7bSml if (start != NULL) 143730ac2e7bSml *start = 143830ac2e7bSml (uint32_t)DB_CKSUMSTART(mp); 143930ac2e7bSml if (stuff != NULL) 144030ac2e7bSml *stuff = 144130ac2e7bSml (uint32_t)DB_CKSUMSTUFF(mp); 144230ac2e7bSml if (end != NULL) 144330ac2e7bSml *end = 144430ac2e7bSml (uint32_t)DB_CKSUMEND(mp); 144530ac2e7bSml } 144630ac2e7bSml } 144730ac2e7bSml } 144830ac2e7bSml } 144930ac2e7bSml } 145030ac2e7bSml 145130ac2e7bSml static void 145230ac2e7bSml nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 145330ac2e7bSml { 145430ac2e7bSml ASSERT(DB_TYPE(mp) == M_DATA); 145530ac2e7bSml 145630ac2e7bSml *mss = 0; 145730ac2e7bSml if (flags != NULL) { 145830ac2e7bSml *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 145930ac2e7bSml if ((*flags != 0) && (mss != NULL)) { 146030ac2e7bSml *mss = (uint32_t)DB_LSOMSS(mp); 146130ac2e7bSml } 146230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 146330ac2e7bSml "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 146430ac2e7bSml *mss, *flags)); 146530ac2e7bSml } 146630ac2e7bSml 146730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 146830ac2e7bSml "<== nxge_lso_info_get: mss %d", *mss)); 146930ac2e7bSml } 147030ac2e7bSml 147130ac2e7bSml /* 147230ac2e7bSml * Do Soft LSO on the oversized packet. 147330ac2e7bSml * 147430ac2e7bSml * 1. Create a chain of message for headers. 147530ac2e7bSml * 2. Fill up header messages with proper information. 147630ac2e7bSml * 3. Copy Eithernet, IP, and TCP headers from the original message to 147730ac2e7bSml * each new message with necessary adjustments. 147830ac2e7bSml * * Unchange the ethernet header for DIX frames. (by default) 147930ac2e7bSml * * IP Total Length field is updated to MSS or less(only for the last one). 148030ac2e7bSml * * IP Identification value is incremented by one for each packet. 148130ac2e7bSml * * TCP sequence Number is recalculated according to the payload length. 148230ac2e7bSml * * Set FIN and/or PSH flags for the *last* packet if applied. 148330ac2e7bSml * * TCP partial Checksum 148430ac2e7bSml * 4. Update LSO information in the first message header. 148530ac2e7bSml * 5. Release the original message header. 148630ac2e7bSml */ 148730ac2e7bSml static mblk_t * 148830ac2e7bSml nxge_do_softlso(mblk_t *mp, uint32_t mss) 148930ac2e7bSml { 149030ac2e7bSml uint32_t hckflags; 149130ac2e7bSml int pktlen; 149230ac2e7bSml int hdrlen; 149330ac2e7bSml int segnum; 149430ac2e7bSml int i; 149530ac2e7bSml struct ether_vlan_header *evh; 149630ac2e7bSml int ehlen, iphlen, tcphlen; 149730ac2e7bSml struct ip *oiph, *niph; 149830ac2e7bSml struct tcphdr *otcph, *ntcph; 149930ac2e7bSml int available, len, left; 150030ac2e7bSml uint16_t ip_id; 150130ac2e7bSml uint32_t tcp_seq; 150230ac2e7bSml #ifdef __sparc 150330ac2e7bSml uint32_t tcp_seq_tmp; 150430ac2e7bSml #endif 150530ac2e7bSml mblk_t *datamp; 150630ac2e7bSml uchar_t *rptr; 150730ac2e7bSml mblk_t *nmp; 150830ac2e7bSml mblk_t *cmp; 150930ac2e7bSml mblk_t *mp_chain; 151030ac2e7bSml boolean_t do_cleanup = B_FALSE; 151130ac2e7bSml t_uscalar_t start_offset = 0; 151230ac2e7bSml t_uscalar_t stuff_offset = 0; 151330ac2e7bSml t_uscalar_t value = 0; 151430ac2e7bSml uint16_t l4_len; 151530ac2e7bSml ipaddr_t src, dst; 151630ac2e7bSml uint32_t cksum, sum, l4cksum; 151730ac2e7bSml 151830ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 151930ac2e7bSml "==> nxge_do_softlso")); 152030ac2e7bSml /* 152130ac2e7bSml * check the length of LSO packet payload and calculate the number of 152230ac2e7bSml * segments to be generated. 152330ac2e7bSml */ 152430ac2e7bSml pktlen = msgsize(mp); 152530ac2e7bSml evh = (struct ether_vlan_header *)mp->b_rptr; 152630ac2e7bSml 152730ac2e7bSml /* VLAN? */ 152830ac2e7bSml if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 152930ac2e7bSml ehlen = sizeof (struct ether_vlan_header); 153030ac2e7bSml else 153130ac2e7bSml ehlen = sizeof (struct ether_header); 153230ac2e7bSml oiph = (struct ip *)(mp->b_rptr + ehlen); 153330ac2e7bSml iphlen = oiph->ip_hl * 4; 153430ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 153530ac2e7bSml tcphlen = otcph->th_off * 4; 153630ac2e7bSml 153730ac2e7bSml l4_len = pktlen - ehlen - iphlen; 153830ac2e7bSml 153930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 154030ac2e7bSml "==> nxge_do_softlso: mss %d oiph $%p " 154130ac2e7bSml "original ip_sum oiph->ip_sum 0x%x " 154230ac2e7bSml "original tcp_sum otcph->th_sum 0x%x " 154330ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d " 154430ac2e7bSml "l4_len %d (0x%x) ip_len - iphlen %d ", 154530ac2e7bSml mss, 154630ac2e7bSml oiph, 154730ac2e7bSml oiph->ip_sum, 154830ac2e7bSml otcph->th_sum, 154930ac2e7bSml ntohs(oiph->ip_len), pktlen, 155030ac2e7bSml ehlen, 155130ac2e7bSml l4_len, 155230ac2e7bSml l4_len, 155330ac2e7bSml ntohs(oiph->ip_len) - iphlen)); 155430ac2e7bSml 155530ac2e7bSml /* IPv4 + TCP */ 155630ac2e7bSml if (!(oiph->ip_v == IPV4_VERSION)) { 155730ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 155830ac2e7bSml "<== nxge_do_softlso: not IPV4 " 155930ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 156030ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 156130ac2e7bSml tcphlen)); 156230ac2e7bSml freemsg(mp); 156330ac2e7bSml return (NULL); 156430ac2e7bSml } 156530ac2e7bSml 156630ac2e7bSml if (!(oiph->ip_p == IPPROTO_TCP)) { 156730ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 156830ac2e7bSml "<== nxge_do_softlso: not TCP " 156930ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 157030ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 157130ac2e7bSml tcphlen)); 157230ac2e7bSml freemsg(mp); 157330ac2e7bSml return (NULL); 157430ac2e7bSml } 157530ac2e7bSml 157630ac2e7bSml if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 157730ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 157830ac2e7bSml "<== nxge_do_softlso: len not matched " 157930ac2e7bSml "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 158030ac2e7bSml ntohs(oiph->ip_len), pktlen, ehlen, 158130ac2e7bSml tcphlen)); 158230ac2e7bSml freemsg(mp); 158330ac2e7bSml return (NULL); 158430ac2e7bSml } 158530ac2e7bSml 158630ac2e7bSml otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 158730ac2e7bSml tcphlen = otcph->th_off * 4; 158830ac2e7bSml 158930ac2e7bSml /* TCP flags can not include URG, RST, or SYN */ 159030ac2e7bSml VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 159130ac2e7bSml 159230ac2e7bSml hdrlen = ehlen + iphlen + tcphlen; 159330ac2e7bSml 159430ac2e7bSml VERIFY(MBLKL(mp) >= hdrlen); 159530ac2e7bSml 159630ac2e7bSml if (MBLKL(mp) > hdrlen) { 159730ac2e7bSml datamp = mp; 159830ac2e7bSml rptr = mp->b_rptr + hdrlen; 159930ac2e7bSml } else { /* = */ 160030ac2e7bSml datamp = mp->b_cont; 160130ac2e7bSml rptr = datamp->b_rptr; 160230ac2e7bSml } 160330ac2e7bSml 160430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 160530ac2e7bSml "nxge_do_softlso: otcph $%p pktlen: %d, " 160630ac2e7bSml "hdrlen %d ehlen %d iphlen %d tcphlen %d " 160730ac2e7bSml "mblkl(mp): %d, mblkl(datamp): %d", 160830ac2e7bSml otcph, 160930ac2e7bSml pktlen, hdrlen, ehlen, iphlen, tcphlen, 161030ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 161130ac2e7bSml 161230ac2e7bSml hckflags = 0; 161330ac2e7bSml nxge_hcksum_retrieve(mp, 161430ac2e7bSml &start_offset, &stuff_offset, &value, NULL, &hckflags); 161530ac2e7bSml 161630ac2e7bSml dst = oiph->ip_dst.s_addr; 161730ac2e7bSml src = oiph->ip_src.s_addr; 161830ac2e7bSml 161930ac2e7bSml cksum = (dst >> 16) + (dst & 0xFFFF) + 162030ac2e7bSml (src >> 16) + (src & 0xFFFF); 162130ac2e7bSml l4cksum = cksum + IP_TCP_CSUM_COMP; 162230ac2e7bSml 162330ac2e7bSml sum = l4_len + l4cksum; 162430ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 162530ac2e7bSml 162630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 162730ac2e7bSml "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 162830ac2e7bSml "hckflags 0x%x start_offset %d stuff_offset %d " 162930ac2e7bSml "value (original) 0x%x th_sum 0x%x " 163030ac2e7bSml "pktlen %d l4_len %d (0x%x) " 163130ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 163230ac2e7bSml dst, src, 163330ac2e7bSml (sum & 0xffff), (~sum & 0xffff), 163430ac2e7bSml hckflags, start_offset, stuff_offset, 163530ac2e7bSml value, otcph->th_sum, 163630ac2e7bSml pktlen, 163730ac2e7bSml l4_len, 163830ac2e7bSml l4_len, 163930ac2e7bSml ntohs(oiph->ip_len) - (int)MBLKL(mp), 164030ac2e7bSml (int)MBLKL(datamp), 164130ac2e7bSml nxge_dump_packet((char *)evh, 12))); 164230ac2e7bSml 164330ac2e7bSml /* 164430ac2e7bSml * Start to process. 164530ac2e7bSml */ 164630ac2e7bSml available = pktlen - hdrlen; 164730ac2e7bSml segnum = (available - 1) / mss + 1; 164830ac2e7bSml 164930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 165030ac2e7bSml "==> nxge_do_softlso: pktlen %d " 165130ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 165230ac2e7bSml "available %d mss %d segnum %d", 165330ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 165430ac2e7bSml available, 165530ac2e7bSml mss, 165630ac2e7bSml segnum)); 165730ac2e7bSml 165830ac2e7bSml VERIFY(segnum >= 2); 165930ac2e7bSml 166030ac2e7bSml /* 166130ac2e7bSml * Try to pre-allocate all header messages 166230ac2e7bSml */ 166330ac2e7bSml mp_chain = NULL; 166430ac2e7bSml for (i = 0; i < segnum; i++) { 166530ac2e7bSml if ((nmp = allocb(hdrlen, 0)) == NULL) { 166630ac2e7bSml /* Clean up the mp_chain */ 166730ac2e7bSml while (mp_chain != NULL) { 166830ac2e7bSml nmp = mp_chain; 166930ac2e7bSml mp_chain = mp_chain->b_next; 167030ac2e7bSml freemsg(nmp); 167130ac2e7bSml } 167230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 167330ac2e7bSml "<== nxge_do_softlso: " 167430ac2e7bSml "Could not allocate enough messages for headers!")); 167530ac2e7bSml freemsg(mp); 167630ac2e7bSml return (NULL); 167730ac2e7bSml } 167830ac2e7bSml nmp->b_next = mp_chain; 167930ac2e7bSml mp_chain = nmp; 168030ac2e7bSml 168130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168230ac2e7bSml "==> nxge_do_softlso: " 168330ac2e7bSml "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 168430ac2e7bSml mp, nmp, mp_chain, mp_chain->b_next)); 168530ac2e7bSml } 168630ac2e7bSml 168730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 168830ac2e7bSml "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 168930ac2e7bSml mp, nmp, mp_chain)); 169030ac2e7bSml 169130ac2e7bSml /* 169230ac2e7bSml * Associate payload with new packets 169330ac2e7bSml */ 169430ac2e7bSml cmp = mp_chain; 169530ac2e7bSml left = available; 169630ac2e7bSml while (cmp != NULL) { 169730ac2e7bSml nmp = dupb(datamp); 169830ac2e7bSml if (nmp == NULL) { 169930ac2e7bSml do_cleanup = B_TRUE; 170030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 170130ac2e7bSml "==>nxge_do_softlso: " 170230ac2e7bSml "Can not dupb(datamp), have to do clean up")); 170330ac2e7bSml goto cleanup_allocated_msgs; 170430ac2e7bSml } 170530ac2e7bSml 170630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 170730ac2e7bSml "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 170830ac2e7bSml "dupb nmp $%p len %d left %d msd %d ", 170930ac2e7bSml mp, cmp, nmp, len, left, mss)); 171030ac2e7bSml 171130ac2e7bSml cmp->b_cont = nmp; 171230ac2e7bSml nmp->b_rptr = rptr; 171330ac2e7bSml len = (left < mss) ? left : mss; 171430ac2e7bSml left -= len; 171530ac2e7bSml 171630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 171730ac2e7bSml "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 171830ac2e7bSml "dupb nmp $%p len %d left %d mss %d ", 171930ac2e7bSml mp, cmp, nmp, len, left, mss)); 172030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 172130ac2e7bSml "nxge_do_softlso: before available: %d, " 172230ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 172330ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 172430ac2e7bSml 172530ac2e7bSml len -= MBLKL(nmp); 172630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 172730ac2e7bSml "nxge_do_softlso: after available: %d, " 172830ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 172930ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 173030ac2e7bSml 173130ac2e7bSml while (len > 0) { 173230ac2e7bSml mblk_t *mmp = NULL; 173330ac2e7bSml 173430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 173530ac2e7bSml "nxge_do_softlso: (4) len > 0 available: %d, " 173630ac2e7bSml "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 173730ac2e7bSml available, left, len, segnum, (int)MBLKL(nmp))); 173830ac2e7bSml 173930ac2e7bSml if (datamp->b_cont != NULL) { 174030ac2e7bSml datamp = datamp->b_cont; 174130ac2e7bSml rptr = datamp->b_rptr; 174230ac2e7bSml mmp = dupb(datamp); 174330ac2e7bSml if (mmp == NULL) { 174430ac2e7bSml do_cleanup = B_TRUE; 174530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 174630ac2e7bSml "==> nxge_do_softlso: " 1747678453a8Sspeer "Can not dupb(datamp) (1), :" 174830ac2e7bSml "have to do clean up")); 174930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 175030ac2e7bSml "==> nxge_do_softlso: " 175130ac2e7bSml "available: %d, left: %d, " 175230ac2e7bSml "len: %d, MBLKL(nmp): %d", 175330ac2e7bSml available, left, len, 175430ac2e7bSml (int)MBLKL(nmp))); 175530ac2e7bSml goto cleanup_allocated_msgs; 175630ac2e7bSml } 175730ac2e7bSml } else { 175830ac2e7bSml NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 175930ac2e7bSml "==> nxge_do_softlso: " 176030ac2e7bSml "(1)available: %d, left: %d, " 176130ac2e7bSml "len: %d, MBLKL(nmp): %d", 176230ac2e7bSml available, left, len, 176330ac2e7bSml (int)MBLKL(nmp))); 176430ac2e7bSml cmn_err(CE_PANIC, 176530ac2e7bSml "==> nxge_do_softlso: " 176630ac2e7bSml "Pointers must have been corrupted!\n" 176730ac2e7bSml "datamp: $%p, nmp: $%p, rptr: $%p", 176830ac2e7bSml (void *)datamp, 176930ac2e7bSml (void *)nmp, 177030ac2e7bSml (void *)rptr); 177130ac2e7bSml } 177230ac2e7bSml nmp->b_cont = mmp; 177330ac2e7bSml nmp = mmp; 177430ac2e7bSml len -= MBLKL(nmp); 177530ac2e7bSml } 177630ac2e7bSml if (len < 0) { 177730ac2e7bSml nmp->b_wptr += len; 177830ac2e7bSml rptr = nmp->b_wptr; 177930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 178030ac2e7bSml "(5) len < 0 (less than 0)" 178130ac2e7bSml "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 178230ac2e7bSml available, left, len, (int)MBLKL(nmp))); 178330ac2e7bSml 178430ac2e7bSml } else if (len == 0) { 178530ac2e7bSml if (datamp->b_cont != NULL) { 178630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 178730ac2e7bSml "(5) len == 0" 178830ac2e7bSml "available: %d, left: %d, len: %d, " 178930ac2e7bSml "MBLKL(nmp): %d", 179030ac2e7bSml available, left, len, (int)MBLKL(nmp))); 179130ac2e7bSml datamp = datamp->b_cont; 179230ac2e7bSml rptr = datamp->b_rptr; 179330ac2e7bSml } else { 179430ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 179530ac2e7bSml "(6)available b_cont == NULL : %d, " 179630ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 179730ac2e7bSml available, left, len, (int)MBLKL(nmp))); 179830ac2e7bSml 179930ac2e7bSml VERIFY(cmp->b_next == NULL); 180030ac2e7bSml VERIFY(left == 0); 180130ac2e7bSml break; /* Done! */ 180230ac2e7bSml } 180330ac2e7bSml } 180430ac2e7bSml cmp = cmp->b_next; 180530ac2e7bSml 180630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 180730ac2e7bSml "(7) do_softlso: " 180830ac2e7bSml "next mp in mp_chain available len != 0 : %d, " 180930ac2e7bSml "left: %d, len: %d, MBLKL(nmp): %d", 181030ac2e7bSml available, left, len, (int)MBLKL(nmp))); 181130ac2e7bSml } 181230ac2e7bSml 181330ac2e7bSml /* 181430ac2e7bSml * From now, start to fill up all headers for the first message 181530ac2e7bSml * Hardware checksum flags need to be updated separately for FULLCKSUM 181630ac2e7bSml * and PARTIALCKSUM cases. For full checksum, copy the original flags 181730ac2e7bSml * into every new packet is enough. But for HCK_PARTIALCKSUM, all 181830ac2e7bSml * required fields need to be updated properly. 181930ac2e7bSml */ 182030ac2e7bSml nmp = mp_chain; 182130ac2e7bSml bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 182230ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 182330ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 182430ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 182530ac2e7bSml ip_id = ntohs(niph->ip_id); 182630ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 182730ac2e7bSml #ifdef __sparc 182830ac2e7bSml bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 182930ac2e7bSml tcp_seq = ntohl(tcp_seq_tmp); 183030ac2e7bSml #else 183130ac2e7bSml tcp_seq = ntohl(ntcph->th_seq); 183230ac2e7bSml #endif 183330ac2e7bSml 183430ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 183530ac2e7bSml 183630ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 183730ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 183830ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 183930ac2e7bSml 184030ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 184130ac2e7bSml niph->ip_sum = 0; 184230ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 184330ac2e7bSml 184430ac2e7bSml l4_len = mss + tcphlen; 184530ac2e7bSml sum = htons(l4_len) + l4cksum; 184630ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 184730ac2e7bSml ntcph->th_sum = (sum & 0xffff); 184830ac2e7bSml 184930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 185030ac2e7bSml "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 185130ac2e7bSml "mss %d pktlen %d l4_len %d (0x%x) " 185230ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d " 185330ac2e7bSml "ip_sum 0x%x " 185430ac2e7bSml "th_sum 0x%x sum 0x%x ) " 185530ac2e7bSml "dump first ip->tcp %s", 185630ac2e7bSml nmp, mp_chain, 185730ac2e7bSml mss, 185830ac2e7bSml pktlen, 185930ac2e7bSml l4_len, 186030ac2e7bSml l4_len, 186130ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp), 186230ac2e7bSml niph->ip_sum, 186330ac2e7bSml ntcph->th_sum, 186430ac2e7bSml sum, 186530ac2e7bSml nxge_dump_packet((char *)niph, 52))); 186630ac2e7bSml 186730ac2e7bSml cmp = nmp; 186830ac2e7bSml while ((nmp = nmp->b_next)->b_next != NULL) { 186930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 187030ac2e7bSml "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 187130ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 187230ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 187330ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 187430ac2e7bSml niph->ip_id = htons(++ip_id); 187530ac2e7bSml niph->ip_len = htons(mss + iphlen + tcphlen); 187630ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 187730ac2e7bSml tcp_seq += mss; 187830ac2e7bSml 187930ac2e7bSml ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 188030ac2e7bSml 188130ac2e7bSml #ifdef __sparc 188230ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 188330ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 188430ac2e7bSml #else 188530ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 188630ac2e7bSml #endif 188730ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 188830ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 188930ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 189030ac2e7bSml 189130ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 189230ac2e7bSml niph->ip_sum = 0; 189330ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 189430ac2e7bSml ntcph->th_sum = (sum & 0xffff); 189530ac2e7bSml 189630ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 189730ac2e7bSml "==> nxge_do_softlso: middle ip_sum 0x%x " 189830ac2e7bSml "th_sum 0x%x " 189930ac2e7bSml " mp $%p (mp_chain $%p) pktlen %d " 190030ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 190130ac2e7bSml niph->ip_sum, 190230ac2e7bSml ntcph->th_sum, 190330ac2e7bSml nmp, mp_chain, 190430ac2e7bSml pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 190530ac2e7bSml } 190630ac2e7bSml 190730ac2e7bSml /* Last segment */ 190830ac2e7bSml /* 190930ac2e7bSml * Set FIN and/or PSH flags if present only in the last packet. 191030ac2e7bSml * The ip_len could be different from prior packets. 191130ac2e7bSml */ 191230ac2e7bSml bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 191330ac2e7bSml nmp->b_wptr = nmp->b_rptr + hdrlen; 191430ac2e7bSml niph = (struct ip *)(nmp->b_rptr + ehlen); 191530ac2e7bSml niph->ip_id = htons(++ip_id); 191630ac2e7bSml niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 191730ac2e7bSml ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 191830ac2e7bSml tcp_seq += mss; 191930ac2e7bSml #ifdef __sparc 192030ac2e7bSml tcp_seq_tmp = htonl(tcp_seq); 192130ac2e7bSml bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 192230ac2e7bSml #else 192330ac2e7bSml ntcph->th_seq = htonl(tcp_seq); 192430ac2e7bSml #endif 192530ac2e7bSml ntcph->th_flags = (otcph->th_flags & ~TH_URG); 192630ac2e7bSml 192730ac2e7bSml DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 192830ac2e7bSml DB_CKSUMSTART(nmp) = start_offset; 192930ac2e7bSml DB_CKSUMSTUFF(nmp) = stuff_offset; 193030ac2e7bSml 193130ac2e7bSml /* calculate IP checksum and TCP pseudo header checksum */ 193230ac2e7bSml niph->ip_sum = 0; 193330ac2e7bSml niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 193430ac2e7bSml 193530ac2e7bSml l4_len = ntohs(niph->ip_len) - iphlen; 193630ac2e7bSml sum = htons(l4_len) + l4cksum; 193730ac2e7bSml sum = (sum & 0xFFFF) + (sum >> 16); 193830ac2e7bSml ntcph->th_sum = (sum & 0xffff); 193930ac2e7bSml 194030ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 194130ac2e7bSml "==> nxge_do_softlso: last next " 194230ac2e7bSml "niph->ip_sum 0x%x " 194330ac2e7bSml "ntcph->th_sum 0x%x sum 0x%x " 194430ac2e7bSml "dump last ip->tcp %s " 194530ac2e7bSml "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 194630ac2e7bSml "l4_len %d (0x%x) " 194730ac2e7bSml "MBLKL(mp): %d, MBLKL(datamp): %d ", 194830ac2e7bSml niph->ip_sum, 194930ac2e7bSml ntcph->th_sum, sum, 195030ac2e7bSml nxge_dump_packet((char *)niph, 52), 195130ac2e7bSml cmp, nmp, mp_chain, 195230ac2e7bSml pktlen, pktlen, 195330ac2e7bSml l4_len, 195430ac2e7bSml l4_len, 195530ac2e7bSml (int)MBLKL(mp), (int)MBLKL(datamp))); 195630ac2e7bSml 195730ac2e7bSml cleanup_allocated_msgs: 195830ac2e7bSml if (do_cleanup) { 195930ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 196030ac2e7bSml "==> nxge_do_softlso: " 196130ac2e7bSml "Failed allocating messages, " 196230ac2e7bSml "have to clean up and fail!")); 196330ac2e7bSml while (mp_chain != NULL) { 196430ac2e7bSml nmp = mp_chain; 196530ac2e7bSml mp_chain = mp_chain->b_next; 196630ac2e7bSml freemsg(nmp); 196730ac2e7bSml } 196830ac2e7bSml } 196930ac2e7bSml /* 197030ac2e7bSml * We're done here, so just free the original message and return the 197130ac2e7bSml * new message chain, that could be NULL if failed, back to the caller. 197230ac2e7bSml */ 197330ac2e7bSml freemsg(mp); 197430ac2e7bSml 197530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 197630ac2e7bSml "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 197730ac2e7bSml return (mp_chain); 197830ac2e7bSml } 197930ac2e7bSml 198030ac2e7bSml /* 198130ac2e7bSml * Will be called before NIC driver do further operation on the message. 198230ac2e7bSml * The input message may include LSO information, if so, go to softlso logic 198330ac2e7bSml * to eliminate the oversized LSO packet for the incapable underlying h/w. 198430ac2e7bSml * The return could be the same non-LSO message or a message chain for LSO case. 198530ac2e7bSml * 198630ac2e7bSml * The driver needs to call this function per packet and process the whole chain 198730ac2e7bSml * if applied. 198830ac2e7bSml */ 198930ac2e7bSml static mblk_t * 199030ac2e7bSml nxge_lso_eliminate(mblk_t *mp) 199130ac2e7bSml { 199230ac2e7bSml uint32_t lsoflags; 199330ac2e7bSml uint32_t mss; 199430ac2e7bSml 199530ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 199630ac2e7bSml "==>nxge_lso_eliminate:")); 199730ac2e7bSml nxge_lso_info_get(mp, &mss, &lsoflags); 199830ac2e7bSml 199930ac2e7bSml if (lsoflags & HW_LSO) { 200030ac2e7bSml mblk_t *nmp; 200130ac2e7bSml 200230ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 200330ac2e7bSml "==>nxge_lso_eliminate:" 200430ac2e7bSml "HW_LSO:mss %d mp $%p", 200530ac2e7bSml mss, mp)); 200630ac2e7bSml if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 200730ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 200830ac2e7bSml "<== nxge_lso_eliminate: " 200930ac2e7bSml "LSO: nmp not NULL nmp $%p mss %d mp $%p", 201030ac2e7bSml nmp, mss, mp)); 201130ac2e7bSml return (nmp); 201230ac2e7bSml } else { 201330ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 201430ac2e7bSml "<== nxge_lso_eliminate_ " 201530ac2e7bSml "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 201630ac2e7bSml nmp, mss, mp)); 201730ac2e7bSml return (NULL); 201830ac2e7bSml } 201930ac2e7bSml } 202030ac2e7bSml 202130ac2e7bSml NXGE_DEBUG_MSG((NULL, TX_CTL, 202230ac2e7bSml "<== nxge_lso_eliminate")); 202330ac2e7bSml return (mp); 202430ac2e7bSml } 202530ac2e7bSml 202630ac2e7bSml static uint32_t 202730ac2e7bSml nxge_csgen(uint16_t *adr, int len) 202830ac2e7bSml { 202930ac2e7bSml int i, odd; 203030ac2e7bSml uint32_t sum = 0; 203130ac2e7bSml uint32_t c = 0; 203230ac2e7bSml 203330ac2e7bSml odd = len % 2; 203430ac2e7bSml for (i = 0; i < (len / 2); i++) { 203530ac2e7bSml sum += (adr[i] & 0xffff); 203630ac2e7bSml } 203730ac2e7bSml if (odd) { 203830ac2e7bSml sum += adr[len / 2] & 0xff00; 203930ac2e7bSml } 204030ac2e7bSml while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 204130ac2e7bSml sum &= 0xffff; 204230ac2e7bSml sum += c; 204330ac2e7bSml } 204430ac2e7bSml return (~sum & 0xffff); 204530ac2e7bSml } 2046