144961713Sgirish /* 244961713Sgirish * CDDL HEADER START 344961713Sgirish * 444961713Sgirish * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 744961713Sgirish * 844961713Sgirish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 944961713Sgirish * or http://www.opensolaris.org/os/licensing. 1044961713Sgirish * See the License for the specific language governing permissions 1144961713Sgirish * and limitations under the License. 1244961713Sgirish * 1344961713Sgirish * When distributing Covered Code, include this CDDL HEADER in each 1444961713Sgirish * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1544961713Sgirish * If applicable, add the following below this CDDL HEADER, with the 1644961713Sgirish * fields enclosed by brackets "[]" replaced with your own identifying 1744961713Sgirish * information: Portions Copyright [yyyy] [name of copyright owner] 1844961713Sgirish * 1944961713Sgirish * CDDL HEADER END 2044961713Sgirish */ 2144961713Sgirish /* 22a3c5bd6dSspeer * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 2344961713Sgirish * Use is subject to license terms. 2444961713Sgirish */ 2544961713Sgirish 2644961713Sgirish #pragma ident "%Z%%M% %I% %E% SMI" 2744961713Sgirish 28a3c5bd6dSspeer #include <sys/nxge/nxge_impl.h> 2944961713Sgirish 3044961713Sgirish extern uint32_t nxge_reclaim_pending; 3144961713Sgirish extern uint32_t nxge_bcopy_thresh; 3244961713Sgirish extern uint32_t nxge_dvma_thresh; 3344961713Sgirish extern uint32_t nxge_dma_stream_thresh; 3444961713Sgirish extern uint32_t nxge_tx_minfree; 3544961713Sgirish extern uint32_t nxge_tx_intr_thres; 3644961713Sgirish extern uint32_t nxge_tx_max_gathers; 3744961713Sgirish extern uint32_t nxge_tx_tiny_pack; 3844961713Sgirish extern uint32_t nxge_tx_use_bcopy; 3944961713Sgirish extern uint32_t nxge_tx_lb_policy; 4044961713Sgirish extern uint32_t nxge_no_tx_lb; 411f8914d5Sml extern nxge_tx_mode_t nxge_tx_scheme; 4244961713Sgirish 4344961713Sgirish typedef struct _mac_tx_hint { 4444961713Sgirish uint16_t sap; 4544961713Sgirish uint16_t vid; 4644961713Sgirish void *hash; 4744961713Sgirish } mac_tx_hint_t, *p_mac_tx_hint_t; 4844961713Sgirish 4944961713Sgirish int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 5044961713Sgirish 5144961713Sgirish int 5244961713Sgirish nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 5344961713Sgirish { 5444961713Sgirish int status = 0; 5544961713Sgirish p_tx_desc_t tx_desc_ring_vp; 5644961713Sgirish npi_handle_t npi_desc_handle; 5744961713Sgirish nxge_os_dma_handle_t tx_desc_dma_handle; 5844961713Sgirish p_tx_desc_t tx_desc_p; 5944961713Sgirish p_tx_msg_t tx_msg_ring; 6044961713Sgirish p_tx_msg_t tx_msg_p; 6144961713Sgirish tx_desc_t tx_desc, *tmp_desc_p; 6244961713Sgirish tx_desc_t sop_tx_desc, *sop_tx_desc_p; 6344961713Sgirish p_tx_pkt_header_t hdrp; 6444961713Sgirish p_tx_pkt_hdr_all_t pkthdrp; 6544961713Sgirish uint8_t npads = 0; 6644961713Sgirish uint64_t dma_ioaddr; 6744961713Sgirish uint32_t dma_flags; 6844961713Sgirish int last_bidx; 6944961713Sgirish uint8_t *b_rptr; 7044961713Sgirish caddr_t kaddr; 7144961713Sgirish uint32_t nmblks; 7244961713Sgirish uint32_t ngathers; 7344961713Sgirish uint32_t clen; 7444961713Sgirish int len; 7544961713Sgirish uint32_t pkt_len, pack_len, min_len; 7644961713Sgirish uint32_t bcopy_thresh; 7744961713Sgirish int i, cur_index, sop_index; 7844961713Sgirish uint16_t tail_index; 7944961713Sgirish boolean_t tail_wrap = B_FALSE; 8044961713Sgirish nxge_dma_common_t desc_area; 8144961713Sgirish nxge_os_dma_handle_t dma_handle; 8244961713Sgirish ddi_dma_cookie_t dma_cookie; 8344961713Sgirish npi_handle_t npi_handle; 8444961713Sgirish p_mblk_t nmp; 8544961713Sgirish p_mblk_t t_mp; 8644961713Sgirish uint32_t ncookies; 8744961713Sgirish boolean_t good_packet; 8844961713Sgirish boolean_t mark_mode = B_FALSE; 8944961713Sgirish p_nxge_stats_t statsp; 9044961713Sgirish p_nxge_tx_ring_stats_t tdc_stats; 9144961713Sgirish t_uscalar_t start_offset = 0; 9244961713Sgirish t_uscalar_t stuff_offset = 0; 9344961713Sgirish t_uscalar_t end_offset = 0; 9444961713Sgirish t_uscalar_t value = 0; 9544961713Sgirish t_uscalar_t cksum_flags = 0; 9644961713Sgirish boolean_t cksum_on = B_FALSE; 9744961713Sgirish uint32_t boff = 0; 9844961713Sgirish uint64_t tot_xfer_len = 0, tmp_len = 0; 9944961713Sgirish boolean_t header_set = B_FALSE; 10044961713Sgirish #ifdef NXGE_DEBUG 10144961713Sgirish p_tx_desc_t tx_desc_ring_pp; 10244961713Sgirish p_tx_desc_t tx_desc_pp; 10344961713Sgirish tx_desc_t *save_desc_p; 10444961713Sgirish int dump_len; 10544961713Sgirish int sad_len; 10644961713Sgirish uint64_t sad; 10744961713Sgirish int xfer_len; 10844961713Sgirish uint32_t msgsize; 10944961713Sgirish #endif 11044961713Sgirish 11144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 11244961713Sgirish "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 11344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 11444961713Sgirish "==> nxge_start: Starting tdc %d desc pending %d", 11544961713Sgirish tx_ring_p->tdc, tx_ring_p->descs_pending)); 11644961713Sgirish 11744961713Sgirish statsp = nxgep->statsp; 11844961713Sgirish 11944961713Sgirish if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 12014ea4bb7Ssd if (!statsp->mac_stats.link_up) { 12144961713Sgirish freemsg(mp); 12244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 12344961713Sgirish "link not up or LB mode")); 12444961713Sgirish goto nxge_start_fail1; 12544961713Sgirish } 12644961713Sgirish } 12744961713Sgirish 12844961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 12944961713Sgirish &stuff_offset, &end_offset, &value, &cksum_flags); 13044961713Sgirish if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 13144961713Sgirish start_offset += sizeof (ether_header_t); 13244961713Sgirish stuff_offset += sizeof (ether_header_t); 13344961713Sgirish } else { 13444961713Sgirish start_offset += sizeof (struct ether_vlan_header); 13544961713Sgirish stuff_offset += sizeof (struct ether_vlan_header); 13644961713Sgirish } 13744961713Sgirish 13844961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 13944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 14044961713Sgirish "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 14144961713Sgirish cksum_flags)); 14244961713Sgirish cksum_on = B_TRUE; 14344961713Sgirish } 14444961713Sgirish 14544961713Sgirish #ifdef NXGE_DEBUG 14644961713Sgirish if (tx_ring_p->descs_pending) { 14744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 14844961713Sgirish "desc pending %d ", tx_ring_p->descs_pending)); 14944961713Sgirish } 15044961713Sgirish 15144961713Sgirish dump_len = (int)(MBLKL(mp)); 15244961713Sgirish dump_len = (dump_len > 128) ? 128: dump_len; 15344961713Sgirish 15444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 15544961713Sgirish "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 15644961713Sgirish "(Before header reserve: ORIGINAL LEN %d)", 15744961713Sgirish tx_ring_p->tdc, 15844961713Sgirish mp->b_rptr, 15944961713Sgirish dump_len)); 16044961713Sgirish 16144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 16244961713Sgirish "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 16344961713Sgirish nxge_dump_packet((char *)mp->b_rptr, dump_len))); 16444961713Sgirish #endif 16544961713Sgirish 16644961713Sgirish MUTEX_ENTER(&tx_ring_p->lock); 16744961713Sgirish tdc_stats = tx_ring_p->tdc_stats; 16844961713Sgirish mark_mode = (tx_ring_p->descs_pending && 16944961713Sgirish ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 17044961713Sgirish < nxge_tx_minfree)); 17144961713Sgirish 17244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 17344961713Sgirish "TX Descriptor ring is channel %d mark mode %d", 17444961713Sgirish tx_ring_p->tdc, mark_mode)); 17544961713Sgirish 17644961713Sgirish if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 17744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 17844961713Sgirish "TX Descriptor ring is full: channel %d", 17944961713Sgirish tx_ring_p->tdc)); 18044961713Sgirish cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 18144961713Sgirish tdc_stats->tx_no_desc++; 18244961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 18344961713Sgirish if (nxgep->resched_needed && !nxgep->resched_running) { 18444961713Sgirish nxgep->resched_running = B_TRUE; 18544961713Sgirish ddi_trigger_softintr(nxgep->resched_id); 18644961713Sgirish } 18744961713Sgirish status = 1; 18844961713Sgirish goto nxge_start_fail1; 18944961713Sgirish } 19044961713Sgirish 19144961713Sgirish nmp = mp; 19244961713Sgirish i = sop_index = tx_ring_p->wr_index; 19344961713Sgirish nmblks = 0; 19444961713Sgirish ngathers = 0; 19544961713Sgirish pkt_len = 0; 19644961713Sgirish pack_len = 0; 19744961713Sgirish clen = 0; 19844961713Sgirish last_bidx = -1; 19944961713Sgirish good_packet = B_TRUE; 20044961713Sgirish 20144961713Sgirish desc_area = tx_ring_p->tdc_desc; 20244961713Sgirish npi_handle = desc_area.npi_handle; 20344961713Sgirish npi_desc_handle.regh = (nxge_os_acc_handle_t) 20444961713Sgirish DMA_COMMON_ACC_HANDLE(desc_area); 20544961713Sgirish tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 20644961713Sgirish #ifdef NXGE_DEBUG 20744961713Sgirish tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 20844961713Sgirish #endif 20944961713Sgirish tx_desc_dma_handle = (nxge_os_dma_handle_t) 21044961713Sgirish DMA_COMMON_HANDLE(desc_area); 21144961713Sgirish tx_msg_ring = tx_ring_p->tx_msg_ring; 21244961713Sgirish 21344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 21444961713Sgirish sop_index, i)); 21544961713Sgirish 21644961713Sgirish #ifdef NXGE_DEBUG 21744961713Sgirish msgsize = msgdsize(nmp); 21844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 21944961713Sgirish "==> nxge_start(1): wr_index %d i %d msgdsize %d", 22044961713Sgirish sop_index, i, msgsize)); 22144961713Sgirish #endif 22244961713Sgirish /* 22344961713Sgirish * The first 16 bytes of the premapped buffer are reserved 22444961713Sgirish * for header. No padding will be used. 22544961713Sgirish */ 22644961713Sgirish pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 2271f8914d5Sml if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 22844961713Sgirish bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 22944961713Sgirish } else { 23044961713Sgirish bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 23144961713Sgirish } 23244961713Sgirish while (nmp) { 23344961713Sgirish good_packet = B_TRUE; 23444961713Sgirish b_rptr = nmp->b_rptr; 23544961713Sgirish len = MBLKL(nmp); 23644961713Sgirish if (len <= 0) { 23744961713Sgirish nmp = nmp->b_cont; 23844961713Sgirish continue; 23944961713Sgirish } 24044961713Sgirish nmblks++; 24144961713Sgirish 24244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 24344961713Sgirish "len %d pkt_len %d pack_len %d", 24444961713Sgirish nmblks, len, pkt_len, pack_len)); 24544961713Sgirish /* 24614ea4bb7Ssd * Hardware limits the transfer length to 4K for NIU and 24714ea4bb7Ssd * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 24814ea4bb7Ssd * use TX_MAX_TRANSFER_LENGTH as the limit for both. 24914ea4bb7Ssd * If len is longer than the limit, then we break nmp into 25014ea4bb7Ssd * two chunks: Make the first chunk equal to the limit and 25114ea4bb7Ssd * the second chunk for the remaining data. If the second 25214ea4bb7Ssd * chunk is still larger than the limit, then it will be 25314ea4bb7Ssd * broken into two in the next pass. 25444961713Sgirish */ 25514ea4bb7Ssd if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 256*53f3d8ecSyc if ((t_mp = dupb(nmp)) != NULL) { 257*53f3d8ecSyc nmp->b_wptr = nmp->b_rptr + 258*53f3d8ecSyc (TX_MAX_TRANSFER_LENGTH 259*53f3d8ecSyc - TX_PKT_HEADER_SIZE); 260*53f3d8ecSyc t_mp->b_rptr = nmp->b_wptr; 261*53f3d8ecSyc t_mp->b_cont = nmp->b_cont; 262*53f3d8ecSyc nmp->b_cont = t_mp; 263*53f3d8ecSyc len = MBLKL(nmp); 264*53f3d8ecSyc } else { 265*53f3d8ecSyc good_packet = B_FALSE; 266*53f3d8ecSyc goto nxge_start_fail2; 267*53f3d8ecSyc } 26844961713Sgirish } 26944961713Sgirish tx_desc.value = 0; 27044961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 27144961713Sgirish #ifdef NXGE_DEBUG 27244961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 27344961713Sgirish #endif 27444961713Sgirish tx_msg_p = &tx_msg_ring[i]; 275adfcba55Sjoycey #if defined(__i386) 276adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 277adfcba55Sjoycey #else 27844961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 279adfcba55Sjoycey #endif 28044961713Sgirish if (!header_set && 28144961713Sgirish ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 28244961713Sgirish (len >= bcopy_thresh))) { 28344961713Sgirish header_set = B_TRUE; 28444961713Sgirish bcopy_thresh += TX_PKT_HEADER_SIZE; 28544961713Sgirish boff = 0; 28644961713Sgirish pack_len = 0; 28744961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 28844961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 28944961713Sgirish clen = pkt_len; 29044961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 29144961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 29244961713Sgirish (void) ddi_dma_sync(dma_handle, 29344961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 29444961713Sgirish DDI_DMA_SYNC_FORDEV); 29544961713Sgirish 29644961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 29744961713Sgirish goto nxge_start_control_header_only; 29844961713Sgirish } 29944961713Sgirish 30044961713Sgirish pkt_len += len; 30144961713Sgirish pack_len += len; 30244961713Sgirish 30344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 30444961713Sgirish "desc entry %d " 30544961713Sgirish "DESC IOADDR $%p " 30644961713Sgirish "desc_vp $%p tx_desc_p $%p " 30744961713Sgirish "desc_pp $%p tx_desc_pp $%p " 30844961713Sgirish "len %d pkt_len %d pack_len %d", 30944961713Sgirish i, 31044961713Sgirish DMA_COMMON_IOADDR(desc_area), 31144961713Sgirish tx_desc_ring_vp, tx_desc_p, 31244961713Sgirish tx_desc_ring_pp, tx_desc_pp, 31344961713Sgirish len, pkt_len, pack_len)); 31444961713Sgirish 31544961713Sgirish if (len < bcopy_thresh) { 31644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 31744961713Sgirish "USE BCOPY: ")); 31844961713Sgirish if (nxge_tx_tiny_pack) { 31944961713Sgirish uint32_t blst = 32044961713Sgirish TXDMA_DESC_NEXT_INDEX(i, -1, 32144961713Sgirish tx_ring_p->tx_wrap_mask); 32244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 32344961713Sgirish "==> nxge_start(5): pack")); 32444961713Sgirish if ((pack_len <= bcopy_thresh) && 32544961713Sgirish (last_bidx == blst)) { 32644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 32744961713Sgirish "==> nxge_start: pack(6) " 32844961713Sgirish "(pkt_len %d pack_len %d)", 32944961713Sgirish pkt_len, pack_len)); 33044961713Sgirish i = blst; 33144961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 33244961713Sgirish #ifdef NXGE_DEBUG 33344961713Sgirish tx_desc_pp = &tx_desc_ring_pp[i]; 33444961713Sgirish #endif 33544961713Sgirish tx_msg_p = &tx_msg_ring[i]; 33644961713Sgirish boff = pack_len - len; 33744961713Sgirish ngathers--; 338a3c5bd6dSspeer } else if (pack_len > bcopy_thresh && 339a3c5bd6dSspeer header_set) { 34044961713Sgirish pack_len = len; 34144961713Sgirish boff = 0; 34244961713Sgirish bcopy_thresh = nxge_bcopy_thresh; 34344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 34444961713Sgirish "==> nxge_start(7): > max NEW " 34544961713Sgirish "bcopy thresh %d " 34644961713Sgirish "pkt_len %d pack_len %d(next)", 34744961713Sgirish bcopy_thresh, 34844961713Sgirish pkt_len, pack_len)); 34944961713Sgirish } 35044961713Sgirish last_bidx = i; 35144961713Sgirish } 35244961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 35344961713Sgirish if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 35444961713Sgirish hdrp = (p_tx_pkt_header_t)kaddr; 35544961713Sgirish header_set = B_TRUE; 35644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 35744961713Sgirish "==> nxge_start(7_x2): " 35844961713Sgirish "pkt_len %d pack_len %d (new hdrp $%p)", 35944961713Sgirish pkt_len, pack_len, hdrp)); 36044961713Sgirish } 36144961713Sgirish tx_msg_p->flags.dma_type = USE_BCOPY; 36244961713Sgirish kaddr += boff; 36344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 36444961713Sgirish "USE BCOPY: before bcopy " 36544961713Sgirish "DESC IOADDR $%p entry %d " 36644961713Sgirish "bcopy packets %d " 36744961713Sgirish "bcopy kaddr $%p " 36844961713Sgirish "bcopy ioaddr (SAD) $%p " 36944961713Sgirish "bcopy clen %d " 37044961713Sgirish "bcopy boff %d", 37144961713Sgirish DMA_COMMON_IOADDR(desc_area), i, 37244961713Sgirish tdc_stats->tx_hdr_pkts, 37344961713Sgirish kaddr, 37444961713Sgirish dma_ioaddr, 37544961713Sgirish clen, 37644961713Sgirish boff)); 37744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 37844961713Sgirish "1USE BCOPY: ")); 37944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 38044961713Sgirish "2USE BCOPY: ")); 38144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 38244961713Sgirish "last USE BCOPY: copy from b_rptr $%p " 38344961713Sgirish "to KADDR $%p (len %d offset %d", 38444961713Sgirish b_rptr, kaddr, len, boff)); 38544961713Sgirish 38644961713Sgirish bcopy(b_rptr, kaddr, len); 38744961713Sgirish 38844961713Sgirish #ifdef NXGE_DEBUG 38944961713Sgirish dump_len = (len > 128) ? 128: len; 39044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 39144961713Sgirish "==> nxge_start: dump packets " 39244961713Sgirish "(After BCOPY len %d)" 39344961713Sgirish "(b_rptr $%p): %s", len, nmp->b_rptr, 39444961713Sgirish nxge_dump_packet((char *)nmp->b_rptr, 39544961713Sgirish dump_len))); 39644961713Sgirish #endif 39744961713Sgirish 39844961713Sgirish dma_handle = tx_msg_p->buf_dma_handle; 39944961713Sgirish dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 40044961713Sgirish (void) ddi_dma_sync(dma_handle, 40144961713Sgirish i * nxge_bcopy_thresh, nxge_bcopy_thresh, 40244961713Sgirish DDI_DMA_SYNC_FORDEV); 40344961713Sgirish clen = len + boff; 40444961713Sgirish tdc_stats->tx_hdr_pkts++; 40544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 40644961713Sgirish "USE BCOPY: " 40744961713Sgirish "DESC IOADDR $%p entry %d " 40844961713Sgirish "bcopy packets %d " 40944961713Sgirish "bcopy kaddr $%p " 41044961713Sgirish "bcopy ioaddr (SAD) $%p " 41144961713Sgirish "bcopy clen %d " 41244961713Sgirish "bcopy boff %d", 41344961713Sgirish DMA_COMMON_IOADDR(desc_area), 41444961713Sgirish i, 41544961713Sgirish tdc_stats->tx_hdr_pkts, 41644961713Sgirish kaddr, 41744961713Sgirish dma_ioaddr, 41844961713Sgirish clen, 41944961713Sgirish boff)); 42044961713Sgirish } else { 42144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 42244961713Sgirish "USE DVMA: len %d", len)); 42344961713Sgirish tx_msg_p->flags.dma_type = USE_DMA; 42444961713Sgirish dma_flags = DDI_DMA_WRITE; 42544961713Sgirish if (len < nxge_dma_stream_thresh) { 42644961713Sgirish dma_flags |= DDI_DMA_CONSISTENT; 42744961713Sgirish } else { 42844961713Sgirish dma_flags |= DDI_DMA_STREAMING; 42944961713Sgirish } 43044961713Sgirish 43144961713Sgirish dma_handle = tx_msg_p->dma_handle; 43244961713Sgirish status = ddi_dma_addr_bind_handle(dma_handle, NULL, 43344961713Sgirish (caddr_t)b_rptr, len, dma_flags, 43444961713Sgirish DDI_DMA_DONTWAIT, NULL, 43544961713Sgirish &dma_cookie, &ncookies); 43644961713Sgirish if (status == DDI_DMA_MAPPED) { 43744961713Sgirish dma_ioaddr = dma_cookie.dmac_laddress; 43844961713Sgirish len = (int)dma_cookie.dmac_size; 43944961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 44044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 44144961713Sgirish "==> nxge_start(12_1): " 44244961713Sgirish "USE DVMA: len %d clen %d " 44344961713Sgirish "ngathers %d", 44444961713Sgirish len, clen, 44544961713Sgirish ngathers)); 446adfcba55Sjoycey #if defined(__i386) 447adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 448adfcba55Sjoycey #else 44944961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 450adfcba55Sjoycey #endif 45144961713Sgirish while (ncookies > 1) { 45244961713Sgirish ngathers++; 45344961713Sgirish /* 45444961713Sgirish * this is the fix for multiple 45544961713Sgirish * cookies, which are basicaly 45644961713Sgirish * a descriptor entry, we don't set 45744961713Sgirish * SOP bit as well as related fields 45844961713Sgirish */ 45944961713Sgirish 46044961713Sgirish (void) npi_txdma_desc_gather_set( 46144961713Sgirish npi_desc_handle, 46244961713Sgirish &tx_desc, 46344961713Sgirish (ngathers -1), 46444961713Sgirish mark_mode, 46544961713Sgirish ngathers, 46644961713Sgirish dma_ioaddr, 46744961713Sgirish clen); 46844961713Sgirish 46944961713Sgirish tx_msg_p->tx_msg_size = clen; 47044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 47144961713Sgirish "==> nxge_start: DMA " 47244961713Sgirish "ncookie %d " 47344961713Sgirish "ngathers %d " 47444961713Sgirish "dma_ioaddr $%p len %d" 47544961713Sgirish "desc $%p descp $%p (%d)", 47644961713Sgirish ncookies, 47744961713Sgirish ngathers, 47844961713Sgirish dma_ioaddr, clen, 47944961713Sgirish *tx_desc_p, tx_desc_p, i)); 48044961713Sgirish 48144961713Sgirish ddi_dma_nextcookie(dma_handle, 48244961713Sgirish &dma_cookie); 48344961713Sgirish dma_ioaddr = 48444961713Sgirish dma_cookie.dmac_laddress; 48544961713Sgirish 48644961713Sgirish len = (int)dma_cookie.dmac_size; 48744961713Sgirish clen = (uint32_t)dma_cookie.dmac_size; 48844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 48944961713Sgirish "==> nxge_start(12_2): " 49044961713Sgirish "USE DVMA: len %d clen %d ", 49144961713Sgirish len, clen)); 49244961713Sgirish 49344961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, 49444961713Sgirish tx_ring_p->tx_wrap_mask); 49544961713Sgirish tx_desc_p = &tx_desc_ring_vp[i]; 49644961713Sgirish 49744961713Sgirish npi_desc_handle.regp = 498adfcba55Sjoycey #if defined(__i386) 499adfcba55Sjoycey (uint32_t)tx_desc_p; 500adfcba55Sjoycey #else 50144961713Sgirish (uint64_t)tx_desc_p; 502adfcba55Sjoycey #endif 50344961713Sgirish tx_msg_p = &tx_msg_ring[i]; 50444961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 50544961713Sgirish tx_desc.value = 0; 50644961713Sgirish 50744961713Sgirish ncookies--; 50844961713Sgirish } 50944961713Sgirish tdc_stats->tx_ddi_pkts++; 51044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 51144961713Sgirish "DMA: ddi packets %d", 51244961713Sgirish tdc_stats->tx_ddi_pkts)); 51344961713Sgirish } else { 51444961713Sgirish NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 51544961713Sgirish "dma mapping failed for %d " 51644961713Sgirish "bytes addr $%p flags %x (%d)", 51744961713Sgirish len, b_rptr, status, status)); 51844961713Sgirish good_packet = B_FALSE; 51944961713Sgirish tdc_stats->tx_dma_bind_fail++; 52044961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 52144961713Sgirish goto nxge_start_fail2; 52244961713Sgirish } 52344961713Sgirish } /* ddi dvma */ 52444961713Sgirish 52544961713Sgirish nmp = nmp->b_cont; 52644961713Sgirish nxge_start_control_header_only: 527adfcba55Sjoycey #if defined(__i386) 528adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 529adfcba55Sjoycey #else 53044961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 531adfcba55Sjoycey #endif 53244961713Sgirish ngathers++; 53344961713Sgirish 53444961713Sgirish if (ngathers == 1) { 53544961713Sgirish #ifdef NXGE_DEBUG 53644961713Sgirish save_desc_p = &sop_tx_desc; 53744961713Sgirish #endif 53844961713Sgirish sop_tx_desc_p = &sop_tx_desc; 53944961713Sgirish sop_tx_desc_p->value = 0; 54044961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = clen; 54144961713Sgirish sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 54244961713Sgirish sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 54344961713Sgirish } else { 54444961713Sgirish #ifdef NXGE_DEBUG 54544961713Sgirish save_desc_p = &tx_desc; 54644961713Sgirish #endif 54744961713Sgirish tmp_desc_p = &tx_desc; 54844961713Sgirish tmp_desc_p->value = 0; 54944961713Sgirish tmp_desc_p->bits.hdw.tr_len = clen; 55044961713Sgirish tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 55144961713Sgirish tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 55244961713Sgirish 55344961713Sgirish tx_desc_p->value = tmp_desc_p->value; 55444961713Sgirish } 55544961713Sgirish 55644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 55744961713Sgirish "Desc_entry %d ngathers %d " 55844961713Sgirish "desc_vp $%p tx_desc_p $%p " 55944961713Sgirish "len %d clen %d pkt_len %d pack_len %d nmblks %d " 56044961713Sgirish "dma_ioaddr (SAD) $%p mark %d", 56144961713Sgirish i, ngathers, 56244961713Sgirish tx_desc_ring_vp, tx_desc_p, 56344961713Sgirish len, clen, pkt_len, pack_len, nmblks, 56444961713Sgirish dma_ioaddr, mark_mode)); 56544961713Sgirish 56644961713Sgirish #ifdef NXGE_DEBUG 56744961713Sgirish npi_desc_handle.nxgep = nxgep; 56844961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 56944961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 57044961713Sgirish sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 57144961713Sgirish xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 57244961713Sgirish TX_PKT_DESC_TR_LEN_SHIFT); 57344961713Sgirish 57444961713Sgirish 57544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 57644961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 57744961713Sgirish "mark %d sop %d\n", 57844961713Sgirish save_desc_p->value, 57944961713Sgirish sad, 58044961713Sgirish save_desc_p->bits.hdw.tr_len, 58144961713Sgirish xfer_len, 58244961713Sgirish save_desc_p->bits.hdw.num_ptr, 58344961713Sgirish save_desc_p->bits.hdw.mark, 58444961713Sgirish save_desc_p->bits.hdw.sop)); 58544961713Sgirish 58644961713Sgirish npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 58744961713Sgirish #endif 58844961713Sgirish 58944961713Sgirish tx_msg_p->tx_msg_size = clen; 59044961713Sgirish i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 59144961713Sgirish if (ngathers > nxge_tx_max_gathers) { 59244961713Sgirish good_packet = B_FALSE; 59344961713Sgirish hcksum_retrieve(mp, NULL, NULL, &start_offset, 59444961713Sgirish &stuff_offset, &end_offset, &value, 59544961713Sgirish &cksum_flags); 59644961713Sgirish 59744961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 59844961713Sgirish "==> nxge_start(14): pull msg - " 59944961713Sgirish "len %d pkt_len %d ngathers %d", 60044961713Sgirish len, pkt_len, ngathers)); 60144961713Sgirish /* Pull all message blocks from b_cont */ 60244961713Sgirish if ((msgpullup(mp, -1)) == NULL) { 60344961713Sgirish goto nxge_start_fail2; 60444961713Sgirish } 60544961713Sgirish goto nxge_start_fail2; 60644961713Sgirish } 60744961713Sgirish } /* while (nmp) */ 60844961713Sgirish 60944961713Sgirish tx_msg_p->tx_message = mp; 61044961713Sgirish tx_desc_p = &tx_desc_ring_vp[sop_index]; 611adfcba55Sjoycey #if defined(__i386) 612adfcba55Sjoycey npi_desc_handle.regp = (uint32_t)tx_desc_p; 613adfcba55Sjoycey #else 61444961713Sgirish npi_desc_handle.regp = (uint64_t)tx_desc_p; 615adfcba55Sjoycey #endif 61644961713Sgirish 61744961713Sgirish pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 61844961713Sgirish pkthdrp->reserved = 0; 61944961713Sgirish hdrp->value = 0; 62044961713Sgirish (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 62144961713Sgirish (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 62244961713Sgirish 62344961713Sgirish if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 62444961713Sgirish tdc_stats->tx_jumbo_pkts++; 62544961713Sgirish } 62644961713Sgirish 62744961713Sgirish min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 62844961713Sgirish if (pkt_len < min_len) { 62944961713Sgirish /* Assume we use bcopy to premapped buffers */ 63044961713Sgirish kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 63144961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 63244961713Sgirish "==> nxge_start(14-1): < (msg_min + 16)" 63344961713Sgirish "len %d pkt_len %d min_len %d bzero %d ngathers %d", 63444961713Sgirish len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 63544961713Sgirish bzero((kaddr + pkt_len), (min_len - pkt_len)); 63644961713Sgirish pkt_len = tx_msg_p->tx_msg_size = min_len; 63744961713Sgirish 63844961713Sgirish sop_tx_desc_p->bits.hdw.tr_len = min_len; 63944961713Sgirish 64044961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 64144961713Sgirish tx_desc_p->value = sop_tx_desc_p->value; 64244961713Sgirish 64344961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, 64444961713Sgirish "==> nxge_start(14-2): < msg_min - " 64544961713Sgirish "len %d pkt_len %d min_len %d ngathers %d", 64644961713Sgirish len, pkt_len, min_len, ngathers)); 64744961713Sgirish } 64844961713Sgirish 64944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 65044961713Sgirish cksum_flags)); 65144961713Sgirish if (cksum_flags & HCK_PARTIALCKSUM) { 65244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 65344961713Sgirish "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 65444961713Sgirish cksum_flags)); 65544961713Sgirish cksum_on = B_TRUE; 65644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 65744961713Sgirish "==> nxge_start: from IP cksum_flags 0x%x " 65844961713Sgirish "(partial checksum) " 65944961713Sgirish "start_offset %d stuff_offset %d", 66044961713Sgirish cksum_flags, start_offset, stuff_offset)); 66144961713Sgirish tmp_len = (uint64_t)(start_offset >> 1); 66244961713Sgirish hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 66344961713Sgirish tmp_len = (uint64_t)(stuff_offset >> 1); 66444961713Sgirish hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 66544961713Sgirish 66644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 66744961713Sgirish "==> nxge_start: from IP cksum_flags 0x%x " 66844961713Sgirish "(partial checksum) " 66944961713Sgirish "after SHIFT start_offset %d stuff_offset %d", 67044961713Sgirish cksum_flags, start_offset, stuff_offset)); 67144961713Sgirish } 67244961713Sgirish { 67344961713Sgirish uint64_t tmp_len; 67444961713Sgirish 67544961713Sgirish /* pkt_len already includes 16 + paddings!! */ 67644961713Sgirish /* Update the control header length */ 67744961713Sgirish tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 67844961713Sgirish tmp_len = hdrp->value | 67944961713Sgirish (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 68044961713Sgirish 68144961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 68244961713Sgirish "==> nxge_start(15_x1): setting SOP " 68344961713Sgirish "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 68444961713Sgirish "0x%llx hdrp->value 0x%llx", 68544961713Sgirish tot_xfer_len, tot_xfer_len, pkt_len, 68644961713Sgirish tmp_len, hdrp->value)); 68744961713Sgirish #if defined(_BIG_ENDIAN) 68844961713Sgirish hdrp->value = ddi_swap64(tmp_len); 68944961713Sgirish #else 69044961713Sgirish hdrp->value = tmp_len; 69144961713Sgirish #endif 69244961713Sgirish NXGE_DEBUG_MSG((nxgep, 69344961713Sgirish TX_CTL, "==> nxge_start(15_x2): setting SOP " 69444961713Sgirish "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 69544961713Sgirish "tmp_len 0x%llx hdrp->value 0x%llx", 69644961713Sgirish tot_xfer_len, pkt_len, 69744961713Sgirish tmp_len, hdrp->value)); 69844961713Sgirish } 69944961713Sgirish 70044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 70144961713Sgirish "wr_index %d " 70244961713Sgirish "tot_xfer_len (%d) pkt_len %d npads %d", 70344961713Sgirish sop_index, 70444961713Sgirish tot_xfer_len, pkt_len, 70544961713Sgirish npads)); 70644961713Sgirish 70744961713Sgirish sop_tx_desc_p->bits.hdw.sop = 1; 70844961713Sgirish sop_tx_desc_p->bits.hdw.mark = mark_mode; 70944961713Sgirish sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 71044961713Sgirish 71144961713Sgirish NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 71244961713Sgirish 71344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 71444961713Sgirish 71544961713Sgirish #ifdef NXGE_DEBUG 71644961713Sgirish npi_desc_handle.nxgep = nxgep; 71744961713Sgirish npi_desc_handle.function.function = nxgep->function_num; 71844961713Sgirish npi_desc_handle.function.instance = nxgep->instance; 71944961713Sgirish 72044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 72144961713Sgirish "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 72244961713Sgirish save_desc_p->value, 72344961713Sgirish sad, 72444961713Sgirish save_desc_p->bits.hdw.tr_len, 72544961713Sgirish xfer_len, 72644961713Sgirish save_desc_p->bits.hdw.num_ptr, 72744961713Sgirish save_desc_p->bits.hdw.mark, 72844961713Sgirish save_desc_p->bits.hdw.sop)); 72944961713Sgirish (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 73044961713Sgirish 73144961713Sgirish dump_len = (pkt_len > 128) ? 128: pkt_len; 73244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 73344961713Sgirish "==> nxge_start: dump packets(17) (after sop set, len " 73444961713Sgirish " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 73544961713Sgirish "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 73644961713Sgirish (char *)hdrp, 73744961713Sgirish nxge_dump_packet((char *)hdrp, dump_len))); 73844961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 73944961713Sgirish "==> nxge_start(18): TX desc sync: sop_index %d", 74044961713Sgirish sop_index)); 74144961713Sgirish #endif 74244961713Sgirish 74344961713Sgirish if ((ngathers == 1) || tx_ring_p->wr_index < i) { 74444961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 74544961713Sgirish sop_index * sizeof (tx_desc_t), 74644961713Sgirish ngathers * sizeof (tx_desc_t), 74744961713Sgirish DDI_DMA_SYNC_FORDEV); 74844961713Sgirish 74944961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 75044961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 75144961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 75244961713Sgirish stuff_offset, start_offset, 75344961713Sgirish pkt_len, ngathers, sop_index)); 75444961713Sgirish } else { /* more than one descriptor and wrap around */ 75544961713Sgirish uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 75644961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 75744961713Sgirish sop_index * sizeof (tx_desc_t), 75844961713Sgirish nsdescs * sizeof (tx_desc_t), 75944961713Sgirish DDI_DMA_SYNC_FORDEV); 76044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 76144961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 76244961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 76344961713Sgirish stuff_offset, start_offset, 76444961713Sgirish pkt_len, ngathers, sop_index)); 76544961713Sgirish 76644961713Sgirish (void) ddi_dma_sync(tx_desc_dma_handle, 76744961713Sgirish 0, 76844961713Sgirish (ngathers - nsdescs) * sizeof (tx_desc_t), 76944961713Sgirish DDI_DMA_SYNC_FORDEV); 77044961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 77144961713Sgirish "cs_off = 0x%02X cs_s_off = 0x%02X " 77244961713Sgirish "pkt_len %d ngathers %d sop_index %d\n", 77344961713Sgirish stuff_offset, start_offset, 77444961713Sgirish pkt_len, ngathers, sop_index)); 77544961713Sgirish } 77644961713Sgirish 77744961713Sgirish tail_index = tx_ring_p->wr_index; 77844961713Sgirish tail_wrap = tx_ring_p->wr_index_wrap; 77944961713Sgirish 78044961713Sgirish tx_ring_p->wr_index = i; 78144961713Sgirish if (tx_ring_p->wr_index <= tail_index) { 78244961713Sgirish tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 78344961713Sgirish B_FALSE : B_TRUE); 78444961713Sgirish } 78544961713Sgirish 78644961713Sgirish tx_ring_p->descs_pending += ngathers; 78744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 78844961713Sgirish "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 78944961713Sgirish tx_ring_p->tdc, 79044961713Sgirish tx_ring_p->wr_index, 79144961713Sgirish tx_ring_p->wr_index_wrap, 79244961713Sgirish ngathers, 79344961713Sgirish tx_ring_p->descs_pending)); 79444961713Sgirish 79544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 79644961713Sgirish 79744961713Sgirish { 79844961713Sgirish tx_ring_kick_t kick; 79944961713Sgirish 80044961713Sgirish kick.value = 0; 80144961713Sgirish kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 80244961713Sgirish kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 80344961713Sgirish 80444961713Sgirish /* Kick start the Transmit kick register */ 80544961713Sgirish TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 80644961713Sgirish TX_RING_KICK_REG, 80744961713Sgirish (uint8_t)tx_ring_p->tdc, 80844961713Sgirish kick.value); 80944961713Sgirish } 81044961713Sgirish 81144961713Sgirish tdc_stats->tx_starts++; 81244961713Sgirish 81344961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 81444961713Sgirish 81544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 81644961713Sgirish 81744961713Sgirish return (status); 81844961713Sgirish 81944961713Sgirish nxge_start_fail2: 82044961713Sgirish if (good_packet == B_FALSE) { 82144961713Sgirish cur_index = sop_index; 82244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 82344961713Sgirish for (i = 0; i < ngathers; i++) { 82444961713Sgirish tx_desc_p = &tx_desc_ring_vp[cur_index]; 825adfcba55Sjoycey #if defined(__i386) 826adfcba55Sjoycey npi_handle.regp = (uint32_t)tx_desc_p; 827adfcba55Sjoycey #else 82844961713Sgirish npi_handle.regp = (uint64_t)tx_desc_p; 829adfcba55Sjoycey #endif 83044961713Sgirish tx_msg_p = &tx_msg_ring[cur_index]; 83144961713Sgirish (void) npi_txdma_desc_set_zero(npi_handle, 1); 83244961713Sgirish if (tx_msg_p->flags.dma_type == USE_DVMA) { 83344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, 834*53f3d8ecSyc "tx_desc_p = %X index = %d", 835*53f3d8ecSyc tx_desc_p, tx_ring_p->rd_index)); 836*53f3d8ecSyc (void) dvma_unload(tx_msg_p->dvma_handle, 837*53f3d8ecSyc 0, -1); 83844961713Sgirish tx_msg_p->dvma_handle = NULL; 83944961713Sgirish if (tx_ring_p->dvma_wr_index == 840*53f3d8ecSyc tx_ring_p->dvma_wrap_mask) 84144961713Sgirish tx_ring_p->dvma_wr_index = 0; 84244961713Sgirish else 84344961713Sgirish tx_ring_p->dvma_wr_index++; 84444961713Sgirish tx_ring_p->dvma_pending--; 845*53f3d8ecSyc } else if (tx_msg_p->flags.dma_type == USE_DMA) { 84644961713Sgirish if (ddi_dma_unbind_handle( 847*53f3d8ecSyc tx_msg_p->dma_handle)) { 84844961713Sgirish cmn_err(CE_WARN, "!nxge_start: " 849*53f3d8ecSyc "ddi_dma_unbind_handle failed"); 850*53f3d8ecSyc } 85144961713Sgirish } 85244961713Sgirish tx_msg_p->flags.dma_type = USE_NONE; 85344961713Sgirish cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 85444961713Sgirish tx_ring_p->tx_wrap_mask); 85544961713Sgirish 85644961713Sgirish } 85744961713Sgirish 85844961713Sgirish nxgep->resched_needed = B_TRUE; 85944961713Sgirish } 86044961713Sgirish 86144961713Sgirish MUTEX_EXIT(&tx_ring_p->lock); 86244961713Sgirish 86344961713Sgirish nxge_start_fail1: 86444961713Sgirish /* Add FMA to check the access handle nxge_hregh */ 86544961713Sgirish 86644961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 86744961713Sgirish 86844961713Sgirish return (status); 86944961713Sgirish } 87044961713Sgirish 8711f8914d5Sml int 8721f8914d5Sml nxge_serial_tx(mblk_t *mp, void *arg) 8731f8914d5Sml { 8741f8914d5Sml p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 8751f8914d5Sml p_nxge_t nxgep = tx_ring_p->nxgep; 8761f8914d5Sml 8771f8914d5Sml return (nxge_start(nxgep, tx_ring_p, mp)); 8781f8914d5Sml } 8791f8914d5Sml 88044961713Sgirish boolean_t 88144961713Sgirish nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 88244961713Sgirish { 88344961713Sgirish p_tx_ring_t *tx_rings; 88444961713Sgirish uint8_t ring_index; 8851f8914d5Sml p_tx_ring_t tx_ring_p; 88644961713Sgirish 88744961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 88844961713Sgirish 88944961713Sgirish ASSERT(mp->b_next == NULL); 89044961713Sgirish 89144961713Sgirish ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 89244961713Sgirish tx_rings = nxgep->tx_rings->rings; 89344961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 89444961713Sgirish tx_rings)); 89544961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 89644961713Sgirish "ring_index %d", nxgep->max_tdcs, ring_index)); 89744961713Sgirish 8981f8914d5Sml switch (nxge_tx_scheme) { 8991f8914d5Sml case NXGE_USE_START: 9001f8914d5Sml if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 9011f8914d5Sml NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 9021f8914d5Sml "ring index %d", ring_index)); 9031f8914d5Sml return (B_FALSE); 9041f8914d5Sml } 9051f8914d5Sml break; 9061f8914d5Sml 9071f8914d5Sml case NXGE_USE_SERIAL: 9081f8914d5Sml default: 9091f8914d5Sml tx_ring_p = tx_rings[ring_index]; 9101f8914d5Sml nxge_serialize_enter(tx_ring_p->serial, mp); 9111f8914d5Sml break; 91244961713Sgirish } 91344961713Sgirish 91444961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 91544961713Sgirish ring_index)); 91644961713Sgirish 91744961713Sgirish return (B_TRUE); 91844961713Sgirish } 91944961713Sgirish 92044961713Sgirish /* 92144961713Sgirish * nxge_m_tx() - send a chain of packets 92244961713Sgirish */ 92344961713Sgirish mblk_t * 92444961713Sgirish nxge_m_tx(void *arg, mblk_t *mp) 92544961713Sgirish { 92644961713Sgirish p_nxge_t nxgep = (p_nxge_t)arg; 92744961713Sgirish mblk_t *next; 92844961713Sgirish mac_tx_hint_t hint; 92944961713Sgirish 93044961713Sgirish if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 93144961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 93244961713Sgirish "==> nxge_m_tx: hardware not initialized")); 93344961713Sgirish NXGE_DEBUG_MSG((nxgep, DDI_CTL, 93444961713Sgirish "<== nxge_m_tx")); 93544961713Sgirish return (mp); 93644961713Sgirish } 93744961713Sgirish 93844961713Sgirish hint.hash = NULL; 93944961713Sgirish hint.vid = 0; 94044961713Sgirish hint.sap = 0; 94144961713Sgirish 94244961713Sgirish while (mp != NULL) { 94344961713Sgirish next = mp->b_next; 94444961713Sgirish mp->b_next = NULL; 94544961713Sgirish 94644961713Sgirish /* 94744961713Sgirish * Until Nemo tx resource works, the mac driver 94844961713Sgirish * does the load balancing based on TCP port, 94944961713Sgirish * or CPU. For debugging, we use a system 95044961713Sgirish * configurable parameter. 95144961713Sgirish */ 95244961713Sgirish if (!nxge_send(nxgep, mp, &hint)) { 95344961713Sgirish mp->b_next = next; 95444961713Sgirish break; 95544961713Sgirish } 95644961713Sgirish 95744961713Sgirish mp = next; 95844961713Sgirish } 95944961713Sgirish 96044961713Sgirish return (mp); 96144961713Sgirish } 96244961713Sgirish 96344961713Sgirish int 96444961713Sgirish nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 96544961713Sgirish { 96644961713Sgirish uint8_t ring_index = 0; 96744961713Sgirish uint8_t *tcp_port; 96844961713Sgirish p_mblk_t nmp; 96944961713Sgirish size_t mblk_len; 97044961713Sgirish size_t iph_len; 97144961713Sgirish size_t hdrs_size; 97244961713Sgirish uint8_t hdrs_buf[sizeof (struct ether_header) + 97344961713Sgirish IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 97444961713Sgirish /* 97544961713Sgirish * allocate space big enough to cover 97644961713Sgirish * the max ip header length and the first 97744961713Sgirish * 4 bytes of the TCP/IP header. 97844961713Sgirish */ 97944961713Sgirish 98044961713Sgirish boolean_t qos = B_FALSE; 98144961713Sgirish 98244961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 98344961713Sgirish 98444961713Sgirish if (hp->vid) { 98544961713Sgirish qos = B_TRUE; 98644961713Sgirish } 98744961713Sgirish switch (nxge_tx_lb_policy) { 98844961713Sgirish case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 98944961713Sgirish default: 99044961713Sgirish tcp_port = mp->b_rptr; 99144961713Sgirish if (!nxge_no_tx_lb && !qos && 99244961713Sgirish (ntohs(((p_ether_header_t)tcp_port)->ether_type) 99344961713Sgirish == ETHERTYPE_IP)) { 99444961713Sgirish nmp = mp; 99544961713Sgirish mblk_len = MBLKL(nmp); 99644961713Sgirish tcp_port = NULL; 99744961713Sgirish if (mblk_len > sizeof (struct ether_header) + 99844961713Sgirish sizeof (uint8_t)) { 99944961713Sgirish tcp_port = nmp->b_rptr + 100044961713Sgirish sizeof (struct ether_header); 100144961713Sgirish mblk_len -= sizeof (struct ether_header); 100244961713Sgirish iph_len = ((*tcp_port) & 0x0f) << 2; 100344961713Sgirish if (mblk_len > (iph_len + sizeof (uint32_t))) { 100444961713Sgirish tcp_port = nmp->b_rptr; 100544961713Sgirish } else { 100644961713Sgirish tcp_port = NULL; 100744961713Sgirish } 100844961713Sgirish } 100944961713Sgirish if (tcp_port == NULL) { 101044961713Sgirish hdrs_size = 0; 101144961713Sgirish ((p_ether_header_t)hdrs_buf)->ether_type = 0; 101244961713Sgirish while ((nmp) && (hdrs_size < 101344961713Sgirish sizeof (hdrs_buf))) { 101444961713Sgirish mblk_len = MBLKL(nmp); 101544961713Sgirish if (mblk_len >= 101644961713Sgirish (sizeof (hdrs_buf) - hdrs_size)) 101744961713Sgirish mblk_len = sizeof (hdrs_buf) - 101844961713Sgirish hdrs_size; 101944961713Sgirish bcopy(nmp->b_rptr, 102044961713Sgirish &hdrs_buf[hdrs_size], mblk_len); 102144961713Sgirish hdrs_size += mblk_len; 102244961713Sgirish nmp = nmp->b_cont; 102344961713Sgirish } 102444961713Sgirish tcp_port = hdrs_buf; 102544961713Sgirish } 102644961713Sgirish tcp_port += sizeof (ether_header_t); 102744961713Sgirish if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1028958cea9eSml switch (tcp_port[9]) { 1029958cea9eSml case IPPROTO_TCP: 1030958cea9eSml case IPPROTO_UDP: 1031958cea9eSml case IPPROTO_ESP: 103244961713Sgirish tcp_port += ((*tcp_port) & 0x0f) << 2; 103344961713Sgirish ring_index = 1034958cea9eSml ((tcp_port[0] ^ 1035958cea9eSml tcp_port[1] ^ 1036958cea9eSml tcp_port[2] ^ 1037958cea9eSml tcp_port[3]) % maxtdcs); 1038958cea9eSml break; 1039958cea9eSml 1040958cea9eSml case IPPROTO_AH: 1041958cea9eSml /* SPI starts at the 4th byte */ 1042958cea9eSml tcp_port += ((*tcp_port) & 0x0f) << 2; 1043958cea9eSml ring_index = 1044958cea9eSml ((tcp_port[4] ^ 1045958cea9eSml tcp_port[5] ^ 1046958cea9eSml tcp_port[6] ^ 1047958cea9eSml tcp_port[7]) % maxtdcs); 1048958cea9eSml break; 1049958cea9eSml 1050958cea9eSml default: 105144961713Sgirish ring_index = tcp_port[19] % maxtdcs; 1052958cea9eSml break; 105344961713Sgirish } 105444961713Sgirish } else { /* fragmented packet */ 105544961713Sgirish ring_index = tcp_port[19] % maxtdcs; 105644961713Sgirish } 105744961713Sgirish } else { 105844961713Sgirish ring_index = mp->b_band % maxtdcs; 105944961713Sgirish } 106044961713Sgirish break; 106144961713Sgirish 106244961713Sgirish case NXGE_TX_LB_HASH: 106344961713Sgirish if (hp->hash) { 1064adfcba55Sjoycey #if defined(__i386) 1065adfcba55Sjoycey ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1066adfcba55Sjoycey #else 106744961713Sgirish ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1068adfcba55Sjoycey #endif 106944961713Sgirish } else { 107044961713Sgirish ring_index = mp->b_band % maxtdcs; 107144961713Sgirish } 107244961713Sgirish break; 107344961713Sgirish 107444961713Sgirish case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 107544961713Sgirish tcp_port = mp->b_rptr; 107644961713Sgirish ring_index = tcp_port[5] % maxtdcs; 107744961713Sgirish break; 107844961713Sgirish } 107944961713Sgirish 108044961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 108144961713Sgirish 108244961713Sgirish return (ring_index); 108344961713Sgirish } 108444961713Sgirish 108544961713Sgirish uint_t 108644961713Sgirish nxge_reschedule(caddr_t arg) 108744961713Sgirish { 108844961713Sgirish p_nxge_t nxgep; 108944961713Sgirish 109044961713Sgirish nxgep = (p_nxge_t)arg; 109144961713Sgirish 109244961713Sgirish NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 109344961713Sgirish 109444961713Sgirish if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 109544961713Sgirish nxgep->resched_needed) { 109644961713Sgirish mac_tx_update(nxgep->mach); 109744961713Sgirish nxgep->resched_needed = B_FALSE; 109844961713Sgirish nxgep->resched_running = B_FALSE; 109944961713Sgirish } 110044961713Sgirish 110144961713Sgirish NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 110244961713Sgirish return (DDI_INTR_CLAIMED); 110344961713Sgirish } 1104