1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 #include <sys/nxge/nxge_hio.h> 30 #include <npi_tx_wr64.h> 31 32 /* Software LSO required header files */ 33 #include <netinet/tcp.h> 34 #include <inet/ip_impl.h> 35 #include <inet/tcp.h> 36 37 static mblk_t *nxge_lso_eliminate(mblk_t *); 38 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 39 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 40 static void nxge_hcksum_retrieve(mblk_t *, 41 uint32_t *, uint32_t *, uint32_t *, 42 uint32_t *, uint32_t *); 43 static uint32_t nxge_csgen(uint16_t *, int); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 p_tx_pkt_hdr_all_t pkthdrp; 81 uint8_t npads = 0; 82 uint64_t dma_ioaddr; 83 uint32_t dma_flags; 84 int last_bidx; 85 uint8_t *b_rptr; 86 caddr_t kaddr; 87 uint32_t nmblks; 88 uint32_t ngathers; 89 uint32_t clen; 90 int len; 91 uint32_t pkt_len, pack_len, min_len; 92 uint32_t bcopy_thresh; 93 int i, cur_index, sop_index; 94 uint16_t tail_index; 95 boolean_t tail_wrap = B_FALSE; 96 nxge_dma_common_t desc_area; 97 nxge_os_dma_handle_t dma_handle; 98 ddi_dma_cookie_t dma_cookie; 99 npi_handle_t npi_handle; 100 p_mblk_t nmp; 101 p_mblk_t t_mp; 102 uint32_t ncookies; 103 boolean_t good_packet; 104 boolean_t mark_mode = B_FALSE; 105 p_nxge_stats_t statsp; 106 p_nxge_tx_ring_stats_t tdc_stats; 107 t_uscalar_t start_offset = 0; 108 t_uscalar_t stuff_offset = 0; 109 t_uscalar_t end_offset = 0; 110 t_uscalar_t value = 0; 111 t_uscalar_t cksum_flags = 0; 112 boolean_t cksum_on = B_FALSE; 113 uint32_t boff = 0; 114 uint64_t tot_xfer_len = 0, tmp_len = 0; 115 boolean_t header_set = B_FALSE; 116 #ifdef NXGE_DEBUG 117 p_tx_desc_t tx_desc_ring_pp; 118 p_tx_desc_t tx_desc_pp; 119 tx_desc_t *save_desc_p; 120 int dump_len; 121 int sad_len; 122 uint64_t sad; 123 int xfer_len; 124 uint32_t msgsize; 125 #endif 126 p_mblk_t mp_chain = NULL; 127 boolean_t is_lso = B_FALSE; 128 boolean_t lso_again; 129 int cur_index_lso; 130 p_mblk_t nmp_lso_save; 131 uint32_t lso_ngathers; 132 boolean_t lso_tail_wrap = B_FALSE; 133 134 NXGE_DEBUG_MSG((nxgep, TX_CTL, 135 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 136 NXGE_DEBUG_MSG((nxgep, TX_CTL, 137 "==> nxge_start: Starting tdc %d desc pending %d", 138 tx_ring_p->tdc, tx_ring_p->descs_pending)); 139 140 statsp = nxgep->statsp; 141 142 if (!isLDOMguest(nxgep)) { 143 switch (nxgep->mac.portmode) { 144 default: 145 if (nxgep->statsp->port_stats.lb_mode == 146 nxge_lb_normal) { 147 if (!statsp->mac_stats.link_up) { 148 freemsg(mp); 149 NXGE_DEBUG_MSG((nxgep, TX_CTL, 150 "==> nxge_start: " 151 "link not up")); 152 goto nxge_start_fail1; 153 } 154 } 155 break; 156 case PORT_10G_FIBER: 157 /* 158 * For the following modes, check the link status 159 * before sending the packet out: 160 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 161 */ 162 if (nxgep->statsp->port_stats.lb_mode < 163 nxge_lb_serdes10g) { 164 if (!statsp->mac_stats.link_up) { 165 freemsg(mp); 166 NXGE_DEBUG_MSG((nxgep, TX_CTL, 167 "==> nxge_start: " 168 "link not up")); 169 goto nxge_start_fail1; 170 } 171 } 172 break; 173 } 174 } 175 176 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 177 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 178 NXGE_DEBUG_MSG((nxgep, TX_CTL, 179 "==> nxge_start: hardware not initialized or stopped")); 180 freemsg(mp); 181 goto nxge_start_fail1; 182 } 183 184 if (nxgep->soft_lso_enable) { 185 mp_chain = nxge_lso_eliminate(mp); 186 NXGE_DEBUG_MSG((nxgep, TX_CTL, 187 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 188 mp, mp_chain)); 189 if (mp_chain == NULL) { 190 NXGE_ERROR_MSG((nxgep, TX_CTL, 191 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 192 mp_chain, mp)); 193 goto nxge_start_fail1; 194 } 195 if (mp_chain != mp) { 196 NXGE_DEBUG_MSG((nxgep, TX_CTL, 197 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 198 mp_chain, mp)); 199 is_lso = B_TRUE; 200 mp = mp_chain; 201 mp_chain = mp_chain->b_next; 202 mp->b_next = NULL; 203 } 204 } 205 206 hcksum_retrieve(mp, NULL, NULL, &start_offset, 207 &stuff_offset, &end_offset, &value, &cksum_flags); 208 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 209 start_offset += sizeof (ether_header_t); 210 stuff_offset += sizeof (ether_header_t); 211 } else { 212 start_offset += sizeof (struct ether_vlan_header); 213 stuff_offset += sizeof (struct ether_vlan_header); 214 } 215 216 if (cksum_flags & HCK_PARTIALCKSUM) { 217 NXGE_DEBUG_MSG((nxgep, TX_CTL, 218 "==> nxge_start: mp $%p len %d " 219 "cksum_flags 0x%x (partial checksum) ", 220 mp, MBLKL(mp), cksum_flags)); 221 cksum_on = B_TRUE; 222 } 223 224 lso_again = B_FALSE; 225 lso_ngathers = 0; 226 227 MUTEX_ENTER(&tx_ring_p->lock); 228 cur_index_lso = tx_ring_p->wr_index; 229 lso_tail_wrap = tx_ring_p->wr_index_wrap; 230 start_again: 231 ngathers = 0; 232 sop_index = tx_ring_p->wr_index; 233 #ifdef NXGE_DEBUG 234 if (tx_ring_p->descs_pending) { 235 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 236 "desc pending %d ", tx_ring_p->descs_pending)); 237 } 238 239 dump_len = (int)(MBLKL(mp)); 240 dump_len = (dump_len > 128) ? 128: dump_len; 241 242 NXGE_DEBUG_MSG((nxgep, TX_CTL, 243 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 244 "(Before header reserve: ORIGINAL LEN %d)", 245 tx_ring_p->tdc, 246 mp->b_rptr, 247 dump_len)); 248 249 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 250 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 251 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 252 #endif 253 254 tdc_stats = tx_ring_p->tdc_stats; 255 mark_mode = (tx_ring_p->descs_pending && 256 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 257 < nxge_tx_minfree)); 258 259 NXGE_DEBUG_MSG((nxgep, TX_CTL, 260 "TX Descriptor ring is channel %d mark mode %d", 261 tx_ring_p->tdc, mark_mode)); 262 263 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 264 NXGE_DEBUG_MSG((nxgep, TX_CTL, 265 "TX Descriptor ring is full: channel %d", 266 tx_ring_p->tdc)); 267 NXGE_DEBUG_MSG((nxgep, TX_CTL, 268 "TX Descriptor ring is full: channel %d", 269 tx_ring_p->tdc)); 270 if (is_lso) { 271 /* free the current mp and mp_chain if not FULL */ 272 tdc_stats->tx_no_desc++; 273 NXGE_DEBUG_MSG((nxgep, TX_CTL, 274 "LSO packet: TX Descriptor ring is full: " 275 "channel %d", 276 tx_ring_p->tdc)); 277 goto nxge_start_fail_lso; 278 } else { 279 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 280 tdc_stats->tx_no_desc++; 281 MUTEX_EXIT(&tx_ring_p->lock); 282 if (nxgep->resched_needed && !nxgep->resched_running) { 283 nxgep->resched_running = B_TRUE; 284 ddi_trigger_softintr(nxgep->resched_id); 285 } 286 status = 1; 287 goto nxge_start_fail1; 288 } 289 } 290 291 nmp = mp; 292 i = sop_index = tx_ring_p->wr_index; 293 nmblks = 0; 294 ngathers = 0; 295 pkt_len = 0; 296 pack_len = 0; 297 clen = 0; 298 last_bidx = -1; 299 good_packet = B_TRUE; 300 301 desc_area = tx_ring_p->tdc_desc; 302 npi_handle = desc_area.npi_handle; 303 npi_desc_handle.regh = (nxge_os_acc_handle_t) 304 DMA_COMMON_ACC_HANDLE(desc_area); 305 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 306 tx_desc_dma_handle = (nxge_os_dma_handle_t) 307 DMA_COMMON_HANDLE(desc_area); 308 tx_msg_ring = tx_ring_p->tx_msg_ring; 309 310 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 311 sop_index, i)); 312 313 #ifdef NXGE_DEBUG 314 msgsize = msgdsize(nmp); 315 NXGE_DEBUG_MSG((nxgep, TX_CTL, 316 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 317 sop_index, i, msgsize)); 318 #endif 319 /* 320 * The first 16 bytes of the premapped buffer are reserved 321 * for header. No padding will be used. 322 */ 323 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 324 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 325 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 326 } else { 327 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 328 } 329 while (nmp) { 330 good_packet = B_TRUE; 331 b_rptr = nmp->b_rptr; 332 len = MBLKL(nmp); 333 if (len <= 0) { 334 nmp = nmp->b_cont; 335 continue; 336 } 337 nmblks++; 338 339 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 340 "len %d pkt_len %d pack_len %d", 341 nmblks, len, pkt_len, pack_len)); 342 /* 343 * Hardware limits the transfer length to 4K for NIU and 344 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 345 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 346 * If len is longer than the limit, then we break nmp into 347 * two chunks: Make the first chunk equal to the limit and 348 * the second chunk for the remaining data. If the second 349 * chunk is still larger than the limit, then it will be 350 * broken into two in the next pass. 351 */ 352 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 353 if ((t_mp = dupb(nmp)) != NULL) { 354 nmp->b_wptr = nmp->b_rptr + 355 (TX_MAX_TRANSFER_LENGTH 356 - TX_PKT_HEADER_SIZE); 357 t_mp->b_rptr = nmp->b_wptr; 358 t_mp->b_cont = nmp->b_cont; 359 nmp->b_cont = t_mp; 360 len = MBLKL(nmp); 361 } else { 362 if (is_lso) { 363 NXGE_DEBUG_MSG((nxgep, TX_CTL, 364 "LSO packet: dupb failed: " 365 "channel %d", 366 tx_ring_p->tdc)); 367 mp = nmp; 368 goto nxge_start_fail_lso; 369 } else { 370 good_packet = B_FALSE; 371 goto nxge_start_fail2; 372 } 373 } 374 } 375 tx_desc.value = 0; 376 tx_desc_p = &tx_desc_ring_vp[i]; 377 #ifdef NXGE_DEBUG 378 tx_desc_pp = &tx_desc_ring_pp[i]; 379 #endif 380 tx_msg_p = &tx_msg_ring[i]; 381 #if defined(__i386) 382 npi_desc_handle.regp = (uint32_t)tx_desc_p; 383 #else 384 npi_desc_handle.regp = (uint64_t)tx_desc_p; 385 #endif 386 if (!header_set && 387 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 388 (len >= bcopy_thresh))) { 389 header_set = B_TRUE; 390 bcopy_thresh += TX_PKT_HEADER_SIZE; 391 boff = 0; 392 pack_len = 0; 393 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 394 hdrp = (p_tx_pkt_header_t)kaddr; 395 clen = pkt_len; 396 dma_handle = tx_msg_p->buf_dma_handle; 397 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 398 (void) ddi_dma_sync(dma_handle, 399 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 400 DDI_DMA_SYNC_FORDEV); 401 402 tx_msg_p->flags.dma_type = USE_BCOPY; 403 goto nxge_start_control_header_only; 404 } 405 406 pkt_len += len; 407 pack_len += len; 408 409 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 410 "desc entry %d " 411 "DESC IOADDR $%p " 412 "desc_vp $%p tx_desc_p $%p " 413 "desc_pp $%p tx_desc_pp $%p " 414 "len %d pkt_len %d pack_len %d", 415 i, 416 DMA_COMMON_IOADDR(desc_area), 417 tx_desc_ring_vp, tx_desc_p, 418 tx_desc_ring_pp, tx_desc_pp, 419 len, pkt_len, pack_len)); 420 421 if (len < bcopy_thresh) { 422 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 423 "USE BCOPY: ")); 424 if (nxge_tx_tiny_pack) { 425 uint32_t blst = 426 TXDMA_DESC_NEXT_INDEX(i, -1, 427 tx_ring_p->tx_wrap_mask); 428 NXGE_DEBUG_MSG((nxgep, TX_CTL, 429 "==> nxge_start(5): pack")); 430 if ((pack_len <= bcopy_thresh) && 431 (last_bidx == blst)) { 432 NXGE_DEBUG_MSG((nxgep, TX_CTL, 433 "==> nxge_start: pack(6) " 434 "(pkt_len %d pack_len %d)", 435 pkt_len, pack_len)); 436 i = blst; 437 tx_desc_p = &tx_desc_ring_vp[i]; 438 #ifdef NXGE_DEBUG 439 tx_desc_pp = &tx_desc_ring_pp[i]; 440 #endif 441 tx_msg_p = &tx_msg_ring[i]; 442 boff = pack_len - len; 443 ngathers--; 444 } else if (pack_len > bcopy_thresh && 445 header_set) { 446 pack_len = len; 447 boff = 0; 448 bcopy_thresh = nxge_bcopy_thresh; 449 NXGE_DEBUG_MSG((nxgep, TX_CTL, 450 "==> nxge_start(7): > max NEW " 451 "bcopy thresh %d " 452 "pkt_len %d pack_len %d(next)", 453 bcopy_thresh, 454 pkt_len, pack_len)); 455 } 456 last_bidx = i; 457 } 458 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 459 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 460 hdrp = (p_tx_pkt_header_t)kaddr; 461 header_set = B_TRUE; 462 NXGE_DEBUG_MSG((nxgep, TX_CTL, 463 "==> nxge_start(7_x2): " 464 "pkt_len %d pack_len %d (new hdrp $%p)", 465 pkt_len, pack_len, hdrp)); 466 } 467 tx_msg_p->flags.dma_type = USE_BCOPY; 468 kaddr += boff; 469 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 470 "USE BCOPY: before bcopy " 471 "DESC IOADDR $%p entry %d " 472 "bcopy packets %d " 473 "bcopy kaddr $%p " 474 "bcopy ioaddr (SAD) $%p " 475 "bcopy clen %d " 476 "bcopy boff %d", 477 DMA_COMMON_IOADDR(desc_area), i, 478 tdc_stats->tx_hdr_pkts, 479 kaddr, 480 dma_ioaddr, 481 clen, 482 boff)); 483 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 484 "1USE BCOPY: ")); 485 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 486 "2USE BCOPY: ")); 487 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 488 "last USE BCOPY: copy from b_rptr $%p " 489 "to KADDR $%p (len %d offset %d", 490 b_rptr, kaddr, len, boff)); 491 492 bcopy(b_rptr, kaddr, len); 493 494 #ifdef NXGE_DEBUG 495 dump_len = (len > 128) ? 128: len; 496 NXGE_DEBUG_MSG((nxgep, TX_CTL, 497 "==> nxge_start: dump packets " 498 "(After BCOPY len %d)" 499 "(b_rptr $%p): %s", len, nmp->b_rptr, 500 nxge_dump_packet((char *)nmp->b_rptr, 501 dump_len))); 502 #endif 503 504 dma_handle = tx_msg_p->buf_dma_handle; 505 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 506 (void) ddi_dma_sync(dma_handle, 507 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 508 DDI_DMA_SYNC_FORDEV); 509 clen = len + boff; 510 tdc_stats->tx_hdr_pkts++; 511 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 512 "USE BCOPY: " 513 "DESC IOADDR $%p entry %d " 514 "bcopy packets %d " 515 "bcopy kaddr $%p " 516 "bcopy ioaddr (SAD) $%p " 517 "bcopy clen %d " 518 "bcopy boff %d", 519 DMA_COMMON_IOADDR(desc_area), 520 i, 521 tdc_stats->tx_hdr_pkts, 522 kaddr, 523 dma_ioaddr, 524 clen, 525 boff)); 526 } else { 527 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 528 "USE DVMA: len %d", len)); 529 tx_msg_p->flags.dma_type = USE_DMA; 530 dma_flags = DDI_DMA_WRITE; 531 if (len < nxge_dma_stream_thresh) { 532 dma_flags |= DDI_DMA_CONSISTENT; 533 } else { 534 dma_flags |= DDI_DMA_STREAMING; 535 } 536 537 dma_handle = tx_msg_p->dma_handle; 538 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 539 (caddr_t)b_rptr, len, dma_flags, 540 DDI_DMA_DONTWAIT, NULL, 541 &dma_cookie, &ncookies); 542 if (status == DDI_DMA_MAPPED) { 543 dma_ioaddr = dma_cookie.dmac_laddress; 544 len = (int)dma_cookie.dmac_size; 545 clen = (uint32_t)dma_cookie.dmac_size; 546 NXGE_DEBUG_MSG((nxgep, TX_CTL, 547 "==> nxge_start(12_1): " 548 "USE DVMA: len %d clen %d " 549 "ngathers %d", 550 len, clen, 551 ngathers)); 552 #if defined(__i386) 553 npi_desc_handle.regp = (uint32_t)tx_desc_p; 554 #else 555 npi_desc_handle.regp = (uint64_t)tx_desc_p; 556 #endif 557 while (ncookies > 1) { 558 ngathers++; 559 /* 560 * this is the fix for multiple 561 * cookies, which are basically 562 * a descriptor entry, we don't set 563 * SOP bit as well as related fields 564 */ 565 566 (void) npi_txdma_desc_gather_set( 567 npi_desc_handle, 568 &tx_desc, 569 (ngathers -1), 570 mark_mode, 571 ngathers, 572 dma_ioaddr, 573 clen); 574 575 tx_msg_p->tx_msg_size = clen; 576 NXGE_DEBUG_MSG((nxgep, TX_CTL, 577 "==> nxge_start: DMA " 578 "ncookie %d " 579 "ngathers %d " 580 "dma_ioaddr $%p len %d" 581 "desc $%p descp $%p (%d)", 582 ncookies, 583 ngathers, 584 dma_ioaddr, clen, 585 *tx_desc_p, tx_desc_p, i)); 586 587 ddi_dma_nextcookie(dma_handle, 588 &dma_cookie); 589 dma_ioaddr = 590 dma_cookie.dmac_laddress; 591 592 len = (int)dma_cookie.dmac_size; 593 clen = (uint32_t)dma_cookie.dmac_size; 594 NXGE_DEBUG_MSG((nxgep, TX_CTL, 595 "==> nxge_start(12_2): " 596 "USE DVMA: len %d clen %d ", 597 len, clen)); 598 599 i = TXDMA_DESC_NEXT_INDEX(i, 1, 600 tx_ring_p->tx_wrap_mask); 601 tx_desc_p = &tx_desc_ring_vp[i]; 602 603 npi_desc_handle.regp = 604 #if defined(__i386) 605 (uint32_t)tx_desc_p; 606 #else 607 (uint64_t)tx_desc_p; 608 #endif 609 tx_msg_p = &tx_msg_ring[i]; 610 tx_msg_p->flags.dma_type = USE_NONE; 611 tx_desc.value = 0; 612 613 ncookies--; 614 } 615 tdc_stats->tx_ddi_pkts++; 616 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 617 "DMA: ddi packets %d", 618 tdc_stats->tx_ddi_pkts)); 619 } else { 620 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 621 "dma mapping failed for %d " 622 "bytes addr $%p flags %x (%d)", 623 len, b_rptr, status, status)); 624 good_packet = B_FALSE; 625 tdc_stats->tx_dma_bind_fail++; 626 tx_msg_p->flags.dma_type = USE_NONE; 627 if (is_lso) { 628 mp = nmp; 629 goto nxge_start_fail_lso; 630 } else { 631 goto nxge_start_fail2; 632 } 633 } 634 } /* ddi dvma */ 635 636 if (is_lso) { 637 nmp_lso_save = nmp; 638 } 639 nmp = nmp->b_cont; 640 nxge_start_control_header_only: 641 #if defined(__i386) 642 npi_desc_handle.regp = (uint32_t)tx_desc_p; 643 #else 644 npi_desc_handle.regp = (uint64_t)tx_desc_p; 645 #endif 646 ngathers++; 647 648 if (ngathers == 1) { 649 #ifdef NXGE_DEBUG 650 save_desc_p = &sop_tx_desc; 651 #endif 652 sop_tx_desc_p = &sop_tx_desc; 653 sop_tx_desc_p->value = 0; 654 sop_tx_desc_p->bits.hdw.tr_len = clen; 655 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 656 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 657 } else { 658 #ifdef NXGE_DEBUG 659 save_desc_p = &tx_desc; 660 #endif 661 tmp_desc_p = &tx_desc; 662 tmp_desc_p->value = 0; 663 tmp_desc_p->bits.hdw.tr_len = clen; 664 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 665 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 666 667 tx_desc_p->value = tmp_desc_p->value; 668 } 669 670 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 671 "Desc_entry %d ngathers %d " 672 "desc_vp $%p tx_desc_p $%p " 673 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 674 "dma_ioaddr (SAD) $%p mark %d", 675 i, ngathers, 676 tx_desc_ring_vp, tx_desc_p, 677 len, clen, pkt_len, pack_len, nmblks, 678 dma_ioaddr, mark_mode)); 679 680 #ifdef NXGE_DEBUG 681 npi_desc_handle.nxgep = nxgep; 682 npi_desc_handle.function.function = nxgep->function_num; 683 npi_desc_handle.function.instance = nxgep->instance; 684 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 685 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 686 TX_PKT_DESC_TR_LEN_SHIFT); 687 688 689 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 690 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 691 "mark %d sop %d\n", 692 save_desc_p->value, 693 sad, 694 save_desc_p->bits.hdw.tr_len, 695 xfer_len, 696 save_desc_p->bits.hdw.num_ptr, 697 save_desc_p->bits.hdw.mark, 698 save_desc_p->bits.hdw.sop)); 699 700 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 701 #endif 702 703 tx_msg_p->tx_msg_size = clen; 704 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 705 if (ngathers > nxge_tx_max_gathers) { 706 good_packet = B_FALSE; 707 hcksum_retrieve(mp, NULL, NULL, &start_offset, 708 &stuff_offset, &end_offset, &value, 709 &cksum_flags); 710 711 NXGE_DEBUG_MSG((NULL, TX_CTL, 712 "==> nxge_start(14): pull msg - " 713 "len %d pkt_len %d ngathers %d", 714 len, pkt_len, ngathers)); 715 /* Pull all message blocks from b_cont */ 716 if (is_lso) { 717 mp = nmp_lso_save; 718 goto nxge_start_fail_lso; 719 } 720 if ((msgpullup(mp, -1)) == NULL) { 721 goto nxge_start_fail2; 722 } 723 goto nxge_start_fail2; 724 } 725 } /* while (nmp) */ 726 727 tx_msg_p->tx_message = mp; 728 tx_desc_p = &tx_desc_ring_vp[sop_index]; 729 #if defined(__i386) 730 npi_desc_handle.regp = (uint32_t)tx_desc_p; 731 #else 732 npi_desc_handle.regp = (uint64_t)tx_desc_p; 733 #endif 734 735 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 736 pkthdrp->reserved = 0; 737 hdrp->value = 0; 738 739 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 740 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 741 742 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 743 tdc_stats->tx_jumbo_pkts++; 744 } 745 746 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 747 if (pkt_len < min_len) { 748 /* Assume we use bcopy to premapped buffers */ 749 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 750 NXGE_DEBUG_MSG((NULL, TX_CTL, 751 "==> nxge_start(14-1): < (msg_min + 16)" 752 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 753 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 754 bzero((kaddr + pkt_len), (min_len - pkt_len)); 755 pkt_len = tx_msg_p->tx_msg_size = min_len; 756 757 sop_tx_desc_p->bits.hdw.tr_len = min_len; 758 759 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 760 tx_desc_p->value = sop_tx_desc_p->value; 761 762 NXGE_DEBUG_MSG((NULL, TX_CTL, 763 "==> nxge_start(14-2): < msg_min - " 764 "len %d pkt_len %d min_len %d ngathers %d", 765 len, pkt_len, min_len, ngathers)); 766 } 767 768 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 769 cksum_flags)); 770 if (cksum_flags & HCK_PARTIALCKSUM) { 771 NXGE_DEBUG_MSG((nxgep, TX_CTL, 772 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 773 cksum_flags)); 774 cksum_on = B_TRUE; 775 NXGE_DEBUG_MSG((nxgep, TX_CTL, 776 "==> nxge_start: from IP cksum_flags 0x%x " 777 "(partial checksum) " 778 "start_offset %d stuff_offset %d", 779 cksum_flags, start_offset, stuff_offset)); 780 tmp_len = (uint64_t)(start_offset >> 1); 781 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 782 tmp_len = (uint64_t)(stuff_offset >> 1); 783 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 784 785 NXGE_DEBUG_MSG((nxgep, TX_CTL, 786 "==> nxge_start: from IP cksum_flags 0x%x " 787 "(partial checksum) " 788 "after SHIFT start_offset %d stuff_offset %d", 789 cksum_flags, start_offset, stuff_offset)); 790 } 791 { 792 uint64_t tmp_len; 793 794 /* pkt_len already includes 16 + paddings!! */ 795 /* Update the control header length */ 796 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 797 tmp_len = hdrp->value | 798 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 799 800 NXGE_DEBUG_MSG((nxgep, TX_CTL, 801 "==> nxge_start(15_x1): setting SOP " 802 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 803 "0x%llx hdrp->value 0x%llx", 804 tot_xfer_len, tot_xfer_len, pkt_len, 805 tmp_len, hdrp->value)); 806 #if defined(_BIG_ENDIAN) 807 hdrp->value = ddi_swap64(tmp_len); 808 #else 809 hdrp->value = tmp_len; 810 #endif 811 NXGE_DEBUG_MSG((nxgep, 812 TX_CTL, "==> nxge_start(15_x2): setting SOP " 813 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 814 "tmp_len 0x%llx hdrp->value 0x%llx", 815 tot_xfer_len, pkt_len, 816 tmp_len, hdrp->value)); 817 } 818 819 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 820 "wr_index %d " 821 "tot_xfer_len (%d) pkt_len %d npads %d", 822 sop_index, 823 tot_xfer_len, pkt_len, 824 npads)); 825 826 sop_tx_desc_p->bits.hdw.sop = 1; 827 sop_tx_desc_p->bits.hdw.mark = mark_mode; 828 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 829 830 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 831 832 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 833 834 #ifdef NXGE_DEBUG 835 npi_desc_handle.nxgep = nxgep; 836 npi_desc_handle.function.function = nxgep->function_num; 837 npi_desc_handle.function.instance = nxgep->instance; 838 839 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 840 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 841 save_desc_p->value, 842 sad, 843 save_desc_p->bits.hdw.tr_len, 844 xfer_len, 845 save_desc_p->bits.hdw.num_ptr, 846 save_desc_p->bits.hdw.mark, 847 save_desc_p->bits.hdw.sop)); 848 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 849 850 dump_len = (pkt_len > 128) ? 128: pkt_len; 851 NXGE_DEBUG_MSG((nxgep, TX_CTL, 852 "==> nxge_start: dump packets(17) (after sop set, len " 853 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 854 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 855 (char *)hdrp, 856 nxge_dump_packet((char *)hdrp, dump_len))); 857 NXGE_DEBUG_MSG((nxgep, TX_CTL, 858 "==> nxge_start(18): TX desc sync: sop_index %d", 859 sop_index)); 860 #endif 861 862 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 863 (void) ddi_dma_sync(tx_desc_dma_handle, 864 sop_index * sizeof (tx_desc_t), 865 ngathers * sizeof (tx_desc_t), 866 DDI_DMA_SYNC_FORDEV); 867 868 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 869 "cs_off = 0x%02X cs_s_off = 0x%02X " 870 "pkt_len %d ngathers %d sop_index %d\n", 871 stuff_offset, start_offset, 872 pkt_len, ngathers, sop_index)); 873 } else { /* more than one descriptor and wrap around */ 874 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 875 (void) ddi_dma_sync(tx_desc_dma_handle, 876 sop_index * sizeof (tx_desc_t), 877 nsdescs * sizeof (tx_desc_t), 878 DDI_DMA_SYNC_FORDEV); 879 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 880 "cs_off = 0x%02X cs_s_off = 0x%02X " 881 "pkt_len %d ngathers %d sop_index %d\n", 882 stuff_offset, start_offset, 883 pkt_len, ngathers, sop_index)); 884 885 (void) ddi_dma_sync(tx_desc_dma_handle, 886 0, 887 (ngathers - nsdescs) * sizeof (tx_desc_t), 888 DDI_DMA_SYNC_FORDEV); 889 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 890 "cs_off = 0x%02X cs_s_off = 0x%02X " 891 "pkt_len %d ngathers %d sop_index %d\n", 892 stuff_offset, start_offset, 893 pkt_len, ngathers, sop_index)); 894 } 895 896 tail_index = tx_ring_p->wr_index; 897 tail_wrap = tx_ring_p->wr_index_wrap; 898 899 tx_ring_p->wr_index = i; 900 if (tx_ring_p->wr_index <= tail_index) { 901 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 902 B_FALSE : B_TRUE); 903 } 904 905 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 906 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 907 tx_ring_p->tdc, 908 tx_ring_p->wr_index, 909 tx_ring_p->wr_index_wrap, 910 ngathers, 911 tx_ring_p->descs_pending)); 912 913 if (is_lso) { 914 lso_ngathers += ngathers; 915 if (mp_chain != NULL) { 916 mp = mp_chain; 917 mp_chain = mp_chain->b_next; 918 mp->b_next = NULL; 919 if (nxge_lso_kick_cnt == lso_ngathers) { 920 tx_ring_p->descs_pending += lso_ngathers; 921 { 922 tx_ring_kick_t kick; 923 924 kick.value = 0; 925 kick.bits.ldw.wrap = 926 tx_ring_p->wr_index_wrap; 927 kick.bits.ldw.tail = 928 (uint16_t)tx_ring_p->wr_index; 929 930 /* Kick the Transmit kick register */ 931 TXDMA_REG_WRITE64( 932 NXGE_DEV_NPI_HANDLE(nxgep), 933 TX_RING_KICK_REG, 934 (uint8_t)tx_ring_p->tdc, 935 kick.value); 936 tdc_stats->tx_starts++; 937 938 NXGE_DEBUG_MSG((nxgep, TX_CTL, 939 "==> nxge_start: more LSO: " 940 "LSO_CNT %d", 941 lso_ngathers)); 942 } 943 lso_ngathers = 0; 944 ngathers = 0; 945 cur_index_lso = sop_index = tx_ring_p->wr_index; 946 lso_tail_wrap = tx_ring_p->wr_index_wrap; 947 } 948 NXGE_DEBUG_MSG((nxgep, TX_CTL, 949 "==> nxge_start: lso again: " 950 "lso_gathers %d ngathers %d cur_index_lso %d " 951 "wr_index %d sop_index %d", 952 lso_ngathers, ngathers, cur_index_lso, 953 tx_ring_p->wr_index, sop_index)); 954 955 NXGE_DEBUG_MSG((nxgep, TX_CTL, 956 "==> nxge_start: next : count %d", 957 lso_ngathers)); 958 lso_again = B_TRUE; 959 goto start_again; 960 } 961 ngathers = lso_ngathers; 962 } 963 964 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 965 966 { 967 tx_ring_kick_t kick; 968 969 kick.value = 0; 970 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 971 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 972 973 /* Kick start the Transmit kick register */ 974 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 975 TX_RING_KICK_REG, 976 (uint8_t)tx_ring_p->tdc, 977 kick.value); 978 } 979 980 tx_ring_p->descs_pending += ngathers; 981 tdc_stats->tx_starts++; 982 983 tx_ring_p->tx_ring_state = TX_RING_STATE_IDLE; 984 985 MUTEX_EXIT(&tx_ring_p->lock); 986 987 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 988 989 return (status); 990 991 nxge_start_fail_lso: 992 status = 0; 993 good_packet = B_FALSE; 994 if (mp != NULL) { 995 freemsg(mp); 996 } 997 if (mp_chain != NULL) { 998 freemsg(mp_chain); 999 } 1000 if (!lso_again && !ngathers) { 1001 MUTEX_EXIT(&tx_ring_p->lock); 1002 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1003 "==> nxge_start: lso exit (nothing changed)")); 1004 goto nxge_start_fail1; 1005 } 1006 1007 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1008 "==> nxge_start (channel %d): before lso " 1009 "lso_gathers %d ngathers %d cur_index_lso %d " 1010 "wr_index %d sop_index %d lso_again %d", 1011 tx_ring_p->tdc, 1012 lso_ngathers, ngathers, cur_index_lso, 1013 tx_ring_p->wr_index, sop_index, lso_again)); 1014 1015 if (lso_again) { 1016 lso_ngathers += ngathers; 1017 ngathers = lso_ngathers; 1018 sop_index = cur_index_lso; 1019 tx_ring_p->wr_index = sop_index; 1020 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1021 } 1022 1023 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1024 "==> nxge_start (channel %d): after lso " 1025 "lso_gathers %d ngathers %d cur_index_lso %d " 1026 "wr_index %d sop_index %d lso_again %d", 1027 tx_ring_p->tdc, 1028 lso_ngathers, ngathers, cur_index_lso, 1029 tx_ring_p->wr_index, sop_index, lso_again)); 1030 1031 nxge_start_fail2: 1032 if (good_packet == B_FALSE) { 1033 cur_index = sop_index; 1034 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1035 for (i = 0; i < ngathers; i++) { 1036 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1037 #if defined(__i386) 1038 npi_handle.regp = (uint32_t)tx_desc_p; 1039 #else 1040 npi_handle.regp = (uint64_t)tx_desc_p; 1041 #endif 1042 tx_msg_p = &tx_msg_ring[cur_index]; 1043 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1044 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1045 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1046 "tx_desc_p = %X index = %d", 1047 tx_desc_p, tx_ring_p->rd_index)); 1048 (void) dvma_unload(tx_msg_p->dvma_handle, 1049 0, -1); 1050 tx_msg_p->dvma_handle = NULL; 1051 if (tx_ring_p->dvma_wr_index == 1052 tx_ring_p->dvma_wrap_mask) 1053 tx_ring_p->dvma_wr_index = 0; 1054 else 1055 tx_ring_p->dvma_wr_index++; 1056 tx_ring_p->dvma_pending--; 1057 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1058 if (ddi_dma_unbind_handle( 1059 tx_msg_p->dma_handle)) { 1060 cmn_err(CE_WARN, "!nxge_start: " 1061 "ddi_dma_unbind_handle failed"); 1062 } 1063 } 1064 tx_msg_p->flags.dma_type = USE_NONE; 1065 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1066 tx_ring_p->tx_wrap_mask); 1067 1068 } 1069 1070 nxgep->resched_needed = B_TRUE; 1071 } 1072 1073 tx_ring_p->tx_ring_state = TX_RING_STATE_IDLE; 1074 1075 MUTEX_EXIT(&tx_ring_p->lock); 1076 1077 nxge_start_fail1: 1078 /* Add FMA to check the access handle nxge_hregh */ 1079 1080 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1081 1082 return (status); 1083 } 1084 1085 int 1086 nxge_serial_tx(mblk_t *mp, void *arg) 1087 { 1088 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1089 p_nxge_t nxgep = tx_ring_p->nxgep; 1090 1091 return (nxge_start(nxgep, tx_ring_p, mp)); 1092 } 1093 1094 boolean_t 1095 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1096 { 1097 p_tx_ring_t *tx_rings; 1098 uint8_t ring_index; 1099 p_tx_ring_t tx_ring_p; 1100 nxge_grp_t *group; 1101 1102 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1103 1104 ASSERT(mp->b_next == NULL); 1105 1106 group = nxgep->tx_set.group[0]; /* The default group */ 1107 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1108 1109 tx_rings = nxgep->tx_rings->rings; 1110 tx_ring_p = tx_rings[group->legend[ring_index]]; 1111 1112 MUTEX_ENTER(&tx_ring_p->lock); 1113 if (tx_ring_p->tx_ring_state == TX_RING_STATE_OFFLINE) { 1114 /* 1115 * OFFLINE means that it is in the process of being 1116 * shared - that is, it has been claimed by the HIO 1117 * code, but hasn't been unlinked from <group> yet. 1118 * So in this case use the first TDC, which always 1119 * belongs to the service domain and can't be shared. 1120 */ 1121 MUTEX_EXIT(&tx_ring_p->lock); 1122 1123 ring_index = 0; 1124 tx_ring_p = tx_rings[group->legend[ring_index]]; 1125 MUTEX_ENTER(&tx_ring_p->lock); 1126 tx_ring_p->tx_ring_state = TX_RING_STATE_BUSY; 1127 } else { 1128 /* 1129 * Otherwise, mark the TDC as BUSY: the HIO code 1130 * will wait until nxge_start() has completed. 1131 */ 1132 tx_ring_p->tx_ring_state = TX_RING_STATE_BUSY; 1133 } 1134 MUTEX_EXIT(&tx_ring_p->lock); 1135 1136 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1137 (int)group->count, group->legend[ring_index], tx_ring_p)); 1138 1139 switch (nxge_tx_scheme) { 1140 case NXGE_USE_START: 1141 if (nxge_start(nxgep, tx_ring_p, mp)) { 1142 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1143 "ring index %d", ring_index)); 1144 return (B_FALSE); 1145 } 1146 break; 1147 1148 case NXGE_USE_SERIAL: 1149 default: 1150 nxge_serialize_enter(tx_ring_p->serial, mp); 1151 break; 1152 } 1153 1154 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1155 ring_index)); 1156 1157 return (B_TRUE); 1158 } 1159 1160 /* 1161 * nxge_m_tx() - send a chain of packets 1162 */ 1163 mblk_t * 1164 nxge_m_tx(void *arg, mblk_t *mp) 1165 { 1166 p_nxge_t nxgep = (p_nxge_t)arg; 1167 mblk_t *next; 1168 mac_tx_hint_t hint; 1169 1170 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1171 1172 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1173 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1174 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1175 "==> nxge_m_tx: hardware not initialized")); 1176 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1177 "<== nxge_m_tx")); 1178 freemsgchain(mp); 1179 mp = NULL; 1180 return (mp); 1181 } 1182 1183 hint.hash = NULL; 1184 hint.vid = 0; 1185 hint.sap = 0; 1186 1187 while (mp != NULL) { 1188 next = mp->b_next; 1189 mp->b_next = NULL; 1190 1191 /* 1192 * Until Nemo tx resource works, the mac driver 1193 * does the load balancing based on TCP port, 1194 * or CPU. For debugging, we use a system 1195 * configurable parameter. 1196 */ 1197 if (!nxge_send(nxgep, mp, &hint)) { 1198 mp->b_next = next; 1199 break; 1200 } 1201 1202 mp = next; 1203 1204 NXGE_DEBUG_MSG((NULL, TX_CTL, 1205 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1206 mp, next)); 1207 } 1208 1209 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1210 return (mp); 1211 } 1212 1213 int 1214 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1215 { 1216 uint8_t ring_index = 0; 1217 uint8_t *tcp_port; 1218 p_mblk_t nmp; 1219 size_t mblk_len; 1220 size_t iph_len; 1221 size_t hdrs_size; 1222 uint8_t hdrs_buf[sizeof (struct ether_header) + 1223 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1224 /* 1225 * allocate space big enough to cover 1226 * the max ip header length and the first 1227 * 4 bytes of the TCP/IP header. 1228 */ 1229 1230 boolean_t qos = B_FALSE; 1231 1232 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1233 1234 if (hp->vid) { 1235 qos = B_TRUE; 1236 } 1237 switch (nxge_tx_lb_policy) { 1238 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1239 default: 1240 tcp_port = mp->b_rptr; 1241 if (!nxge_no_tx_lb && !qos && 1242 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1243 == ETHERTYPE_IP)) { 1244 nmp = mp; 1245 mblk_len = MBLKL(nmp); 1246 tcp_port = NULL; 1247 if (mblk_len > sizeof (struct ether_header) + 1248 sizeof (uint8_t)) { 1249 tcp_port = nmp->b_rptr + 1250 sizeof (struct ether_header); 1251 mblk_len -= sizeof (struct ether_header); 1252 iph_len = ((*tcp_port) & 0x0f) << 2; 1253 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1254 tcp_port = nmp->b_rptr; 1255 } else { 1256 tcp_port = NULL; 1257 } 1258 } 1259 if (tcp_port == NULL) { 1260 hdrs_size = 0; 1261 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1262 while ((nmp) && (hdrs_size < 1263 sizeof (hdrs_buf))) { 1264 mblk_len = MBLKL(nmp); 1265 if (mblk_len >= 1266 (sizeof (hdrs_buf) - hdrs_size)) 1267 mblk_len = sizeof (hdrs_buf) - 1268 hdrs_size; 1269 bcopy(nmp->b_rptr, 1270 &hdrs_buf[hdrs_size], mblk_len); 1271 hdrs_size += mblk_len; 1272 nmp = nmp->b_cont; 1273 } 1274 tcp_port = hdrs_buf; 1275 } 1276 tcp_port += sizeof (ether_header_t); 1277 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1278 switch (tcp_port[9]) { 1279 case IPPROTO_TCP: 1280 case IPPROTO_UDP: 1281 case IPPROTO_ESP: 1282 tcp_port += ((*tcp_port) & 0x0f) << 2; 1283 ring_index = 1284 ((tcp_port[0] ^ 1285 tcp_port[1] ^ 1286 tcp_port[2] ^ 1287 tcp_port[3]) % maxtdcs); 1288 break; 1289 1290 case IPPROTO_AH: 1291 /* SPI starts at the 4th byte */ 1292 tcp_port += ((*tcp_port) & 0x0f) << 2; 1293 ring_index = 1294 ((tcp_port[4] ^ 1295 tcp_port[5] ^ 1296 tcp_port[6] ^ 1297 tcp_port[7]) % maxtdcs); 1298 break; 1299 1300 default: 1301 ring_index = tcp_port[19] % maxtdcs; 1302 break; 1303 } 1304 } else { /* fragmented packet */ 1305 ring_index = tcp_port[19] % maxtdcs; 1306 } 1307 } else { 1308 ring_index = mp->b_band % maxtdcs; 1309 } 1310 break; 1311 1312 case NXGE_TX_LB_HASH: 1313 if (hp->hash) { 1314 #if defined(__i386) 1315 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1316 #else 1317 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1318 #endif 1319 } else { 1320 ring_index = mp->b_band % maxtdcs; 1321 } 1322 break; 1323 1324 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1325 tcp_port = mp->b_rptr; 1326 ring_index = tcp_port[5] % maxtdcs; 1327 break; 1328 } 1329 1330 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1331 1332 return (ring_index); 1333 } 1334 1335 uint_t 1336 nxge_reschedule(caddr_t arg) 1337 { 1338 p_nxge_t nxgep; 1339 1340 nxgep = (p_nxge_t)arg; 1341 1342 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1343 1344 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1345 nxgep->resched_needed) { 1346 if (!isLDOMguest(nxgep)) 1347 mac_tx_update(nxgep->mach); 1348 #if defined(sun4v) 1349 else { /* isLDOMguest(nxgep) */ 1350 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1351 nxgep->nxge_hw_p->hio; 1352 nx_vio_fp_t *vio = &nhd->hio.vio; 1353 1354 /* Call back vnet. */ 1355 if (vio->cb.vio_net_tx_update) { 1356 (*vio->cb.vio_net_tx_update) 1357 (nxgep->hio_vr->vhp); 1358 } 1359 } 1360 #endif 1361 nxgep->resched_needed = B_FALSE; 1362 nxgep->resched_running = B_FALSE; 1363 } 1364 1365 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1366 return (DDI_INTR_CLAIMED); 1367 } 1368 1369 1370 /* Software LSO starts here */ 1371 static void 1372 nxge_hcksum_retrieve(mblk_t *mp, 1373 uint32_t *start, uint32_t *stuff, uint32_t *end, 1374 uint32_t *value, uint32_t *flags) 1375 { 1376 if (mp->b_datap->db_type == M_DATA) { 1377 if (flags != NULL) { 1378 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1379 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1380 HCK_FULLCKSUM_OK); 1381 if ((*flags & (HCK_PARTIALCKSUM | 1382 HCK_FULLCKSUM)) != 0) { 1383 if (value != NULL) 1384 *value = (uint32_t)DB_CKSUM16(mp); 1385 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1386 if (start != NULL) 1387 *start = 1388 (uint32_t)DB_CKSUMSTART(mp); 1389 if (stuff != NULL) 1390 *stuff = 1391 (uint32_t)DB_CKSUMSTUFF(mp); 1392 if (end != NULL) 1393 *end = 1394 (uint32_t)DB_CKSUMEND(mp); 1395 } 1396 } 1397 } 1398 } 1399 } 1400 1401 static void 1402 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1403 { 1404 ASSERT(DB_TYPE(mp) == M_DATA); 1405 1406 *mss = 0; 1407 if (flags != NULL) { 1408 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1409 if ((*flags != 0) && (mss != NULL)) { 1410 *mss = (uint32_t)DB_LSOMSS(mp); 1411 } 1412 NXGE_DEBUG_MSG((NULL, TX_CTL, 1413 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1414 *mss, *flags)); 1415 } 1416 1417 NXGE_DEBUG_MSG((NULL, TX_CTL, 1418 "<== nxge_lso_info_get: mss %d", *mss)); 1419 } 1420 1421 /* 1422 * Do Soft LSO on the oversized packet. 1423 * 1424 * 1. Create a chain of message for headers. 1425 * 2. Fill up header messages with proper information. 1426 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1427 * each new message with necessary adjustments. 1428 * * Unchange the ethernet header for DIX frames. (by default) 1429 * * IP Total Length field is updated to MSS or less(only for the last one). 1430 * * IP Identification value is incremented by one for each packet. 1431 * * TCP sequence Number is recalculated according to the payload length. 1432 * * Set FIN and/or PSH flags for the *last* packet if applied. 1433 * * TCP partial Checksum 1434 * 4. Update LSO information in the first message header. 1435 * 5. Release the original message header. 1436 */ 1437 static mblk_t * 1438 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1439 { 1440 uint32_t hckflags; 1441 int pktlen; 1442 int hdrlen; 1443 int segnum; 1444 int i; 1445 struct ether_vlan_header *evh; 1446 int ehlen, iphlen, tcphlen; 1447 struct ip *oiph, *niph; 1448 struct tcphdr *otcph, *ntcph; 1449 int available, len, left; 1450 uint16_t ip_id; 1451 uint32_t tcp_seq; 1452 #ifdef __sparc 1453 uint32_t tcp_seq_tmp; 1454 #endif 1455 mblk_t *datamp; 1456 uchar_t *rptr; 1457 mblk_t *nmp; 1458 mblk_t *cmp; 1459 mblk_t *mp_chain; 1460 boolean_t do_cleanup = B_FALSE; 1461 t_uscalar_t start_offset = 0; 1462 t_uscalar_t stuff_offset = 0; 1463 t_uscalar_t value = 0; 1464 uint16_t l4_len; 1465 ipaddr_t src, dst; 1466 uint32_t cksum, sum, l4cksum; 1467 1468 NXGE_DEBUG_MSG((NULL, TX_CTL, 1469 "==> nxge_do_softlso")); 1470 /* 1471 * check the length of LSO packet payload and calculate the number of 1472 * segments to be generated. 1473 */ 1474 pktlen = msgsize(mp); 1475 evh = (struct ether_vlan_header *)mp->b_rptr; 1476 1477 /* VLAN? */ 1478 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1479 ehlen = sizeof (struct ether_vlan_header); 1480 else 1481 ehlen = sizeof (struct ether_header); 1482 oiph = (struct ip *)(mp->b_rptr + ehlen); 1483 iphlen = oiph->ip_hl * 4; 1484 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1485 tcphlen = otcph->th_off * 4; 1486 1487 l4_len = pktlen - ehlen - iphlen; 1488 1489 NXGE_DEBUG_MSG((NULL, TX_CTL, 1490 "==> nxge_do_softlso: mss %d oiph $%p " 1491 "original ip_sum oiph->ip_sum 0x%x " 1492 "original tcp_sum otcph->th_sum 0x%x " 1493 "oiph->ip_len %d pktlen %d ehlen %d " 1494 "l4_len %d (0x%x) ip_len - iphlen %d ", 1495 mss, 1496 oiph, 1497 oiph->ip_sum, 1498 otcph->th_sum, 1499 ntohs(oiph->ip_len), pktlen, 1500 ehlen, 1501 l4_len, 1502 l4_len, 1503 ntohs(oiph->ip_len) - iphlen)); 1504 1505 /* IPv4 + TCP */ 1506 if (!(oiph->ip_v == IPV4_VERSION)) { 1507 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1508 "<== nxge_do_softlso: not IPV4 " 1509 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1510 ntohs(oiph->ip_len), pktlen, ehlen, 1511 tcphlen)); 1512 freemsg(mp); 1513 return (NULL); 1514 } 1515 1516 if (!(oiph->ip_p == IPPROTO_TCP)) { 1517 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1518 "<== nxge_do_softlso: not TCP " 1519 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1520 ntohs(oiph->ip_len), pktlen, ehlen, 1521 tcphlen)); 1522 freemsg(mp); 1523 return (NULL); 1524 } 1525 1526 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1527 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1528 "<== nxge_do_softlso: len not matched " 1529 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1530 ntohs(oiph->ip_len), pktlen, ehlen, 1531 tcphlen)); 1532 freemsg(mp); 1533 return (NULL); 1534 } 1535 1536 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1537 tcphlen = otcph->th_off * 4; 1538 1539 /* TCP flags can not include URG, RST, or SYN */ 1540 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1541 1542 hdrlen = ehlen + iphlen + tcphlen; 1543 1544 VERIFY(MBLKL(mp) >= hdrlen); 1545 1546 if (MBLKL(mp) > hdrlen) { 1547 datamp = mp; 1548 rptr = mp->b_rptr + hdrlen; 1549 } else { /* = */ 1550 datamp = mp->b_cont; 1551 rptr = datamp->b_rptr; 1552 } 1553 1554 NXGE_DEBUG_MSG((NULL, TX_CTL, 1555 "nxge_do_softlso: otcph $%p pktlen: %d, " 1556 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1557 "mblkl(mp): %d, mblkl(datamp): %d", 1558 otcph, 1559 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1560 (int)MBLKL(mp), (int)MBLKL(datamp))); 1561 1562 hckflags = 0; 1563 nxge_hcksum_retrieve(mp, 1564 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1565 1566 dst = oiph->ip_dst.s_addr; 1567 src = oiph->ip_src.s_addr; 1568 1569 cksum = (dst >> 16) + (dst & 0xFFFF) + 1570 (src >> 16) + (src & 0xFFFF); 1571 l4cksum = cksum + IP_TCP_CSUM_COMP; 1572 1573 sum = l4_len + l4cksum; 1574 sum = (sum & 0xFFFF) + (sum >> 16); 1575 1576 NXGE_DEBUG_MSG((NULL, TX_CTL, 1577 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1578 "hckflags 0x%x start_offset %d stuff_offset %d " 1579 "value (original) 0x%x th_sum 0x%x " 1580 "pktlen %d l4_len %d (0x%x) " 1581 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1582 dst, src, 1583 (sum & 0xffff), (~sum & 0xffff), 1584 hckflags, start_offset, stuff_offset, 1585 value, otcph->th_sum, 1586 pktlen, 1587 l4_len, 1588 l4_len, 1589 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1590 (int)MBLKL(datamp), 1591 nxge_dump_packet((char *)evh, 12))); 1592 1593 /* 1594 * Start to process. 1595 */ 1596 available = pktlen - hdrlen; 1597 segnum = (available - 1) / mss + 1; 1598 1599 NXGE_DEBUG_MSG((NULL, TX_CTL, 1600 "==> nxge_do_softlso: pktlen %d " 1601 "MBLKL(mp): %d, MBLKL(datamp): %d " 1602 "available %d mss %d segnum %d", 1603 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1604 available, 1605 mss, 1606 segnum)); 1607 1608 VERIFY(segnum >= 2); 1609 1610 /* 1611 * Try to pre-allocate all header messages 1612 */ 1613 mp_chain = NULL; 1614 for (i = 0; i < segnum; i++) { 1615 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1616 /* Clean up the mp_chain */ 1617 while (mp_chain != NULL) { 1618 nmp = mp_chain; 1619 mp_chain = mp_chain->b_next; 1620 freemsg(nmp); 1621 } 1622 NXGE_DEBUG_MSG((NULL, TX_CTL, 1623 "<== nxge_do_softlso: " 1624 "Could not allocate enough messages for headers!")); 1625 freemsg(mp); 1626 return (NULL); 1627 } 1628 nmp->b_next = mp_chain; 1629 mp_chain = nmp; 1630 1631 NXGE_DEBUG_MSG((NULL, TX_CTL, 1632 "==> nxge_do_softlso: " 1633 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1634 mp, nmp, mp_chain, mp_chain->b_next)); 1635 } 1636 1637 NXGE_DEBUG_MSG((NULL, TX_CTL, 1638 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1639 mp, nmp, mp_chain)); 1640 1641 /* 1642 * Associate payload with new packets 1643 */ 1644 cmp = mp_chain; 1645 left = available; 1646 while (cmp != NULL) { 1647 nmp = dupb(datamp); 1648 if (nmp == NULL) { 1649 do_cleanup = B_TRUE; 1650 NXGE_DEBUG_MSG((NULL, TX_CTL, 1651 "==>nxge_do_softlso: " 1652 "Can not dupb(datamp), have to do clean up")); 1653 goto cleanup_allocated_msgs; 1654 } 1655 1656 NXGE_DEBUG_MSG((NULL, TX_CTL, 1657 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1658 "dupb nmp $%p len %d left %d msd %d ", 1659 mp, cmp, nmp, len, left, mss)); 1660 1661 cmp->b_cont = nmp; 1662 nmp->b_rptr = rptr; 1663 len = (left < mss) ? left : mss; 1664 left -= len; 1665 1666 NXGE_DEBUG_MSG((NULL, TX_CTL, 1667 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1668 "dupb nmp $%p len %d left %d mss %d ", 1669 mp, cmp, nmp, len, left, mss)); 1670 NXGE_DEBUG_MSG((NULL, TX_CTL, 1671 "nxge_do_softlso: before available: %d, " 1672 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1673 available, left, len, segnum, (int)MBLKL(nmp))); 1674 1675 len -= MBLKL(nmp); 1676 NXGE_DEBUG_MSG((NULL, TX_CTL, 1677 "nxge_do_softlso: after available: %d, " 1678 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1679 available, left, len, segnum, (int)MBLKL(nmp))); 1680 1681 while (len > 0) { 1682 mblk_t *mmp = NULL; 1683 1684 NXGE_DEBUG_MSG((NULL, TX_CTL, 1685 "nxge_do_softlso: (4) len > 0 available: %d, " 1686 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1687 available, left, len, segnum, (int)MBLKL(nmp))); 1688 1689 if (datamp->b_cont != NULL) { 1690 datamp = datamp->b_cont; 1691 rptr = datamp->b_rptr; 1692 mmp = dupb(datamp); 1693 if (mmp == NULL) { 1694 do_cleanup = B_TRUE; 1695 NXGE_DEBUG_MSG((NULL, TX_CTL, 1696 "==> nxge_do_softlso: " 1697 "Can not dupb(datamp) (1), :" 1698 "have to do clean up")); 1699 NXGE_DEBUG_MSG((NULL, TX_CTL, 1700 "==> nxge_do_softlso: " 1701 "available: %d, left: %d, " 1702 "len: %d, MBLKL(nmp): %d", 1703 available, left, len, 1704 (int)MBLKL(nmp))); 1705 goto cleanup_allocated_msgs; 1706 } 1707 } else { 1708 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1709 "==> nxge_do_softlso: " 1710 "(1)available: %d, left: %d, " 1711 "len: %d, MBLKL(nmp): %d", 1712 available, left, len, 1713 (int)MBLKL(nmp))); 1714 cmn_err(CE_PANIC, 1715 "==> nxge_do_softlso: " 1716 "Pointers must have been corrupted!\n" 1717 "datamp: $%p, nmp: $%p, rptr: $%p", 1718 (void *)datamp, 1719 (void *)nmp, 1720 (void *)rptr); 1721 } 1722 nmp->b_cont = mmp; 1723 nmp = mmp; 1724 len -= MBLKL(nmp); 1725 } 1726 if (len < 0) { 1727 nmp->b_wptr += len; 1728 rptr = nmp->b_wptr; 1729 NXGE_DEBUG_MSG((NULL, TX_CTL, 1730 "(5) len < 0 (less than 0)" 1731 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1732 available, left, len, (int)MBLKL(nmp))); 1733 1734 } else if (len == 0) { 1735 if (datamp->b_cont != NULL) { 1736 NXGE_DEBUG_MSG((NULL, TX_CTL, 1737 "(5) len == 0" 1738 "available: %d, left: %d, len: %d, " 1739 "MBLKL(nmp): %d", 1740 available, left, len, (int)MBLKL(nmp))); 1741 datamp = datamp->b_cont; 1742 rptr = datamp->b_rptr; 1743 } else { 1744 NXGE_DEBUG_MSG((NULL, TX_CTL, 1745 "(6)available b_cont == NULL : %d, " 1746 "left: %d, len: %d, MBLKL(nmp): %d", 1747 available, left, len, (int)MBLKL(nmp))); 1748 1749 VERIFY(cmp->b_next == NULL); 1750 VERIFY(left == 0); 1751 break; /* Done! */ 1752 } 1753 } 1754 cmp = cmp->b_next; 1755 1756 NXGE_DEBUG_MSG((NULL, TX_CTL, 1757 "(7) do_softlso: " 1758 "next mp in mp_chain available len != 0 : %d, " 1759 "left: %d, len: %d, MBLKL(nmp): %d", 1760 available, left, len, (int)MBLKL(nmp))); 1761 } 1762 1763 /* 1764 * From now, start to fill up all headers for the first message 1765 * Hardware checksum flags need to be updated separately for FULLCKSUM 1766 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1767 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1768 * required fields need to be updated properly. 1769 */ 1770 nmp = mp_chain; 1771 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1772 nmp->b_wptr = nmp->b_rptr + hdrlen; 1773 niph = (struct ip *)(nmp->b_rptr + ehlen); 1774 niph->ip_len = htons(mss + iphlen + tcphlen); 1775 ip_id = ntohs(niph->ip_id); 1776 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1777 #ifdef __sparc 1778 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1779 tcp_seq = ntohl(tcp_seq_tmp); 1780 #else 1781 tcp_seq = ntohl(ntcph->th_seq); 1782 #endif 1783 1784 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1785 1786 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1787 DB_CKSUMSTART(nmp) = start_offset; 1788 DB_CKSUMSTUFF(nmp) = stuff_offset; 1789 1790 /* calculate IP checksum and TCP pseudo header checksum */ 1791 niph->ip_sum = 0; 1792 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1793 1794 l4_len = mss + tcphlen; 1795 sum = htons(l4_len) + l4cksum; 1796 sum = (sum & 0xFFFF) + (sum >> 16); 1797 ntcph->th_sum = (sum & 0xffff); 1798 1799 NXGE_DEBUG_MSG((NULL, TX_CTL, 1800 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1801 "mss %d pktlen %d l4_len %d (0x%x) " 1802 "MBLKL(mp): %d, MBLKL(datamp): %d " 1803 "ip_sum 0x%x " 1804 "th_sum 0x%x sum 0x%x ) " 1805 "dump first ip->tcp %s", 1806 nmp, mp_chain, 1807 mss, 1808 pktlen, 1809 l4_len, 1810 l4_len, 1811 (int)MBLKL(mp), (int)MBLKL(datamp), 1812 niph->ip_sum, 1813 ntcph->th_sum, 1814 sum, 1815 nxge_dump_packet((char *)niph, 52))); 1816 1817 cmp = nmp; 1818 while ((nmp = nmp->b_next)->b_next != NULL) { 1819 NXGE_DEBUG_MSG((NULL, TX_CTL, 1820 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1821 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1822 nmp->b_wptr = nmp->b_rptr + hdrlen; 1823 niph = (struct ip *)(nmp->b_rptr + ehlen); 1824 niph->ip_id = htons(++ip_id); 1825 niph->ip_len = htons(mss + iphlen + tcphlen); 1826 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1827 tcp_seq += mss; 1828 1829 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1830 1831 #ifdef __sparc 1832 tcp_seq_tmp = htonl(tcp_seq); 1833 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1834 #else 1835 ntcph->th_seq = htonl(tcp_seq); 1836 #endif 1837 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1838 DB_CKSUMSTART(nmp) = start_offset; 1839 DB_CKSUMSTUFF(nmp) = stuff_offset; 1840 1841 /* calculate IP checksum and TCP pseudo header checksum */ 1842 niph->ip_sum = 0; 1843 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1844 ntcph->th_sum = (sum & 0xffff); 1845 1846 NXGE_DEBUG_MSG((NULL, TX_CTL, 1847 "==> nxge_do_softlso: middle ip_sum 0x%x " 1848 "th_sum 0x%x " 1849 " mp $%p (mp_chain $%p) pktlen %d " 1850 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1851 niph->ip_sum, 1852 ntcph->th_sum, 1853 nmp, mp_chain, 1854 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1855 } 1856 1857 /* Last segment */ 1858 /* 1859 * Set FIN and/or PSH flags if present only in the last packet. 1860 * The ip_len could be different from prior packets. 1861 */ 1862 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1863 nmp->b_wptr = nmp->b_rptr + hdrlen; 1864 niph = (struct ip *)(nmp->b_rptr + ehlen); 1865 niph->ip_id = htons(++ip_id); 1866 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1867 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1868 tcp_seq += mss; 1869 #ifdef __sparc 1870 tcp_seq_tmp = htonl(tcp_seq); 1871 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1872 #else 1873 ntcph->th_seq = htonl(tcp_seq); 1874 #endif 1875 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1876 1877 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1878 DB_CKSUMSTART(nmp) = start_offset; 1879 DB_CKSUMSTUFF(nmp) = stuff_offset; 1880 1881 /* calculate IP checksum and TCP pseudo header checksum */ 1882 niph->ip_sum = 0; 1883 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1884 1885 l4_len = ntohs(niph->ip_len) - iphlen; 1886 sum = htons(l4_len) + l4cksum; 1887 sum = (sum & 0xFFFF) + (sum >> 16); 1888 ntcph->th_sum = (sum & 0xffff); 1889 1890 NXGE_DEBUG_MSG((NULL, TX_CTL, 1891 "==> nxge_do_softlso: last next " 1892 "niph->ip_sum 0x%x " 1893 "ntcph->th_sum 0x%x sum 0x%x " 1894 "dump last ip->tcp %s " 1895 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1896 "l4_len %d (0x%x) " 1897 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1898 niph->ip_sum, 1899 ntcph->th_sum, sum, 1900 nxge_dump_packet((char *)niph, 52), 1901 cmp, nmp, mp_chain, 1902 pktlen, pktlen, 1903 l4_len, 1904 l4_len, 1905 (int)MBLKL(mp), (int)MBLKL(datamp))); 1906 1907 cleanup_allocated_msgs: 1908 if (do_cleanup) { 1909 NXGE_DEBUG_MSG((NULL, TX_CTL, 1910 "==> nxge_do_softlso: " 1911 "Failed allocating messages, " 1912 "have to clean up and fail!")); 1913 while (mp_chain != NULL) { 1914 nmp = mp_chain; 1915 mp_chain = mp_chain->b_next; 1916 freemsg(nmp); 1917 } 1918 } 1919 /* 1920 * We're done here, so just free the original message and return the 1921 * new message chain, that could be NULL if failed, back to the caller. 1922 */ 1923 freemsg(mp); 1924 1925 NXGE_DEBUG_MSG((NULL, TX_CTL, 1926 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1927 return (mp_chain); 1928 } 1929 1930 /* 1931 * Will be called before NIC driver do further operation on the message. 1932 * The input message may include LSO information, if so, go to softlso logic 1933 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1934 * The return could be the same non-LSO message or a message chain for LSO case. 1935 * 1936 * The driver needs to call this function per packet and process the whole chain 1937 * if applied. 1938 */ 1939 static mblk_t * 1940 nxge_lso_eliminate(mblk_t *mp) 1941 { 1942 uint32_t lsoflags; 1943 uint32_t mss; 1944 1945 NXGE_DEBUG_MSG((NULL, TX_CTL, 1946 "==>nxge_lso_eliminate:")); 1947 nxge_lso_info_get(mp, &mss, &lsoflags); 1948 1949 if (lsoflags & HW_LSO) { 1950 mblk_t *nmp; 1951 1952 NXGE_DEBUG_MSG((NULL, TX_CTL, 1953 "==>nxge_lso_eliminate:" 1954 "HW_LSO:mss %d mp $%p", 1955 mss, mp)); 1956 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1957 NXGE_DEBUG_MSG((NULL, TX_CTL, 1958 "<== nxge_lso_eliminate: " 1959 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1960 nmp, mss, mp)); 1961 return (nmp); 1962 } else { 1963 NXGE_DEBUG_MSG((NULL, TX_CTL, 1964 "<== nxge_lso_eliminate_ " 1965 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1966 nmp, mss, mp)); 1967 return (NULL); 1968 } 1969 } 1970 1971 NXGE_DEBUG_MSG((NULL, TX_CTL, 1972 "<== nxge_lso_eliminate")); 1973 return (mp); 1974 } 1975 1976 static uint32_t 1977 nxge_csgen(uint16_t *adr, int len) 1978 { 1979 int i, odd; 1980 uint32_t sum = 0; 1981 uint32_t c = 0; 1982 1983 odd = len % 2; 1984 for (i = 0; i < (len / 2); i++) { 1985 sum += (adr[i] & 0xffff); 1986 } 1987 if (odd) { 1988 sum += adr[len / 2] & 0xff00; 1989 } 1990 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1991 sum &= 0xffff; 1992 sum += c; 1993 } 1994 return (~sum & 0xffff); 1995 } 1996