1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/nxge/nxge_impl.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern uint32_t nxge_reclaim_pending; 44 extern uint32_t nxge_bcopy_thresh; 45 extern uint32_t nxge_dvma_thresh; 46 extern uint32_t nxge_dma_stream_thresh; 47 extern uint32_t nxge_tx_minfree; 48 extern uint32_t nxge_tx_intr_thres; 49 extern uint32_t nxge_tx_max_gathers; 50 extern uint32_t nxge_tx_tiny_pack; 51 extern uint32_t nxge_tx_use_bcopy; 52 extern uint32_t nxge_tx_lb_policy; 53 extern uint32_t nxge_no_tx_lb; 54 extern nxge_tx_mode_t nxge_tx_scheme; 55 uint32_t nxge_lso_kick_cnt = 2; 56 57 typedef struct _mac_tx_hint { 58 uint16_t sap; 59 uint16_t vid; 60 void *hash; 61 } mac_tx_hint_t, *p_mac_tx_hint_t; 62 63 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 64 65 int 66 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 67 { 68 int status = 0; 69 p_tx_desc_t tx_desc_ring_vp; 70 npi_handle_t npi_desc_handle; 71 nxge_os_dma_handle_t tx_desc_dma_handle; 72 p_tx_desc_t tx_desc_p; 73 p_tx_msg_t tx_msg_ring; 74 p_tx_msg_t tx_msg_p; 75 tx_desc_t tx_desc, *tmp_desc_p; 76 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 77 p_tx_pkt_header_t hdrp; 78 p_tx_pkt_hdr_all_t pkthdrp; 79 uint8_t npads = 0; 80 uint64_t dma_ioaddr; 81 uint32_t dma_flags; 82 int last_bidx; 83 uint8_t *b_rptr; 84 caddr_t kaddr; 85 uint32_t nmblks; 86 uint32_t ngathers; 87 uint32_t clen; 88 int len; 89 uint32_t pkt_len, pack_len, min_len; 90 uint32_t bcopy_thresh; 91 int i, cur_index, sop_index; 92 uint16_t tail_index; 93 boolean_t tail_wrap = B_FALSE; 94 nxge_dma_common_t desc_area; 95 nxge_os_dma_handle_t dma_handle; 96 ddi_dma_cookie_t dma_cookie; 97 npi_handle_t npi_handle; 98 p_mblk_t nmp; 99 p_mblk_t t_mp; 100 uint32_t ncookies; 101 boolean_t good_packet; 102 boolean_t mark_mode = B_FALSE; 103 p_nxge_stats_t statsp; 104 p_nxge_tx_ring_stats_t tdc_stats; 105 t_uscalar_t start_offset = 0; 106 t_uscalar_t stuff_offset = 0; 107 t_uscalar_t end_offset = 0; 108 t_uscalar_t value = 0; 109 t_uscalar_t cksum_flags = 0; 110 boolean_t cksum_on = B_FALSE; 111 uint32_t boff = 0; 112 uint64_t tot_xfer_len = 0, tmp_len = 0; 113 boolean_t header_set = B_FALSE; 114 #ifdef NXGE_DEBUG 115 p_tx_desc_t tx_desc_ring_pp; 116 p_tx_desc_t tx_desc_pp; 117 tx_desc_t *save_desc_p; 118 int dump_len; 119 int sad_len; 120 uint64_t sad; 121 int xfer_len; 122 uint32_t msgsize; 123 #endif 124 p_mblk_t mp_chain = NULL; 125 boolean_t is_lso = B_FALSE; 126 boolean_t lso_again; 127 int cur_index_lso; 128 p_mblk_t nmp_lso_save; 129 uint32_t lso_ngathers; 130 boolean_t lso_tail_wrap = B_FALSE; 131 132 NXGE_DEBUG_MSG((nxgep, TX_CTL, 133 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 134 NXGE_DEBUG_MSG((nxgep, TX_CTL, 135 "==> nxge_start: Starting tdc %d desc pending %d", 136 tx_ring_p->tdc, tx_ring_p->descs_pending)); 137 138 statsp = nxgep->statsp; 139 140 switch (nxgep->mac.portmode) { 141 default: 142 if (nxgep->statsp->port_stats.lb_mode == nxge_lb_normal) { 143 if (!statsp->mac_stats.link_up) { 144 freemsg(mp); 145 NXGE_DEBUG_MSG((nxgep, TX_CTL, 146 "==> nxge_start: " 147 "link not up")); 148 goto nxge_start_fail1; 149 } 150 } 151 break; 152 case PORT_10G_FIBER: 153 /* 154 * For the following modes, check the link status 155 * before sending the packet out: 156 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 157 */ 158 if (nxgep->statsp->port_stats.lb_mode < nxge_lb_serdes10g) { 159 if (!statsp->mac_stats.link_up) { 160 freemsg(mp); 161 NXGE_DEBUG_MSG((nxgep, TX_CTL, 162 "==> nxge_start: " 163 "link not up")); 164 goto nxge_start_fail1; 165 } 166 } 167 break; 168 } 169 170 if (nxgep->soft_lso_enable) { 171 mp_chain = nxge_lso_eliminate(mp); 172 NXGE_DEBUG_MSG((nxgep, TX_CTL, 173 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 174 mp, mp_chain)); 175 if (mp_chain == NULL) { 176 NXGE_ERROR_MSG((nxgep, TX_CTL, 177 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 178 mp_chain, mp)); 179 goto nxge_start_fail1; 180 } 181 if (mp_chain != mp) { 182 NXGE_DEBUG_MSG((nxgep, TX_CTL, 183 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 184 mp_chain, mp)); 185 is_lso = B_TRUE; 186 mp = mp_chain; 187 mp_chain = mp_chain->b_next; 188 mp->b_next = NULL; 189 } 190 } 191 192 hcksum_retrieve(mp, NULL, NULL, &start_offset, 193 &stuff_offset, &end_offset, &value, &cksum_flags); 194 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 195 start_offset += sizeof (ether_header_t); 196 stuff_offset += sizeof (ether_header_t); 197 } else { 198 start_offset += sizeof (struct ether_vlan_header); 199 stuff_offset += sizeof (struct ether_vlan_header); 200 } 201 202 if (cksum_flags & HCK_PARTIALCKSUM) { 203 NXGE_DEBUG_MSG((nxgep, TX_CTL, 204 "==> nxge_start: mp $%p len %d " 205 "cksum_flags 0x%x (partial checksum) ", 206 mp, MBLKL(mp), cksum_flags)); 207 cksum_on = B_TRUE; 208 } 209 210 lso_again = B_FALSE; 211 lso_ngathers = 0; 212 213 MUTEX_ENTER(&tx_ring_p->lock); 214 cur_index_lso = tx_ring_p->wr_index; 215 lso_tail_wrap = tx_ring_p->wr_index_wrap; 216 start_again: 217 ngathers = 0; 218 sop_index = tx_ring_p->wr_index; 219 #ifdef NXGE_DEBUG 220 if (tx_ring_p->descs_pending) { 221 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 222 "desc pending %d ", tx_ring_p->descs_pending)); 223 } 224 225 dump_len = (int)(MBLKL(mp)); 226 dump_len = (dump_len > 128) ? 128: dump_len; 227 228 NXGE_DEBUG_MSG((nxgep, TX_CTL, 229 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 230 "(Before header reserve: ORIGINAL LEN %d)", 231 tx_ring_p->tdc, 232 mp->b_rptr, 233 dump_len)); 234 235 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 236 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 237 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 238 #endif 239 240 tdc_stats = tx_ring_p->tdc_stats; 241 mark_mode = (tx_ring_p->descs_pending && 242 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 243 < nxge_tx_minfree)); 244 245 NXGE_DEBUG_MSG((nxgep, TX_CTL, 246 "TX Descriptor ring is channel %d mark mode %d", 247 tx_ring_p->tdc, mark_mode)); 248 249 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, nxge_tx_minfree)) { 250 NXGE_DEBUG_MSG((nxgep, TX_CTL, 251 "TX Descriptor ring is full: channel %d", 252 tx_ring_p->tdc)); 253 NXGE_DEBUG_MSG((nxgep, TX_CTL, 254 "TX Descriptor ring is full: channel %d", 255 tx_ring_p->tdc)); 256 if (is_lso) { 257 /* free the current mp and mp_chain if not FULL */ 258 tdc_stats->tx_no_desc++; 259 NXGE_DEBUG_MSG((nxgep, TX_CTL, 260 "LSO packet: TX Descriptor ring is full: " 261 "channel %d", 262 tx_ring_p->tdc)); 263 goto nxge_start_fail_lso; 264 } else { 265 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 266 tdc_stats->tx_no_desc++; 267 MUTEX_EXIT(&tx_ring_p->lock); 268 if (nxgep->resched_needed && !nxgep->resched_running) { 269 nxgep->resched_running = B_TRUE; 270 ddi_trigger_softintr(nxgep->resched_id); 271 } 272 status = 1; 273 goto nxge_start_fail1; 274 } 275 } 276 277 nmp = mp; 278 i = sop_index = tx_ring_p->wr_index; 279 nmblks = 0; 280 ngathers = 0; 281 pkt_len = 0; 282 pack_len = 0; 283 clen = 0; 284 last_bidx = -1; 285 good_packet = B_TRUE; 286 287 desc_area = tx_ring_p->tdc_desc; 288 npi_handle = desc_area.npi_handle; 289 npi_desc_handle.regh = (nxge_os_acc_handle_t) 290 DMA_COMMON_ACC_HANDLE(desc_area); 291 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 292 #ifdef NXGE_DEBUG 293 tx_desc_ring_pp = (p_tx_desc_t)DMA_COMMON_IOADDR(desc_area); 294 #endif 295 tx_desc_dma_handle = (nxge_os_dma_handle_t) 296 DMA_COMMON_HANDLE(desc_area); 297 tx_msg_ring = tx_ring_p->tx_msg_ring; 298 299 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 300 sop_index, i)); 301 302 #ifdef NXGE_DEBUG 303 msgsize = msgdsize(nmp); 304 NXGE_DEBUG_MSG((nxgep, TX_CTL, 305 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 306 sop_index, i, msgsize)); 307 #endif 308 /* 309 * The first 16 bytes of the premapped buffer are reserved 310 * for header. No padding will be used. 311 */ 312 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 313 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 314 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 315 } else { 316 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 317 } 318 while (nmp) { 319 good_packet = B_TRUE; 320 b_rptr = nmp->b_rptr; 321 len = MBLKL(nmp); 322 if (len <= 0) { 323 nmp = nmp->b_cont; 324 continue; 325 } 326 nmblks++; 327 328 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 329 "len %d pkt_len %d pack_len %d", 330 nmblks, len, pkt_len, pack_len)); 331 /* 332 * Hardware limits the transfer length to 4K for NIU and 333 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 334 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 335 * If len is longer than the limit, then we break nmp into 336 * two chunks: Make the first chunk equal to the limit and 337 * the second chunk for the remaining data. If the second 338 * chunk is still larger than the limit, then it will be 339 * broken into two in the next pass. 340 */ 341 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 342 if ((t_mp = dupb(nmp)) != NULL) { 343 nmp->b_wptr = nmp->b_rptr + 344 (TX_MAX_TRANSFER_LENGTH 345 - TX_PKT_HEADER_SIZE); 346 t_mp->b_rptr = nmp->b_wptr; 347 t_mp->b_cont = nmp->b_cont; 348 nmp->b_cont = t_mp; 349 len = MBLKL(nmp); 350 } else { 351 if (is_lso) { 352 NXGE_DEBUG_MSG((nxgep, TX_CTL, 353 "LSO packet: dupb failed: " 354 "channel %d", 355 tx_ring_p->tdc)); 356 mp = nmp; 357 goto nxge_start_fail_lso; 358 } else { 359 good_packet = B_FALSE; 360 goto nxge_start_fail2; 361 } 362 } 363 } 364 tx_desc.value = 0; 365 tx_desc_p = &tx_desc_ring_vp[i]; 366 #ifdef NXGE_DEBUG 367 tx_desc_pp = &tx_desc_ring_pp[i]; 368 #endif 369 tx_msg_p = &tx_msg_ring[i]; 370 #if defined(__i386) 371 npi_desc_handle.regp = (uint32_t)tx_desc_p; 372 #else 373 npi_desc_handle.regp = (uint64_t)tx_desc_p; 374 #endif 375 if (!header_set && 376 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 377 (len >= bcopy_thresh))) { 378 header_set = B_TRUE; 379 bcopy_thresh += TX_PKT_HEADER_SIZE; 380 boff = 0; 381 pack_len = 0; 382 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 383 hdrp = (p_tx_pkt_header_t)kaddr; 384 clen = pkt_len; 385 dma_handle = tx_msg_p->buf_dma_handle; 386 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 387 (void) ddi_dma_sync(dma_handle, 388 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 389 DDI_DMA_SYNC_FORDEV); 390 391 tx_msg_p->flags.dma_type = USE_BCOPY; 392 goto nxge_start_control_header_only; 393 } 394 395 pkt_len += len; 396 pack_len += len; 397 398 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 399 "desc entry %d " 400 "DESC IOADDR $%p " 401 "desc_vp $%p tx_desc_p $%p " 402 "desc_pp $%p tx_desc_pp $%p " 403 "len %d pkt_len %d pack_len %d", 404 i, 405 DMA_COMMON_IOADDR(desc_area), 406 tx_desc_ring_vp, tx_desc_p, 407 tx_desc_ring_pp, tx_desc_pp, 408 len, pkt_len, pack_len)); 409 410 if (len < bcopy_thresh) { 411 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 412 "USE BCOPY: ")); 413 if (nxge_tx_tiny_pack) { 414 uint32_t blst = 415 TXDMA_DESC_NEXT_INDEX(i, -1, 416 tx_ring_p->tx_wrap_mask); 417 NXGE_DEBUG_MSG((nxgep, TX_CTL, 418 "==> nxge_start(5): pack")); 419 if ((pack_len <= bcopy_thresh) && 420 (last_bidx == blst)) { 421 NXGE_DEBUG_MSG((nxgep, TX_CTL, 422 "==> nxge_start: pack(6) " 423 "(pkt_len %d pack_len %d)", 424 pkt_len, pack_len)); 425 i = blst; 426 tx_desc_p = &tx_desc_ring_vp[i]; 427 #ifdef NXGE_DEBUG 428 tx_desc_pp = &tx_desc_ring_pp[i]; 429 #endif 430 tx_msg_p = &tx_msg_ring[i]; 431 boff = pack_len - len; 432 ngathers--; 433 } else if (pack_len > bcopy_thresh && 434 header_set) { 435 pack_len = len; 436 boff = 0; 437 bcopy_thresh = nxge_bcopy_thresh; 438 NXGE_DEBUG_MSG((nxgep, TX_CTL, 439 "==> nxge_start(7): > max NEW " 440 "bcopy thresh %d " 441 "pkt_len %d pack_len %d(next)", 442 bcopy_thresh, 443 pkt_len, pack_len)); 444 } 445 last_bidx = i; 446 } 447 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 448 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 449 hdrp = (p_tx_pkt_header_t)kaddr; 450 header_set = B_TRUE; 451 NXGE_DEBUG_MSG((nxgep, TX_CTL, 452 "==> nxge_start(7_x2): " 453 "pkt_len %d pack_len %d (new hdrp $%p)", 454 pkt_len, pack_len, hdrp)); 455 } 456 tx_msg_p->flags.dma_type = USE_BCOPY; 457 kaddr += boff; 458 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 459 "USE BCOPY: before bcopy " 460 "DESC IOADDR $%p entry %d " 461 "bcopy packets %d " 462 "bcopy kaddr $%p " 463 "bcopy ioaddr (SAD) $%p " 464 "bcopy clen %d " 465 "bcopy boff %d", 466 DMA_COMMON_IOADDR(desc_area), i, 467 tdc_stats->tx_hdr_pkts, 468 kaddr, 469 dma_ioaddr, 470 clen, 471 boff)); 472 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 473 "1USE BCOPY: ")); 474 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 475 "2USE BCOPY: ")); 476 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 477 "last USE BCOPY: copy from b_rptr $%p " 478 "to KADDR $%p (len %d offset %d", 479 b_rptr, kaddr, len, boff)); 480 481 bcopy(b_rptr, kaddr, len); 482 483 #ifdef NXGE_DEBUG 484 dump_len = (len > 128) ? 128: len; 485 NXGE_DEBUG_MSG((nxgep, TX_CTL, 486 "==> nxge_start: dump packets " 487 "(After BCOPY len %d)" 488 "(b_rptr $%p): %s", len, nmp->b_rptr, 489 nxge_dump_packet((char *)nmp->b_rptr, 490 dump_len))); 491 #endif 492 493 dma_handle = tx_msg_p->buf_dma_handle; 494 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 495 (void) ddi_dma_sync(dma_handle, 496 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 497 DDI_DMA_SYNC_FORDEV); 498 clen = len + boff; 499 tdc_stats->tx_hdr_pkts++; 500 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 501 "USE BCOPY: " 502 "DESC IOADDR $%p entry %d " 503 "bcopy packets %d " 504 "bcopy kaddr $%p " 505 "bcopy ioaddr (SAD) $%p " 506 "bcopy clen %d " 507 "bcopy boff %d", 508 DMA_COMMON_IOADDR(desc_area), 509 i, 510 tdc_stats->tx_hdr_pkts, 511 kaddr, 512 dma_ioaddr, 513 clen, 514 boff)); 515 } else { 516 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 517 "USE DVMA: len %d", len)); 518 tx_msg_p->flags.dma_type = USE_DMA; 519 dma_flags = DDI_DMA_WRITE; 520 if (len < nxge_dma_stream_thresh) { 521 dma_flags |= DDI_DMA_CONSISTENT; 522 } else { 523 dma_flags |= DDI_DMA_STREAMING; 524 } 525 526 dma_handle = tx_msg_p->dma_handle; 527 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 528 (caddr_t)b_rptr, len, dma_flags, 529 DDI_DMA_DONTWAIT, NULL, 530 &dma_cookie, &ncookies); 531 if (status == DDI_DMA_MAPPED) { 532 dma_ioaddr = dma_cookie.dmac_laddress; 533 len = (int)dma_cookie.dmac_size; 534 clen = (uint32_t)dma_cookie.dmac_size; 535 NXGE_DEBUG_MSG((nxgep, TX_CTL, 536 "==> nxge_start(12_1): " 537 "USE DVMA: len %d clen %d " 538 "ngathers %d", 539 len, clen, 540 ngathers)); 541 #if defined(__i386) 542 npi_desc_handle.regp = (uint32_t)tx_desc_p; 543 #else 544 npi_desc_handle.regp = (uint64_t)tx_desc_p; 545 #endif 546 while (ncookies > 1) { 547 ngathers++; 548 /* 549 * this is the fix for multiple 550 * cookies, which are basically 551 * a descriptor entry, we don't set 552 * SOP bit as well as related fields 553 */ 554 555 (void) npi_txdma_desc_gather_set( 556 npi_desc_handle, 557 &tx_desc, 558 (ngathers -1), 559 mark_mode, 560 ngathers, 561 dma_ioaddr, 562 clen); 563 564 tx_msg_p->tx_msg_size = clen; 565 NXGE_DEBUG_MSG((nxgep, TX_CTL, 566 "==> nxge_start: DMA " 567 "ncookie %d " 568 "ngathers %d " 569 "dma_ioaddr $%p len %d" 570 "desc $%p descp $%p (%d)", 571 ncookies, 572 ngathers, 573 dma_ioaddr, clen, 574 *tx_desc_p, tx_desc_p, i)); 575 576 ddi_dma_nextcookie(dma_handle, 577 &dma_cookie); 578 dma_ioaddr = 579 dma_cookie.dmac_laddress; 580 581 len = (int)dma_cookie.dmac_size; 582 clen = (uint32_t)dma_cookie.dmac_size; 583 NXGE_DEBUG_MSG((nxgep, TX_CTL, 584 "==> nxge_start(12_2): " 585 "USE DVMA: len %d clen %d ", 586 len, clen)); 587 588 i = TXDMA_DESC_NEXT_INDEX(i, 1, 589 tx_ring_p->tx_wrap_mask); 590 tx_desc_p = &tx_desc_ring_vp[i]; 591 592 npi_desc_handle.regp = 593 #if defined(__i386) 594 (uint32_t)tx_desc_p; 595 #else 596 (uint64_t)tx_desc_p; 597 #endif 598 tx_msg_p = &tx_msg_ring[i]; 599 tx_msg_p->flags.dma_type = USE_NONE; 600 tx_desc.value = 0; 601 602 ncookies--; 603 } 604 tdc_stats->tx_ddi_pkts++; 605 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 606 "DMA: ddi packets %d", 607 tdc_stats->tx_ddi_pkts)); 608 } else { 609 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 610 "dma mapping failed for %d " 611 "bytes addr $%p flags %x (%d)", 612 len, b_rptr, status, status)); 613 good_packet = B_FALSE; 614 tdc_stats->tx_dma_bind_fail++; 615 tx_msg_p->flags.dma_type = USE_NONE; 616 if (is_lso) { 617 mp = nmp; 618 goto nxge_start_fail_lso; 619 } else { 620 goto nxge_start_fail2; 621 } 622 } 623 } /* ddi dvma */ 624 625 if (is_lso) { 626 nmp_lso_save = nmp; 627 } 628 nmp = nmp->b_cont; 629 nxge_start_control_header_only: 630 #if defined(__i386) 631 npi_desc_handle.regp = (uint32_t)tx_desc_p; 632 #else 633 npi_desc_handle.regp = (uint64_t)tx_desc_p; 634 #endif 635 ngathers++; 636 637 if (ngathers == 1) { 638 #ifdef NXGE_DEBUG 639 save_desc_p = &sop_tx_desc; 640 #endif 641 sop_tx_desc_p = &sop_tx_desc; 642 sop_tx_desc_p->value = 0; 643 sop_tx_desc_p->bits.hdw.tr_len = clen; 644 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 645 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 646 } else { 647 #ifdef NXGE_DEBUG 648 save_desc_p = &tx_desc; 649 #endif 650 tmp_desc_p = &tx_desc; 651 tmp_desc_p->value = 0; 652 tmp_desc_p->bits.hdw.tr_len = clen; 653 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 654 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 655 656 tx_desc_p->value = tmp_desc_p->value; 657 } 658 659 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 660 "Desc_entry %d ngathers %d " 661 "desc_vp $%p tx_desc_p $%p " 662 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 663 "dma_ioaddr (SAD) $%p mark %d", 664 i, ngathers, 665 tx_desc_ring_vp, tx_desc_p, 666 len, clen, pkt_len, pack_len, nmblks, 667 dma_ioaddr, mark_mode)); 668 669 #ifdef NXGE_DEBUG 670 npi_desc_handle.nxgep = nxgep; 671 npi_desc_handle.function.function = nxgep->function_num; 672 npi_desc_handle.function.instance = nxgep->instance; 673 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 674 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 675 TX_PKT_DESC_TR_LEN_SHIFT); 676 677 678 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 679 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 680 "mark %d sop %d\n", 681 save_desc_p->value, 682 sad, 683 save_desc_p->bits.hdw.tr_len, 684 xfer_len, 685 save_desc_p->bits.hdw.num_ptr, 686 save_desc_p->bits.hdw.mark, 687 save_desc_p->bits.hdw.sop)); 688 689 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 690 #endif 691 692 tx_msg_p->tx_msg_size = clen; 693 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 694 if (ngathers > nxge_tx_max_gathers) { 695 good_packet = B_FALSE; 696 hcksum_retrieve(mp, NULL, NULL, &start_offset, 697 &stuff_offset, &end_offset, &value, 698 &cksum_flags); 699 700 NXGE_DEBUG_MSG((NULL, TX_CTL, 701 "==> nxge_start(14): pull msg - " 702 "len %d pkt_len %d ngathers %d", 703 len, pkt_len, ngathers)); 704 /* Pull all message blocks from b_cont */ 705 if (is_lso) { 706 mp = nmp_lso_save; 707 goto nxge_start_fail_lso; 708 } 709 if ((msgpullup(mp, -1)) == NULL) { 710 goto nxge_start_fail2; 711 } 712 goto nxge_start_fail2; 713 } 714 } /* while (nmp) */ 715 716 tx_msg_p->tx_message = mp; 717 tx_desc_p = &tx_desc_ring_vp[sop_index]; 718 #if defined(__i386) 719 npi_desc_handle.regp = (uint32_t)tx_desc_p; 720 #else 721 npi_desc_handle.regp = (uint64_t)tx_desc_p; 722 #endif 723 724 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 725 pkthdrp->reserved = 0; 726 hdrp->value = 0; 727 (void) nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 728 (pkt_len - TX_PKT_HEADER_SIZE), npads, pkthdrp); 729 730 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 731 tdc_stats->tx_jumbo_pkts++; 732 } 733 734 min_len = (nxgep->msg_min + TX_PKT_HEADER_SIZE + (npads * 2)); 735 if (pkt_len < min_len) { 736 /* Assume we use bcopy to premapped buffers */ 737 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 738 NXGE_DEBUG_MSG((NULL, TX_CTL, 739 "==> nxge_start(14-1): < (msg_min + 16)" 740 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 741 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 742 bzero((kaddr + pkt_len), (min_len - pkt_len)); 743 pkt_len = tx_msg_p->tx_msg_size = min_len; 744 745 sop_tx_desc_p->bits.hdw.tr_len = min_len; 746 747 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 748 tx_desc_p->value = sop_tx_desc_p->value; 749 750 NXGE_DEBUG_MSG((NULL, TX_CTL, 751 "==> nxge_start(14-2): < msg_min - " 752 "len %d pkt_len %d min_len %d ngathers %d", 753 len, pkt_len, min_len, ngathers)); 754 } 755 756 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 757 cksum_flags)); 758 if (cksum_flags & HCK_PARTIALCKSUM) { 759 NXGE_DEBUG_MSG((nxgep, TX_CTL, 760 "==> nxge_start: cksum_flags 0x%x (partial checksum) ", 761 cksum_flags)); 762 cksum_on = B_TRUE; 763 NXGE_DEBUG_MSG((nxgep, TX_CTL, 764 "==> nxge_start: from IP cksum_flags 0x%x " 765 "(partial checksum) " 766 "start_offset %d stuff_offset %d", 767 cksum_flags, start_offset, stuff_offset)); 768 tmp_len = (uint64_t)(start_offset >> 1); 769 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4START_SHIFT); 770 tmp_len = (uint64_t)(stuff_offset >> 1); 771 hdrp->value |= (tmp_len << TX_PKT_HEADER_L4STUFF_SHIFT); 772 773 NXGE_DEBUG_MSG((nxgep, TX_CTL, 774 "==> nxge_start: from IP cksum_flags 0x%x " 775 "(partial checksum) " 776 "after SHIFT start_offset %d stuff_offset %d", 777 cksum_flags, start_offset, stuff_offset)); 778 } 779 { 780 uint64_t tmp_len; 781 782 /* pkt_len already includes 16 + paddings!! */ 783 /* Update the control header length */ 784 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 785 tmp_len = hdrp->value | 786 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 787 788 NXGE_DEBUG_MSG((nxgep, TX_CTL, 789 "==> nxge_start(15_x1): setting SOP " 790 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 791 "0x%llx hdrp->value 0x%llx", 792 tot_xfer_len, tot_xfer_len, pkt_len, 793 tmp_len, hdrp->value)); 794 #if defined(_BIG_ENDIAN) 795 hdrp->value = ddi_swap64(tmp_len); 796 #else 797 hdrp->value = tmp_len; 798 #endif 799 NXGE_DEBUG_MSG((nxgep, 800 TX_CTL, "==> nxge_start(15_x2): setting SOP " 801 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 802 "tmp_len 0x%llx hdrp->value 0x%llx", 803 tot_xfer_len, pkt_len, 804 tmp_len, hdrp->value)); 805 } 806 807 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 808 "wr_index %d " 809 "tot_xfer_len (%d) pkt_len %d npads %d", 810 sop_index, 811 tot_xfer_len, pkt_len, 812 npads)); 813 814 sop_tx_desc_p->bits.hdw.sop = 1; 815 sop_tx_desc_p->bits.hdw.mark = mark_mode; 816 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 817 818 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 819 820 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 821 822 #ifdef NXGE_DEBUG 823 npi_desc_handle.nxgep = nxgep; 824 npi_desc_handle.function.function = nxgep->function_num; 825 npi_desc_handle.function.instance = nxgep->instance; 826 827 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 828 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 829 save_desc_p->value, 830 sad, 831 save_desc_p->bits.hdw.tr_len, 832 xfer_len, 833 save_desc_p->bits.hdw.num_ptr, 834 save_desc_p->bits.hdw.mark, 835 save_desc_p->bits.hdw.sop)); 836 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 837 838 dump_len = (pkt_len > 128) ? 128: pkt_len; 839 NXGE_DEBUG_MSG((nxgep, TX_CTL, 840 "==> nxge_start: dump packets(17) (after sop set, len " 841 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 842 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 843 (char *)hdrp, 844 nxge_dump_packet((char *)hdrp, dump_len))); 845 NXGE_DEBUG_MSG((nxgep, TX_CTL, 846 "==> nxge_start(18): TX desc sync: sop_index %d", 847 sop_index)); 848 #endif 849 850 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 851 (void) ddi_dma_sync(tx_desc_dma_handle, 852 sop_index * sizeof (tx_desc_t), 853 ngathers * sizeof (tx_desc_t), 854 DDI_DMA_SYNC_FORDEV); 855 856 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 857 "cs_off = 0x%02X cs_s_off = 0x%02X " 858 "pkt_len %d ngathers %d sop_index %d\n", 859 stuff_offset, start_offset, 860 pkt_len, ngathers, sop_index)); 861 } else { /* more than one descriptor and wrap around */ 862 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 863 (void) ddi_dma_sync(tx_desc_dma_handle, 864 sop_index * sizeof (tx_desc_t), 865 nsdescs * sizeof (tx_desc_t), 866 DDI_DMA_SYNC_FORDEV); 867 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 868 "cs_off = 0x%02X cs_s_off = 0x%02X " 869 "pkt_len %d ngathers %d sop_index %d\n", 870 stuff_offset, start_offset, 871 pkt_len, ngathers, sop_index)); 872 873 (void) ddi_dma_sync(tx_desc_dma_handle, 874 0, 875 (ngathers - nsdescs) * sizeof (tx_desc_t), 876 DDI_DMA_SYNC_FORDEV); 877 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 878 "cs_off = 0x%02X cs_s_off = 0x%02X " 879 "pkt_len %d ngathers %d sop_index %d\n", 880 stuff_offset, start_offset, 881 pkt_len, ngathers, sop_index)); 882 } 883 884 tail_index = tx_ring_p->wr_index; 885 tail_wrap = tx_ring_p->wr_index_wrap; 886 887 tx_ring_p->wr_index = i; 888 if (tx_ring_p->wr_index <= tail_index) { 889 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 890 B_FALSE : B_TRUE); 891 } 892 893 tx_ring_p->descs_pending += ngathers; 894 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 895 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 896 tx_ring_p->tdc, 897 tx_ring_p->wr_index, 898 tx_ring_p->wr_index_wrap, 899 ngathers, 900 tx_ring_p->descs_pending)); 901 902 if (is_lso) { 903 lso_ngathers += ngathers; 904 if (mp_chain != NULL) { 905 mp = mp_chain; 906 mp_chain = mp_chain->b_next; 907 mp->b_next = NULL; 908 if (nxge_lso_kick_cnt == lso_ngathers) { 909 { 910 tx_ring_kick_t kick; 911 912 kick.value = 0; 913 kick.bits.ldw.wrap = 914 tx_ring_p->wr_index_wrap; 915 kick.bits.ldw.tail = 916 (uint16_t)tx_ring_p->wr_index; 917 918 /* Kick the Transmit kick register */ 919 TXDMA_REG_WRITE64( 920 NXGE_DEV_NPI_HANDLE(nxgep), 921 TX_RING_KICK_REG, 922 (uint8_t)tx_ring_p->tdc, 923 kick.value); 924 tdc_stats->tx_starts++; 925 NXGE_DEBUG_MSG((nxgep, TX_CTL, 926 "==> nxge_start: more LSO: " 927 "LSO_CNT %d", 928 lso_gathers)); 929 } 930 lso_ngathers = 0; 931 ngathers = 0; 932 cur_index_lso = sop_index = tx_ring_p->wr_index; 933 lso_tail_wrap = tx_ring_p->wr_index_wrap; 934 } 935 NXGE_DEBUG_MSG((nxgep, TX_CTL, 936 "==> nxge_start: lso again: " 937 "lso_gathers %d ngathers %d cur_index_lso %d " 938 "wr_index %d sop_index %d", 939 lso_ngathers, ngathers, cur_index_lso, 940 tx_ring_p->wr_index, sop_index)); 941 942 NXGE_DEBUG_MSG((nxgep, TX_CTL, 943 "==> nxge_start: next : count %d", 944 lso_gathers)); 945 lso_again = B_TRUE; 946 goto start_again; 947 } 948 } 949 950 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 951 952 { 953 tx_ring_kick_t kick; 954 955 kick.value = 0; 956 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 957 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 958 959 /* Kick start the Transmit kick register */ 960 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 961 TX_RING_KICK_REG, 962 (uint8_t)tx_ring_p->tdc, 963 kick.value); 964 } 965 966 tdc_stats->tx_starts++; 967 968 MUTEX_EXIT(&tx_ring_p->lock); 969 970 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 971 972 return (status); 973 974 nxge_start_fail_lso: 975 status = 0; 976 good_packet = B_FALSE; 977 if (mp != NULL) { 978 freemsg(mp); 979 } 980 if (mp_chain != NULL) { 981 freemsg(mp_chain); 982 } 983 if (!lso_again && !ngathers) { 984 MUTEX_EXIT(&tx_ring_p->lock); 985 NXGE_DEBUG_MSG((nxgep, TX_CTL, 986 "==> nxge_start: lso exit (nothing changed)")); 987 goto nxge_start_fail1; 988 } 989 990 NXGE_DEBUG_MSG((nxgep, TX_CTL, 991 "==> nxge_start (channel %d): before lso " 992 "lso_gathers %d ngathers %d cur_index_lso %d " 993 "wr_index %d sop_index %d lso_again %d", 994 tx_ring_p->tdc, 995 lso_ngathers, ngathers, cur_index_lso, 996 tx_ring_p->wr_index, sop_index, lso_again)); 997 998 if (lso_again) { 999 lso_ngathers += ngathers; 1000 ngathers = lso_ngathers; 1001 sop_index = cur_index_lso; 1002 tx_ring_p->wr_index = sop_index; 1003 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1004 } 1005 1006 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1007 "==> nxge_start (channel %d): after lso " 1008 "lso_gathers %d ngathers %d cur_index_lso %d " 1009 "wr_index %d sop_index %d lso_again %d", 1010 tx_ring_p->tdc, 1011 lso_ngathers, ngathers, cur_index_lso, 1012 tx_ring_p->wr_index, sop_index, lso_again)); 1013 1014 nxge_start_fail2: 1015 if (good_packet == B_FALSE) { 1016 cur_index = sop_index; 1017 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1018 for (i = 0; i < ngathers; i++) { 1019 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1020 #if defined(__i386) 1021 npi_handle.regp = (uint32_t)tx_desc_p; 1022 #else 1023 npi_handle.regp = (uint64_t)tx_desc_p; 1024 #endif 1025 tx_msg_p = &tx_msg_ring[cur_index]; 1026 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1027 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1028 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1029 "tx_desc_p = %X index = %d", 1030 tx_desc_p, tx_ring_p->rd_index)); 1031 (void) dvma_unload(tx_msg_p->dvma_handle, 1032 0, -1); 1033 tx_msg_p->dvma_handle = NULL; 1034 if (tx_ring_p->dvma_wr_index == 1035 tx_ring_p->dvma_wrap_mask) 1036 tx_ring_p->dvma_wr_index = 0; 1037 else 1038 tx_ring_p->dvma_wr_index++; 1039 tx_ring_p->dvma_pending--; 1040 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1041 if (ddi_dma_unbind_handle( 1042 tx_msg_p->dma_handle)) { 1043 cmn_err(CE_WARN, "!nxge_start: " 1044 "ddi_dma_unbind_handle failed"); 1045 } 1046 } 1047 tx_msg_p->flags.dma_type = USE_NONE; 1048 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1049 tx_ring_p->tx_wrap_mask); 1050 1051 } 1052 1053 nxgep->resched_needed = B_TRUE; 1054 } 1055 1056 MUTEX_EXIT(&tx_ring_p->lock); 1057 1058 nxge_start_fail1: 1059 /* Add FMA to check the access handle nxge_hregh */ 1060 1061 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1062 1063 return (status); 1064 } 1065 1066 int 1067 nxge_serial_tx(mblk_t *mp, void *arg) 1068 { 1069 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1070 p_nxge_t nxgep = tx_ring_p->nxgep; 1071 1072 return (nxge_start(nxgep, tx_ring_p, mp)); 1073 } 1074 1075 boolean_t 1076 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1077 { 1078 p_tx_ring_t *tx_rings; 1079 uint8_t ring_index; 1080 p_tx_ring_t tx_ring_p; 1081 1082 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1083 1084 ASSERT(mp->b_next == NULL); 1085 1086 ring_index = nxge_tx_lb_ring_1(mp, nxgep->max_tdcs, hp); 1087 tx_rings = nxgep->tx_rings->rings; 1088 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: tx_rings $%p", 1089 tx_rings)); 1090 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_tx_msg: max_tdcs %d " 1091 "ring_index %d", nxgep->max_tdcs, ring_index)); 1092 1093 switch (nxge_tx_scheme) { 1094 case NXGE_USE_START: 1095 if (nxge_start(nxgep, tx_rings[ring_index], mp)) { 1096 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1097 "ring index %d", ring_index)); 1098 return (B_FALSE); 1099 } 1100 break; 1101 1102 case NXGE_USE_SERIAL: 1103 default: 1104 tx_ring_p = tx_rings[ring_index]; 1105 nxge_serialize_enter(tx_ring_p->serial, mp); 1106 break; 1107 } 1108 1109 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1110 ring_index)); 1111 1112 return (B_TRUE); 1113 } 1114 1115 /* 1116 * nxge_m_tx() - send a chain of packets 1117 */ 1118 mblk_t * 1119 nxge_m_tx(void *arg, mblk_t *mp) 1120 { 1121 p_nxge_t nxgep = (p_nxge_t)arg; 1122 mblk_t *next; 1123 mac_tx_hint_t hint; 1124 1125 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 1126 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1127 "==> nxge_m_tx: hardware not initialized")); 1128 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1129 "<== nxge_m_tx")); 1130 return (mp); 1131 } 1132 1133 hint.hash = NULL; 1134 hint.vid = 0; 1135 hint.sap = 0; 1136 1137 while (mp != NULL) { 1138 next = mp->b_next; 1139 mp->b_next = NULL; 1140 1141 /* 1142 * Until Nemo tx resource works, the mac driver 1143 * does the load balancing based on TCP port, 1144 * or CPU. For debugging, we use a system 1145 * configurable parameter. 1146 */ 1147 if (!nxge_send(nxgep, mp, &hint)) { 1148 mp->b_next = next; 1149 break; 1150 } 1151 1152 mp = next; 1153 1154 NXGE_DEBUG_MSG((NULL, TX_CTL, 1155 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1156 mp, next)); 1157 } 1158 1159 return (mp); 1160 } 1161 1162 int 1163 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1164 { 1165 uint8_t ring_index = 0; 1166 uint8_t *tcp_port; 1167 p_mblk_t nmp; 1168 size_t mblk_len; 1169 size_t iph_len; 1170 size_t hdrs_size; 1171 uint8_t hdrs_buf[sizeof (struct ether_header) + 1172 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1173 /* 1174 * allocate space big enough to cover 1175 * the max ip header length and the first 1176 * 4 bytes of the TCP/IP header. 1177 */ 1178 1179 boolean_t qos = B_FALSE; 1180 1181 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1182 1183 if (hp->vid) { 1184 qos = B_TRUE; 1185 } 1186 switch (nxge_tx_lb_policy) { 1187 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1188 default: 1189 tcp_port = mp->b_rptr; 1190 if (!nxge_no_tx_lb && !qos && 1191 (ntohs(((p_ether_header_t)tcp_port)->ether_type) 1192 == ETHERTYPE_IP)) { 1193 nmp = mp; 1194 mblk_len = MBLKL(nmp); 1195 tcp_port = NULL; 1196 if (mblk_len > sizeof (struct ether_header) + 1197 sizeof (uint8_t)) { 1198 tcp_port = nmp->b_rptr + 1199 sizeof (struct ether_header); 1200 mblk_len -= sizeof (struct ether_header); 1201 iph_len = ((*tcp_port) & 0x0f) << 2; 1202 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1203 tcp_port = nmp->b_rptr; 1204 } else { 1205 tcp_port = NULL; 1206 } 1207 } 1208 if (tcp_port == NULL) { 1209 hdrs_size = 0; 1210 ((p_ether_header_t)hdrs_buf)->ether_type = 0; 1211 while ((nmp) && (hdrs_size < 1212 sizeof (hdrs_buf))) { 1213 mblk_len = MBLKL(nmp); 1214 if (mblk_len >= 1215 (sizeof (hdrs_buf) - hdrs_size)) 1216 mblk_len = sizeof (hdrs_buf) - 1217 hdrs_size; 1218 bcopy(nmp->b_rptr, 1219 &hdrs_buf[hdrs_size], mblk_len); 1220 hdrs_size += mblk_len; 1221 nmp = nmp->b_cont; 1222 } 1223 tcp_port = hdrs_buf; 1224 } 1225 tcp_port += sizeof (ether_header_t); 1226 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1227 switch (tcp_port[9]) { 1228 case IPPROTO_TCP: 1229 case IPPROTO_UDP: 1230 case IPPROTO_ESP: 1231 tcp_port += ((*tcp_port) & 0x0f) << 2; 1232 ring_index = 1233 ((tcp_port[0] ^ 1234 tcp_port[1] ^ 1235 tcp_port[2] ^ 1236 tcp_port[3]) % maxtdcs); 1237 break; 1238 1239 case IPPROTO_AH: 1240 /* SPI starts at the 4th byte */ 1241 tcp_port += ((*tcp_port) & 0x0f) << 2; 1242 ring_index = 1243 ((tcp_port[4] ^ 1244 tcp_port[5] ^ 1245 tcp_port[6] ^ 1246 tcp_port[7]) % maxtdcs); 1247 break; 1248 1249 default: 1250 ring_index = tcp_port[19] % maxtdcs; 1251 break; 1252 } 1253 } else { /* fragmented packet */ 1254 ring_index = tcp_port[19] % maxtdcs; 1255 } 1256 } else { 1257 ring_index = mp->b_band % maxtdcs; 1258 } 1259 break; 1260 1261 case NXGE_TX_LB_HASH: 1262 if (hp->hash) { 1263 #if defined(__i386) 1264 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1265 #else 1266 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1267 #endif 1268 } else { 1269 ring_index = mp->b_band % maxtdcs; 1270 } 1271 break; 1272 1273 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1274 tcp_port = mp->b_rptr; 1275 ring_index = tcp_port[5] % maxtdcs; 1276 break; 1277 } 1278 1279 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1280 1281 return (ring_index); 1282 } 1283 1284 uint_t 1285 nxge_reschedule(caddr_t arg) 1286 { 1287 p_nxge_t nxgep; 1288 1289 nxgep = (p_nxge_t)arg; 1290 1291 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1292 1293 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1294 nxgep->resched_needed) { 1295 mac_tx_update(nxgep->mach); 1296 nxgep->resched_needed = B_FALSE; 1297 nxgep->resched_running = B_FALSE; 1298 } 1299 1300 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1301 return (DDI_INTR_CLAIMED); 1302 } 1303 1304 1305 /* Software LSO starts here */ 1306 static void 1307 nxge_hcksum_retrieve(mblk_t *mp, 1308 uint32_t *start, uint32_t *stuff, uint32_t *end, 1309 uint32_t *value, uint32_t *flags) 1310 { 1311 if (mp->b_datap->db_type == M_DATA) { 1312 if (flags != NULL) { 1313 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1314 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1315 HCK_FULLCKSUM_OK); 1316 if ((*flags & (HCK_PARTIALCKSUM | 1317 HCK_FULLCKSUM)) != 0) { 1318 if (value != NULL) 1319 *value = (uint32_t)DB_CKSUM16(mp); 1320 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1321 if (start != NULL) 1322 *start = 1323 (uint32_t)DB_CKSUMSTART(mp); 1324 if (stuff != NULL) 1325 *stuff = 1326 (uint32_t)DB_CKSUMSTUFF(mp); 1327 if (end != NULL) 1328 *end = 1329 (uint32_t)DB_CKSUMEND(mp); 1330 } 1331 } 1332 } 1333 } 1334 } 1335 1336 static void 1337 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1338 { 1339 ASSERT(DB_TYPE(mp) == M_DATA); 1340 1341 *mss = 0; 1342 if (flags != NULL) { 1343 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1344 if ((*flags != 0) && (mss != NULL)) { 1345 *mss = (uint32_t)DB_LSOMSS(mp); 1346 } 1347 NXGE_DEBUG_MSG((NULL, TX_CTL, 1348 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1349 *mss, *flags)); 1350 } 1351 1352 NXGE_DEBUG_MSG((NULL, TX_CTL, 1353 "<== nxge_lso_info_get: mss %d", *mss)); 1354 } 1355 1356 /* 1357 * Do Soft LSO on the oversized packet. 1358 * 1359 * 1. Create a chain of message for headers. 1360 * 2. Fill up header messages with proper information. 1361 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1362 * each new message with necessary adjustments. 1363 * * Unchange the ethernet header for DIX frames. (by default) 1364 * * IP Total Length field is updated to MSS or less(only for the last one). 1365 * * IP Identification value is incremented by one for each packet. 1366 * * TCP sequence Number is recalculated according to the payload length. 1367 * * Set FIN and/or PSH flags for the *last* packet if applied. 1368 * * TCP partial Checksum 1369 * 4. Update LSO information in the first message header. 1370 * 5. Release the original message header. 1371 */ 1372 static mblk_t * 1373 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1374 { 1375 uint32_t hckflags; 1376 int pktlen; 1377 int hdrlen; 1378 int segnum; 1379 int i; 1380 struct ether_vlan_header *evh; 1381 int ehlen, iphlen, tcphlen; 1382 struct ip *oiph, *niph; 1383 struct tcphdr *otcph, *ntcph; 1384 int available, len, left; 1385 uint16_t ip_id; 1386 uint32_t tcp_seq; 1387 #ifdef __sparc 1388 uint32_t tcp_seq_tmp; 1389 #endif 1390 mblk_t *datamp; 1391 uchar_t *rptr; 1392 mblk_t *nmp; 1393 mblk_t *cmp; 1394 mblk_t *mp_chain; 1395 boolean_t do_cleanup = B_FALSE; 1396 t_uscalar_t start_offset = 0; 1397 t_uscalar_t stuff_offset = 0; 1398 t_uscalar_t value = 0; 1399 uint16_t l4_len; 1400 ipaddr_t src, dst; 1401 uint32_t cksum, sum, l4cksum; 1402 1403 NXGE_DEBUG_MSG((NULL, TX_CTL, 1404 "==> nxge_do_softlso")); 1405 /* 1406 * check the length of LSO packet payload and calculate the number of 1407 * segments to be generated. 1408 */ 1409 pktlen = msgsize(mp); 1410 evh = (struct ether_vlan_header *)mp->b_rptr; 1411 1412 /* VLAN? */ 1413 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1414 ehlen = sizeof (struct ether_vlan_header); 1415 else 1416 ehlen = sizeof (struct ether_header); 1417 oiph = (struct ip *)(mp->b_rptr + ehlen); 1418 iphlen = oiph->ip_hl * 4; 1419 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1420 tcphlen = otcph->th_off * 4; 1421 1422 l4_len = pktlen - ehlen - iphlen; 1423 1424 NXGE_DEBUG_MSG((NULL, TX_CTL, 1425 "==> nxge_do_softlso: mss %d oiph $%p " 1426 "original ip_sum oiph->ip_sum 0x%x " 1427 "original tcp_sum otcph->th_sum 0x%x " 1428 "oiph->ip_len %d pktlen %d ehlen %d " 1429 "l4_len %d (0x%x) ip_len - iphlen %d ", 1430 mss, 1431 oiph, 1432 oiph->ip_sum, 1433 otcph->th_sum, 1434 ntohs(oiph->ip_len), pktlen, 1435 ehlen, 1436 l4_len, 1437 l4_len, 1438 ntohs(oiph->ip_len) - iphlen)); 1439 1440 /* IPv4 + TCP */ 1441 if (!(oiph->ip_v == IPV4_VERSION)) { 1442 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1443 "<== nxge_do_softlso: not IPV4 " 1444 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1445 ntohs(oiph->ip_len), pktlen, ehlen, 1446 tcphlen)); 1447 freemsg(mp); 1448 return (NULL); 1449 } 1450 1451 if (!(oiph->ip_p == IPPROTO_TCP)) { 1452 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1453 "<== nxge_do_softlso: not TCP " 1454 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1455 ntohs(oiph->ip_len), pktlen, ehlen, 1456 tcphlen)); 1457 freemsg(mp); 1458 return (NULL); 1459 } 1460 1461 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1462 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1463 "<== nxge_do_softlso: len not matched " 1464 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1465 ntohs(oiph->ip_len), pktlen, ehlen, 1466 tcphlen)); 1467 freemsg(mp); 1468 return (NULL); 1469 } 1470 1471 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1472 tcphlen = otcph->th_off * 4; 1473 1474 /* TCP flags can not include URG, RST, or SYN */ 1475 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1476 1477 hdrlen = ehlen + iphlen + tcphlen; 1478 1479 VERIFY(MBLKL(mp) >= hdrlen); 1480 1481 if (MBLKL(mp) > hdrlen) { 1482 datamp = mp; 1483 rptr = mp->b_rptr + hdrlen; 1484 } else { /* = */ 1485 datamp = mp->b_cont; 1486 rptr = datamp->b_rptr; 1487 } 1488 1489 NXGE_DEBUG_MSG((NULL, TX_CTL, 1490 "nxge_do_softlso: otcph $%p pktlen: %d, " 1491 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1492 "mblkl(mp): %d, mblkl(datamp): %d", 1493 otcph, 1494 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1495 (int)MBLKL(mp), (int)MBLKL(datamp))); 1496 1497 hckflags = 0; 1498 nxge_hcksum_retrieve(mp, 1499 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1500 1501 dst = oiph->ip_dst.s_addr; 1502 src = oiph->ip_src.s_addr; 1503 1504 cksum = (dst >> 16) + (dst & 0xFFFF) + 1505 (src >> 16) + (src & 0xFFFF); 1506 l4cksum = cksum + IP_TCP_CSUM_COMP; 1507 1508 sum = l4_len + l4cksum; 1509 sum = (sum & 0xFFFF) + (sum >> 16); 1510 1511 NXGE_DEBUG_MSG((NULL, TX_CTL, 1512 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1513 "hckflags 0x%x start_offset %d stuff_offset %d " 1514 "value (original) 0x%x th_sum 0x%x " 1515 "pktlen %d l4_len %d (0x%x) " 1516 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1517 dst, src, 1518 (sum & 0xffff), (~sum & 0xffff), 1519 hckflags, start_offset, stuff_offset, 1520 value, otcph->th_sum, 1521 pktlen, 1522 l4_len, 1523 l4_len, 1524 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1525 (int)MBLKL(datamp), 1526 nxge_dump_packet((char *)evh, 12))); 1527 1528 /* 1529 * Start to process. 1530 */ 1531 available = pktlen - hdrlen; 1532 segnum = (available - 1) / mss + 1; 1533 1534 NXGE_DEBUG_MSG((NULL, TX_CTL, 1535 "==> nxge_do_softlso: pktlen %d " 1536 "MBLKL(mp): %d, MBLKL(datamp): %d " 1537 "available %d mss %d segnum %d", 1538 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1539 available, 1540 mss, 1541 segnum)); 1542 1543 VERIFY(segnum >= 2); 1544 1545 /* 1546 * Try to pre-allocate all header messages 1547 */ 1548 mp_chain = NULL; 1549 for (i = 0; i < segnum; i++) { 1550 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1551 /* Clean up the mp_chain */ 1552 while (mp_chain != NULL) { 1553 nmp = mp_chain; 1554 mp_chain = mp_chain->b_next; 1555 freemsg(nmp); 1556 } 1557 NXGE_DEBUG_MSG((NULL, TX_CTL, 1558 "<== nxge_do_softlso: " 1559 "Could not allocate enough messages for headers!")); 1560 freemsg(mp); 1561 return (NULL); 1562 } 1563 nmp->b_next = mp_chain; 1564 mp_chain = nmp; 1565 1566 NXGE_DEBUG_MSG((NULL, TX_CTL, 1567 "==> nxge_do_softlso: " 1568 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1569 mp, nmp, mp_chain, mp_chain->b_next)); 1570 } 1571 1572 NXGE_DEBUG_MSG((NULL, TX_CTL, 1573 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1574 mp, nmp, mp_chain)); 1575 1576 /* 1577 * Associate payload with new packets 1578 */ 1579 cmp = mp_chain; 1580 left = available; 1581 while (cmp != NULL) { 1582 nmp = dupb(datamp); 1583 if (nmp == NULL) { 1584 do_cleanup = B_TRUE; 1585 NXGE_DEBUG_MSG((NULL, TX_CTL, 1586 "==>nxge_do_softlso: " 1587 "Can not dupb(datamp), have to do clean up")); 1588 goto cleanup_allocated_msgs; 1589 } 1590 1591 NXGE_DEBUG_MSG((NULL, TX_CTL, 1592 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1593 "dupb nmp $%p len %d left %d msd %d ", 1594 mp, cmp, nmp, len, left, mss)); 1595 1596 cmp->b_cont = nmp; 1597 nmp->b_rptr = rptr; 1598 len = (left < mss) ? left : mss; 1599 left -= len; 1600 1601 NXGE_DEBUG_MSG((NULL, TX_CTL, 1602 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1603 "dupb nmp $%p len %d left %d mss %d ", 1604 mp, cmp, nmp, len, left, mss)); 1605 NXGE_DEBUG_MSG((NULL, TX_CTL, 1606 "nxge_do_softlso: before available: %d, " 1607 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1608 available, left, len, segnum, (int)MBLKL(nmp))); 1609 1610 len -= MBLKL(nmp); 1611 NXGE_DEBUG_MSG((NULL, TX_CTL, 1612 "nxge_do_softlso: after available: %d, " 1613 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1614 available, left, len, segnum, (int)MBLKL(nmp))); 1615 1616 while (len > 0) { 1617 mblk_t *mmp = NULL; 1618 1619 NXGE_DEBUG_MSG((NULL, TX_CTL, 1620 "nxge_do_softlso: (4) len > 0 available: %d, " 1621 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1622 available, left, len, segnum, (int)MBLKL(nmp))); 1623 1624 if (datamp->b_cont != NULL) { 1625 datamp = datamp->b_cont; 1626 rptr = datamp->b_rptr; 1627 mmp = dupb(datamp); 1628 if (mmp == NULL) { 1629 do_cleanup = B_TRUE; 1630 NXGE_DEBUG_MSG((NULL, TX_CTL, 1631 "==> nxge_do_softlso: " 1632 "Can not dupb(datamp) (1), : 1633 "have to do clean up")); 1634 NXGE_DEBUG_MSG((NULL, TX_CTL, 1635 "==> nxge_do_softlso: " 1636 "available: %d, left: %d, " 1637 "len: %d, MBLKL(nmp): %d", 1638 available, left, len, 1639 (int)MBLKL(nmp))); 1640 goto cleanup_allocated_msgs; 1641 } 1642 } else { 1643 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1644 "==> nxge_do_softlso: " 1645 "(1)available: %d, left: %d, " 1646 "len: %d, MBLKL(nmp): %d", 1647 available, left, len, 1648 (int)MBLKL(nmp))); 1649 cmn_err(CE_PANIC, 1650 "==> nxge_do_softlso: " 1651 "Pointers must have been corrupted!\n" 1652 "datamp: $%p, nmp: $%p, rptr: $%p", 1653 (void *)datamp, 1654 (void *)nmp, 1655 (void *)rptr); 1656 } 1657 nmp->b_cont = mmp; 1658 nmp = mmp; 1659 len -= MBLKL(nmp); 1660 } 1661 if (len < 0) { 1662 nmp->b_wptr += len; 1663 rptr = nmp->b_wptr; 1664 NXGE_DEBUG_MSG((NULL, TX_CTL, 1665 "(5) len < 0 (less than 0)" 1666 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1667 available, left, len, (int)MBLKL(nmp))); 1668 1669 } else if (len == 0) { 1670 if (datamp->b_cont != NULL) { 1671 NXGE_DEBUG_MSG((NULL, TX_CTL, 1672 "(5) len == 0" 1673 "available: %d, left: %d, len: %d, " 1674 "MBLKL(nmp): %d", 1675 available, left, len, (int)MBLKL(nmp))); 1676 datamp = datamp->b_cont; 1677 rptr = datamp->b_rptr; 1678 } else { 1679 NXGE_DEBUG_MSG((NULL, TX_CTL, 1680 "(6)available b_cont == NULL : %d, " 1681 "left: %d, len: %d, MBLKL(nmp): %d", 1682 available, left, len, (int)MBLKL(nmp))); 1683 1684 VERIFY(cmp->b_next == NULL); 1685 VERIFY(left == 0); 1686 break; /* Done! */ 1687 } 1688 } 1689 cmp = cmp->b_next; 1690 1691 NXGE_DEBUG_MSG((NULL, TX_CTL, 1692 "(7) do_softlso: " 1693 "next mp in mp_chain available len != 0 : %d, " 1694 "left: %d, len: %d, MBLKL(nmp): %d", 1695 available, left, len, (int)MBLKL(nmp))); 1696 } 1697 1698 /* 1699 * From now, start to fill up all headers for the first message 1700 * Hardware checksum flags need to be updated separately for FULLCKSUM 1701 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1702 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1703 * required fields need to be updated properly. 1704 */ 1705 nmp = mp_chain; 1706 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1707 nmp->b_wptr = nmp->b_rptr + hdrlen; 1708 niph = (struct ip *)(nmp->b_rptr + ehlen); 1709 niph->ip_len = htons(mss + iphlen + tcphlen); 1710 ip_id = ntohs(niph->ip_id); 1711 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1712 #ifdef __sparc 1713 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1714 tcp_seq = ntohl(tcp_seq_tmp); 1715 #else 1716 tcp_seq = ntohl(ntcph->th_seq); 1717 #endif 1718 1719 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1720 1721 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1722 DB_CKSUMSTART(nmp) = start_offset; 1723 DB_CKSUMSTUFF(nmp) = stuff_offset; 1724 1725 /* calculate IP checksum and TCP pseudo header checksum */ 1726 niph->ip_sum = 0; 1727 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1728 1729 l4_len = mss + tcphlen; 1730 sum = htons(l4_len) + l4cksum; 1731 sum = (sum & 0xFFFF) + (sum >> 16); 1732 ntcph->th_sum = (sum & 0xffff); 1733 1734 NXGE_DEBUG_MSG((NULL, TX_CTL, 1735 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1736 "mss %d pktlen %d l4_len %d (0x%x) " 1737 "MBLKL(mp): %d, MBLKL(datamp): %d " 1738 "ip_sum 0x%x " 1739 "th_sum 0x%x sum 0x%x ) " 1740 "dump first ip->tcp %s", 1741 nmp, mp_chain, 1742 mss, 1743 pktlen, 1744 l4_len, 1745 l4_len, 1746 (int)MBLKL(mp), (int)MBLKL(datamp), 1747 niph->ip_sum, 1748 ntcph->th_sum, 1749 sum, 1750 nxge_dump_packet((char *)niph, 52))); 1751 1752 cmp = nmp; 1753 while ((nmp = nmp->b_next)->b_next != NULL) { 1754 NXGE_DEBUG_MSG((NULL, TX_CTL, 1755 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1756 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1757 nmp->b_wptr = nmp->b_rptr + hdrlen; 1758 niph = (struct ip *)(nmp->b_rptr + ehlen); 1759 niph->ip_id = htons(++ip_id); 1760 niph->ip_len = htons(mss + iphlen + tcphlen); 1761 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1762 tcp_seq += mss; 1763 1764 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1765 1766 #ifdef __sparc 1767 tcp_seq_tmp = htonl(tcp_seq); 1768 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1769 #else 1770 ntcph->th_seq = htonl(tcp_seq); 1771 #endif 1772 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1773 DB_CKSUMSTART(nmp) = start_offset; 1774 DB_CKSUMSTUFF(nmp) = stuff_offset; 1775 1776 /* calculate IP checksum and TCP pseudo header checksum */ 1777 niph->ip_sum = 0; 1778 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1779 ntcph->th_sum = (sum & 0xffff); 1780 1781 NXGE_DEBUG_MSG((NULL, TX_CTL, 1782 "==> nxge_do_softlso: middle ip_sum 0x%x " 1783 "th_sum 0x%x " 1784 " mp $%p (mp_chain $%p) pktlen %d " 1785 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1786 niph->ip_sum, 1787 ntcph->th_sum, 1788 nmp, mp_chain, 1789 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1790 } 1791 1792 /* Last segment */ 1793 /* 1794 * Set FIN and/or PSH flags if present only in the last packet. 1795 * The ip_len could be different from prior packets. 1796 */ 1797 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1798 nmp->b_wptr = nmp->b_rptr + hdrlen; 1799 niph = (struct ip *)(nmp->b_rptr + ehlen); 1800 niph->ip_id = htons(++ip_id); 1801 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1802 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1803 tcp_seq += mss; 1804 #ifdef __sparc 1805 tcp_seq_tmp = htonl(tcp_seq); 1806 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1807 #else 1808 ntcph->th_seq = htonl(tcp_seq); 1809 #endif 1810 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1811 1812 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1813 DB_CKSUMSTART(nmp) = start_offset; 1814 DB_CKSUMSTUFF(nmp) = stuff_offset; 1815 1816 /* calculate IP checksum and TCP pseudo header checksum */ 1817 niph->ip_sum = 0; 1818 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1819 1820 l4_len = ntohs(niph->ip_len) - iphlen; 1821 sum = htons(l4_len) + l4cksum; 1822 sum = (sum & 0xFFFF) + (sum >> 16); 1823 ntcph->th_sum = (sum & 0xffff); 1824 1825 NXGE_DEBUG_MSG((NULL, TX_CTL, 1826 "==> nxge_do_softlso: last next " 1827 "niph->ip_sum 0x%x " 1828 "ntcph->th_sum 0x%x sum 0x%x " 1829 "dump last ip->tcp %s " 1830 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1831 "l4_len %d (0x%x) " 1832 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1833 niph->ip_sum, 1834 ntcph->th_sum, sum, 1835 nxge_dump_packet((char *)niph, 52), 1836 cmp, nmp, mp_chain, 1837 pktlen, pktlen, 1838 l4_len, 1839 l4_len, 1840 (int)MBLKL(mp), (int)MBLKL(datamp))); 1841 1842 cleanup_allocated_msgs: 1843 if (do_cleanup) { 1844 NXGE_DEBUG_MSG((NULL, TX_CTL, 1845 "==> nxge_do_softlso: " 1846 "Failed allocating messages, " 1847 "have to clean up and fail!")); 1848 while (mp_chain != NULL) { 1849 nmp = mp_chain; 1850 mp_chain = mp_chain->b_next; 1851 freemsg(nmp); 1852 } 1853 } 1854 /* 1855 * We're done here, so just free the original message and return the 1856 * new message chain, that could be NULL if failed, back to the caller. 1857 */ 1858 freemsg(mp); 1859 1860 NXGE_DEBUG_MSG((NULL, TX_CTL, 1861 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1862 return (mp_chain); 1863 } 1864 1865 /* 1866 * Will be called before NIC driver do further operation on the message. 1867 * The input message may include LSO information, if so, go to softlso logic 1868 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1869 * The return could be the same non-LSO message or a message chain for LSO case. 1870 * 1871 * The driver needs to call this function per packet and process the whole chain 1872 * if applied. 1873 */ 1874 static mblk_t * 1875 nxge_lso_eliminate(mblk_t *mp) 1876 { 1877 uint32_t lsoflags; 1878 uint32_t mss; 1879 1880 NXGE_DEBUG_MSG((NULL, TX_CTL, 1881 "==>nxge_lso_eliminate:")); 1882 nxge_lso_info_get(mp, &mss, &lsoflags); 1883 1884 if (lsoflags & HW_LSO) { 1885 mblk_t *nmp; 1886 1887 NXGE_DEBUG_MSG((NULL, TX_CTL, 1888 "==>nxge_lso_eliminate:" 1889 "HW_LSO:mss %d mp $%p", 1890 mss, mp)); 1891 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 1892 NXGE_DEBUG_MSG((NULL, TX_CTL, 1893 "<== nxge_lso_eliminate: " 1894 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 1895 nmp, mss, mp)); 1896 return (nmp); 1897 } else { 1898 NXGE_DEBUG_MSG((NULL, TX_CTL, 1899 "<== nxge_lso_eliminate_ " 1900 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 1901 nmp, mss, mp)); 1902 return (NULL); 1903 } 1904 } 1905 1906 NXGE_DEBUG_MSG((NULL, TX_CTL, 1907 "<== nxge_lso_eliminate")); 1908 return (mp); 1909 } 1910 1911 static uint32_t 1912 nxge_csgen(uint16_t *adr, int len) 1913 { 1914 int i, odd; 1915 uint32_t sum = 0; 1916 uint32_t c = 0; 1917 1918 odd = len % 2; 1919 for (i = 0; i < (len / 2); i++) { 1920 sum += (adr[i] & 0xffff); 1921 } 1922 if (odd) { 1923 sum += adr[len / 2] & 0xff00; 1924 } 1925 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 1926 sum &= 0xffff; 1927 sum += c; 1928 } 1929 return (~sum & 0xffff); 1930 } 1931