1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/nxge/nxge_impl.h> 27 #include <sys/nxge/nxge_hio.h> 28 #include <npi_tx_wr64.h> 29 30 /* Software LSO required header files */ 31 #include <netinet/tcp.h> 32 #include <inet/ip_impl.h> 33 #include <inet/tcp.h> 34 35 static mblk_t *nxge_lso_eliminate(mblk_t *); 36 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss); 37 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *); 38 static void nxge_hcksum_retrieve(mblk_t *, 39 uint32_t *, uint32_t *, uint32_t *, 40 uint32_t *, uint32_t *); 41 static uint32_t nxge_csgen(uint16_t *, int); 42 43 extern void nxge_txdma_freemsg_task(p_tx_ring_t ringp); 44 45 extern uint32_t nxge_reclaim_pending; 46 extern uint32_t nxge_bcopy_thresh; 47 extern uint32_t nxge_dvma_thresh; 48 extern uint32_t nxge_dma_stream_thresh; 49 extern uint32_t nxge_tx_minfree; 50 extern uint32_t nxge_tx_intr_thres; 51 extern uint32_t nxge_tx_max_gathers; 52 extern uint32_t nxge_tx_tiny_pack; 53 extern uint32_t nxge_tx_use_bcopy; 54 extern uint32_t nxge_tx_lb_policy; 55 extern uint32_t nxge_no_tx_lb; 56 extern nxge_tx_mode_t nxge_tx_scheme; 57 uint32_t nxge_lso_kick_cnt = 2; 58 59 typedef struct _mac_tx_hint { 60 uint16_t sap; 61 uint16_t vid; 62 void *hash; 63 } mac_tx_hint_t, *p_mac_tx_hint_t; 64 65 int nxge_tx_lb_ring_1(p_mblk_t, uint32_t, p_mac_tx_hint_t); 66 67 int 68 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp) 69 { 70 int status = 0; 71 p_tx_desc_t tx_desc_ring_vp; 72 npi_handle_t npi_desc_handle; 73 nxge_os_dma_handle_t tx_desc_dma_handle; 74 p_tx_desc_t tx_desc_p; 75 p_tx_msg_t tx_msg_ring; 76 p_tx_msg_t tx_msg_p; 77 tx_desc_t tx_desc, *tmp_desc_p; 78 tx_desc_t sop_tx_desc, *sop_tx_desc_p; 79 p_tx_pkt_header_t hdrp; 80 tx_pkt_header_t tmp_hdrp; 81 p_tx_pkt_hdr_all_t pkthdrp; 82 uint8_t npads = 0; 83 uint64_t dma_ioaddr; 84 uint32_t dma_flags; 85 int last_bidx; 86 uint8_t *b_rptr; 87 caddr_t kaddr; 88 uint32_t nmblks; 89 uint32_t ngathers; 90 uint32_t clen; 91 int len; 92 uint32_t pkt_len, pack_len, min_len; 93 uint32_t bcopy_thresh; 94 int i, cur_index, sop_index; 95 uint16_t tail_index; 96 boolean_t tail_wrap = B_FALSE; 97 nxge_dma_common_t desc_area; 98 nxge_os_dma_handle_t dma_handle; 99 ddi_dma_cookie_t dma_cookie; 100 npi_handle_t npi_handle; 101 p_mblk_t nmp; 102 p_mblk_t t_mp; 103 uint32_t ncookies; 104 boolean_t good_packet; 105 boolean_t mark_mode = B_FALSE; 106 p_nxge_stats_t statsp; 107 p_nxge_tx_ring_stats_t tdc_stats; 108 t_uscalar_t start_offset = 0; 109 t_uscalar_t stuff_offset = 0; 110 t_uscalar_t end_offset = 0; 111 t_uscalar_t value = 0; 112 t_uscalar_t cksum_flags = 0; 113 boolean_t cksum_on = B_FALSE; 114 uint32_t boff = 0; 115 uint64_t tot_xfer_len = 0; 116 boolean_t header_set = B_FALSE; 117 #ifdef NXGE_DEBUG 118 p_tx_desc_t tx_desc_ring_pp; 119 p_tx_desc_t tx_desc_pp; 120 tx_desc_t *save_desc_p; 121 int dump_len; 122 int sad_len; 123 uint64_t sad; 124 int xfer_len; 125 uint32_t msgsize; 126 #endif 127 p_mblk_t mp_chain = NULL; 128 boolean_t is_lso = B_FALSE; 129 boolean_t lso_again; 130 int cur_index_lso; 131 p_mblk_t nmp_lso_save; 132 uint32_t lso_ngathers; 133 boolean_t lso_tail_wrap = B_FALSE; 134 135 NXGE_DEBUG_MSG((nxgep, TX_CTL, 136 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc)); 137 NXGE_DEBUG_MSG((nxgep, TX_CTL, 138 "==> nxge_start: Starting tdc %d desc pending %d", 139 tx_ring_p->tdc, tx_ring_p->descs_pending)); 140 141 statsp = nxgep->statsp; 142 143 if (!isLDOMguest(nxgep)) { 144 switch (nxgep->mac.portmode) { 145 default: 146 if (nxgep->statsp->port_stats.lb_mode == 147 nxge_lb_normal) { 148 if (!statsp->mac_stats.link_up) { 149 freemsg(mp); 150 NXGE_DEBUG_MSG((nxgep, TX_CTL, 151 "==> nxge_start: " 152 "link not up")); 153 goto nxge_start_fail1; 154 } 155 } 156 break; 157 case PORT_10G_FIBER: 158 /* 159 * For the following modes, check the link status 160 * before sending the packet out: 161 * nxge_lb_normal, nxge_lb_ext10g, nxge_lb_phy10g 162 */ 163 if (nxgep->statsp->port_stats.lb_mode < 164 nxge_lb_serdes10g) { 165 if (!statsp->mac_stats.link_up) { 166 freemsg(mp); 167 NXGE_DEBUG_MSG((nxgep, TX_CTL, 168 "==> nxge_start: " 169 "link not up")); 170 goto nxge_start_fail1; 171 } 172 } 173 break; 174 } 175 } 176 177 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 178 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 179 NXGE_DEBUG_MSG((nxgep, TX_CTL, 180 "==> nxge_start: hardware not initialized or stopped")); 181 freemsg(mp); 182 goto nxge_start_fail1; 183 } 184 185 if (nxgep->soft_lso_enable) { 186 mp_chain = nxge_lso_eliminate(mp); 187 NXGE_DEBUG_MSG((nxgep, TX_CTL, 188 "==> nxge_start(0): LSO mp $%p mp_chain $%p", 189 mp, mp_chain)); 190 if (mp_chain == NULL) { 191 NXGE_ERROR_MSG((nxgep, TX_CTL, 192 "==> nxge_send(0): NULL mp_chain $%p != mp $%p", 193 mp_chain, mp)); 194 goto nxge_start_fail1; 195 } 196 if (mp_chain != mp) { 197 NXGE_DEBUG_MSG((nxgep, TX_CTL, 198 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p", 199 mp_chain, mp)); 200 is_lso = B_TRUE; 201 mp = mp_chain; 202 mp_chain = mp_chain->b_next; 203 mp->b_next = NULL; 204 } 205 } 206 207 hcksum_retrieve(mp, NULL, NULL, &start_offset, 208 &stuff_offset, &end_offset, &value, &cksum_flags); 209 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) { 210 start_offset += sizeof (ether_header_t); 211 stuff_offset += sizeof (ether_header_t); 212 } else { 213 start_offset += sizeof (struct ether_vlan_header); 214 stuff_offset += sizeof (struct ether_vlan_header); 215 } 216 217 if (cksum_flags & HCK_PARTIALCKSUM) { 218 NXGE_DEBUG_MSG((nxgep, TX_CTL, 219 "==> nxge_start: mp $%p len %d " 220 "cksum_flags 0x%x (partial checksum) ", 221 mp, MBLKL(mp), cksum_flags)); 222 cksum_on = B_TRUE; 223 } 224 225 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp; 226 pkthdrp->reserved = 0; 227 tmp_hdrp.value = 0; 228 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on, 229 0, 0, pkthdrp, 230 start_offset, stuff_offset); 231 232 lso_again = B_FALSE; 233 lso_ngathers = 0; 234 235 MUTEX_ENTER(&tx_ring_p->lock); 236 237 if (isLDOMservice(nxgep)) { 238 tx_ring_p->tx_ring_busy = B_TRUE; 239 if (tx_ring_p->tx_ring_offline) { 240 freemsg(mp); 241 tx_ring_p->tx_ring_busy = B_FALSE; 242 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 243 NXGE_TX_RING_OFFLINED); 244 MUTEX_EXIT(&tx_ring_p->lock); 245 return (status); 246 } 247 } 248 249 cur_index_lso = tx_ring_p->wr_index; 250 lso_tail_wrap = tx_ring_p->wr_index_wrap; 251 start_again: 252 ngathers = 0; 253 sop_index = tx_ring_p->wr_index; 254 #ifdef NXGE_DEBUG 255 if (tx_ring_p->descs_pending) { 256 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 257 "desc pending %d ", tx_ring_p->descs_pending)); 258 } 259 260 dump_len = (int)(MBLKL(mp)); 261 dump_len = (dump_len > 128) ? 128: dump_len; 262 263 NXGE_DEBUG_MSG((nxgep, TX_CTL, 264 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p " 265 "(Before header reserve: ORIGINAL LEN %d)", 266 tx_ring_p->tdc, 267 mp->b_rptr, 268 dump_len)); 269 270 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets " 271 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr, 272 nxge_dump_packet((char *)mp->b_rptr, dump_len))); 273 #endif 274 275 tdc_stats = tx_ring_p->tdc_stats; 276 mark_mode = (tx_ring_p->descs_pending && 277 ((tx_ring_p->tx_ring_size - tx_ring_p->descs_pending) 278 < nxge_tx_minfree)); 279 280 NXGE_DEBUG_MSG((nxgep, TX_CTL, 281 "TX Descriptor ring is channel %d mark mode %d", 282 tx_ring_p->tdc, mark_mode)); 283 284 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) { 285 if (!nxge_txdma_reclaim(nxgep, tx_ring_p, 286 (nxge_tx_minfree + lso_ngathers))) { 287 NXGE_DEBUG_MSG((nxgep, TX_CTL, 288 "TX Descriptor ring is full: channel %d", 289 tx_ring_p->tdc)); 290 NXGE_DEBUG_MSG((nxgep, TX_CTL, 291 "TX Descriptor ring is full: channel %d", 292 tx_ring_p->tdc)); 293 if (is_lso) { 294 /* 295 * free the current mp and mp_chain if not FULL. 296 */ 297 tdc_stats->tx_no_desc++; 298 NXGE_DEBUG_MSG((nxgep, TX_CTL, 299 "LSO packet: TX Descriptor ring is full: " 300 "channel %d", 301 tx_ring_p->tdc)); 302 goto nxge_start_fail_lso; 303 } else { 304 boolean_t skip_sched = B_FALSE; 305 306 cas32((uint32_t *)&tx_ring_p->queueing, 0, 1); 307 tdc_stats->tx_no_desc++; 308 309 if (isLDOMservice(nxgep)) { 310 tx_ring_p->tx_ring_busy = B_FALSE; 311 if (tx_ring_p->tx_ring_offline) { 312 (void) atomic_swap_32( 313 &tx_ring_p->tx_ring_offline, 314 NXGE_TX_RING_OFFLINED); 315 skip_sched = B_TRUE; 316 } 317 } 318 319 MUTEX_EXIT(&tx_ring_p->lock); 320 if (nxgep->resched_needed && 321 !nxgep->resched_running && !skip_sched) { 322 nxgep->resched_running = B_TRUE; 323 ddi_trigger_softintr(nxgep->resched_id); 324 } 325 status = 1; 326 goto nxge_start_fail1; 327 } 328 } 329 } 330 331 nmp = mp; 332 i = sop_index = tx_ring_p->wr_index; 333 nmblks = 0; 334 ngathers = 0; 335 pkt_len = 0; 336 pack_len = 0; 337 clen = 0; 338 last_bidx = -1; 339 good_packet = B_TRUE; 340 341 desc_area = tx_ring_p->tdc_desc; 342 npi_handle = desc_area.npi_handle; 343 npi_desc_handle.regh = (nxge_os_acc_handle_t) 344 DMA_COMMON_ACC_HANDLE(desc_area); 345 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area); 346 tx_desc_dma_handle = (nxge_os_dma_handle_t) 347 DMA_COMMON_HANDLE(desc_area); 348 tx_msg_ring = tx_ring_p->tx_msg_ring; 349 350 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d", 351 sop_index, i)); 352 353 #ifdef NXGE_DEBUG 354 msgsize = msgdsize(nmp); 355 NXGE_DEBUG_MSG((nxgep, TX_CTL, 356 "==> nxge_start(1): wr_index %d i %d msgdsize %d", 357 sop_index, i, msgsize)); 358 #endif 359 /* 360 * The first 16 bytes of the premapped buffer are reserved 361 * for header. No padding will be used. 362 */ 363 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE; 364 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) { 365 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE); 366 } else { 367 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE); 368 } 369 while (nmp) { 370 good_packet = B_TRUE; 371 b_rptr = nmp->b_rptr; 372 len = MBLKL(nmp); 373 if (len <= 0) { 374 nmp = nmp->b_cont; 375 continue; 376 } 377 nmblks++; 378 379 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d " 380 "len %d pkt_len %d pack_len %d", 381 nmblks, len, pkt_len, pack_len)); 382 /* 383 * Hardware limits the transfer length to 4K for NIU and 384 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just 385 * use TX_MAX_TRANSFER_LENGTH as the limit for both. 386 * If len is longer than the limit, then we break nmp into 387 * two chunks: Make the first chunk equal to the limit and 388 * the second chunk for the remaining data. If the second 389 * chunk is still larger than the limit, then it will be 390 * broken into two in the next pass. 391 */ 392 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) { 393 if ((t_mp = dupb(nmp)) != NULL) { 394 nmp->b_wptr = nmp->b_rptr + 395 (TX_MAX_TRANSFER_LENGTH 396 - TX_PKT_HEADER_SIZE); 397 t_mp->b_rptr = nmp->b_wptr; 398 t_mp->b_cont = nmp->b_cont; 399 nmp->b_cont = t_mp; 400 len = MBLKL(nmp); 401 } else { 402 if (is_lso) { 403 NXGE_DEBUG_MSG((nxgep, TX_CTL, 404 "LSO packet: dupb failed: " 405 "channel %d", 406 tx_ring_p->tdc)); 407 mp = nmp; 408 goto nxge_start_fail_lso; 409 } else { 410 good_packet = B_FALSE; 411 goto nxge_start_fail2; 412 } 413 } 414 } 415 tx_desc.value = 0; 416 tx_desc_p = &tx_desc_ring_vp[i]; 417 #ifdef NXGE_DEBUG 418 tx_desc_pp = &tx_desc_ring_pp[i]; 419 #endif 420 tx_msg_p = &tx_msg_ring[i]; 421 #if defined(__i386) 422 npi_desc_handle.regp = (uint32_t)tx_desc_p; 423 #else 424 npi_desc_handle.regp = (uint64_t)tx_desc_p; 425 #endif 426 if (!header_set && 427 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) || 428 (len >= bcopy_thresh))) { 429 header_set = B_TRUE; 430 bcopy_thresh += TX_PKT_HEADER_SIZE; 431 boff = 0; 432 pack_len = 0; 433 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 434 hdrp = (p_tx_pkt_header_t)kaddr; 435 clen = pkt_len; 436 dma_handle = tx_msg_p->buf_dma_handle; 437 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 438 (void) ddi_dma_sync(dma_handle, 439 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 440 DDI_DMA_SYNC_FORDEV); 441 442 tx_msg_p->flags.dma_type = USE_BCOPY; 443 goto nxge_start_control_header_only; 444 } 445 446 pkt_len += len; 447 pack_len += len; 448 449 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): " 450 "desc entry %d " 451 "DESC IOADDR $%p " 452 "desc_vp $%p tx_desc_p $%p " 453 "desc_pp $%p tx_desc_pp $%p " 454 "len %d pkt_len %d pack_len %d", 455 i, 456 DMA_COMMON_IOADDR(desc_area), 457 tx_desc_ring_vp, tx_desc_p, 458 tx_desc_ring_pp, tx_desc_pp, 459 len, pkt_len, pack_len)); 460 461 if (len < bcopy_thresh) { 462 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): " 463 "USE BCOPY: ")); 464 if (nxge_tx_tiny_pack) { 465 uint32_t blst = 466 TXDMA_DESC_NEXT_INDEX(i, -1, 467 tx_ring_p->tx_wrap_mask); 468 NXGE_DEBUG_MSG((nxgep, TX_CTL, 469 "==> nxge_start(5): pack")); 470 if ((pack_len <= bcopy_thresh) && 471 (last_bidx == blst)) { 472 NXGE_DEBUG_MSG((nxgep, TX_CTL, 473 "==> nxge_start: pack(6) " 474 "(pkt_len %d pack_len %d)", 475 pkt_len, pack_len)); 476 i = blst; 477 tx_desc_p = &tx_desc_ring_vp[i]; 478 #ifdef NXGE_DEBUG 479 tx_desc_pp = &tx_desc_ring_pp[i]; 480 #endif 481 tx_msg_p = &tx_msg_ring[i]; 482 boff = pack_len - len; 483 ngathers--; 484 } else if (pack_len > bcopy_thresh && 485 header_set) { 486 pack_len = len; 487 boff = 0; 488 bcopy_thresh = nxge_bcopy_thresh; 489 NXGE_DEBUG_MSG((nxgep, TX_CTL, 490 "==> nxge_start(7): > max NEW " 491 "bcopy thresh %d " 492 "pkt_len %d pack_len %d(next)", 493 bcopy_thresh, 494 pkt_len, pack_len)); 495 } 496 last_bidx = i; 497 } 498 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 499 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) { 500 hdrp = (p_tx_pkt_header_t)kaddr; 501 header_set = B_TRUE; 502 NXGE_DEBUG_MSG((nxgep, TX_CTL, 503 "==> nxge_start(7_x2): " 504 "pkt_len %d pack_len %d (new hdrp $%p)", 505 pkt_len, pack_len, hdrp)); 506 } 507 tx_msg_p->flags.dma_type = USE_BCOPY; 508 kaddr += boff; 509 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): " 510 "USE BCOPY: before bcopy " 511 "DESC IOADDR $%p entry %d " 512 "bcopy packets %d " 513 "bcopy kaddr $%p " 514 "bcopy ioaddr (SAD) $%p " 515 "bcopy clen %d " 516 "bcopy boff %d", 517 DMA_COMMON_IOADDR(desc_area), i, 518 tdc_stats->tx_hdr_pkts, 519 kaddr, 520 dma_ioaddr, 521 clen, 522 boff)); 523 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 524 "1USE BCOPY: ")); 525 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 526 "2USE BCOPY: ")); 527 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: " 528 "last USE BCOPY: copy from b_rptr $%p " 529 "to KADDR $%p (len %d offset %d", 530 b_rptr, kaddr, len, boff)); 531 532 bcopy(b_rptr, kaddr, len); 533 534 #ifdef NXGE_DEBUG 535 dump_len = (len > 128) ? 128: len; 536 NXGE_DEBUG_MSG((nxgep, TX_CTL, 537 "==> nxge_start: dump packets " 538 "(After BCOPY len %d)" 539 "(b_rptr $%p): %s", len, nmp->b_rptr, 540 nxge_dump_packet((char *)nmp->b_rptr, 541 dump_len))); 542 #endif 543 544 dma_handle = tx_msg_p->buf_dma_handle; 545 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma); 546 (void) ddi_dma_sync(dma_handle, 547 i * nxge_bcopy_thresh, nxge_bcopy_thresh, 548 DDI_DMA_SYNC_FORDEV); 549 clen = len + boff; 550 tdc_stats->tx_hdr_pkts++; 551 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): " 552 "USE BCOPY: " 553 "DESC IOADDR $%p entry %d " 554 "bcopy packets %d " 555 "bcopy kaddr $%p " 556 "bcopy ioaddr (SAD) $%p " 557 "bcopy clen %d " 558 "bcopy boff %d", 559 DMA_COMMON_IOADDR(desc_area), 560 i, 561 tdc_stats->tx_hdr_pkts, 562 kaddr, 563 dma_ioaddr, 564 clen, 565 boff)); 566 } else { 567 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): " 568 "USE DVMA: len %d", len)); 569 tx_msg_p->flags.dma_type = USE_DMA; 570 dma_flags = DDI_DMA_WRITE; 571 if (len < nxge_dma_stream_thresh) { 572 dma_flags |= DDI_DMA_CONSISTENT; 573 } else { 574 dma_flags |= DDI_DMA_STREAMING; 575 } 576 577 dma_handle = tx_msg_p->dma_handle; 578 status = ddi_dma_addr_bind_handle(dma_handle, NULL, 579 (caddr_t)b_rptr, len, dma_flags, 580 DDI_DMA_DONTWAIT, NULL, 581 &dma_cookie, &ncookies); 582 if (status == DDI_DMA_MAPPED) { 583 dma_ioaddr = dma_cookie.dmac_laddress; 584 len = (int)dma_cookie.dmac_size; 585 clen = (uint32_t)dma_cookie.dmac_size; 586 NXGE_DEBUG_MSG((nxgep, TX_CTL, 587 "==> nxge_start(12_1): " 588 "USE DVMA: len %d clen %d " 589 "ngathers %d", 590 len, clen, 591 ngathers)); 592 #if defined(__i386) 593 npi_desc_handle.regp = (uint32_t)tx_desc_p; 594 #else 595 npi_desc_handle.regp = (uint64_t)tx_desc_p; 596 #endif 597 while (ncookies > 1) { 598 ngathers++; 599 /* 600 * this is the fix for multiple 601 * cookies, which are basically 602 * a descriptor entry, we don't set 603 * SOP bit as well as related fields 604 */ 605 606 (void) npi_txdma_desc_gather_set( 607 npi_desc_handle, 608 &tx_desc, 609 (ngathers -1), 610 mark_mode, 611 ngathers, 612 dma_ioaddr, 613 clen); 614 615 tx_msg_p->tx_msg_size = clen; 616 NXGE_DEBUG_MSG((nxgep, TX_CTL, 617 "==> nxge_start: DMA " 618 "ncookie %d " 619 "ngathers %d " 620 "dma_ioaddr $%p len %d" 621 "desc $%p descp $%p (%d)", 622 ncookies, 623 ngathers, 624 dma_ioaddr, clen, 625 *tx_desc_p, tx_desc_p, i)); 626 627 ddi_dma_nextcookie(dma_handle, 628 &dma_cookie); 629 dma_ioaddr = 630 dma_cookie.dmac_laddress; 631 632 len = (int)dma_cookie.dmac_size; 633 clen = (uint32_t)dma_cookie.dmac_size; 634 NXGE_DEBUG_MSG((nxgep, TX_CTL, 635 "==> nxge_start(12_2): " 636 "USE DVMA: len %d clen %d ", 637 len, clen)); 638 639 i = TXDMA_DESC_NEXT_INDEX(i, 1, 640 tx_ring_p->tx_wrap_mask); 641 tx_desc_p = &tx_desc_ring_vp[i]; 642 643 #if defined(__i386) 644 npi_desc_handle.regp = 645 (uint32_t)tx_desc_p; 646 #else 647 npi_desc_handle.regp = 648 (uint64_t)tx_desc_p; 649 #endif 650 tx_msg_p = &tx_msg_ring[i]; 651 tx_msg_p->flags.dma_type = USE_NONE; 652 tx_desc.value = 0; 653 654 ncookies--; 655 } 656 tdc_stats->tx_ddi_pkts++; 657 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:" 658 "DMA: ddi packets %d", 659 tdc_stats->tx_ddi_pkts)); 660 } else { 661 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 662 "dma mapping failed for %d " 663 "bytes addr $%p flags %x (%d)", 664 len, b_rptr, status, status)); 665 good_packet = B_FALSE; 666 tdc_stats->tx_dma_bind_fail++; 667 tx_msg_p->flags.dma_type = USE_NONE; 668 if (is_lso) { 669 mp = nmp; 670 goto nxge_start_fail_lso; 671 } else { 672 goto nxge_start_fail2; 673 } 674 } 675 } /* ddi dvma */ 676 677 if (is_lso) { 678 nmp_lso_save = nmp; 679 } 680 nmp = nmp->b_cont; 681 nxge_start_control_header_only: 682 #if defined(__i386) 683 npi_desc_handle.regp = (uint32_t)tx_desc_p; 684 #else 685 npi_desc_handle.regp = (uint64_t)tx_desc_p; 686 #endif 687 ngathers++; 688 689 if (ngathers == 1) { 690 #ifdef NXGE_DEBUG 691 save_desc_p = &sop_tx_desc; 692 #endif 693 sop_tx_desc_p = &sop_tx_desc; 694 sop_tx_desc_p->value = 0; 695 sop_tx_desc_p->bits.hdw.tr_len = clen; 696 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 697 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 698 } else { 699 #ifdef NXGE_DEBUG 700 save_desc_p = &tx_desc; 701 #endif 702 tmp_desc_p = &tx_desc; 703 tmp_desc_p->value = 0; 704 tmp_desc_p->bits.hdw.tr_len = clen; 705 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32; 706 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff; 707 708 tx_desc_p->value = tmp_desc_p->value; 709 } 710 711 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): " 712 "Desc_entry %d ngathers %d " 713 "desc_vp $%p tx_desc_p $%p " 714 "len %d clen %d pkt_len %d pack_len %d nmblks %d " 715 "dma_ioaddr (SAD) $%p mark %d", 716 i, ngathers, 717 tx_desc_ring_vp, tx_desc_p, 718 len, clen, pkt_len, pack_len, nmblks, 719 dma_ioaddr, mark_mode)); 720 721 #ifdef NXGE_DEBUG 722 npi_desc_handle.nxgep = nxgep; 723 npi_desc_handle.function.function = nxgep->function_num; 724 npi_desc_handle.function.instance = nxgep->instance; 725 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK); 726 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >> 727 TX_PKT_DESC_TR_LEN_SHIFT); 728 729 730 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 731 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t" 732 "mark %d sop %d\n", 733 save_desc_p->value, 734 sad, 735 save_desc_p->bits.hdw.tr_len, 736 xfer_len, 737 save_desc_p->bits.hdw.num_ptr, 738 save_desc_p->bits.hdw.mark, 739 save_desc_p->bits.hdw.sop)); 740 741 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i); 742 #endif 743 744 tx_msg_p->tx_msg_size = clen; 745 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask); 746 if (ngathers > nxge_tx_max_gathers) { 747 good_packet = B_FALSE; 748 hcksum_retrieve(mp, NULL, NULL, &start_offset, 749 &stuff_offset, &end_offset, &value, 750 &cksum_flags); 751 752 NXGE_DEBUG_MSG((NULL, TX_CTL, 753 "==> nxge_start(14): pull msg - " 754 "len %d pkt_len %d ngathers %d", 755 len, pkt_len, ngathers)); 756 /* Pull all message blocks from b_cont */ 757 if (is_lso) { 758 mp = nmp_lso_save; 759 goto nxge_start_fail_lso; 760 } 761 if ((msgpullup(mp, -1)) == NULL) { 762 goto nxge_start_fail2; 763 } 764 goto nxge_start_fail2; 765 } 766 } /* while (nmp) */ 767 768 tx_msg_p->tx_message = mp; 769 tx_desc_p = &tx_desc_ring_vp[sop_index]; 770 #if defined(__i386) 771 npi_desc_handle.regp = (uint32_t)tx_desc_p; 772 #else 773 npi_desc_handle.regp = (uint64_t)tx_desc_p; 774 #endif 775 776 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp; 777 pkthdrp->reserved = 0; 778 hdrp->value = 0; 779 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t)); 780 781 if (pkt_len > NXGE_MTU_DEFAULT_MAX) { 782 tdc_stats->tx_jumbo_pkts++; 783 } 784 785 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2)); 786 if (pkt_len < min_len) { 787 /* Assume we use bcopy to premapped buffers */ 788 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma); 789 NXGE_DEBUG_MSG((NULL, TX_CTL, 790 "==> nxge_start(14-1): < (msg_min + 16)" 791 "len %d pkt_len %d min_len %d bzero %d ngathers %d", 792 len, pkt_len, min_len, (min_len - pkt_len), ngathers)); 793 bzero((kaddr + pkt_len), (min_len - pkt_len)); 794 pkt_len = tx_msg_p->tx_msg_size = min_len; 795 796 sop_tx_desc_p->bits.hdw.tr_len = min_len; 797 798 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 799 tx_desc_p->value = sop_tx_desc_p->value; 800 801 NXGE_DEBUG_MSG((NULL, TX_CTL, 802 "==> nxge_start(14-2): < msg_min - " 803 "len %d pkt_len %d min_len %d ngathers %d", 804 len, pkt_len, min_len, ngathers)); 805 } 806 807 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ", 808 cksum_flags)); 809 { 810 uint64_t tmp_len; 811 812 /* pkt_len already includes 16 + paddings!! */ 813 /* Update the control header length */ 814 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE); 815 tmp_len = hdrp->value | 816 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT); 817 818 NXGE_DEBUG_MSG((nxgep, TX_CTL, 819 "==> nxge_start(15_x1): setting SOP " 820 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len " 821 "0x%llx hdrp->value 0x%llx", 822 tot_xfer_len, tot_xfer_len, pkt_len, 823 tmp_len, hdrp->value)); 824 #if defined(_BIG_ENDIAN) 825 hdrp->value = ddi_swap64(tmp_len); 826 #else 827 hdrp->value = tmp_len; 828 #endif 829 NXGE_DEBUG_MSG((nxgep, 830 TX_CTL, "==> nxge_start(15_x2): setting SOP " 831 "after SWAP: tot_xfer_len 0x%llx pkt_len %d " 832 "tmp_len 0x%llx hdrp->value 0x%llx", 833 tot_xfer_len, pkt_len, 834 tmp_len, hdrp->value)); 835 } 836 837 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP " 838 "wr_index %d " 839 "tot_xfer_len (%d) pkt_len %d npads %d", 840 sop_index, 841 tot_xfer_len, pkt_len, 842 npads)); 843 844 sop_tx_desc_p->bits.hdw.sop = 1; 845 sop_tx_desc_p->bits.hdw.mark = mark_mode; 846 sop_tx_desc_p->bits.hdw.num_ptr = ngathers; 847 848 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value); 849 850 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done")); 851 852 #ifdef NXGE_DEBUG 853 npi_desc_handle.nxgep = nxgep; 854 npi_desc_handle.function.function = nxgep->function_num; 855 npi_desc_handle.function.instance = nxgep->instance; 856 857 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n" 858 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n", 859 save_desc_p->value, 860 sad, 861 save_desc_p->bits.hdw.tr_len, 862 xfer_len, 863 save_desc_p->bits.hdw.num_ptr, 864 save_desc_p->bits.hdw.mark, 865 save_desc_p->bits.hdw.sop)); 866 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index); 867 868 dump_len = (pkt_len > 128) ? 128: pkt_len; 869 NXGE_DEBUG_MSG((nxgep, TX_CTL, 870 "==> nxge_start: dump packets(17) (after sop set, len " 871 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n" 872 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len, 873 (char *)hdrp, 874 nxge_dump_packet((char *)hdrp, dump_len))); 875 NXGE_DEBUG_MSG((nxgep, TX_CTL, 876 "==> nxge_start(18): TX desc sync: sop_index %d", 877 sop_index)); 878 #endif 879 880 if ((ngathers == 1) || tx_ring_p->wr_index < i) { 881 (void) ddi_dma_sync(tx_desc_dma_handle, 882 sop_index * sizeof (tx_desc_t), 883 ngathers * sizeof (tx_desc_t), 884 DDI_DMA_SYNC_FORDEV); 885 886 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 " 887 "cs_off = 0x%02X cs_s_off = 0x%02X " 888 "pkt_len %d ngathers %d sop_index %d\n", 889 stuff_offset, start_offset, 890 pkt_len, ngathers, sop_index)); 891 } else { /* more than one descriptor and wrap around */ 892 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index; 893 (void) ddi_dma_sync(tx_desc_dma_handle, 894 sop_index * sizeof (tx_desc_t), 895 nsdescs * sizeof (tx_desc_t), 896 DDI_DMA_SYNC_FORDEV); 897 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 " 898 "cs_off = 0x%02X cs_s_off = 0x%02X " 899 "pkt_len %d ngathers %d sop_index %d\n", 900 stuff_offset, start_offset, 901 pkt_len, ngathers, sop_index)); 902 903 (void) ddi_dma_sync(tx_desc_dma_handle, 904 0, 905 (ngathers - nsdescs) * sizeof (tx_desc_t), 906 DDI_DMA_SYNC_FORDEV); 907 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 " 908 "cs_off = 0x%02X cs_s_off = 0x%02X " 909 "pkt_len %d ngathers %d sop_index %d\n", 910 stuff_offset, start_offset, 911 pkt_len, ngathers, sop_index)); 912 } 913 914 tail_index = tx_ring_p->wr_index; 915 tail_wrap = tx_ring_p->wr_index_wrap; 916 917 tx_ring_p->wr_index = i; 918 if (tx_ring_p->wr_index <= tail_index) { 919 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ? 920 B_FALSE : B_TRUE); 921 } 922 923 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: " 924 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d", 925 tx_ring_p->tdc, 926 tx_ring_p->wr_index, 927 tx_ring_p->wr_index_wrap, 928 ngathers, 929 tx_ring_p->descs_pending)); 930 931 if (is_lso) { 932 lso_ngathers += ngathers; 933 if (mp_chain != NULL) { 934 mp = mp_chain; 935 mp_chain = mp_chain->b_next; 936 mp->b_next = NULL; 937 if (nxge_lso_kick_cnt == lso_ngathers) { 938 tx_ring_p->descs_pending += lso_ngathers; 939 { 940 tx_ring_kick_t kick; 941 942 kick.value = 0; 943 kick.bits.ldw.wrap = 944 tx_ring_p->wr_index_wrap; 945 kick.bits.ldw.tail = 946 (uint16_t)tx_ring_p->wr_index; 947 948 /* Kick the Transmit kick register */ 949 TXDMA_REG_WRITE64( 950 NXGE_DEV_NPI_HANDLE(nxgep), 951 TX_RING_KICK_REG, 952 (uint8_t)tx_ring_p->tdc, 953 kick.value); 954 tdc_stats->tx_starts++; 955 956 NXGE_DEBUG_MSG((nxgep, TX_CTL, 957 "==> nxge_start: more LSO: " 958 "LSO_CNT %d", 959 lso_ngathers)); 960 } 961 lso_ngathers = 0; 962 ngathers = 0; 963 cur_index_lso = sop_index = tx_ring_p->wr_index; 964 lso_tail_wrap = tx_ring_p->wr_index_wrap; 965 } 966 NXGE_DEBUG_MSG((nxgep, TX_CTL, 967 "==> nxge_start: lso again: " 968 "lso_gathers %d ngathers %d cur_index_lso %d " 969 "wr_index %d sop_index %d", 970 lso_ngathers, ngathers, cur_index_lso, 971 tx_ring_p->wr_index, sop_index)); 972 973 NXGE_DEBUG_MSG((nxgep, TX_CTL, 974 "==> nxge_start: next : count %d", 975 lso_ngathers)); 976 lso_again = B_TRUE; 977 goto start_again; 978 } 979 ngathers = lso_ngathers; 980 } 981 982 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: ")); 983 984 { 985 tx_ring_kick_t kick; 986 987 kick.value = 0; 988 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap; 989 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index; 990 991 /* Kick start the Transmit kick register */ 992 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep), 993 TX_RING_KICK_REG, 994 (uint8_t)tx_ring_p->tdc, 995 kick.value); 996 } 997 998 tx_ring_p->descs_pending += ngathers; 999 tdc_stats->tx_starts++; 1000 1001 if (isLDOMservice(nxgep)) { 1002 tx_ring_p->tx_ring_busy = B_FALSE; 1003 if (tx_ring_p->tx_ring_offline) { 1004 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1005 NXGE_TX_RING_OFFLINED); 1006 } 1007 } 1008 1009 MUTEX_EXIT(&tx_ring_p->lock); 1010 1011 nxge_txdma_freemsg_task(tx_ring_p); 1012 1013 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1014 1015 return (status); 1016 1017 nxge_start_fail_lso: 1018 status = 0; 1019 good_packet = B_FALSE; 1020 if (mp != NULL) { 1021 freemsg(mp); 1022 } 1023 if (mp_chain != NULL) { 1024 freemsg(mp_chain); 1025 } 1026 if (!lso_again && !ngathers) { 1027 if (isLDOMservice(nxgep)) { 1028 tx_ring_p->tx_ring_busy = B_FALSE; 1029 if (tx_ring_p->tx_ring_offline) { 1030 (void) atomic_swap_32( 1031 &tx_ring_p->tx_ring_offline, 1032 NXGE_TX_RING_OFFLINED); 1033 } 1034 } 1035 1036 MUTEX_EXIT(&tx_ring_p->lock); 1037 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1038 "==> nxge_start: lso exit (nothing changed)")); 1039 goto nxge_start_fail1; 1040 } 1041 1042 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1043 "==> nxge_start (channel %d): before lso " 1044 "lso_gathers %d ngathers %d cur_index_lso %d " 1045 "wr_index %d sop_index %d lso_again %d", 1046 tx_ring_p->tdc, 1047 lso_ngathers, ngathers, cur_index_lso, 1048 tx_ring_p->wr_index, sop_index, lso_again)); 1049 1050 if (lso_again) { 1051 lso_ngathers += ngathers; 1052 ngathers = lso_ngathers; 1053 sop_index = cur_index_lso; 1054 tx_ring_p->wr_index = sop_index; 1055 tx_ring_p->wr_index_wrap = lso_tail_wrap; 1056 } 1057 1058 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1059 "==> nxge_start (channel %d): after lso " 1060 "lso_gathers %d ngathers %d cur_index_lso %d " 1061 "wr_index %d sop_index %d lso_again %d", 1062 tx_ring_p->tdc, 1063 lso_ngathers, ngathers, cur_index_lso, 1064 tx_ring_p->wr_index, sop_index, lso_again)); 1065 1066 nxge_start_fail2: 1067 if (good_packet == B_FALSE) { 1068 cur_index = sop_index; 1069 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up")); 1070 for (i = 0; i < ngathers; i++) { 1071 tx_desc_p = &tx_desc_ring_vp[cur_index]; 1072 #if defined(__i386) 1073 npi_handle.regp = (uint32_t)tx_desc_p; 1074 #else 1075 npi_handle.regp = (uint64_t)tx_desc_p; 1076 #endif 1077 tx_msg_p = &tx_msg_ring[cur_index]; 1078 (void) npi_txdma_desc_set_zero(npi_handle, 1); 1079 if (tx_msg_p->flags.dma_type == USE_DVMA) { 1080 NXGE_DEBUG_MSG((nxgep, TX_CTL, 1081 "tx_desc_p = %X index = %d", 1082 tx_desc_p, tx_ring_p->rd_index)); 1083 (void) dvma_unload(tx_msg_p->dvma_handle, 1084 0, -1); 1085 tx_msg_p->dvma_handle = NULL; 1086 if (tx_ring_p->dvma_wr_index == 1087 tx_ring_p->dvma_wrap_mask) 1088 tx_ring_p->dvma_wr_index = 0; 1089 else 1090 tx_ring_p->dvma_wr_index++; 1091 tx_ring_p->dvma_pending--; 1092 } else if (tx_msg_p->flags.dma_type == USE_DMA) { 1093 if (ddi_dma_unbind_handle( 1094 tx_msg_p->dma_handle)) { 1095 cmn_err(CE_WARN, "!nxge_start: " 1096 "ddi_dma_unbind_handle failed"); 1097 } 1098 } 1099 tx_msg_p->flags.dma_type = USE_NONE; 1100 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1, 1101 tx_ring_p->tx_wrap_mask); 1102 1103 } 1104 1105 nxgep->resched_needed = B_TRUE; 1106 } 1107 1108 if (isLDOMservice(nxgep)) { 1109 tx_ring_p->tx_ring_busy = B_FALSE; 1110 if (tx_ring_p->tx_ring_offline) { 1111 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline, 1112 NXGE_TX_RING_OFFLINED); 1113 } 1114 } 1115 1116 MUTEX_EXIT(&tx_ring_p->lock); 1117 1118 nxge_start_fail1: 1119 /* Add FMA to check the access handle nxge_hregh */ 1120 1121 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start")); 1122 1123 return (status); 1124 } 1125 1126 int 1127 nxge_serial_tx(mblk_t *mp, void *arg) 1128 { 1129 p_tx_ring_t tx_ring_p = (p_tx_ring_t)arg; 1130 p_nxge_t nxgep = tx_ring_p->nxgep; 1131 int status = 0; 1132 1133 if (isLDOMservice(nxgep)) { 1134 if (tx_ring_p->tx_ring_offline) { 1135 freemsg(mp); 1136 return (status); 1137 } 1138 } 1139 1140 status = nxge_start(nxgep, tx_ring_p, mp); 1141 return (status); 1142 } 1143 1144 boolean_t 1145 nxge_send(p_nxge_t nxgep, mblk_t *mp, p_mac_tx_hint_t hp) 1146 { 1147 p_tx_ring_t *tx_rings; 1148 uint8_t ring_index; 1149 p_tx_ring_t tx_ring_p; 1150 nxge_grp_t *group; 1151 1152 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_send")); 1153 1154 ASSERT(mp->b_next == NULL); 1155 1156 group = nxgep->tx_set.group[0]; /* The default group */ 1157 ring_index = nxge_tx_lb_ring_1(mp, group->count, hp); 1158 1159 tx_rings = nxgep->tx_rings->rings; 1160 tx_ring_p = tx_rings[group->legend[ring_index]]; 1161 1162 if (isLDOMservice(nxgep)) { 1163 if (tx_ring_p->tx_ring_offline) { 1164 /* 1165 * OFFLINE means that it is in the process of being 1166 * shared - that is, it has been claimed by the HIO 1167 * code, but hasn't been unlinked from <group> yet. 1168 * So in this case use the first TDC, which always 1169 * belongs to the service domain and can't be shared. 1170 */ 1171 ring_index = 0; 1172 tx_ring_p = tx_rings[group->legend[ring_index]]; 1173 } 1174 } 1175 1176 NXGE_DEBUG_MSG((nxgep, TX_CTL, "count %d, tx_rings[%d] = %p", 1177 (int)group->count, group->legend[ring_index], tx_ring_p)); 1178 1179 switch (nxge_tx_scheme) { 1180 case NXGE_USE_START: 1181 if (nxge_start(nxgep, tx_ring_p, mp)) { 1182 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: failed " 1183 "ring index %d", ring_index)); 1184 return (B_FALSE); 1185 } 1186 break; 1187 1188 case NXGE_USE_SERIAL: 1189 default: 1190 nxge_serialize_enter(tx_ring_p->serial, mp); 1191 break; 1192 } 1193 1194 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_send: ring index %d", 1195 ring_index)); 1196 1197 return (B_TRUE); 1198 } 1199 1200 /* 1201 * nxge_m_tx() - send a chain of packets 1202 */ 1203 mblk_t * 1204 nxge_m_tx(void *arg, mblk_t *mp) 1205 { 1206 p_nxge_t nxgep = (p_nxge_t)arg; 1207 mblk_t *next; 1208 mac_tx_hint_t hint; 1209 1210 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_m_tx")); 1211 1212 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 1213 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 1214 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1215 "==> nxge_m_tx: hardware not initialized")); 1216 NXGE_DEBUG_MSG((nxgep, DDI_CTL, 1217 "<== nxge_m_tx")); 1218 freemsgchain(mp); 1219 mp = NULL; 1220 return (mp); 1221 } 1222 1223 hint.hash = NULL; 1224 hint.vid = 0; 1225 hint.sap = 0; 1226 1227 while (mp != NULL) { 1228 next = mp->b_next; 1229 mp->b_next = NULL; 1230 1231 /* 1232 * Until Nemo tx resource works, the mac driver 1233 * does the load balancing based on TCP port, 1234 * or CPU. For debugging, we use a system 1235 * configurable parameter. 1236 */ 1237 if (!nxge_send(nxgep, mp, &hint)) { 1238 mp->b_next = next; 1239 break; 1240 } 1241 1242 mp = next; 1243 1244 NXGE_DEBUG_MSG((NULL, TX_CTL, 1245 "==> nxge_m_tx: (go back to loop) mp $%p next $%p", 1246 mp, next)); 1247 } 1248 1249 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_m_tx")); 1250 return (mp); 1251 } 1252 1253 int 1254 nxge_tx_lb_ring_1(p_mblk_t mp, uint32_t maxtdcs, p_mac_tx_hint_t hp) 1255 { 1256 uint8_t ring_index = 0; 1257 uint8_t *tcp_port; 1258 p_mblk_t nmp; 1259 size_t mblk_len; 1260 size_t iph_len; 1261 size_t hdrs_size; 1262 uint8_t hdrs_buf[sizeof (struct ether_vlan_header) + 1263 IP_MAX_HDR_LENGTH + sizeof (uint32_t)]; 1264 /* 1265 * allocate space big enough to cover 1266 * the max ip header length and the first 1267 * 4 bytes of the TCP/IP header. 1268 */ 1269 1270 boolean_t qos = B_FALSE; 1271 ushort_t eth_type; 1272 size_t eth_hdr_size; 1273 1274 NXGE_DEBUG_MSG((NULL, TX_CTL, "==> nxge_tx_lb_ring")); 1275 1276 if (hp->vid) { 1277 qos = B_TRUE; 1278 } 1279 switch (nxge_tx_lb_policy) { 1280 case NXGE_TX_LB_TCPUDP: /* default IPv4 TCP/UDP */ 1281 default: 1282 tcp_port = mp->b_rptr; 1283 eth_type = ntohs(((struct ether_header *)tcp_port)->ether_type); 1284 if (eth_type == VLAN_ETHERTYPE) { 1285 eth_type = ntohs(((struct ether_vlan_header *) 1286 tcp_port)->ether_type); 1287 eth_hdr_size = sizeof (struct ether_vlan_header); 1288 } else { 1289 eth_hdr_size = sizeof (struct ether_header); 1290 } 1291 1292 if (!nxge_no_tx_lb && !qos && eth_type == ETHERTYPE_IP) { 1293 nmp = mp; 1294 mblk_len = MBLKL(nmp); 1295 tcp_port = NULL; 1296 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) { 1297 tcp_port = nmp->b_rptr + eth_hdr_size; 1298 mblk_len -= eth_hdr_size; 1299 iph_len = ((*tcp_port) & 0x0f) << 2; 1300 if (mblk_len > (iph_len + sizeof (uint32_t))) { 1301 tcp_port = nmp->b_rptr; 1302 } else { 1303 tcp_port = NULL; 1304 } 1305 } 1306 if (tcp_port == NULL) { 1307 hdrs_size = 0; 1308 while ((nmp) && (hdrs_size < 1309 sizeof (hdrs_buf))) { 1310 mblk_len = MBLKL(nmp); 1311 if (mblk_len >= 1312 (sizeof (hdrs_buf) - hdrs_size)) 1313 mblk_len = sizeof (hdrs_buf) - 1314 hdrs_size; 1315 bcopy(nmp->b_rptr, 1316 &hdrs_buf[hdrs_size], mblk_len); 1317 hdrs_size += mblk_len; 1318 nmp = nmp->b_cont; 1319 } 1320 tcp_port = hdrs_buf; 1321 } 1322 tcp_port += eth_hdr_size; 1323 if (!(tcp_port[6] & 0x3f) && !(tcp_port[7] & 0xff)) { 1324 switch (tcp_port[9]) { 1325 case IPPROTO_TCP: 1326 case IPPROTO_UDP: 1327 case IPPROTO_ESP: 1328 tcp_port += ((*tcp_port) & 0x0f) << 2; 1329 ring_index = 1330 ((tcp_port[0] ^ 1331 tcp_port[1] ^ 1332 tcp_port[2] ^ 1333 tcp_port[3]) % maxtdcs); 1334 break; 1335 1336 case IPPROTO_AH: 1337 /* SPI starts at the 4th byte */ 1338 tcp_port += ((*tcp_port) & 0x0f) << 2; 1339 ring_index = 1340 ((tcp_port[4] ^ 1341 tcp_port[5] ^ 1342 tcp_port[6] ^ 1343 tcp_port[7]) % maxtdcs); 1344 break; 1345 1346 default: 1347 ring_index = tcp_port[19] % maxtdcs; 1348 break; 1349 } 1350 } else { /* fragmented packet */ 1351 ring_index = tcp_port[19] % maxtdcs; 1352 } 1353 } else { 1354 ring_index = mp->b_band % maxtdcs; 1355 } 1356 break; 1357 1358 case NXGE_TX_LB_HASH: 1359 if (hp->hash) { 1360 #if defined(__i386) 1361 ring_index = ((uint32_t)(hp->hash) % maxtdcs); 1362 #else 1363 ring_index = ((uint64_t)(hp->hash) % maxtdcs); 1364 #endif 1365 } else { 1366 ring_index = mp->b_band % maxtdcs; 1367 } 1368 break; 1369 1370 case NXGE_TX_LB_DEST_MAC: /* Use destination MAC address */ 1371 tcp_port = mp->b_rptr; 1372 ring_index = tcp_port[5] % maxtdcs; 1373 break; 1374 } 1375 1376 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_tx_lb_ring")); 1377 1378 return (ring_index); 1379 } 1380 1381 uint_t 1382 nxge_reschedule(caddr_t arg) 1383 { 1384 p_nxge_t nxgep; 1385 1386 nxgep = (p_nxge_t)arg; 1387 1388 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_reschedule")); 1389 1390 if (nxgep->nxge_mac_state == NXGE_MAC_STARTED && 1391 nxgep->resched_needed) { 1392 if (!isLDOMguest(nxgep)) 1393 mac_tx_update(nxgep->mach); 1394 #if defined(sun4v) 1395 else { /* isLDOMguest(nxgep) */ 1396 nxge_hio_data_t *nhd = (nxge_hio_data_t *) 1397 nxgep->nxge_hw_p->hio; 1398 nx_vio_fp_t *vio = &nhd->hio.vio; 1399 1400 /* Call back vnet. */ 1401 if (vio->cb.vio_net_tx_update) { 1402 (*vio->cb.vio_net_tx_update) 1403 (nxgep->hio_vr->vhp); 1404 } 1405 } 1406 #endif 1407 nxgep->resched_needed = B_FALSE; 1408 nxgep->resched_running = B_FALSE; 1409 } 1410 1411 NXGE_DEBUG_MSG((NULL, TX_CTL, "<== nxge_reschedule")); 1412 return (DDI_INTR_CLAIMED); 1413 } 1414 1415 1416 /* Software LSO starts here */ 1417 static void 1418 nxge_hcksum_retrieve(mblk_t *mp, 1419 uint32_t *start, uint32_t *stuff, uint32_t *end, 1420 uint32_t *value, uint32_t *flags) 1421 { 1422 if (mp->b_datap->db_type == M_DATA) { 1423 if (flags != NULL) { 1424 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM | 1425 HCK_PARTIALCKSUM | HCK_FULLCKSUM | 1426 HCK_FULLCKSUM_OK); 1427 if ((*flags & (HCK_PARTIALCKSUM | 1428 HCK_FULLCKSUM)) != 0) { 1429 if (value != NULL) 1430 *value = (uint32_t)DB_CKSUM16(mp); 1431 if ((*flags & HCK_PARTIALCKSUM) != 0) { 1432 if (start != NULL) 1433 *start = 1434 (uint32_t)DB_CKSUMSTART(mp); 1435 if (stuff != NULL) 1436 *stuff = 1437 (uint32_t)DB_CKSUMSTUFF(mp); 1438 if (end != NULL) 1439 *end = 1440 (uint32_t)DB_CKSUMEND(mp); 1441 } 1442 } 1443 } 1444 } 1445 } 1446 1447 static void 1448 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags) 1449 { 1450 ASSERT(DB_TYPE(mp) == M_DATA); 1451 1452 *mss = 0; 1453 if (flags != NULL) { 1454 *flags = DB_CKSUMFLAGS(mp) & HW_LSO; 1455 if ((*flags != 0) && (mss != NULL)) { 1456 *mss = (uint32_t)DB_LSOMSS(mp); 1457 } 1458 NXGE_DEBUG_MSG((NULL, TX_CTL, 1459 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x", 1460 *mss, *flags)); 1461 } 1462 1463 NXGE_DEBUG_MSG((NULL, TX_CTL, 1464 "<== nxge_lso_info_get: mss %d", *mss)); 1465 } 1466 1467 /* 1468 * Do Soft LSO on the oversized packet. 1469 * 1470 * 1. Create a chain of message for headers. 1471 * 2. Fill up header messages with proper information. 1472 * 3. Copy Eithernet, IP, and TCP headers from the original message to 1473 * each new message with necessary adjustments. 1474 * * Unchange the ethernet header for DIX frames. (by default) 1475 * * IP Total Length field is updated to MSS or less(only for the last one). 1476 * * IP Identification value is incremented by one for each packet. 1477 * * TCP sequence Number is recalculated according to the payload length. 1478 * * Set FIN and/or PSH flags for the *last* packet if applied. 1479 * * TCP partial Checksum 1480 * 4. Update LSO information in the first message header. 1481 * 5. Release the original message header. 1482 */ 1483 static mblk_t * 1484 nxge_do_softlso(mblk_t *mp, uint32_t mss) 1485 { 1486 uint32_t hckflags; 1487 int pktlen; 1488 int hdrlen; 1489 int segnum; 1490 int i; 1491 struct ether_vlan_header *evh; 1492 int ehlen, iphlen, tcphlen; 1493 struct ip *oiph, *niph; 1494 struct tcphdr *otcph, *ntcph; 1495 int available, len, left; 1496 uint16_t ip_id; 1497 uint32_t tcp_seq; 1498 #ifdef __sparc 1499 uint32_t tcp_seq_tmp; 1500 #endif 1501 mblk_t *datamp; 1502 uchar_t *rptr; 1503 mblk_t *nmp; 1504 mblk_t *cmp; 1505 mblk_t *mp_chain; 1506 boolean_t do_cleanup = B_FALSE; 1507 t_uscalar_t start_offset = 0; 1508 t_uscalar_t stuff_offset = 0; 1509 t_uscalar_t value = 0; 1510 uint16_t l4_len; 1511 ipaddr_t src, dst; 1512 uint32_t cksum, sum, l4cksum; 1513 1514 NXGE_DEBUG_MSG((NULL, TX_CTL, 1515 "==> nxge_do_softlso")); 1516 /* 1517 * check the length of LSO packet payload and calculate the number of 1518 * segments to be generated. 1519 */ 1520 pktlen = msgsize(mp); 1521 evh = (struct ether_vlan_header *)mp->b_rptr; 1522 1523 /* VLAN? */ 1524 if (evh->ether_tpid == htons(ETHERTYPE_VLAN)) 1525 ehlen = sizeof (struct ether_vlan_header); 1526 else 1527 ehlen = sizeof (struct ether_header); 1528 oiph = (struct ip *)(mp->b_rptr + ehlen); 1529 iphlen = oiph->ip_hl * 4; 1530 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1531 tcphlen = otcph->th_off * 4; 1532 1533 l4_len = pktlen - ehlen - iphlen; 1534 1535 NXGE_DEBUG_MSG((NULL, TX_CTL, 1536 "==> nxge_do_softlso: mss %d oiph $%p " 1537 "original ip_sum oiph->ip_sum 0x%x " 1538 "original tcp_sum otcph->th_sum 0x%x " 1539 "oiph->ip_len %d pktlen %d ehlen %d " 1540 "l4_len %d (0x%x) ip_len - iphlen %d ", 1541 mss, 1542 oiph, 1543 oiph->ip_sum, 1544 otcph->th_sum, 1545 ntohs(oiph->ip_len), pktlen, 1546 ehlen, 1547 l4_len, 1548 l4_len, 1549 ntohs(oiph->ip_len) - iphlen)); 1550 1551 /* IPv4 + TCP */ 1552 if (!(oiph->ip_v == IPV4_VERSION)) { 1553 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1554 "<== nxge_do_softlso: not IPV4 " 1555 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1556 ntohs(oiph->ip_len), pktlen, ehlen, 1557 tcphlen)); 1558 freemsg(mp); 1559 return (NULL); 1560 } 1561 1562 if (!(oiph->ip_p == IPPROTO_TCP)) { 1563 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1564 "<== nxge_do_softlso: not TCP " 1565 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1566 ntohs(oiph->ip_len), pktlen, ehlen, 1567 tcphlen)); 1568 freemsg(mp); 1569 return (NULL); 1570 } 1571 1572 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) { 1573 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1574 "<== nxge_do_softlso: len not matched " 1575 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d", 1576 ntohs(oiph->ip_len), pktlen, ehlen, 1577 tcphlen)); 1578 freemsg(mp); 1579 return (NULL); 1580 } 1581 1582 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen); 1583 tcphlen = otcph->th_off * 4; 1584 1585 /* TCP flags can not include URG, RST, or SYN */ 1586 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0); 1587 1588 hdrlen = ehlen + iphlen + tcphlen; 1589 1590 VERIFY(MBLKL(mp) >= hdrlen); 1591 1592 if (MBLKL(mp) > hdrlen) { 1593 datamp = mp; 1594 rptr = mp->b_rptr + hdrlen; 1595 } else { /* = */ 1596 datamp = mp->b_cont; 1597 rptr = datamp->b_rptr; 1598 } 1599 1600 NXGE_DEBUG_MSG((NULL, TX_CTL, 1601 "nxge_do_softlso: otcph $%p pktlen: %d, " 1602 "hdrlen %d ehlen %d iphlen %d tcphlen %d " 1603 "mblkl(mp): %d, mblkl(datamp): %d", 1604 otcph, 1605 pktlen, hdrlen, ehlen, iphlen, tcphlen, 1606 (int)MBLKL(mp), (int)MBLKL(datamp))); 1607 1608 hckflags = 0; 1609 nxge_hcksum_retrieve(mp, 1610 &start_offset, &stuff_offset, &value, NULL, &hckflags); 1611 1612 dst = oiph->ip_dst.s_addr; 1613 src = oiph->ip_src.s_addr; 1614 1615 cksum = (dst >> 16) + (dst & 0xFFFF) + 1616 (src >> 16) + (src & 0xFFFF); 1617 l4cksum = cksum + IP_TCP_CSUM_COMP; 1618 1619 sum = l4_len + l4cksum; 1620 sum = (sum & 0xFFFF) + (sum >> 16); 1621 1622 NXGE_DEBUG_MSG((NULL, TX_CTL, 1623 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x " 1624 "hckflags 0x%x start_offset %d stuff_offset %d " 1625 "value (original) 0x%x th_sum 0x%x " 1626 "pktlen %d l4_len %d (0x%x) " 1627 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s", 1628 dst, src, 1629 (sum & 0xffff), (~sum & 0xffff), 1630 hckflags, start_offset, stuff_offset, 1631 value, otcph->th_sum, 1632 pktlen, 1633 l4_len, 1634 l4_len, 1635 ntohs(oiph->ip_len) - (int)MBLKL(mp), 1636 (int)MBLKL(datamp), 1637 nxge_dump_packet((char *)evh, 12))); 1638 1639 /* 1640 * Start to process. 1641 */ 1642 available = pktlen - hdrlen; 1643 segnum = (available - 1) / mss + 1; 1644 1645 NXGE_DEBUG_MSG((NULL, TX_CTL, 1646 "==> nxge_do_softlso: pktlen %d " 1647 "MBLKL(mp): %d, MBLKL(datamp): %d " 1648 "available %d mss %d segnum %d", 1649 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp), 1650 available, 1651 mss, 1652 segnum)); 1653 1654 VERIFY(segnum >= 2); 1655 1656 /* 1657 * Try to pre-allocate all header messages 1658 */ 1659 mp_chain = NULL; 1660 for (i = 0; i < segnum; i++) { 1661 if ((nmp = allocb(hdrlen, 0)) == NULL) { 1662 /* Clean up the mp_chain */ 1663 while (mp_chain != NULL) { 1664 nmp = mp_chain; 1665 mp_chain = mp_chain->b_next; 1666 freemsg(nmp); 1667 } 1668 NXGE_DEBUG_MSG((NULL, TX_CTL, 1669 "<== nxge_do_softlso: " 1670 "Could not allocate enough messages for headers!")); 1671 freemsg(mp); 1672 return (NULL); 1673 } 1674 nmp->b_next = mp_chain; 1675 mp_chain = nmp; 1676 1677 NXGE_DEBUG_MSG((NULL, TX_CTL, 1678 "==> nxge_do_softlso: " 1679 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p", 1680 mp, nmp, mp_chain, mp_chain->b_next)); 1681 } 1682 1683 NXGE_DEBUG_MSG((NULL, TX_CTL, 1684 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p", 1685 mp, nmp, mp_chain)); 1686 1687 /* 1688 * Associate payload with new packets 1689 */ 1690 cmp = mp_chain; 1691 left = available; 1692 while (cmp != NULL) { 1693 nmp = dupb(datamp); 1694 if (nmp == NULL) { 1695 do_cleanup = B_TRUE; 1696 NXGE_DEBUG_MSG((NULL, TX_CTL, 1697 "==>nxge_do_softlso: " 1698 "Can not dupb(datamp), have to do clean up")); 1699 goto cleanup_allocated_msgs; 1700 } 1701 1702 NXGE_DEBUG_MSG((NULL, TX_CTL, 1703 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p " 1704 "dupb nmp $%p len %d left %d msd %d ", 1705 mp, cmp, nmp, len, left, mss)); 1706 1707 cmp->b_cont = nmp; 1708 nmp->b_rptr = rptr; 1709 len = (left < mss) ? left : mss; 1710 left -= len; 1711 1712 NXGE_DEBUG_MSG((NULL, TX_CTL, 1713 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p " 1714 "dupb nmp $%p len %d left %d mss %d ", 1715 mp, cmp, nmp, len, left, mss)); 1716 NXGE_DEBUG_MSG((NULL, TX_CTL, 1717 "nxge_do_softlso: before available: %d, " 1718 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1719 available, left, len, segnum, (int)MBLKL(nmp))); 1720 1721 len -= MBLKL(nmp); 1722 NXGE_DEBUG_MSG((NULL, TX_CTL, 1723 "nxge_do_softlso: after available: %d, " 1724 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1725 available, left, len, segnum, (int)MBLKL(nmp))); 1726 1727 while (len > 0) { 1728 mblk_t *mmp = NULL; 1729 1730 NXGE_DEBUG_MSG((NULL, TX_CTL, 1731 "nxge_do_softlso: (4) len > 0 available: %d, " 1732 "left: %d, len: %d, segnum: %d MBLK(nmp): %d", 1733 available, left, len, segnum, (int)MBLKL(nmp))); 1734 1735 if (datamp->b_cont != NULL) { 1736 datamp = datamp->b_cont; 1737 rptr = datamp->b_rptr; 1738 mmp = dupb(datamp); 1739 if (mmp == NULL) { 1740 do_cleanup = B_TRUE; 1741 NXGE_DEBUG_MSG((NULL, TX_CTL, 1742 "==> nxge_do_softlso: " 1743 "Can not dupb(datamp) (1), :" 1744 "have to do clean up")); 1745 NXGE_DEBUG_MSG((NULL, TX_CTL, 1746 "==> nxge_do_softlso: " 1747 "available: %d, left: %d, " 1748 "len: %d, MBLKL(nmp): %d", 1749 available, left, len, 1750 (int)MBLKL(nmp))); 1751 goto cleanup_allocated_msgs; 1752 } 1753 } else { 1754 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 1755 "==> nxge_do_softlso: " 1756 "(1)available: %d, left: %d, " 1757 "len: %d, MBLKL(nmp): %d", 1758 available, left, len, 1759 (int)MBLKL(nmp))); 1760 cmn_err(CE_PANIC, 1761 "==> nxge_do_softlso: " 1762 "Pointers must have been corrupted!\n" 1763 "datamp: $%p, nmp: $%p, rptr: $%p", 1764 (void *)datamp, 1765 (void *)nmp, 1766 (void *)rptr); 1767 } 1768 nmp->b_cont = mmp; 1769 nmp = mmp; 1770 len -= MBLKL(nmp); 1771 } 1772 if (len < 0) { 1773 nmp->b_wptr += len; 1774 rptr = nmp->b_wptr; 1775 NXGE_DEBUG_MSG((NULL, TX_CTL, 1776 "(5) len < 0 (less than 0)" 1777 "available: %d, left: %d, len: %d, MBLKL(nmp): %d", 1778 available, left, len, (int)MBLKL(nmp))); 1779 1780 } else if (len == 0) { 1781 if (datamp->b_cont != NULL) { 1782 NXGE_DEBUG_MSG((NULL, TX_CTL, 1783 "(5) len == 0" 1784 "available: %d, left: %d, len: %d, " 1785 "MBLKL(nmp): %d", 1786 available, left, len, (int)MBLKL(nmp))); 1787 datamp = datamp->b_cont; 1788 rptr = datamp->b_rptr; 1789 } else { 1790 NXGE_DEBUG_MSG((NULL, TX_CTL, 1791 "(6)available b_cont == NULL : %d, " 1792 "left: %d, len: %d, MBLKL(nmp): %d", 1793 available, left, len, (int)MBLKL(nmp))); 1794 1795 VERIFY(cmp->b_next == NULL); 1796 VERIFY(left == 0); 1797 break; /* Done! */ 1798 } 1799 } 1800 cmp = cmp->b_next; 1801 1802 NXGE_DEBUG_MSG((NULL, TX_CTL, 1803 "(7) do_softlso: " 1804 "next mp in mp_chain available len != 0 : %d, " 1805 "left: %d, len: %d, MBLKL(nmp): %d", 1806 available, left, len, (int)MBLKL(nmp))); 1807 } 1808 1809 /* 1810 * From now, start to fill up all headers for the first message 1811 * Hardware checksum flags need to be updated separately for FULLCKSUM 1812 * and PARTIALCKSUM cases. For full checksum, copy the original flags 1813 * into every new packet is enough. But for HCK_PARTIALCKSUM, all 1814 * required fields need to be updated properly. 1815 */ 1816 nmp = mp_chain; 1817 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen); 1818 nmp->b_wptr = nmp->b_rptr + hdrlen; 1819 niph = (struct ip *)(nmp->b_rptr + ehlen); 1820 niph->ip_len = htons(mss + iphlen + tcphlen); 1821 ip_id = ntohs(niph->ip_id); 1822 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1823 #ifdef __sparc 1824 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4); 1825 tcp_seq = ntohl(tcp_seq_tmp); 1826 #else 1827 tcp_seq = ntohl(ntcph->th_seq); 1828 #endif 1829 1830 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST); 1831 1832 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1833 DB_CKSUMSTART(nmp) = start_offset; 1834 DB_CKSUMSTUFF(nmp) = stuff_offset; 1835 1836 /* calculate IP checksum and TCP pseudo header checksum */ 1837 niph->ip_sum = 0; 1838 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1839 1840 l4_len = mss + tcphlen; 1841 sum = htons(l4_len) + l4cksum; 1842 sum = (sum & 0xFFFF) + (sum >> 16); 1843 ntcph->th_sum = (sum & 0xffff); 1844 1845 NXGE_DEBUG_MSG((NULL, TX_CTL, 1846 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) " 1847 "mss %d pktlen %d l4_len %d (0x%x) " 1848 "MBLKL(mp): %d, MBLKL(datamp): %d " 1849 "ip_sum 0x%x " 1850 "th_sum 0x%x sum 0x%x ) " 1851 "dump first ip->tcp %s", 1852 nmp, mp_chain, 1853 mss, 1854 pktlen, 1855 l4_len, 1856 l4_len, 1857 (int)MBLKL(mp), (int)MBLKL(datamp), 1858 niph->ip_sum, 1859 ntcph->th_sum, 1860 sum, 1861 nxge_dump_packet((char *)niph, 52))); 1862 1863 cmp = nmp; 1864 while ((nmp = nmp->b_next)->b_next != NULL) { 1865 NXGE_DEBUG_MSG((NULL, TX_CTL, 1866 "==>nxge_do_softlso: middle l4_len %d ", l4_len)); 1867 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1868 nmp->b_wptr = nmp->b_rptr + hdrlen; 1869 niph = (struct ip *)(nmp->b_rptr + ehlen); 1870 niph->ip_id = htons(++ip_id); 1871 niph->ip_len = htons(mss + iphlen + tcphlen); 1872 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1873 tcp_seq += mss; 1874 1875 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG); 1876 1877 #ifdef __sparc 1878 tcp_seq_tmp = htonl(tcp_seq); 1879 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1880 #else 1881 ntcph->th_seq = htonl(tcp_seq); 1882 #endif 1883 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1884 DB_CKSUMSTART(nmp) = start_offset; 1885 DB_CKSUMSTUFF(nmp) = stuff_offset; 1886 1887 /* calculate IP checksum and TCP pseudo header checksum */ 1888 niph->ip_sum = 0; 1889 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1890 ntcph->th_sum = (sum & 0xffff); 1891 1892 NXGE_DEBUG_MSG((NULL, TX_CTL, 1893 "==> nxge_do_softlso: middle ip_sum 0x%x " 1894 "th_sum 0x%x " 1895 " mp $%p (mp_chain $%p) pktlen %d " 1896 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1897 niph->ip_sum, 1898 ntcph->th_sum, 1899 nmp, mp_chain, 1900 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp))); 1901 } 1902 1903 /* Last segment */ 1904 /* 1905 * Set FIN and/or PSH flags if present only in the last packet. 1906 * The ip_len could be different from prior packets. 1907 */ 1908 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen); 1909 nmp->b_wptr = nmp->b_rptr + hdrlen; 1910 niph = (struct ip *)(nmp->b_rptr + ehlen); 1911 niph->ip_id = htons(++ip_id); 1912 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen); 1913 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen); 1914 tcp_seq += mss; 1915 #ifdef __sparc 1916 tcp_seq_tmp = htonl(tcp_seq); 1917 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4); 1918 #else 1919 ntcph->th_seq = htonl(tcp_seq); 1920 #endif 1921 ntcph->th_flags = (otcph->th_flags & ~TH_URG); 1922 1923 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags; 1924 DB_CKSUMSTART(nmp) = start_offset; 1925 DB_CKSUMSTUFF(nmp) = stuff_offset; 1926 1927 /* calculate IP checksum and TCP pseudo header checksum */ 1928 niph->ip_sum = 0; 1929 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen); 1930 1931 l4_len = ntohs(niph->ip_len) - iphlen; 1932 sum = htons(l4_len) + l4cksum; 1933 sum = (sum & 0xFFFF) + (sum >> 16); 1934 ntcph->th_sum = (sum & 0xffff); 1935 1936 NXGE_DEBUG_MSG((NULL, TX_CTL, 1937 "==> nxge_do_softlso: last next " 1938 "niph->ip_sum 0x%x " 1939 "ntcph->th_sum 0x%x sum 0x%x " 1940 "dump last ip->tcp %s " 1941 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) " 1942 "l4_len %d (0x%x) " 1943 "MBLKL(mp): %d, MBLKL(datamp): %d ", 1944 niph->ip_sum, 1945 ntcph->th_sum, sum, 1946 nxge_dump_packet((char *)niph, 52), 1947 cmp, nmp, mp_chain, 1948 pktlen, pktlen, 1949 l4_len, 1950 l4_len, 1951 (int)MBLKL(mp), (int)MBLKL(datamp))); 1952 1953 cleanup_allocated_msgs: 1954 if (do_cleanup) { 1955 NXGE_DEBUG_MSG((NULL, TX_CTL, 1956 "==> nxge_do_softlso: " 1957 "Failed allocating messages, " 1958 "have to clean up and fail!")); 1959 while (mp_chain != NULL) { 1960 nmp = mp_chain; 1961 mp_chain = mp_chain->b_next; 1962 freemsg(nmp); 1963 } 1964 } 1965 /* 1966 * We're done here, so just free the original message and return the 1967 * new message chain, that could be NULL if failed, back to the caller. 1968 */ 1969 freemsg(mp); 1970 1971 NXGE_DEBUG_MSG((NULL, TX_CTL, 1972 "<== nxge_do_softlso:mp_chain $%p", mp_chain)); 1973 return (mp_chain); 1974 } 1975 1976 /* 1977 * Will be called before NIC driver do further operation on the message. 1978 * The input message may include LSO information, if so, go to softlso logic 1979 * to eliminate the oversized LSO packet for the incapable underlying h/w. 1980 * The return could be the same non-LSO message or a message chain for LSO case. 1981 * 1982 * The driver needs to call this function per packet and process the whole chain 1983 * if applied. 1984 */ 1985 static mblk_t * 1986 nxge_lso_eliminate(mblk_t *mp) 1987 { 1988 uint32_t lsoflags; 1989 uint32_t mss; 1990 1991 NXGE_DEBUG_MSG((NULL, TX_CTL, 1992 "==>nxge_lso_eliminate:")); 1993 nxge_lso_info_get(mp, &mss, &lsoflags); 1994 1995 if (lsoflags & HW_LSO) { 1996 mblk_t *nmp; 1997 1998 NXGE_DEBUG_MSG((NULL, TX_CTL, 1999 "==>nxge_lso_eliminate:" 2000 "HW_LSO:mss %d mp $%p", 2001 mss, mp)); 2002 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) { 2003 NXGE_DEBUG_MSG((NULL, TX_CTL, 2004 "<== nxge_lso_eliminate: " 2005 "LSO: nmp not NULL nmp $%p mss %d mp $%p", 2006 nmp, mss, mp)); 2007 return (nmp); 2008 } else { 2009 NXGE_DEBUG_MSG((NULL, TX_CTL, 2010 "<== nxge_lso_eliminate_ " 2011 "LSO: failed nmp NULL nmp $%p mss %d mp $%p", 2012 nmp, mss, mp)); 2013 return (NULL); 2014 } 2015 } 2016 2017 NXGE_DEBUG_MSG((NULL, TX_CTL, 2018 "<== nxge_lso_eliminate")); 2019 return (mp); 2020 } 2021 2022 static uint32_t 2023 nxge_csgen(uint16_t *adr, int len) 2024 { 2025 int i, odd; 2026 uint32_t sum = 0; 2027 uint32_t c = 0; 2028 2029 odd = len % 2; 2030 for (i = 0; i < (len / 2); i++) { 2031 sum += (adr[i] & 0xffff); 2032 } 2033 if (odd) { 2034 sum += adr[len / 2] & 0xff00; 2035 } 2036 while ((c = ((sum & 0xffff0000) >> 16)) != 0) { 2037 sum &= 0xffff; 2038 sum += c; 2039 } 2040 return (~sum & 0xffff); 2041 } 2042