1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Emulex. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Source file containing the implementation of the Transmit 29 * Path 30 */ 31 32 #include <oce_impl.h> 33 34 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed); 35 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, 36 mblk_t *mp); 37 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 38 uint32_t pkt_len); 39 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 40 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, 41 size_t size, int flags); 42 static oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq); 43 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 44 45 static void oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed); 46 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 47 static inline oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq); 48 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq); 49 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 50 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed); 51 static void oce_remove_vtag(mblk_t *mp); 52 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag); 53 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm); 54 55 56 static ddi_dma_attr_t tx_map_dma_attr = { 57 DMA_ATTR_V0, /* version number */ 58 0x0000000000000000ull, /* low address */ 59 0xFFFFFFFFFFFFFFFFull, /* high address */ 60 0x0000000000010000ull, /* dma counter max */ 61 OCE_TXMAP_ALIGN, /* alignment */ 62 0x7FF, /* burst sizes */ 63 0x00000001, /* minimum transfer size */ 64 0x00000000FFFFFFFFull, /* maximum transfer size */ 65 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 66 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */ 67 0x00000001, /* granularity */ 68 0 /* DMA flags */ 69 }; 70 71 /* 72 * WQ map handle destructor 73 * 74 * wq - Pointer to WQ structure 75 * wqmd - pointer to WQE mapping handle descriptor 76 * 77 * return none 78 */ 79 80 static void 81 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 82 { 83 _NOTE(ARGUNUSED(wq)); 84 /* Free the DMA handle */ 85 if (wqmd->dma_handle != NULL) 86 (void) ddi_dma_free_handle(&(wqmd->dma_handle)); 87 wqmd->dma_handle = NULL; 88 } /* oce_wqm_dtor */ 89 90 /* 91 * WQ map handles contructor 92 * 93 * wqmd - pointer to WQE mapping handle descriptor 94 * wq - Pointer to WQ structure 95 * 96 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 97 */ 98 static int 99 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq) 100 { 101 struct oce_dev *dev; 102 int ret; 103 104 dev = wq->parent; 105 /* Allocate DMA handle */ 106 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr, 107 KM_SLEEP, NULL, &wqmd->dma_handle); 108 109 return (ret); 110 } /* oce_wqm_ctor */ 111 112 /* 113 * function to create WQ mapping handles cache 114 * 115 * wq - pointer to WQ structure 116 * 117 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 118 */ 119 int 120 oce_wqm_cache_create(struct oce_wq *wq) 121 { 122 struct oce_dev *dev = wq->parent; 123 int size; 124 int cnt; 125 int ret; 126 127 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t); 128 wq->wq_mdesc_array = kmem_zalloc(size, KM_SLEEP); 129 130 /* Create the free buffer list */ 131 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri)); 132 133 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) { 134 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq); 135 if (ret != DDI_SUCCESS) { 136 goto wqm_fail; 137 } 138 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, 139 &wq->wq_mdesc_array[cnt]); 140 } 141 return (DDI_SUCCESS); 142 143 wqm_fail: 144 oce_wqm_cache_destroy(wq); 145 return (DDI_FAILURE); 146 } 147 148 /* 149 * function to destroy WQ mapping handles cache 150 * 151 * wq - pointer to WQ structure 152 * 153 * return none 154 */ 155 void 156 oce_wqm_cache_destroy(struct oce_wq *wq) 157 { 158 oce_wq_mdesc_t *wqmd; 159 160 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) { 161 oce_wqm_dtor(wq, wqmd); 162 } 163 164 kmem_free(wq->wq_mdesc_array, 165 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t)); 166 167 OCE_LIST_DESTROY(&wq->wq_mdesc_list); 168 } 169 170 /* 171 * function to create WQ buffer cache 172 * 173 * wq - pointer to WQ structure 174 * buf_size - size of the buffer 175 * 176 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 177 */ 178 int 179 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size) 180 { 181 struct oce_dev *dev = wq->parent; 182 int size; 183 int cnt; 184 int ret; 185 186 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t); 187 wq->wq_bdesc_array = kmem_zalloc(size, KM_SLEEP); 188 189 /* Create the free buffer list */ 190 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri)); 191 192 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) { 193 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt], 194 wq, buf_size, DDI_DMA_STREAMING); 195 if (ret != DDI_SUCCESS) { 196 goto wqb_fail; 197 } 198 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, 199 &wq->wq_bdesc_array[cnt]); 200 } 201 return (DDI_SUCCESS); 202 203 wqb_fail: 204 oce_wqb_cache_destroy(wq); 205 return (DDI_FAILURE); 206 } 207 208 /* 209 * function to destroy WQ buffer cache 210 * 211 * wq - pointer to WQ structure 212 * 213 * return none 214 */ 215 void 216 oce_wqb_cache_destroy(struct oce_wq *wq) 217 { 218 oce_wq_bdesc_t *wqbd; 219 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) { 220 oce_wqb_dtor(wq, wqbd); 221 } 222 kmem_free(wq->wq_bdesc_array, 223 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t)); 224 OCE_LIST_DESTROY(&wq->wq_buf_list); 225 } 226 227 /* 228 * WQ buffer constructor 229 * 230 * wqbd - pointer to WQ buffer descriptor 231 * wq - pointer to WQ structure 232 * size - size of the buffer 233 * flags - KM_SLEEP or KM_NOSLEEP 234 * 235 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 236 */ 237 static int 238 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags) 239 { 240 struct oce_dev *dev; 241 dev = wq->parent; 242 wqbd->wqb = oce_alloc_dma_buffer(dev, size, flags); 243 if (wqbd->wqb == NULL) { 244 return (DDI_FAILURE); 245 } 246 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr); 247 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr); 248 return (DDI_SUCCESS); 249 } 250 251 /* 252 * WQ buffer destructor 253 * 254 * wq - pointer to WQ structure 255 * wqbd - pointer to WQ buffer descriptor 256 * 257 * return none 258 */ 259 static void 260 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 261 { 262 oce_free_dma_buffer(wq->parent, wqbd->wqb); 263 } 264 265 /* 266 * function to alloc WQE buffer descriptor 267 * 268 * wq - pointer to WQ structure 269 * 270 * return pointer to WQE buffer descriptor 271 */ 272 static inline oce_wq_bdesc_t * 273 oce_wqb_alloc(struct oce_wq *wq) 274 { 275 oce_wq_bdesc_t *wqbd; 276 wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list); 277 return (wqbd); 278 } 279 280 /* 281 * function to free WQE buffer descriptor 282 * 283 * wq - pointer to WQ structure 284 * wqbd - pointer to WQ buffer descriptor 285 * 286 * return none 287 */ 288 static inline void 289 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 290 { 291 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd); 292 } /* oce_wqb_free */ 293 294 /* 295 * function to allocate WQE mapping descriptor 296 * 297 * wq - pointer to WQ structure 298 * 299 * return pointer to WQE mapping descriptor 300 */ 301 static inline oce_wq_mdesc_t * 302 oce_wqm_alloc(struct oce_wq *wq) 303 { 304 oce_wq_mdesc_t *wqmd; 305 wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list); 306 return (wqmd); 307 308 } /* oce_wqm_alloc */ 309 310 /* 311 * function to insert WQE mapping descriptor to the list 312 * 313 * wq - pointer to WQ structure 314 * wqmd - Pointer to WQ mapping descriptor 315 * 316 * return none 317 */ 318 static inline void 319 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 320 { 321 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd); 322 } 323 324 /* 325 * function to free WQE mapping descriptor 326 * 327 * wq - pointer to WQ structure 328 * wqmd - Pointer to WQ mapping descriptor 329 * 330 * return none 331 */ 332 static void 333 oce_wqmd_free(struct oce_wq *wq, oce_wqe_desc_t *wqed) 334 { 335 int ndesc; 336 oce_wq_mdesc_t *wqmd; 337 338 if (wqed == NULL) { 339 return; 340 } 341 for (ndesc = 0; ndesc < wqed->nhdl; ndesc++) { 342 wqmd = wqed->hdesc[ndesc].hdl; 343 (void) ddi_dma_unbind_handle(wqmd->dma_handle); 344 oce_wqm_free(wq, wqmd); 345 } 346 } 347 348 /* 349 * WQED kmem_cache constructor 350 * 351 * buf - pointer to WQE descriptor 352 * 353 * return DDI_SUCCESS 354 */ 355 int 356 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags) 357 { 358 oce_wqe_desc_t *wqed = (oce_wqe_desc_t *)buf; 359 360 _NOTE(ARGUNUSED(arg)); 361 _NOTE(ARGUNUSED(kmflags)); 362 363 bzero(wqed, sizeof (oce_wqe_desc_t)); 364 return (DDI_SUCCESS); 365 } 366 367 /* 368 * WQED kmem_cache destructor 369 * 370 * buf - pointer to WQE descriptor 371 * 372 * return none 373 */ 374 void 375 oce_wqe_desc_dtor(void *buf, void *arg) 376 { 377 _NOTE(ARGUNUSED(buf)); 378 _NOTE(ARGUNUSED(arg)); 379 } 380 381 /* 382 * function to choose a WQ given a mblk depending on priority, flowID etc. 383 * 384 * dev - software handle to device 385 * pkt - the mblk to send 386 * 387 * return pointer to the WQ selected 388 */ 389 struct oce_wq * 390 oce_get_wq(struct oce_dev *dev, mblk_t *pkt) 391 { 392 _NOTE(ARGUNUSED(pkt)); 393 /* for the time being hardcode */ 394 return (dev->wq[0]); 395 } /* oce_get_wq */ 396 397 /* 398 * function to populate the single WQE 399 * 400 * wq - pointer to wq 401 * wqed - pointer to WQ entry descriptor 402 * 403 * return none 404 */ 405 #pragma inline(oce_fill_ring_descs) 406 static void 407 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed) 408 { 409 410 struct oce_nic_frag_wqe *wqe; 411 int i; 412 /* Copy the precreate WQE descs to the ring desc */ 413 for (i = 0; i < wqed->wqe_cnt; i++) { 414 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring, 415 struct oce_nic_frag_wqe); 416 417 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE); 418 RING_PUT(wq->ring, 1); 419 } 420 } /* oce_fill_ring_descs */ 421 422 /* 423 * function to copy the packet to preallocated Tx buffer 424 * 425 * wq - pointer to WQ 426 * wqed - Pointer to WQE descriptor 427 * mp - Pointer to packet chain 428 * pktlen - Size of the packet 429 * 430 * return 0=>success, error code otherwise 431 */ 432 static int 433 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 434 uint32_t pkt_len) 435 { 436 oce_wq_bdesc_t *wqbd; 437 caddr_t buf_va; 438 struct oce_dev *dev = wq->parent; 439 440 wqbd = oce_wqb_alloc(wq); 441 if (wqbd == NULL) { 442 atomic_inc_32(&dev->tx_noxmtbuf); 443 oce_log(dev, CE_WARN, MOD_TX, "%s", 444 "wqb pool empty"); 445 return (ENOMEM); 446 } 447 448 /* create a fragment wqe for the packet */ 449 wqed->frag[1].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi; 450 wqed->frag[1].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo; 451 buf_va = DBUF_VA(wqbd->wqb); 452 453 /* copy pkt into buffer */ 454 for (; mp != NULL; mp = mp->b_cont) { 455 bcopy(mp->b_rptr, buf_va, MBLKL(mp)); 456 buf_va += MBLKL(mp); 457 } 458 459 wqed->frag[1].u0.s.frag_len = pkt_len; 460 wqed->hdesc[0].hdl = (void *)(wqbd); 461 462 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len, 463 DDI_DMA_SYNC_FORDEV); 464 465 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) { 466 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 467 /* Free the buffer */ 468 oce_wqb_free(wq, wqbd); 469 return (EIO); 470 } 471 wqed->frag_cnt = 2; 472 wqed->nhdl = 1; 473 wqed->type = COPY_WQE; 474 return (0); 475 } /* oce_bcopy_wqe */ 476 477 /* 478 * function to copy the packet or dma map on the fly depending on size 479 * 480 * wq - pointer to WQ 481 * wqed - Pointer to WQE descriptor 482 * mp - Pointer to packet chain 483 * 484 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 485 */ 486 static int 487 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp) 488 { 489 ddi_dma_cookie_t cookie; 490 oce_wq_mdesc_t *wqmd; 491 int32_t nfrag = 1; 492 uint32_t ncookies; 493 int ret; 494 uint32_t len; 495 struct oce_dev *dev = wq->parent; 496 497 wqed->nhdl = 0; 498 wqed->mp = mp; 499 500 for (; mp != NULL; mp = mp->b_cont) { 501 len = MBLKL(mp); 502 if (len == 0) { 503 oce_log(dev, CE_NOTE, MOD_TX, "%s", 504 "Zero len MBLK "); 505 continue; 506 } 507 508 wqmd = oce_wqm_alloc(wq); 509 if (wqmd == NULL) { 510 oce_log(dev, CE_WARN, MOD_TX, "%s", 511 "wqm pool empty"); 512 ret = ENOMEM; 513 goto map_fail; 514 } 515 516 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, 517 (struct as *)0, (caddr_t)mp->b_rptr, 518 len, DDI_DMA_WRITE | DDI_DMA_STREAMING, 519 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); 520 if (ret != DDI_DMA_MAPPED) { 521 oce_log(dev, CE_WARN, MOD_TX, "%s", 522 "Failed to Map SGL"); 523 /* free the last one */ 524 oce_wqm_free(wq, wqmd); 525 goto map_fail; 526 } 527 528 do { 529 wqed->frag[nfrag].u0.s.frag_pa_hi = 530 ADDR_HI(cookie.dmac_laddress); 531 wqed->frag[nfrag].u0.s.frag_pa_lo = 532 ADDR_LO(cookie.dmac_laddress); 533 wqed->frag[nfrag].u0.s.frag_len = 534 (uint32_t)cookie.dmac_size; 535 nfrag++; 536 if (--ncookies > 0) 537 ddi_dma_nextcookie(wqmd->dma_handle, 538 &cookie); 539 else break; 540 } while (ncookies > 0); 541 542 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; 543 wqed->nhdl++; 544 } 545 wqed->frag_cnt = nfrag; 546 wqed->type = MAPPED_WQE; 547 return (0); 548 549 map_fail: 550 wqed->mp = NULL; 551 oce_wqmd_free(wq, wqed); 552 return (ret); 553 } /* oce_map_wqe */ 554 555 static inline int 556 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm) 557 { 558 struct oce_nic_tx_cqe *cqe; 559 uint16_t num_cqe = 0; 560 struct oce_cq *cq; 561 oce_wqe_desc_t *wqed; 562 int wqe_freed = 0; 563 struct oce_dev *dev; 564 565 cq = wq->cq; 566 dev = wq->parent; 567 mutex_enter(&wq->txc_lock); 568 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 569 while (WQ_CQE_VALID(cqe)) { 570 571 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe)); 572 573 /* update stats */ 574 if (cqe->u0.s.status != 0) { 575 atomic_inc_32(&dev->tx_errors); 576 } 577 578 /* complete the WQEs */ 579 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list); 580 581 wqe_freed = wqed->wqe_cnt; 582 oce_free_wqed(wq, wqed); 583 RING_GET(wq->ring, wqe_freed); 584 atomic_add_32(&wq->wq_free, wqe_freed); 585 /* clear the valid bit and progress cqe */ 586 WQ_CQE_INVALIDATE(cqe); 587 RING_GET(cq->ring, 1); 588 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 589 struct oce_nic_tx_cqe); 590 num_cqe++; 591 } /* for all valid CQE */ 592 mutex_exit(&wq->txc_lock); 593 if (num_cqe) 594 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm); 595 return (num_cqe); 596 } /* oce_process_tx_completion */ 597 598 /* 599 * function to drain a TxCQ and process its CQEs 600 * 601 * dev - software handle to the device 602 * cq - pointer to the cq to drain 603 * 604 * return the number of CQEs processed 605 */ 606 uint16_t 607 oce_drain_wq_cq(void *arg) 608 { 609 uint16_t num_cqe = 0; 610 struct oce_dev *dev; 611 struct oce_wq *wq; 612 613 wq = (struct oce_wq *)arg; 614 dev = wq->parent; 615 616 /* do while we do not reach a cqe that is not valid */ 617 num_cqe = oce_process_tx_compl(wq, B_FALSE); 618 619 /* check if we need to restart Tx */ 620 if (wq->resched && num_cqe) { 621 wq->resched = B_FALSE; 622 mac_tx_update(dev->mac_handle); 623 } 624 625 return (num_cqe); 626 } /* oce_process_wq_cqe */ 627 628 /* 629 * function to insert vtag to packet 630 * 631 * mp - mblk pointer 632 * vlan_tag - tag to be inserted 633 * 634 * return none 635 */ 636 static inline void 637 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag) 638 { 639 struct ether_vlan_header *evh; 640 (void) memmove(mp->b_rptr - VLAN_TAGSZ, 641 mp->b_rptr, 2 * ETHERADDRL); 642 mp->b_rptr -= VLAN_TAGSZ; 643 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 644 evh->ether_tpid = htons(VLAN_TPID); 645 evh->ether_tci = htons(vlan_tag); 646 } 647 648 /* 649 * function to strip vtag from packet 650 * 651 * mp - mblk pointer 652 * 653 * return none 654 */ 655 656 static inline void 657 oce_remove_vtag(mblk_t *mp) 658 { 659 (void) memmove(mp->b_rptr + VLAN_TAGSZ, mp->b_rptr, 660 ETHERADDRL * 2); 661 mp->b_rptr += VLAN_TAGSZ; 662 } 663 664 /* 665 * function to xmit Single packet over the wire 666 * 667 * wq - pointer to WQ 668 * mp - Pointer to packet chain 669 * 670 * return pointer to the packet 671 */ 672 mblk_t * 673 oce_send_packet(struct oce_wq *wq, mblk_t *mp) 674 { 675 676 struct oce_nic_hdr_wqe *wqeh; 677 struct oce_dev *dev; 678 struct ether_header *eh; 679 struct ether_vlan_header *evh; 680 int32_t num_wqes; 681 uint16_t etype; 682 uint32_t ip_offset; 683 uint32_t csum_flags = 0; 684 boolean_t use_copy = B_FALSE; 685 boolean_t tagged = B_FALSE; 686 uint16_t vlan_tag; 687 uint32_t reg_value = 0; 688 oce_wqe_desc_t *wqed = NULL; 689 mblk_t *nmp = NULL; 690 mblk_t *tmp = NULL; 691 uint32_t pkt_len = 0; 692 int num_mblks = 0; 693 int ret = 0; 694 uint32_t flags = 0; 695 uint32_t mss = 0; 696 697 /* retrieve the adap priv struct ptr */ 698 dev = wq->parent; 699 700 /* check if we have enough free slots */ 701 if (wq->wq_free < wq->cfg.q_len/2) { 702 (void) oce_process_tx_compl(wq, B_FALSE); 703 } 704 if (wq->wq_free < OCE_MAX_TX_HDL) { 705 return (mp); 706 } 707 708 /* check if we should copy */ 709 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) { 710 pkt_len += MBLKL(tmp); 711 num_mblks++; 712 } 713 714 /* Retrieve LSO info */ 715 mac_lso_get(mp, &mss, &flags); 716 717 /* get the offload flags */ 718 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags); 719 720 /* Limit should be always less than Tx Buffer Size */ 721 if (pkt_len < dev->tx_bcopy_limit) { 722 use_copy = B_TRUE; 723 } else { 724 /* restrict the mapped segment to wat we support */ 725 if (num_mblks > OCE_MAX_TX_HDL) { 726 nmp = msgpullup(mp, -1); 727 if (nmp == NULL) { 728 atomic_inc_32(&wq->pkt_drops); 729 freemsg(mp); 730 return (NULL); 731 } 732 /* Reset it to new collapsed mp */ 733 freemsg(mp); 734 mp = nmp; 735 } 736 } 737 738 /* Get the packet descriptor for Tx */ 739 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP); 740 if (wqed == NULL) { 741 atomic_inc_32(&wq->pkt_drops); 742 freemsg(mp); 743 return (NULL); 744 } 745 eh = (struct ether_header *)(void *)mp->b_rptr; 746 if (ntohs(eh->ether_type) == VLAN_TPID) { 747 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 748 tagged = B_TRUE; 749 etype = ntohs(evh->ether_type); 750 ip_offset = sizeof (struct ether_vlan_header); 751 pkt_len -= VLAN_TAGSZ; 752 vlan_tag = ntohs(evh->ether_tci); 753 oce_remove_vtag(mp); 754 } else { 755 etype = ntohs(eh->ether_type); 756 ip_offset = sizeof (struct ether_header); 757 } 758 bzero(wqed, sizeof (oce_wqe_desc_t)); 759 760 /* Save the WQ pointer */ 761 wqed->wq = wq; 762 if (use_copy == B_TRUE) { 763 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len); 764 } else { 765 ret = oce_map_wqe(wq, wqed, mp); 766 } 767 768 /* 769 * Any failure other than insufficient Q entries 770 * drop the packet 771 */ 772 if (ret != 0) { 773 kmem_cache_free(wq->wqed_cache, wqed); 774 /* drop the packet */ 775 atomic_inc_32(&wq->pkt_drops); 776 freemsg(mp); 777 return (NULL); 778 } 779 780 /* increment pending wqed to be scheduled */ 781 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0]; 782 783 /* fill rest of wqe header fields based on packet */ 784 if (flags & HW_LSO) { 785 wqeh->u0.s.lso = B_TRUE; 786 wqeh->u0.s.lso_mss = mss; 787 } 788 if (csum_flags & HCK_FULLCKSUM) { 789 uint8_t *proto; 790 if (etype == ETHERTYPE_IP) { 791 proto = (uint8_t *)(void *) 792 (mp->b_rptr + ip_offset); 793 if (proto[9] == 6) 794 /* IPPROTO_TCP */ 795 wqeh->u0.s.tcpcs = B_TRUE; 796 else if (proto[9] == 17) 797 /* IPPROTO_UDP */ 798 wqeh->u0.s.udpcs = B_TRUE; 799 } 800 } 801 802 if (csum_flags & HCK_IPV4_HDRCKSUM) 803 wqeh->u0.s.ipcs = B_TRUE; 804 if (tagged) { 805 wqeh->u0.s.vlan = B_TRUE; 806 wqeh->u0.s.vlan_tag = vlan_tag; 807 } 808 809 wqeh->u0.s.complete = B_TRUE; 810 wqeh->u0.s.event = B_TRUE; 811 wqeh->u0.s.crc = B_TRUE; 812 wqeh->u0.s.total_length = pkt_len; 813 814 /* frag count + header wqe */ 815 num_wqes = wqed->frag_cnt; 816 817 /* h/w expects even no. of WQEs */ 818 if (num_wqes & 0x1) 819 num_wqes++; 820 wqed->wqe_cnt = (uint16_t)num_wqes; 821 wqeh->u0.s.num_wqe = num_wqes; 822 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE)); 823 824 mutex_enter(&wq->tx_lock); 825 if (num_wqes > wq->wq_free) { 826 atomic_inc_32(&wq->tx_deferd); 827 mutex_exit(&wq->tx_lock); 828 goto wqe_fail; 829 } 830 atomic_add_32(&wq->wq_free, -num_wqes); 831 wqed->wq_start_idx = wq->ring->pidx; 832 833 /* fill the wq for adapter */ 834 oce_fill_ring_descs(wq, wqed); 835 836 /* Add the packet desc to list to be retrieved during cmpl */ 837 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed); 838 839 /* ring tx doorbell */ 840 reg_value = (num_wqes << 16) | wq->wq_id; 841 /* Ring the door bell */ 842 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value); 843 mutex_exit(&wq->tx_lock); 844 845 /* free mp if copied or packet chain collapsed */ 846 if (use_copy == B_TRUE) { 847 freemsg(mp); 848 } 849 return (NULL); 850 851 wqe_fail: 852 853 if (tagged) { 854 oce_insert_vtag(mp, vlan_tag); 855 } 856 857 /* set it to null in case map_wqe has set it */ 858 wqed->mp = NULL; 859 oce_free_wqed(wq, wqed); 860 return (mp); 861 } /* oce_send_packet */ 862 863 /* 864 * function to free the WQE descriptor 865 * 866 * wq - pointer to WQ 867 * wqed - Pointer to WQE descriptor 868 * 869 * return none 870 */ 871 #pragma inline(oce_free_wqed) 872 static void 873 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed) 874 { 875 if (wqed == NULL) { 876 return; 877 } 878 879 if (wqed->type == COPY_WQE) { 880 oce_wqb_free(wq, wqed->hdesc[0].hdl); 881 } else if (wqed->type == MAPPED_WQE) { 882 oce_wqmd_free(wq, wqed); 883 } else ASSERT(0); 884 885 if (wqed->mp) 886 freemsg(wqed->mp); 887 kmem_cache_free(wq->wqed_cache, wqed); 888 } /* oce_free_wqed */ 889 890 /* 891 * function to start the WQ 892 * 893 * wq - pointer to WQ 894 * 895 * return DDI_SUCCESS 896 */ 897 898 int 899 oce_start_wq(struct oce_wq *wq) 900 { 901 _NOTE(ARGUNUSED(wq)); 902 return (DDI_SUCCESS); 903 } /* oce_start_wq */ 904 905 /* 906 * function to stop the WQ 907 * 908 * wq - pointer to WQ 909 * 910 * return none 911 */ 912 void 913 oce_clean_wq(struct oce_wq *wq) 914 { 915 oce_wqe_desc_t *wqed; 916 int ti; 917 918 /* Wait for already posted Tx to complete */ 919 920 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) { 921 (void) oce_process_tx_compl(wq, B_FALSE); 922 OCE_MSDELAY(1); 923 } 924 925 /* Free the remaining descriptors */ 926 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) { 927 atomic_add_32(&wq->wq_free, wqed->wqe_cnt); 928 oce_free_wqed(wq, wqed); 929 } 930 oce_drain_eq(wq->cq->eq); 931 } /* oce_stop_wq */ 932 933 /* 934 * function to set the tx mapping handle fma attr 935 * 936 * fm_caps - capability flags 937 * 938 * return none 939 */ 940 941 void 942 oce_set_tx_map_dma_fma_flags(int fm_caps) 943 { 944 if (fm_caps == DDI_FM_NOT_CAPABLE) { 945 return; 946 } 947 948 if (DDI_FM_DMA_ERR_CAP(fm_caps)) { 949 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 950 } else { 951 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 952 } 953 } /* oce_set_tx_map_dma_fma_flags */ 954