1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Emulex. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Source file containing the implementation of the Transmit 29 * Path 30 */ 31 32 #include <oce_impl.h> 33 34 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed); 35 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, 36 mblk_t *mp, uint32_t pkt_len); 37 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 38 uint32_t pkt_len); 39 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 40 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, 41 size_t size, int flags); 42 static inline oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq); 43 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd); 44 45 static void oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 46 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 47 static oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq); 48 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq); 49 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd); 50 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed); 51 static void oce_remove_vtag(mblk_t *mp); 52 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag); 53 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm); 54 55 56 static ddi_dma_attr_t tx_map_dma_attr = { 57 DMA_ATTR_V0, /* version number */ 58 0x0000000000000000ull, /* low address */ 59 0xFFFFFFFFFFFFFFFFull, /* high address */ 60 0x0000000000010000ull, /* dma counter max */ 61 OCE_TXMAP_ALIGN, /* alignment */ 62 0x7FF, /* burst sizes */ 63 0x00000001, /* minimum transfer size */ 64 0x00000000FFFFFFFFull, /* maximum transfer size */ 65 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 66 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */ 67 0x00000001, /* granularity */ 68 DDI_DMA_FLAGERR /* dma_attr_flags */ 69 }; 70 71 72 ddi_dma_attr_t oce_tx_dma_buf_attr = { 73 DMA_ATTR_V0, /* version number */ 74 0x0000000000000000ull, /* low address */ 75 0xFFFFFFFFFFFFFFFFull, /* high address */ 76 0x00000000FFFFFFFFull, /* dma counter max */ 77 OCE_DMA_ALIGNMENT, /* alignment */ 78 0x000007FF, /* burst sizes */ 79 0x00000001, /* minimum transfer size */ 80 0x00000000FFFFFFFFull, /* maximum transfer size */ 81 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */ 82 1, /* scatter/gather list length */ 83 0x00000001, /* granularity */ 84 DDI_DMA_FLAGERR /* dma_attr_flags */ 85 }; 86 87 /* 88 * WQ map handle destructor 89 * 90 * wq - Pointer to WQ structure 91 * wqmd - pointer to WQE mapping handle descriptor 92 * 93 * return none 94 */ 95 96 static void 97 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 98 { 99 _NOTE(ARGUNUSED(wq)); 100 /* Free the DMA handle */ 101 if (wqmd->dma_handle != NULL) 102 (void) ddi_dma_free_handle(&(wqmd->dma_handle)); 103 wqmd->dma_handle = NULL; 104 } /* oce_wqm_dtor */ 105 106 /* 107 * WQ map handles contructor 108 * 109 * wqmd - pointer to WQE mapping handle descriptor 110 * wq - Pointer to WQ structure 111 * 112 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 113 */ 114 static int 115 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq) 116 { 117 struct oce_dev *dev; 118 int ret; 119 120 dev = wq->parent; 121 /* Allocate DMA handle */ 122 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr, 123 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle); 124 125 return (ret); 126 } /* oce_wqm_ctor */ 127 128 /* 129 * function to create WQ mapping handles cache 130 * 131 * wq - pointer to WQ structure 132 * 133 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 134 */ 135 int 136 oce_wqm_cache_create(struct oce_wq *wq) 137 { 138 struct oce_dev *dev = wq->parent; 139 int size; 140 int cnt; 141 int ret; 142 143 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t); 144 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP); 145 if (wq->wq_mdesc_array == NULL) { 146 return (DDI_FAILURE); 147 } 148 149 /* Create the free buffer list */ 150 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri)); 151 152 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) { 153 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq); 154 if (ret != DDI_SUCCESS) { 155 goto wqm_fail; 156 } 157 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, 158 &wq->wq_mdesc_array[cnt]); 159 } 160 return (DDI_SUCCESS); 161 162 wqm_fail: 163 oce_wqm_cache_destroy(wq); 164 return (DDI_FAILURE); 165 } 166 167 /* 168 * function to destroy WQ mapping handles cache 169 * 170 * wq - pointer to WQ structure 171 * 172 * return none 173 */ 174 void 175 oce_wqm_cache_destroy(struct oce_wq *wq) 176 { 177 oce_wq_mdesc_t *wqmd; 178 179 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) { 180 oce_wqm_dtor(wq, wqmd); 181 } 182 183 kmem_free(wq->wq_mdesc_array, 184 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t)); 185 186 OCE_LIST_DESTROY(&wq->wq_mdesc_list); 187 } 188 189 /* 190 * function to create WQ buffer cache 191 * 192 * wq - pointer to WQ structure 193 * buf_size - size of the buffer 194 * 195 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 196 */ 197 int 198 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size) 199 { 200 struct oce_dev *dev = wq->parent; 201 int size; 202 int cnt; 203 int ret; 204 205 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t); 206 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP); 207 if (wq->wq_bdesc_array == NULL) { 208 return (DDI_FAILURE); 209 } 210 211 /* Create the free buffer list */ 212 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri)); 213 214 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) { 215 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt], 216 wq, buf_size, DDI_DMA_STREAMING); 217 if (ret != DDI_SUCCESS) { 218 goto wqb_fail; 219 } 220 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, 221 &wq->wq_bdesc_array[cnt]); 222 } 223 return (DDI_SUCCESS); 224 225 wqb_fail: 226 oce_wqb_cache_destroy(wq); 227 return (DDI_FAILURE); 228 } 229 230 /* 231 * function to destroy WQ buffer cache 232 * 233 * wq - pointer to WQ structure 234 * 235 * return none 236 */ 237 void 238 oce_wqb_cache_destroy(struct oce_wq *wq) 239 { 240 oce_wq_bdesc_t *wqbd; 241 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) { 242 oce_wqb_dtor(wq, wqbd); 243 } 244 kmem_free(wq->wq_bdesc_array, 245 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t)); 246 OCE_LIST_DESTROY(&wq->wq_buf_list); 247 } 248 249 /* 250 * WQ buffer constructor 251 * 252 * wqbd - pointer to WQ buffer descriptor 253 * wq - pointer to WQ structure 254 * size - size of the buffer 255 * flags - KM_SLEEP or KM_NOSLEEP 256 * 257 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 258 */ 259 static int 260 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags) 261 { 262 struct oce_dev *dev; 263 dev = wq->parent; 264 265 wqbd->wqb = oce_alloc_dma_buffer(dev, size, &oce_tx_dma_buf_attr, 266 flags); 267 if (wqbd->wqb == NULL) { 268 return (DDI_FAILURE); 269 } 270 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr); 271 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr); 272 return (DDI_SUCCESS); 273 } 274 275 /* 276 * WQ buffer destructor 277 * 278 * wq - pointer to WQ structure 279 * wqbd - pointer to WQ buffer descriptor 280 * 281 * return none 282 */ 283 static void 284 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 285 { 286 oce_free_dma_buffer(wq->parent, wqbd->wqb); 287 } 288 289 /* 290 * function to alloc WQE buffer descriptor 291 * 292 * wq - pointer to WQ structure 293 * 294 * return pointer to WQE buffer descriptor 295 */ 296 static inline oce_wq_bdesc_t * 297 oce_wqb_alloc(struct oce_wq *wq) 298 { 299 return (OCE_LIST_REM_HEAD(&wq->wq_buf_list)); 300 } 301 302 /* 303 * function to free WQE buffer descriptor 304 * 305 * wq - pointer to WQ structure 306 * wqbd - pointer to WQ buffer descriptor 307 * 308 * return none 309 */ 310 static inline void 311 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd) 312 { 313 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd); 314 } /* oce_wqb_free */ 315 316 /* 317 * function to allocate WQE mapping descriptor 318 * 319 * wq - pointer to WQ structure 320 * 321 * return pointer to WQE mapping descriptor 322 */ 323 static inline oce_wq_mdesc_t * 324 oce_wqm_alloc(struct oce_wq *wq) 325 { 326 return (OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)); 327 } /* oce_wqm_alloc */ 328 329 /* 330 * function to insert WQE mapping descriptor to the list 331 * 332 * wq - pointer to WQ structure 333 * wqmd - Pointer to WQ mapping descriptor 334 * 335 * return none 336 */ 337 static inline void 338 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 339 { 340 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd); 341 } 342 343 /* 344 * function to free WQE mapping descriptor 345 * 346 * wq - pointer to WQ structure 347 * wqmd - Pointer to WQ mapping descriptor 348 * 349 * return none 350 */ 351 static void 352 oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd) 353 { 354 if (wqmd == NULL) { 355 return; 356 } 357 (void) ddi_dma_unbind_handle(wqmd->dma_handle); 358 oce_wqm_free(wq, wqmd); 359 } 360 361 /* 362 * WQED kmem_cache constructor 363 * 364 * buf - pointer to WQE descriptor 365 * 366 * return DDI_SUCCESS 367 */ 368 int 369 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags) 370 { 371 _NOTE(ARGUNUSED(buf)); 372 _NOTE(ARGUNUSED(arg)); 373 _NOTE(ARGUNUSED(kmflags)); 374 375 return (DDI_SUCCESS); 376 } 377 378 /* 379 * WQED kmem_cache destructor 380 * 381 * buf - pointer to WQE descriptor 382 * 383 * return none 384 */ 385 void 386 oce_wqe_desc_dtor(void *buf, void *arg) 387 { 388 _NOTE(ARGUNUSED(buf)); 389 _NOTE(ARGUNUSED(arg)); 390 } 391 392 /* 393 * function to choose a WQ given a mblk depending on priority, flowID etc. 394 * 395 * dev - software handle to device 396 * mp - the mblk to send 397 * 398 * return pointer to the WQ selected 399 */ 400 static uint8_t oce_tx_hash_policy = 0x4; 401 struct oce_wq * 402 oce_get_wq(struct oce_dev *dev, mblk_t *mp) 403 { 404 struct oce_wq *wq; 405 int qidx = 0; 406 if (dev->nwqs > 1) { 407 qidx = mac_pkt_hash(DL_ETHER, mp, oce_tx_hash_policy, B_TRUE); 408 qidx = qidx % dev->nwqs; 409 410 } else { 411 qidx = 0; 412 } 413 wq = dev->wq[qidx]; 414 /* for the time being hardcode */ 415 return (wq); 416 } /* oce_get_wq */ 417 418 /* 419 * function to populate the single WQE 420 * 421 * wq - pointer to wq 422 * wqed - pointer to WQ entry descriptor 423 * 424 * return none 425 */ 426 #pragma inline(oce_fill_ring_descs) 427 static void 428 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed) 429 { 430 431 struct oce_nic_frag_wqe *wqe; 432 int i; 433 /* Copy the precreate WQE descs to the ring desc */ 434 for (i = 0; i < wqed->wqe_cnt; i++) { 435 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring, 436 struct oce_nic_frag_wqe); 437 438 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE); 439 RING_PUT(wq->ring, 1); 440 } 441 } /* oce_fill_ring_descs */ 442 443 /* 444 * function to copy the packet to preallocated Tx buffer 445 * 446 * wq - pointer to WQ 447 * wqed - Pointer to WQE descriptor 448 * mp - Pointer to packet chain 449 * pktlen - Size of the packet 450 * 451 * return 0=>success, error code otherwise 452 */ 453 static int 454 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 455 uint32_t pkt_len) 456 { 457 oce_wq_bdesc_t *wqbd; 458 caddr_t buf_va; 459 struct oce_dev *dev = wq->parent; 460 int len = 0; 461 462 wqbd = oce_wqb_alloc(wq); 463 if (wqbd == NULL) { 464 atomic_inc_32(&dev->tx_noxmtbuf); 465 oce_log(dev, CE_WARN, MOD_TX, "%s", 466 "wqb pool empty"); 467 return (ENOMEM); 468 } 469 470 /* create a fragment wqe for the packet */ 471 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi; 472 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo; 473 buf_va = DBUF_VA(wqbd->wqb); 474 475 /* copy pkt into buffer */ 476 for (len = 0; mp != NULL && len < pkt_len; mp = mp->b_cont) { 477 bcopy(mp->b_rptr, buf_va, MBLKL(mp)); 478 buf_va += MBLKL(mp); 479 len += MBLKL(mp); 480 } 481 482 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len, 483 DDI_DMA_SYNC_FORDEV); 484 485 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) { 486 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 487 /* Free the buffer */ 488 oce_wqb_free(wq, wqbd); 489 return (EIO); 490 } 491 wqed->frag[wqed->frag_idx].u0.s.frag_len = pkt_len; 492 wqed->hdesc[wqed->nhdl].hdl = (void *)(wqbd); 493 wqed->hdesc[wqed->nhdl].type = COPY_WQE; 494 wqed->frag_cnt++; 495 wqed->frag_idx++; 496 wqed->nhdl++; 497 return (0); 498 } /* oce_bcopy_wqe */ 499 500 /* 501 * function to copy the packet or dma map on the fly depending on size 502 * 503 * wq - pointer to WQ 504 * wqed - Pointer to WQE descriptor 505 * mp - Pointer to packet chain 506 * 507 * return DDI_SUCCESS=>success, DDI_FAILURE=>error 508 */ 509 static int 510 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp, 511 uint32_t pkt_len) 512 { 513 ddi_dma_cookie_t cookie; 514 oce_wq_mdesc_t *wqmd; 515 uint32_t ncookies; 516 int ret; 517 struct oce_dev *dev = wq->parent; 518 519 wqmd = oce_wqm_alloc(wq); 520 if (wqmd == NULL) { 521 oce_log(dev, CE_WARN, MOD_TX, "%s", 522 "wqm pool empty"); 523 return (ENOMEM); 524 } 525 526 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle, 527 (struct as *)0, (caddr_t)mp->b_rptr, 528 pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING, 529 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies); 530 if (ret != DDI_DMA_MAPPED) { 531 oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d", 532 ret); 533 /* free the last one */ 534 oce_wqm_free(wq, wqmd); 535 return (ENOMEM); 536 } 537 do { 538 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = 539 ADDR_HI(cookie.dmac_laddress); 540 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = 541 ADDR_LO(cookie.dmac_laddress); 542 wqed->frag[wqed->frag_idx].u0.s.frag_len = 543 (uint32_t)cookie.dmac_size; 544 wqed->frag_cnt++; 545 wqed->frag_idx++; 546 if (--ncookies > 0) 547 ddi_dma_nextcookie(wqmd->dma_handle, 548 &cookie); 549 else break; 550 } while (ncookies > 0); 551 552 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd; 553 wqed->hdesc[wqed->nhdl].type = MAPPED_WQE; 554 wqed->nhdl++; 555 return (0); 556 } /* oce_map_wqe */ 557 558 static inline int 559 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm) 560 { 561 struct oce_nic_tx_cqe *cqe; 562 uint16_t num_cqe = 0; 563 struct oce_cq *cq; 564 oce_wqe_desc_t *wqed; 565 int wqe_freed = 0; 566 struct oce_dev *dev; 567 568 cq = wq->cq; 569 dev = wq->parent; 570 (void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0, 571 DDI_DMA_SYNC_FORKERNEL); 572 573 mutex_enter(&wq->txc_lock); 574 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 575 while (WQ_CQE_VALID(cqe)) { 576 577 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe)); 578 579 /* update stats */ 580 if (cqe->u0.s.status != 0) { 581 atomic_inc_32(&dev->tx_errors); 582 } 583 584 /* complete the WQEs */ 585 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list); 586 587 wqe_freed = wqed->wqe_cnt; 588 oce_free_wqed(wq, wqed); 589 RING_GET(wq->ring, wqe_freed); 590 atomic_add_32(&wq->wq_free, wqe_freed); 591 /* clear the valid bit and progress cqe */ 592 WQ_CQE_INVALIDATE(cqe); 593 RING_GET(cq->ring, 1); 594 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 595 struct oce_nic_tx_cqe); 596 num_cqe++; 597 } /* for all valid CQE */ 598 mutex_exit(&wq->txc_lock); 599 if (num_cqe) 600 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm); 601 return (num_cqe); 602 } /* oce_process_tx_completion */ 603 604 /* 605 * function to drain a TxCQ and process its CQEs 606 * 607 * dev - software handle to the device 608 * cq - pointer to the cq to drain 609 * 610 * return the number of CQEs processed 611 */ 612 uint16_t 613 oce_drain_wq_cq(void *arg) 614 { 615 uint16_t num_cqe = 0; 616 struct oce_dev *dev; 617 struct oce_wq *wq; 618 619 wq = (struct oce_wq *)arg; 620 dev = wq->parent; 621 622 /* do while we do not reach a cqe that is not valid */ 623 num_cqe = oce_process_tx_compl(wq, B_FALSE); 624 625 /* check if we need to restart Tx */ 626 if (wq->resched && num_cqe) { 627 wq->resched = B_FALSE; 628 mac_tx_update(dev->mac_handle); 629 } 630 631 return (num_cqe); 632 } /* oce_process_wq_cqe */ 633 634 /* 635 * function to insert vtag to packet 636 * 637 * mp - mblk pointer 638 * vlan_tag - tag to be inserted 639 * 640 * return none 641 */ 642 static inline void 643 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag) 644 { 645 struct ether_vlan_header *evh; 646 (void) memmove(mp->b_rptr - VLAN_TAGSZ, 647 mp->b_rptr, 2 * ETHERADDRL); 648 mp->b_rptr -= VLAN_TAGSZ; 649 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 650 evh->ether_tpid = htons(VLAN_TPID); 651 evh->ether_tci = htons(vlan_tag); 652 } 653 654 /* 655 * function to strip vtag from packet 656 * 657 * mp - mblk pointer 658 * 659 * return none 660 */ 661 662 static inline void 663 oce_remove_vtag(mblk_t *mp) 664 { 665 (void) memmove(mp->b_rptr + VLAN_TAGSZ, mp->b_rptr, 666 ETHERADDRL * 2); 667 mp->b_rptr += VLAN_TAGSZ; 668 } 669 670 /* 671 * function to xmit Single packet over the wire 672 * 673 * wq - pointer to WQ 674 * mp - Pointer to packet chain 675 * 676 * return pointer to the packet 677 */ 678 mblk_t * 679 oce_send_packet(struct oce_wq *wq, mblk_t *mp) 680 { 681 682 struct oce_nic_hdr_wqe *wqeh; 683 struct oce_dev *dev; 684 struct ether_header *eh; 685 struct ether_vlan_header *evh; 686 int32_t num_wqes; 687 uint16_t etype; 688 uint32_t ip_offset; 689 uint32_t csum_flags = 0; 690 boolean_t use_copy = B_FALSE; 691 boolean_t tagged = B_FALSE; 692 uint16_t vlan_tag; 693 uint32_t reg_value = 0; 694 oce_wqe_desc_t *wqed = NULL; 695 mblk_t *nmp = NULL; 696 mblk_t *tmp = NULL; 697 uint32_t pkt_len = 0; 698 int num_mblks = 0; 699 int ret = 0; 700 uint32_t mss = 0; 701 uint32_t flags = 0; 702 int len = 0; 703 704 /* retrieve the adap priv struct ptr */ 705 dev = wq->parent; 706 707 /* check if we have enough free slots */ 708 if (wq->wq_free < dev->tx_reclaim_threshold) { 709 (void) oce_process_tx_compl(wq, B_FALSE); 710 } 711 if (wq->wq_free < OCE_MAX_TX_HDL) { 712 return (mp); 713 } 714 715 /* check if we should copy */ 716 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) { 717 pkt_len += MBLKL(tmp); 718 num_mblks++; 719 } 720 721 if (pkt_len == 0 || num_mblks == 0) { 722 freemsg(mp); 723 return (NULL); 724 } 725 726 /* retrieve LSO information */ 727 mac_lso_get(mp, &mss, &flags); 728 729 /* get the offload flags */ 730 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags); 731 732 /* restrict the mapped segment to wat we support */ 733 if (num_mblks > OCE_MAX_TX_HDL) { 734 nmp = msgpullup(mp, -1); 735 if (nmp == NULL) { 736 atomic_inc_32(&wq->pkt_drops); 737 freemsg(mp); 738 return (NULL); 739 } 740 /* Reset it to new collapsed mp */ 741 freemsg(mp); 742 mp = nmp; 743 } 744 745 /* Get the packet descriptor for Tx */ 746 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP); 747 if (wqed == NULL) { 748 atomic_inc_32(&wq->pkt_drops); 749 freemsg(mp); 750 return (NULL); 751 } 752 eh = (struct ether_header *)(void *)mp->b_rptr; 753 if (ntohs(eh->ether_type) == VLAN_TPID) { 754 evh = (struct ether_vlan_header *)(void *)mp->b_rptr; 755 tagged = B_TRUE; 756 etype = ntohs(evh->ether_type); 757 ip_offset = sizeof (struct ether_vlan_header); 758 pkt_len -= VLAN_TAGSZ; 759 vlan_tag = ntohs(evh->ether_tci); 760 oce_remove_vtag(mp); 761 } else { 762 etype = ntohs(eh->ether_type); 763 ip_offset = sizeof (struct ether_header); 764 } 765 766 /* Save the WQ pointer */ 767 wqed->wq = wq; 768 wqed->frag_idx = 1; /* index zero is always header */ 769 wqed->frag_cnt = 0; 770 wqed->nhdl = 0; 771 wqed->mp = NULL; 772 OCE_LIST_LINK_INIT(&wqed->link); 773 774 /* If entire packet is less than the copy limit just do copy */ 775 if (pkt_len < dev->tx_bcopy_limit) { 776 use_copy = B_TRUE; 777 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len); 778 } else { 779 /* copy or dma map the individual fragments */ 780 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) { 781 len = MBLKL(nmp); 782 if (len == 0) { 783 continue; 784 } 785 if (len < dev->tx_bcopy_limit) { 786 ret = oce_bcopy_wqe(wq, wqed, nmp, len); 787 } else { 788 ret = oce_map_wqe(wq, wqed, nmp, len); 789 } 790 if (ret != 0) 791 break; 792 } 793 } 794 795 /* 796 * Any failure other than insufficient Q entries 797 * drop the packet 798 */ 799 if (ret != 0) { 800 oce_free_wqed(wq, wqed); 801 atomic_inc_32(&wq->pkt_drops); 802 freemsg(mp); 803 return (NULL); 804 } 805 806 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0]; 807 bzero(wqeh, sizeof (struct oce_nic_hdr_wqe)); 808 809 /* fill rest of wqe header fields based on packet */ 810 if (flags & HW_LSO) { 811 wqeh->u0.s.lso = B_TRUE; 812 wqeh->u0.s.lso_mss = mss; 813 } 814 if (csum_flags & HCK_FULLCKSUM) { 815 uint8_t *proto; 816 if (etype == ETHERTYPE_IP) { 817 proto = (uint8_t *)(void *) 818 (mp->b_rptr + ip_offset); 819 if (proto[9] == 6) 820 /* IPPROTO_TCP */ 821 wqeh->u0.s.tcpcs = B_TRUE; 822 else if (proto[9] == 17) 823 /* IPPROTO_UDP */ 824 wqeh->u0.s.udpcs = B_TRUE; 825 } 826 } 827 828 if (csum_flags & HCK_IPV4_HDRCKSUM) 829 wqeh->u0.s.ipcs = B_TRUE; 830 if (tagged) { 831 wqeh->u0.s.vlan = B_TRUE; 832 wqeh->u0.s.vlan_tag = vlan_tag; 833 } 834 835 wqeh->u0.s.complete = B_TRUE; 836 wqeh->u0.s.event = B_TRUE; 837 wqeh->u0.s.crc = B_TRUE; 838 wqeh->u0.s.total_length = pkt_len; 839 840 num_wqes = wqed->frag_cnt + 1; 841 842 /* h/w expects even no. of WQEs */ 843 if (num_wqes & 0x1) { 844 bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe)); 845 num_wqes++; 846 } 847 wqed->wqe_cnt = (uint16_t)num_wqes; 848 wqeh->u0.s.num_wqe = num_wqes; 849 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE)); 850 851 mutex_enter(&wq->tx_lock); 852 if (num_wqes > wq->wq_free) { 853 atomic_inc_32(&wq->tx_deferd); 854 mutex_exit(&wq->tx_lock); 855 goto wqe_fail; 856 } 857 atomic_add_32(&wq->wq_free, -num_wqes); 858 859 /* fill the wq for adapter */ 860 oce_fill_ring_descs(wq, wqed); 861 862 /* Add the packet desc to list to be retrieved during cmpl */ 863 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed); 864 (void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0, 865 DDI_DMA_SYNC_FORDEV); 866 867 /* ring tx doorbell */ 868 reg_value = (num_wqes << 16) | wq->wq_id; 869 /* Ring the door bell */ 870 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value); 871 mutex_exit(&wq->tx_lock); 872 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) { 873 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED); 874 } 875 876 /* free mp if copied or packet chain collapsed */ 877 if (use_copy == B_TRUE) { 878 freemsg(mp); 879 } else 880 wqed->mp = mp; 881 return (NULL); 882 883 wqe_fail: 884 885 if (tagged) { 886 oce_insert_vtag(mp, vlan_tag); 887 } 888 oce_free_wqed(wq, wqed); 889 return (mp); 890 } /* oce_send_packet */ 891 892 /* 893 * function to free the WQE descriptor 894 * 895 * wq - pointer to WQ 896 * wqed - Pointer to WQE descriptor 897 * 898 * return none 899 */ 900 #pragma inline(oce_free_wqed) 901 static void 902 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed) 903 { 904 int i = 0; 905 if (wqed == NULL) { 906 return; 907 } 908 909 for (i = 0; i < wqed->nhdl; i++) { 910 if (wqed->hdesc[i].type == COPY_WQE) { 911 oce_wqb_free(wq, wqed->hdesc[i].hdl); 912 } else if (wqed->hdesc[i].type == MAPPED_WQE) { 913 oce_wqmd_free(wq, wqed->hdesc[i].hdl); 914 } 915 } 916 if (wqed->mp) 917 freemsg(wqed->mp); 918 kmem_cache_free(wq->wqed_cache, wqed); 919 } /* oce_free_wqed */ 920 921 /* 922 * function to start the WQ 923 * 924 * wq - pointer to WQ 925 * 926 * return DDI_SUCCESS 927 */ 928 929 int 930 oce_start_wq(struct oce_wq *wq) 931 { 932 _NOTE(ARGUNUSED(wq)); 933 return (DDI_SUCCESS); 934 } /* oce_start_wq */ 935 936 /* 937 * function to stop the WQ 938 * 939 * wq - pointer to WQ 940 * 941 * return none 942 */ 943 void 944 oce_clean_wq(struct oce_wq *wq) 945 { 946 oce_wqe_desc_t *wqed; 947 int ti; 948 949 /* Wait for already posted Tx to complete */ 950 951 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) { 952 (void) oce_process_tx_compl(wq, B_FALSE); 953 OCE_MSDELAY(1); 954 } 955 956 /* Free the remaining descriptors */ 957 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) { 958 atomic_add_32(&wq->wq_free, wqed->wqe_cnt); 959 oce_free_wqed(wq, wqed); 960 } 961 oce_drain_eq(wq->cq->eq); 962 } /* oce_stop_wq */ 963 964 /* 965 * function to set the tx mapping handle fma attr 966 * 967 * fm_caps - capability flags 968 * 969 * return none 970 */ 971 972 void 973 oce_set_tx_map_dma_fma_flags(int fm_caps) 974 { 975 if (fm_caps == DDI_FM_NOT_CAPABLE) { 976 return; 977 } 978 979 if (DDI_FM_DMA_ERR_CAP(fm_caps)) { 980 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 981 } else { 982 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR; 983 } 984 } /* oce_set_tx_map_dma_fma_flags */ 985